547 lines
16 KiB
TypeScript
547 lines
16 KiB
TypeScript
import OpenAI from "openai";
|
||
import { z } from "zod";
|
||
import { ASR_CONTENT_TAGS, normalizeAsrContentTags } from "@/lib/asr-tags";
|
||
import { downloadFile } from "@/lib/minio";
|
||
import { prisma } from "@/lib/prisma";
|
||
import {
|
||
type ExtractedAudioSegment,
|
||
type ExtractedVideoFrame,
|
||
extractAudio,
|
||
extractAudioSegments,
|
||
extractVideoFrames,
|
||
} from "../media";
|
||
import prompt from "./prompt.md";
|
||
|
||
export type AsrLogger = (message: string) => void;
|
||
|
||
type TranscriptAwemeOptions = {
|
||
log?: AsrLogger;
|
||
};
|
||
|
||
type TranscriptAudioOptions = {
|
||
segment?: {
|
||
index: number;
|
||
total: number;
|
||
startMs: number;
|
||
endMs: number;
|
||
};
|
||
};
|
||
|
||
let qwenClient: OpenAI | null = null;
|
||
|
||
const AUDIO_DATA_URL_PREFIX = "data:;base64,";
|
||
const DEFAULT_QWEN_MAX_STRING_LENGTH = 28_000_000;
|
||
const DEFAULT_SAFE_AUDIO_DATA_URL_LENGTH = 24_000_000;
|
||
const DEFAULT_AUDIO_BITRATE_KBPS = 128;
|
||
const DEFAULT_AUDIO_CHUNK_MS = 10 * 60 * 1000;
|
||
const MIN_AUDIO_CHUNK_MS = 60 * 1000;
|
||
const CHUNK_REEXTRACT_LIMIT = 5;
|
||
|
||
function getQwenClient() {
|
||
if (qwenClient) return qwenClient;
|
||
|
||
const apiKey = process.env.QWEN_API_KEY || process.env.DASHSCOPE_API_KEY;
|
||
if (!apiKey) {
|
||
throw new Error("Missing QWEN_API_KEY environment variable");
|
||
}
|
||
|
||
qwenClient = new OpenAI({
|
||
apiKey,
|
||
baseURL:
|
||
process.env.QWEN_API_BASE_URL ||
|
||
"https://dashscope.aliyuncs.com/compatible-mode/v1",
|
||
});
|
||
|
||
return qwenClient;
|
||
}
|
||
|
||
// Zod schema aligned with prompt.md
|
||
export const AsrSchema = z
|
||
.object({
|
||
speech_detected: z.boolean(),
|
||
language: z.string().min(1).nullable(),
|
||
audio_type: z.string().nullable(),
|
||
transcript: z.array(z.string()),
|
||
non_speech_summary: z.string().nullable(),
|
||
content_tags: z.array(z.enum(ASR_CONTENT_TAGS)).min(1).max(3),
|
||
})
|
||
.strict();
|
||
|
||
const RawAsrSchema = z.object({
|
||
speech_detected: z.boolean(),
|
||
language: z.string().min(1).nullable(),
|
||
audio_type: z.string().nullable(),
|
||
transcript: z.array(z.string()),
|
||
non_speech_summary: z.string().nullable(),
|
||
content_tags: z.unknown().optional(),
|
||
});
|
||
|
||
export type AsrResult = z.infer<typeof AsrSchema>;
|
||
|
||
function extractJsonObject(text: string) {
|
||
const trimmed = text.trim();
|
||
const fenced = trimmed.match(/```(?:json)?\s*([\s\S]*?)\s*```/i);
|
||
if (fenced?.[1]) return fenced[1].trim();
|
||
|
||
const start = trimmed.indexOf("{");
|
||
const end = trimmed.lastIndexOf("}");
|
||
if (start >= 0 && end > start) {
|
||
return trimmed.slice(start, end + 1);
|
||
}
|
||
|
||
return trimmed;
|
||
}
|
||
|
||
function parseAsrResult(content: string): AsrResult {
|
||
const parsed = RawAsrSchema.parse(JSON.parse(extractJsonObject(content)));
|
||
return AsrSchema.parse({
|
||
...parsed,
|
||
content_tags: normalizeAsrContentTags(parsed.content_tags),
|
||
});
|
||
}
|
||
|
||
function getPositiveIntEnv(name: string, fallback: number) {
|
||
const value = Number.parseInt(process.env[name] ?? "", 10);
|
||
return Number.isFinite(value) && value > 0 ? value : fallback;
|
||
}
|
||
|
||
function getAudioBitrateKbps() {
|
||
return getPositiveIntEnv(
|
||
"ASR_AUDIO_BITRATE_KBPS",
|
||
DEFAULT_AUDIO_BITRATE_KBPS,
|
||
);
|
||
}
|
||
|
||
function getAudioChunkMs() {
|
||
return Math.max(
|
||
MIN_AUDIO_CHUNK_MS,
|
||
getPositiveIntEnv(
|
||
"ASR_AUDIO_CHUNK_SECONDS",
|
||
DEFAULT_AUDIO_CHUNK_MS / 1000,
|
||
) * 1000,
|
||
);
|
||
}
|
||
|
||
function getMaxAudioDataUrlLength() {
|
||
const providerLimit = getPositiveIntEnv(
|
||
"QWEN_MAX_STRING_LENGTH",
|
||
DEFAULT_QWEN_MAX_STRING_LENGTH,
|
||
);
|
||
const configuredLimit = getPositiveIntEnv(
|
||
"ASR_MAX_AUDIO_DATA_URL_LENGTH",
|
||
DEFAULT_SAFE_AUDIO_DATA_URL_LENGTH,
|
||
);
|
||
return Math.max(
|
||
1_000_000,
|
||
Math.min(configuredLimit, providerLimit - 1_000_000),
|
||
);
|
||
}
|
||
|
||
function getBase64Length(byteLength: number) {
|
||
return Math.ceil(byteLength / 3) * 4;
|
||
}
|
||
|
||
function getAudioDataUrlLength(audio: Buffer) {
|
||
return AUDIO_DATA_URL_PREFIX.length + getBase64Length(audio.length);
|
||
}
|
||
|
||
function estimateAudioDataUrlLength(durationMs: number, bitrateKbps: number) {
|
||
const estimatedBytes = Math.ceil((durationMs / 1000) * bitrateKbps * 125);
|
||
return AUDIO_DATA_URL_PREFIX.length + getBase64Length(estimatedBytes);
|
||
}
|
||
|
||
function formatTimestamp(ms: number) {
|
||
const totalSeconds = Math.max(0, Math.floor(ms / 1000));
|
||
const hours = Math.floor(totalSeconds / 3600);
|
||
const minutes = Math.floor((totalSeconds % 3600) / 60);
|
||
const seconds = totalSeconds % 60;
|
||
if (hours > 0) {
|
||
return `${hours}:${minutes.toString().padStart(2, "0")}:${seconds
|
||
.toString()
|
||
.padStart(2, "0")}`;
|
||
}
|
||
return `${minutes}:${seconds.toString().padStart(2, "0")}`;
|
||
}
|
||
|
||
function getSegmentInstruction(segment?: TranscriptAudioOptions["segment"]) {
|
||
if (!segment) return "";
|
||
|
||
return `\n\n当前音频是长视频的第 ${segment.index + 1}/${segment.total} 段，时间范围 ${formatTimestamp(segment.startMs)}-${formatTimestamp(segment.endMs)}。请只转写这一段音频，不要猜测其他分片未出现的台词；content_tags 仍结合整条视频三帧、视频简介和当前分片内容判断。`;
|
||
}
|
||
|
||
function shouldChunkBeforeFullExtract(durationMs: number) {
|
||
if (!durationMs || durationMs <= 0) return false;
|
||
return (
|
||
estimateAudioDataUrlLength(durationMs, getAudioBitrateKbps()) >
|
||
getMaxAudioDataUrlLength()
|
||
);
|
||
}
|
||
|
||
function mergeContentTags(results: AsrResult[]) {
|
||
const counts = new Map<string, { count: number; firstSeen: number }>();
|
||
let order = 0;
|
||
for (const result of results) {
|
||
for (const tag of result.content_tags) {
|
||
const current = counts.get(tag);
|
||
if (current) {
|
||
current.count += 1;
|
||
} else {
|
||
counts.set(tag, { count: 1, firstSeen: order });
|
||
order += 1;
|
||
}
|
||
}
|
||
}
|
||
|
||
const sortedTags = [...counts.entries()].sort(
|
||
(left, right) =>
|
||
right[1].count - left[1].count || left[1].firstSeen - right[1].firstSeen,
|
||
);
|
||
const preferredTags = sortedTags.filter(([tag]) => tag !== "其他");
|
||
const tags = (preferredTags.length > 0 ? preferredTags : sortedTags)
|
||
.slice(0, 3)
|
||
.map(([tag]) => tag);
|
||
|
||
return normalizeAsrContentTags(tags);
|
||
}
|
||
|
||
function mergeLanguages(results: AsrResult[]) {
|
||
const languages = results
|
||
.flatMap((result) =>
|
||
(result.language ?? "")
|
||
.split(/[,，、\s]+/)
|
||
.map((item) => item.trim())
|
||
.filter(Boolean),
|
||
)
|
||
.filter((language, index, array) => array.indexOf(language) === index);
|
||
|
||
return languages.length > 0 ? languages.join(",") : null;
|
||
}
|
||
|
||
function getMostCommonAudioType(results: AsrResult[]) {
|
||
const counts = new Map<string, { count: number; firstSeen: number }>();
|
||
let order = 0;
|
||
for (const result of results) {
|
||
if (!result.audio_type) continue;
|
||
const current = counts.get(result.audio_type);
|
||
if (current) {
|
||
current.count += 1;
|
||
} else {
|
||
counts.set(result.audio_type, { count: 1, firstSeen: order });
|
||
order += 1;
|
||
}
|
||
}
|
||
|
||
return (
|
||
[...counts.entries()].sort(
|
||
(left, right) =>
|
||
right[1].count - left[1].count ||
|
||
left[1].firstSeen - right[1].firstSeen,
|
||
)[0]?.[0] ?? null
|
||
);
|
||
}
|
||
|
||
function mergeTranscriptLines(results: AsrResult[]) {
|
||
const lines: string[] = [];
|
||
for (const result of results) {
|
||
for (const line of result.transcript) {
|
||
const normalizedLine = line.trim();
|
||
if (!normalizedLine) continue;
|
||
if (lines.at(-1) === normalizedLine) continue;
|
||
lines.push(normalizedLine);
|
||
}
|
||
}
|
||
return lines;
|
||
}
|
||
|
||
function mergeAsrResults(results: AsrResult[]): AsrResult {
|
||
if (results.length === 0) {
|
||
throw new Error("No ASR chunk results returned from Qwen Omni");
|
||
}
|
||
|
||
const speechDetected = results.some((result) => result.speech_detected);
|
||
const nonSpeechSummaries = results
|
||
.map((result) => result.non_speech_summary?.trim())
|
||
.filter((summary): summary is string => Boolean(summary));
|
||
|
||
return AsrSchema.parse({
|
||
speech_detected: speechDetected,
|
||
language: speechDetected ? mergeLanguages(results) : null,
|
||
audio_type: speechDetected ? null : getMostCommonAudioType(results),
|
||
transcript: mergeTranscriptLines(results),
|
||
non_speech_summary:
|
||
speechDetected || nonSpeechSummaries.length === 0
|
||
? null
|
||
: nonSpeechSummaries.join("；"),
|
||
content_tags: mergeContentTags(results),
|
||
});
|
||
}
|
||
|
||
async function transcriptAudio(
|
||
audio: Buffer,
|
||
frames: ExtractedVideoFrame[],
|
||
videoDesc: string,
|
||
log?: AsrLogger,
|
||
options: TranscriptAudioOptions = {},
|
||
) {
|
||
const base64Audio = Buffer.from(audio).toString("base64");
|
||
const client = getQwenClient();
|
||
const model = process.env.QWEN_OMNI_MODEL || "qwen3.5-omni-plus";
|
||
const frameContentParts = frames.map((frame, index) => ({
|
||
type: "image_url" as const,
|
||
image_url: {
|
||
url: `data:${frame.contentType};base64,${frame.buffer.toString("base64")}`,
|
||
},
|
||
index,
|
||
}));
|
||
|
||
log?.(
|
||
`调用 Qwen Omni: model=${model}, audio=${Math.round(audio.length / 1024)}KB, audioDataUrl=${getAudioDataUrlLength(audio)}, frames=${frames.length}`,
|
||
);
|
||
|
||
const completion = await client.chat.completions.create({
|
||
model,
|
||
messages: [
|
||
{
|
||
role: "user",
|
||
content: [
|
||
...frameContentParts.map(({ type, image_url }) => ({
|
||
type,
|
||
image_url,
|
||
})),
|
||
{
|
||
type: "input_audio",
|
||
input_audio: {
|
||
data: `${AUDIO_DATA_URL_PREFIX}${base64Audio}`,
|
||
format: "mp3",
|
||
},
|
||
},
|
||
{
|
||
type: "text",
|
||
text: `${prompt}${getSegmentInstruction(options.segment)}\n\n视频简介：\n${videoDesc || "无"}\n\n已附带视频三帧，时间点约为：${
|
||
frames
|
||
.map((frame) => `${(frame.timestampMs / 1000).toFixed(1)}s`)
|
||
.join("、") || "无"
|
||
}。请结合画面和音频判断内容标签。`,
|
||
},
|
||
],
|
||
},
|
||
],
|
||
modalities: ["text"],
|
||
stream: true,
|
||
stream_options: { include_usage: true },
|
||
});
|
||
|
||
let content = "";
|
||
for await (const chunk of completion) {
|
||
const deltaContent = chunk.choices?.[0]?.delta?.content;
|
||
if (typeof deltaContent === "string") content += deltaContent;
|
||
}
|
||
|
||
log?.(`Qwen Omni 原始输出: ${content}`);
|
||
|
||
if (!content.trim()) {
|
||
throw new Error("No ASR result returned from Qwen Omni");
|
||
}
|
||
|
||
try {
|
||
return parseAsrResult(content);
|
||
} catch {
|
||
throw new Error("Failed to parse ASR result JSON");
|
||
}
|
||
}
|
||
|
||
async function extractSafeAudioSegments(
|
||
videoBuffer: Buffer,
|
||
durationMs: number,
|
||
log?: AsrLogger,
|
||
) {
|
||
if (!durationMs || durationMs <= 0) {
|
||
throw new Error(
|
||
"Audio is too large for a single request and video duration is missing",
|
||
);
|
||
}
|
||
|
||
const maxAudioDataUrlLength = getMaxAudioDataUrlLength();
|
||
let segmentDurationMs = Math.min(getAudioChunkMs(), durationMs);
|
||
|
||
for (let attempt = 1; attempt <= CHUNK_REEXTRACT_LIMIT; attempt++) {
|
||
log?.(
|
||
`提取音频分片: chunk=${Math.round(segmentDurationMs / 1000)}s, bitrate=${getAudioBitrateKbps()}kbps`,
|
||
);
|
||
const segments = await extractAudioSegments(videoBuffer, {
|
||
durationMs,
|
||
segmentDurationMs,
|
||
format: "mp3",
|
||
bitrateKbps: getAudioBitrateKbps(),
|
||
});
|
||
|
||
if (segments.length === 0) {
|
||
throw new Error("Failed to extract audio segments from video");
|
||
}
|
||
|
||
const largestSegmentLength = Math.max(
|
||
...segments.map((segment) => getAudioDataUrlLength(segment.buffer)),
|
||
);
|
||
log?.(
|
||
`音频分片提取完成: count=${segments.length}, largestDataUrl=${largestSegmentLength}`,
|
||
);
|
||
|
||
if (largestSegmentLength <= maxAudioDataUrlLength) return segments;
|
||
|
||
if (segmentDurationMs <= MIN_AUDIO_CHUNK_MS) {
|
||
throw new Error(
|
||
`Audio segment is still too large after splitting: ${largestSegmentLength}`,
|
||
);
|
||
}
|
||
|
||
const shrinkRatio = Math.max(
|
||
0.25,
|
||
Math.min(0.85, (maxAudioDataUrlLength / largestSegmentLength) * 0.8),
|
||
);
|
||
segmentDurationMs = Math.max(
|
||
MIN_AUDIO_CHUNK_MS,
|
||
Math.floor(segmentDurationMs * shrinkRatio),
|
||
);
|
||
log?.(
|
||
`音频分片仍过大，缩短分片后重试: ${Math.round(segmentDurationMs / 1000)}s`,
|
||
);
|
||
}
|
||
|
||
throw new Error("Failed to extract safe audio segments from video");
|
||
}
|
||
|
||
async function transcriptAudioSegments(
|
||
segments: ExtractedAudioSegment[],
|
||
frames: ExtractedVideoFrame[],
|
||
videoDesc: string,
|
||
log?: AsrLogger,
|
||
) {
|
||
const results: AsrResult[] = [];
|
||
|
||
for (const segment of segments) {
|
||
log?.(
|
||
`识别音频分片 ${segment.index + 1}/${segments.length}: ${formatTimestamp(segment.startMs)}-${formatTimestamp(segment.endMs)}`,
|
||
);
|
||
const result = await transcriptAudio(
|
||
segment.buffer,
|
||
frames,
|
||
videoDesc,
|
||
log,
|
||
{
|
||
segment: {
|
||
index: segment.index,
|
||
total: segments.length,
|
||
startMs: segment.startMs,
|
||
endMs: segment.endMs,
|
||
},
|
||
},
|
||
);
|
||
results.push(result);
|
||
}
|
||
|
||
const mergedResult = mergeAsrResults(results);
|
||
log?.(
|
||
`分片结果合并完成: chunks=${results.length}, transcript=${mergedResult.transcript.length} 段, tags=${mergedResult.content_tags.join("、")}`,
|
||
);
|
||
return mergedResult;
|
||
}
|
||
|
||
export async function transcriptAweme(
|
||
awemeId: string,
|
||
options: TranscriptAwemeOptions = {},
|
||
): Promise<AsrResult> {
|
||
const log = options.log;
|
||
log?.(`查询视频: ${awemeId}`);
|
||
const aweme = await prisma.video.findUnique({
|
||
where: { aweme_id: awemeId },
|
||
});
|
||
if (!aweme) {
|
||
throw new Error("Aweme not found or aweme is not a video post");
|
||
}
|
||
const vPath = aweme.video_url;
|
||
log?.(`下载视频文件: ${vPath}`);
|
||
const buffer = await downloadFile(vPath);
|
||
log?.(`视频下载完成: ${Math.round(buffer.length / 1024)}KB`);
|
||
|
||
log?.("提取视频三帧");
|
||
const frames = await extractVideoFrames(buffer, {
|
||
count: 3,
|
||
durationMs: aweme.duration_ms,
|
||
});
|
||
log?.(
|
||
`视频帧提取完成: ${frames
|
||
.map((frame) => `${(frame.timestampMs / 1000).toFixed(1)}s`)
|
||
.join("、")}`,
|
||
);
|
||
|
||
let result: AsrResult;
|
||
const maxAudioDataUrlLength = getMaxAudioDataUrlLength();
|
||
if (shouldChunkBeforeFullExtract(aweme.duration_ms)) {
|
||
log?.(
|
||
`预计整段音频会超过安全阈值 ${maxAudioDataUrlLength}，启用长视频分片识别`,
|
||
);
|
||
const segments = await extractSafeAudioSegments(
|
||
buffer,
|
||
aweme.duration_ms,
|
||
log,
|
||
);
|
||
result = await transcriptAudioSegments(segments, frames, aweme.desc, log);
|
||
} else {
|
||
log?.("提取音频");
|
||
const audioDat = await extractAudio(buffer, {
|
||
format: "mp3",
|
||
bitrateKbps: getAudioBitrateKbps(),
|
||
});
|
||
|
||
if (!audioDat?.buffer) {
|
||
throw new Error("Failed to extract audio from video");
|
||
}
|
||
|
||
const audioDataUrlLength = getAudioDataUrlLength(audioDat.buffer);
|
||
log?.(
|
||
`音频提取完成: ${Math.round(audioDat.buffer.length / 1024)}KB, dataUrl=${audioDataUrlLength}`,
|
||
);
|
||
|
||
if (audioDataUrlLength > maxAudioDataUrlLength) {
|
||
log?.(
|
||
`整段音频 data URL 超过安全阈值 ${maxAudioDataUrlLength}，改用分片识别`,
|
||
);
|
||
const segments = await extractSafeAudioSegments(
|
||
buffer,
|
||
aweme.duration_ms,
|
||
log,
|
||
);
|
||
result = await transcriptAudioSegments(segments, frames, aweme.desc, log);
|
||
} else {
|
||
result = await transcriptAudio(audioDat.buffer, frames, aweme.desc, log);
|
||
}
|
||
}
|
||
log?.(`解析完成: tags=${result.content_tags.join("、")}`);
|
||
|
||
// 将转写结果持久化到数据库
|
||
log?.("写入数据库 VideoTranscript");
|
||
await prisma.videoTranscript.upsert({
|
||
where: { videoId: awemeId },
|
||
update: {
|
||
speech_detected: result.speech_detected,
|
||
language: result.language,
|
||
audio_type: result.audio_type,
|
||
transcript: result.transcript,
|
||
non_speech_summary: result.non_speech_summary,
|
||
content_tags: result.content_tags,
|
||
},
|
||
create: {
|
||
videoId: awemeId,
|
||
speech_detected: result.speech_detected,
|
||
language: result.language,
|
||
audio_type: result.audio_type,
|
||
transcript: result.transcript,
|
||
non_speech_summary: result.non_speech_summary,
|
||
content_tags: result.content_tags,
|
||
},
|
||
});
|
||
log?.("数据库写入完成");
|
||
|
||
return result;
|
||
}
|