import OpenAI from "openai"; import { z } from "zod"; import { ASR_CONTENT_TAGS, normalizeAsrContentTags } from "@/lib/asr-tags"; import { downloadFile } from "@/lib/minio"; import { prisma } from "@/lib/prisma"; import { type ExtractedAudioSegment, type ExtractedVideoFrame, extractAudio, extractAudioSegments, extractVideoFrames, } from "../media"; import prompt from "./prompt.md"; export type AsrLogger = (message: string) => void; type TranscriptAwemeOptions = { log?: AsrLogger; }; type TranscriptAudioOptions = { segment?: { index: number; total: number; startMs: number; endMs: number; }; }; let qwenClient: OpenAI | null = null; const AUDIO_DATA_URL_PREFIX = "data:;base64,"; const DEFAULT_QWEN_MAX_STRING_LENGTH = 28_000_000; const DEFAULT_SAFE_AUDIO_DATA_URL_LENGTH = 24_000_000; const DEFAULT_AUDIO_BITRATE_KBPS = 128; const DEFAULT_AUDIO_CHUNK_MS = 10 * 60 * 1000; const MIN_AUDIO_CHUNK_MS = 60 * 1000; const CHUNK_REEXTRACT_LIMIT = 5; function getQwenClient() { if (qwenClient) return qwenClient; const apiKey = process.env.QWEN_API_KEY || process.env.DASHSCOPE_API_KEY; if (!apiKey) { throw new Error("Missing QWEN_API_KEY environment variable"); } qwenClient = new OpenAI({ apiKey, baseURL: process.env.QWEN_API_BASE_URL || "https://dashscope.aliyuncs.com/compatible-mode/v1", }); return qwenClient; } // Zod schema aligned with prompt.md export const AsrSchema = z .object({ speech_detected: z.boolean(), language: z.string().min(1).nullable(), audio_type: z.string().nullable(), transcript: z.array(z.string()), non_speech_summary: z.string().nullable(), content_tags: z.array(z.enum(ASR_CONTENT_TAGS)).min(1).max(3), }) .strict(); const RawAsrSchema = z.object({ speech_detected: z.boolean(), language: z.string().min(1).nullable(), audio_type: z.string().nullable(), transcript: z.array(z.string()), non_speech_summary: z.string().nullable(), content_tags: z.unknown().optional(), }); export type AsrResult = z.infer; function extractJsonObject(text: string) { const trimmed = text.trim(); const fenced = trimmed.match(/```(?:json)?\s*([\s\S]*?)\s*```/i); if (fenced?.[1]) return fenced[1].trim(); const start = trimmed.indexOf("{"); const end = trimmed.lastIndexOf("}"); if (start >= 0 && end > start) { return trimmed.slice(start, end + 1); } return trimmed; } function parseAsrResult(content: string): AsrResult { const parsed = RawAsrSchema.parse(JSON.parse(extractJsonObject(content))); return AsrSchema.parse({ ...parsed, content_tags: normalizeAsrContentTags(parsed.content_tags), }); } function getPositiveIntEnv(name: string, fallback: number) { const value = Number.parseInt(process.env[name] ?? "", 10); return Number.isFinite(value) && value > 0 ? value : fallback; } function getAudioBitrateKbps() { return getPositiveIntEnv( "ASR_AUDIO_BITRATE_KBPS", DEFAULT_AUDIO_BITRATE_KBPS, ); } function getAudioChunkMs() { return Math.max( MIN_AUDIO_CHUNK_MS, getPositiveIntEnv( "ASR_AUDIO_CHUNK_SECONDS", DEFAULT_AUDIO_CHUNK_MS / 1000, ) * 1000, ); } function getMaxAudioDataUrlLength() { const providerLimit = getPositiveIntEnv( "QWEN_MAX_STRING_LENGTH", DEFAULT_QWEN_MAX_STRING_LENGTH, ); const configuredLimit = getPositiveIntEnv( "ASR_MAX_AUDIO_DATA_URL_LENGTH", DEFAULT_SAFE_AUDIO_DATA_URL_LENGTH, ); return Math.max( 1_000_000, Math.min(configuredLimit, providerLimit - 1_000_000), ); } function getBase64Length(byteLength: number) { return Math.ceil(byteLength / 3) * 4; } function getAudioDataUrlLength(audio: Buffer) { return AUDIO_DATA_URL_PREFIX.length + getBase64Length(audio.length); } function estimateAudioDataUrlLength(durationMs: number, bitrateKbps: number) { const estimatedBytes = Math.ceil((durationMs / 1000) * bitrateKbps * 125); return AUDIO_DATA_URL_PREFIX.length + getBase64Length(estimatedBytes); } function formatTimestamp(ms: number) { const totalSeconds = Math.max(0, Math.floor(ms / 1000)); const hours = Math.floor(totalSeconds / 3600); const minutes = Math.floor((totalSeconds % 3600) / 60); const seconds = totalSeconds % 60; if (hours > 0) { return `${hours}:${minutes.toString().padStart(2, "0")}:${seconds .toString() .padStart(2, "0")}`; } return `${minutes}:${seconds.toString().padStart(2, "0")}`; } function getSegmentInstruction(segment?: TranscriptAudioOptions["segment"]) { if (!segment) return ""; return `\n\n当前音频是长视频的第 ${segment.index + 1}/${segment.total} 段,时间范围 ${formatTimestamp(segment.startMs)}-${formatTimestamp(segment.endMs)}。请只转写这一段音频,不要猜测其他分片未出现的台词;content_tags 仍结合整条视频三帧、视频简介和当前分片内容判断。`; } function shouldChunkBeforeFullExtract(durationMs: number) { if (!durationMs || durationMs <= 0) return false; return ( estimateAudioDataUrlLength(durationMs, getAudioBitrateKbps()) > getMaxAudioDataUrlLength() ); } function mergeContentTags(results: AsrResult[]) { const counts = new Map(); let order = 0; for (const result of results) { for (const tag of result.content_tags) { const current = counts.get(tag); if (current) { current.count += 1; } else { counts.set(tag, { count: 1, firstSeen: order }); order += 1; } } } const sortedTags = [...counts.entries()].sort( (left, right) => right[1].count - left[1].count || left[1].firstSeen - right[1].firstSeen, ); const preferredTags = sortedTags.filter(([tag]) => tag !== "其他"); const tags = (preferredTags.length > 0 ? preferredTags : sortedTags) .slice(0, 3) .map(([tag]) => tag); return normalizeAsrContentTags(tags); } function mergeLanguages(results: AsrResult[]) { const languages = results .flatMap((result) => (result.language ?? "") .split(/[,,、\s]+/) .map((item) => item.trim()) .filter(Boolean), ) .filter((language, index, array) => array.indexOf(language) === index); return languages.length > 0 ? languages.join(",") : null; } function getMostCommonAudioType(results: AsrResult[]) { const counts = new Map(); let order = 0; for (const result of results) { if (!result.audio_type) continue; const current = counts.get(result.audio_type); if (current) { current.count += 1; } else { counts.set(result.audio_type, { count: 1, firstSeen: order }); order += 1; } } return ( [...counts.entries()].sort( (left, right) => right[1].count - left[1].count || left[1].firstSeen - right[1].firstSeen, )[0]?.[0] ?? null ); } function mergeTranscriptLines(results: AsrResult[]) { const lines: string[] = []; for (const result of results) { for (const line of result.transcript) { const normalizedLine = line.trim(); if (!normalizedLine) continue; if (lines.at(-1) === normalizedLine) continue; lines.push(normalizedLine); } } return lines; } function mergeAsrResults(results: AsrResult[]): AsrResult { if (results.length === 0) { throw new Error("No ASR chunk results returned from Qwen Omni"); } const speechDetected = results.some((result) => result.speech_detected); const nonSpeechSummaries = results .map((result) => result.non_speech_summary?.trim()) .filter((summary): summary is string => Boolean(summary)); return AsrSchema.parse({ speech_detected: speechDetected, language: speechDetected ? mergeLanguages(results) : null, audio_type: speechDetected ? null : getMostCommonAudioType(results), transcript: mergeTranscriptLines(results), non_speech_summary: speechDetected || nonSpeechSummaries.length === 0 ? null : nonSpeechSummaries.join(";"), content_tags: mergeContentTags(results), }); } async function transcriptAudio( audio: Buffer, frames: ExtractedVideoFrame[], videoDesc: string, log?: AsrLogger, options: TranscriptAudioOptions = {}, ) { const base64Audio = Buffer.from(audio).toString("base64"); const client = getQwenClient(); const model = process.env.QWEN_OMNI_MODEL || "qwen3.5-omni-plus"; const frameContentParts = frames.map((frame, index) => ({ type: "image_url" as const, image_url: { url: `data:${frame.contentType};base64,${frame.buffer.toString("base64")}`, }, index, })); log?.( `调用 Qwen Omni: model=${model}, audio=${Math.round(audio.length / 1024)}KB, audioDataUrl=${getAudioDataUrlLength(audio)}, frames=${frames.length}`, ); const completion = await client.chat.completions.create({ model, messages: [ { role: "user", content: [ ...frameContentParts.map(({ type, image_url }) => ({ type, image_url, })), { type: "input_audio", input_audio: { data: `${AUDIO_DATA_URL_PREFIX}${base64Audio}`, format: "mp3", }, }, { type: "text", text: `${prompt}${getSegmentInstruction(options.segment)}\n\n视频简介:\n${videoDesc || "无"}\n\n已附带视频三帧,时间点约为:${ frames .map((frame) => `${(frame.timestampMs / 1000).toFixed(1)}s`) .join("、") || "无" }。请结合画面和音频判断内容标签。`, }, ], }, ], modalities: ["text"], stream: true, stream_options: { include_usage: true }, }); let content = ""; for await (const chunk of completion) { const deltaContent = chunk.choices?.[0]?.delta?.content; if (typeof deltaContent === "string") content += deltaContent; } log?.(`Qwen Omni 原始输出: ${content}`); if (!content.trim()) { throw new Error("No ASR result returned from Qwen Omni"); } try { return parseAsrResult(content); } catch { throw new Error("Failed to parse ASR result JSON"); } } async function extractSafeAudioSegments( videoBuffer: Buffer, durationMs: number, log?: AsrLogger, ) { if (!durationMs || durationMs <= 0) { throw new Error( "Audio is too large for a single request and video duration is missing", ); } const maxAudioDataUrlLength = getMaxAudioDataUrlLength(); let segmentDurationMs = Math.min(getAudioChunkMs(), durationMs); for (let attempt = 1; attempt <= CHUNK_REEXTRACT_LIMIT; attempt++) { log?.( `提取音频分片: chunk=${Math.round(segmentDurationMs / 1000)}s, bitrate=${getAudioBitrateKbps()}kbps`, ); const segments = await extractAudioSegments(videoBuffer, { durationMs, segmentDurationMs, format: "mp3", bitrateKbps: getAudioBitrateKbps(), }); if (segments.length === 0) { throw new Error("Failed to extract audio segments from video"); } const largestSegmentLength = Math.max( ...segments.map((segment) => getAudioDataUrlLength(segment.buffer)), ); log?.( `音频分片提取完成: count=${segments.length}, largestDataUrl=${largestSegmentLength}`, ); if (largestSegmentLength <= maxAudioDataUrlLength) return segments; if (segmentDurationMs <= MIN_AUDIO_CHUNK_MS) { throw new Error( `Audio segment is still too large after splitting: ${largestSegmentLength}`, ); } const shrinkRatio = Math.max( 0.25, Math.min(0.85, (maxAudioDataUrlLength / largestSegmentLength) * 0.8), ); segmentDurationMs = Math.max( MIN_AUDIO_CHUNK_MS, Math.floor(segmentDurationMs * shrinkRatio), ); log?.( `音频分片仍过大,缩短分片后重试: ${Math.round(segmentDurationMs / 1000)}s`, ); } throw new Error("Failed to extract safe audio segments from video"); } async function transcriptAudioSegments( segments: ExtractedAudioSegment[], frames: ExtractedVideoFrame[], videoDesc: string, log?: AsrLogger, ) { const results: AsrResult[] = []; for (const segment of segments) { log?.( `识别音频分片 ${segment.index + 1}/${segments.length}: ${formatTimestamp(segment.startMs)}-${formatTimestamp(segment.endMs)}`, ); const result = await transcriptAudio( segment.buffer, frames, videoDesc, log, { segment: { index: segment.index, total: segments.length, startMs: segment.startMs, endMs: segment.endMs, }, }, ); results.push(result); } const mergedResult = mergeAsrResults(results); log?.( `分片结果合并完成: chunks=${results.length}, transcript=${mergedResult.transcript.length} 段, tags=${mergedResult.content_tags.join("、")}`, ); return mergedResult; } export async function transcriptAweme( awemeId: string, options: TranscriptAwemeOptions = {}, ): Promise { const log = options.log; log?.(`查询视频: ${awemeId}`); const aweme = await prisma.video.findUnique({ where: { aweme_id: awemeId }, }); if (!aweme) { throw new Error("Aweme not found or aweme is not a video post"); } const vPath = aweme.video_url; log?.(`下载视频文件: ${vPath}`); const buffer = await downloadFile(vPath); log?.(`视频下载完成: ${Math.round(buffer.length / 1024)}KB`); log?.("提取视频三帧"); const frames = await extractVideoFrames(buffer, { count: 3, durationMs: aweme.duration_ms, }); log?.( `视频帧提取完成: ${frames .map((frame) => `${(frame.timestampMs / 1000).toFixed(1)}s`) .join("、")}`, ); let result: AsrResult; const maxAudioDataUrlLength = getMaxAudioDataUrlLength(); if (shouldChunkBeforeFullExtract(aweme.duration_ms)) { log?.( `预计整段音频会超过安全阈值 ${maxAudioDataUrlLength},启用长视频分片识别`, ); const segments = await extractSafeAudioSegments( buffer, aweme.duration_ms, log, ); result = await transcriptAudioSegments(segments, frames, aweme.desc, log); } else { log?.("提取音频"); const audioDat = await extractAudio(buffer, { format: "mp3", bitrateKbps: getAudioBitrateKbps(), }); if (!audioDat?.buffer) { throw new Error("Failed to extract audio from video"); } const audioDataUrlLength = getAudioDataUrlLength(audioDat.buffer); log?.( `音频提取完成: ${Math.round(audioDat.buffer.length / 1024)}KB, dataUrl=${audioDataUrlLength}`, ); if (audioDataUrlLength > maxAudioDataUrlLength) { log?.( `整段音频 data URL 超过安全阈值 ${maxAudioDataUrlLength},改用分片识别`, ); const segments = await extractSafeAudioSegments( buffer, aweme.duration_ms, log, ); result = await transcriptAudioSegments(segments, frames, aweme.desc, log); } else { result = await transcriptAudio(audioDat.buffer, frames, aweme.desc, log); } } log?.(`解析完成: tags=${result.content_tags.join("、")}`); // 将转写结果持久化到数据库 log?.("写入数据库 VideoTranscript"); await prisma.videoTranscript.upsert({ where: { videoId: awemeId }, update: { speech_detected: result.speech_detected, language: result.language, audio_type: result.audio_type, transcript: result.transcript, non_speech_summary: result.non_speech_summary, content_tags: result.content_tags, }, create: { videoId: awemeId, speech_detected: result.speech_detected, language: result.language, audio_type: result.audio_type, transcript: result.transcript, non_speech_summary: result.non_speech_summary, content_tags: result.content_tags, }, }); log?.("数据库写入完成"); return result; }