import OpenAI from "openai"; import { z } from "zod"; import { ASR_CONTENT_TAGS, normalizeAsrContentTags } from "@/lib/asr-tags"; import { downloadFile } from "@/lib/minio"; import { prisma } from "@/lib/prisma"; import { type ExtractedVideoFrame, extractAudio, extractVideoFrames, } from "../media"; import prompt from "./prompt.md"; export type AsrLogger = (message: string) => void; type TranscriptAwemeOptions = { log?: AsrLogger; }; let qwenClient: OpenAI | null = null; function getQwenClient() { if (qwenClient) return qwenClient; const apiKey = process.env.QWEN_API_KEY || process.env.DASHSCOPE_API_KEY; if (!apiKey) { throw new Error("Missing QWEN_API_KEY environment variable"); } qwenClient = new OpenAI({ apiKey, baseURL: process.env.QWEN_API_BASE_URL || "https://dashscope.aliyuncs.com/compatible-mode/v1", }); return qwenClient; } // Zod schema aligned with prompt.md export const AsrSchema = z .object({ speech_detected: z.boolean(), language: z.string().min(1).nullable(), audio_type: z.string().nullable(), transcript: z.array(z.string()), non_speech_summary: z.string().nullable(), content_tags: z.array(z.enum(ASR_CONTENT_TAGS)).min(1).max(3), }) .strict(); const RawAsrSchema = z.object({ speech_detected: z.boolean(), language: z.string().min(1).nullable(), audio_type: z.string().nullable(), transcript: z.array(z.string()), non_speech_summary: z.string().nullable(), content_tags: z.unknown().optional(), }); export type AsrResult = z.infer; function extractJsonObject(text: string) { const trimmed = text.trim(); const fenced = trimmed.match(/```(?:json)?\s*([\s\S]*?)\s*```/i); if (fenced?.[1]) return fenced[1].trim(); const start = trimmed.indexOf("{"); const end = trimmed.lastIndexOf("}"); if (start >= 0 && end > start) { return trimmed.slice(start, end + 1); } return trimmed; } function parseAsrResult(content: string): AsrResult { const parsed = RawAsrSchema.parse(JSON.parse(extractJsonObject(content))); return AsrSchema.parse({ ...parsed, content_tags: normalizeAsrContentTags(parsed.content_tags), }); } async function transcriptAudio( audio: Buffer, frames: ExtractedVideoFrame[], videoDesc: string, log?: AsrLogger, ) { const base64Audio = Buffer.from(audio).toString("base64"); const client = getQwenClient(); const model = process.env.QWEN_OMNI_MODEL || "qwen3.5-omni-plus"; const frameContentParts = frames.map((frame, index) => ({ type: "image_url" as const, image_url: { url: `data:${frame.contentType};base64,${frame.buffer.toString("base64")}`, }, index, })); log?.( `调用 Qwen Omni: model=${model}, audio=${Math.round(audio.length / 1024)}KB, frames=${frames.length}`, ); const completion = await client.chat.completions.create({ model, messages: [ { role: "user", content: [ ...frameContentParts.map(({ type, image_url }) => ({ type, image_url, })), { type: "input_audio", input_audio: { data: `data:;base64,${base64Audio}`, format: "mp3" }, }, { type: "text", text: `${prompt}\n\n视频简介:\n${videoDesc || "无"}\n\n已附带视频三帧,时间点约为:${ frames .map((frame) => `${(frame.timestampMs / 1000).toFixed(1)}s`) .join("、") || "无" }。请结合画面和音频判断内容标签。`, }, ], }, ], modalities: ["text"], stream: true, stream_options: { include_usage: true }, }); let content = ""; for await (const chunk of completion) { const deltaContent = chunk.choices?.[0]?.delta?.content; if (typeof deltaContent === "string") content += deltaContent; } log?.(`Qwen Omni 原始输出: ${content}`); if (!content.trim()) { throw new Error("No ASR result returned from Qwen Omni"); } try { return parseAsrResult(content); } catch { throw new Error("Failed to parse ASR result JSON"); } } export async function transcriptAweme( awemeId: string, options: TranscriptAwemeOptions = {}, ): Promise { const log = options.log; log?.(`查询视频: ${awemeId}`); const aweme = await prisma.video.findUnique({ where: { aweme_id: awemeId }, }); if (!aweme) { throw new Error("Aweme not found or aweme is not a video post"); } const vPath = aweme.video_url; log?.(`下载视频文件: ${vPath}`); const buffer = await downloadFile(vPath); log?.(`视频下载完成: ${Math.round(buffer.length / 1024)}KB`); log?.("提取视频三帧"); const frames = await extractVideoFrames(buffer, { count: 3, durationMs: aweme.duration_ms, }); log?.( `视频帧提取完成: ${frames .map((frame) => `${(frame.timestampMs / 1000).toFixed(1)}s`) .join("、")}`, ); log?.("提取音频"); const audioDat = await extractAudio(buffer, { format: "mp3", bitrateKbps: 128, }); if (!audioDat || !audioDat.buffer) { throw new Error("Failed to extract audio from video"); } log?.(`音频提取完成: ${Math.round(audioDat.buffer.length / 1024)}KB`); // 调用 Qwen Omni 生成转写与固定分类标签 const result = await transcriptAudio( audioDat.buffer, frames, aweme.desc, log, ); log?.(`解析完成: tags=${result.content_tags.join("、")}`); // 将转写结果持久化到数据库 log?.("写入数据库 VideoTranscript"); await prisma.videoTranscript.upsert({ where: { videoId: awemeId }, update: { speech_detected: result.speech_detected, language: result.language, audio_type: result.audio_type, transcript: result.transcript, non_speech_summary: result.non_speech_summary, content_tags: result.content_tags, }, create: { videoId: awemeId, speech_detected: result.speech_detected, language: result.language, audio_type: result.audio_type, transcript: result.transcript, non_speech_summary: result.non_speech_summary, content_tags: result.content_tags, }, }); log?.("数据库写入完成"); return result; }