使用 qwen 进行语音识别，加入 tag

Co-authored-by: Copilot <copilot@github.com>
2026-04-29 20:10:08 +08:00
parent 040273586b
commit 206a561ee7
26 changed files with 3720 additions and 443 deletions
@@ -0,0 +1,227 @@
+import OpenAI from "openai";
+import { z } from "zod";
+import { ASR_CONTENT_TAGS, normalizeAsrContentTags } from "@/lib/asr-tags";
+import { downloadFile } from "@/lib/minio";
+import { prisma } from "@/lib/prisma";
+import {
+  type ExtractedVideoFrame,
+  extractAudio,
+  extractVideoFrames,
+} from "../media";
+import prompt from "./prompt.md";
+
+export type AsrLogger = (message: string) => void;
+
+type TranscriptAwemeOptions = {
+  log?: AsrLogger;
+};
+
+let qwenClient: OpenAI | null = null;
+
+function getQwenClient() {
+  if (qwenClient) return qwenClient;
+
+  const apiKey = process.env.QWEN_API_KEY || process.env.DASHSCOPE_API_KEY;
+  if (!apiKey) {
+    throw new Error("Missing QWEN_API_KEY environment variable");
+  }
+
+  qwenClient = new OpenAI({
+    apiKey,
+    baseURL:
+      process.env.QWEN_API_BASE_URL ||
+      "https://dashscope.aliyuncs.com/compatible-mode/v1",
+  });
+
+  return qwenClient;
+}
+
+// Zod schema aligned with prompt.md
+export const AsrSchema = z
+  .object({
+    speech_detected: z.boolean(),
+    language: z.string().min(1).nullable(),
+    audio_type: z.string().nullable(),
+    transcript: z.array(z.string()),
+    non_speech_summary: z.string().nullable(),
+    content_tags: z.array(z.enum(ASR_CONTENT_TAGS)).min(1).max(3),
+  })
+  .strict();
+
+const RawAsrSchema = z.object({
+  speech_detected: z.boolean(),
+  language: z.string().min(1).nullable(),
+  audio_type: z.string().nullable(),
+  transcript: z.array(z.string()),
+  non_speech_summary: z.string().nullable(),
+  content_tags: z.unknown().optional(),
+});
+
+export type AsrResult = z.infer<typeof AsrSchema>;
+
+function extractJsonObject(text: string) {
+  const trimmed = text.trim();
+  const fenced = trimmed.match(/```(?:json)?\s*([\s\S]*?)\s*```/i);
+  if (fenced?.[1]) return fenced[1].trim();
+
+  const start = trimmed.indexOf("{");
+  const end = trimmed.lastIndexOf("}");
+  if (start >= 0 && end > start) {
+    return trimmed.slice(start, end + 1);
+  }
+
+  return trimmed;
+}
+
+function parseAsrResult(content: string): AsrResult {
+  const parsed = RawAsrSchema.parse(JSON.parse(extractJsonObject(content)));
+  return AsrSchema.parse({
+    ...parsed,
+    content_tags: normalizeAsrContentTags(parsed.content_tags),
+  });
+}
+
+async function transcriptAudio(
+  audio: Buffer,
+  frames: ExtractedVideoFrame[],
+  videoDesc: string,
+  log?: AsrLogger,
+) {
+  const base64Audio = Buffer.from(audio).toString("base64");
+  const client = getQwenClient();
+  const model = process.env.QWEN_OMNI_MODEL || "qwen3.5-omni-plus";
+  const frameContentParts = frames.map((frame, index) => ({
+    type: "image_url" as const,
+    image_url: {
+      url: `data:${frame.contentType};base64,${frame.buffer.toString("base64")}`,
+    },
+    index,
+  }));
+
+  log?.(
+    `调用 Qwen Omni: model=${model}, audio=${Math.round(audio.length / 1024)}KB, frames=${frames.length}`,
+  );
+
+  const completion = await client.chat.completions.create({
+    model,
+    messages: [
+      {
+        role: "user",
+        content: [
+          ...frameContentParts.map(({ type, image_url }) => ({
+            type,
+            image_url,
+          })),
+          {
+            type: "input_audio",
+            input_audio: { data: `data:;base64,${base64Audio}`, format: "mp3" },
+          },
+          {
+            type: "text",
+            text: `${prompt}\n\n视频简介：\n${videoDesc || "无"}\n\n已附带视频三帧，时间点约为：${
+              frames
+                .map((frame) => `${(frame.timestampMs / 1000).toFixed(1)}s`)
+                .join("、") || "无"
+            }。请结合画面和音频判断内容标签。`,
+          },
+        ],
+      },
+    ],
+    modalities: ["text"],
+    stream: true,
+    stream_options: { include_usage: true },
+  });
+
+  let content = "";
+  for await (const chunk of completion) {
+    const deltaContent = chunk.choices?.[0]?.delta?.content;
+    if (typeof deltaContent === "string") content += deltaContent;
+  }
+
+  log?.(`Qwen Omni 原始输出: ${content}`);
+
+  if (!content.trim()) {
+    throw new Error("No ASR result returned from Qwen Omni");
+  }
+
+  try {
+    return parseAsrResult(content);
+  } catch {
+    throw new Error("Failed to parse ASR result JSON");
+  }
+}
+
+export async function transcriptAweme(
+  awemeId: string,
+  options: TranscriptAwemeOptions = {},
+): Promise<AsrResult> {
+  const log = options.log;
+  log?.(`查询视频: ${awemeId}`);
+  const aweme = await prisma.video.findUnique({
+    where: { aweme_id: awemeId },
+  });
+  if (!aweme) {
+    throw new Error("Aweme not found or aweme is not a video post");
+  }
+  const vPath = aweme.video_url;
+  log?.(`下载视频文件: ${vPath}`);
+  const buffer = await downloadFile(vPath);
+  log?.(`视频下载完成: ${Math.round(buffer.length / 1024)}KB`);
+
+  log?.("提取视频三帧");
+  const frames = await extractVideoFrames(buffer, {
+    count: 3,
+    durationMs: aweme.duration_ms,
+  });
+  log?.(
+    `视频帧提取完成: ${frames
+      .map((frame) => `${(frame.timestampMs / 1000).toFixed(1)}s`)
+      .join("、")}`,
+  );
+
+  log?.("提取音频");
+  const audioDat = await extractAudio(buffer, {
+    format: "mp3",
+    bitrateKbps: 128,
+  });
+
+  if (!audioDat || !audioDat.buffer) {
+    throw new Error("Failed to extract audio from video");
+  }
+  log?.(`音频提取完成: ${Math.round(audioDat.buffer.length / 1024)}KB`);
+
+  // 调用 Qwen Omni 生成转写与固定分类标签
+  const result = await transcriptAudio(
+    audioDat.buffer,
+    frames,
+    aweme.desc,
+    log,
+  );
+  log?.(`解析完成: tags=${result.content_tags.join("、")}`);
+
+  // 将转写结果持久化到数据库
+  log?.("写入数据库 VideoTranscript");
+  await prisma.videoTranscript.upsert({
+    where: { videoId: awemeId },
+    update: {
+      speech_detected: result.speech_detected,
+      language: result.language,
+      audio_type: result.audio_type,
+      transcript: result.transcript,
+      non_speech_summary: result.non_speech_summary,
+      content_tags: result.content_tags,
+    },
+    create: {
+      videoId: awemeId,
+      speech_detected: result.speech_detected,
+      language: result.language,
+      audio_type: result.audio_type,
+      transcript: result.transcript,
+      non_speech_summary: result.non_speech_summary,
+      content_tags: result.content_tags,
+    },
+  });
+  log?.("数据库写入完成");
+
+  return result;
+}
@@ -0,0 +1,39 @@
+你将接收一段音频、视频中固定抽取的三帧画面，并可能收到一段视频描述作为辅助信息。请完成：语音活动检测（VAD）、声源分类、语音转文字，以及内容标签分类。
+
+只能输出一个合法 JSON 对象，不要输出 Markdown、解释文字或代码块。
+
+`content_tags` 必须从以下固定标签中选择 1 到 3 个，不能自造标签：
+
+美女、游戏、时政、美食、旅行、体育、科技、教育、宠物、音乐、影视、财经、生活、其他
+
+**输出 JSON Schema（示例）**
+
+```json
+{
+  "speech_detected": true,
+  "language": "zh-CN",
+  "audio_type": null,
+  "transcript": ["大家好，我是xxx", "欢迎来到今天的视频", "今天我们来聊聊AI的未来"],
+  "non_speech_summary": null,
+  "content_tags": ["科技", "教育"]
+}
+```
+
+**当无发言时返回：** 
+
+```json
+{
+  "speech_detected": false,
+  "language": null,
+  "audio_type": "music | ambience | animal | mechanical | other",
+  "transcript": [],
+  "non_speech_summary": "示例：纯音乐-钢琴独奏，节奏舒缓；或 环境声-雨声伴随雷鸣。",
+  "content_tags": ["音乐"]
+}
+```
+
+** 规则补充 **
+
+* 允许`language` 为多标签（如 "zh-CN,en"）或为 `null`（无发言时）。
+* `transcript` 按自然语义分句，每段尽量保持完整短句。
+* 必须结合三帧画面、音频内容和辅助视频描述判断标签；信息不足时使用 `其他`。
@@ -0,0 +1,22 @@
+export const runtime = "nodejs";
+
+import { type NextRequest, NextResponse } from "next/server";
+import { transcriptAweme } from ".";
+
+export async function GET(req: NextRequest) {
+  const { searchParams } = new URL(req.url);
+  const awemeId = searchParams.get("awemeId");
+  if (!awemeId) {
+    return NextResponse.json(
+      { error: "Missing awemeId parameter" },
+      { status: 400 },
+    );
+  }
+  try {
+    const result = await transcriptAweme(awemeId, /* {log: (msg) => console.log(`[ASR] ${msg}`)} */);
+    return NextResponse.json(result);
+  } catch (error) {
+    console.error("ASR failed:", error);
+    return NextResponse.json({ error: "ASR failed" }, { status: 500 });
+  }
+}