允许后端任务

2026-06-17 17:58:00 +08:00
parent eac6ad2343
commit b9af979582
6 changed files with 1874 additions and 304 deletions
@@ -0,0 +1,55 @@
+export const runtime = "nodejs";
+export const dynamic = "force-dynamic";
+
+import { type NextRequest, NextResponse } from "next/server";
+import {
+  AsrBatchJobError,
+  controlAsrBatchJob,
+  getAsrBatchJob,
+} from "@/lib/asr-batch-jobs";
+
+type RouteContext = {
+  params: Promise<{ jobId: string }>;
+};
+
+export async function GET(_req: NextRequest, { params }: RouteContext) {
+  const { jobId } = await params;
+  const job = getAsrBatchJob(jobId, true);
+  if (!job) {
+    return NextResponse.json({ error: "任务不存在" }, { status: 404 });
+  }
+
+  return NextResponse.json({ job });
+}
+
+export async function PATCH(req: NextRequest, { params }: RouteContext) {
+  const { jobId } = await params;
+  let body: unknown;
+  try {
+    body = await req.json();
+  } catch {
+    return NextResponse.json({ error: "Invalid JSON body" }, { status: 400 });
+  }
+
+  const action =
+    body && typeof body === "object" && "action" in body
+      ? (body as { action?: unknown }).action
+      : undefined;
+
+  try {
+    const job = controlAsrBatchJob(jobId, action);
+    return NextResponse.json({ job });
+  } catch (error) {
+    if (error instanceof AsrBatchJobError) {
+      return NextResponse.json(
+        { error: error.message },
+        { status: error.status },
+      );
+    }
+    console.error("Failed to control ASR batch job:", error);
+    return NextResponse.json(
+      { error: "Failed to control ASR batch job" },
+      { status: 500 },
+    );
+  }
+}
@@ -0,0 +1,45 @@
+export const runtime = "nodejs";
+export const dynamic = "force-dynamic";
+
+import { type NextRequest, NextResponse } from "next/server";
+import {
+  AsrBatchJobError,
+  createAsrBatchJob,
+  getActiveAsrBatchJob,
+  listAsrBatchJobs,
+} from "@/lib/asr-batch-jobs";
+
+export async function GET() {
+  return NextResponse.json({
+    jobs: listAsrBatchJobs(),
+    activeJob: getActiveAsrBatchJob(),
+  });
+}
+
+export async function POST(req: NextRequest) {
+  let body: unknown;
+  try {
+    body = await req.json();
+  } catch {
+    return NextResponse.json({ error: "Invalid JSON body" }, { status: 400 });
+  }
+
+  try {
+    const job = await createAsrBatchJob(
+      body && typeof body === "object" ? body : {},
+    );
+    return NextResponse.json({ job }, { status: 201 });
+  } catch (error) {
+    if (error instanceof AsrBatchJobError) {
+      return NextResponse.json(
+        { error: error.message },
+        { status: error.status },
+      );
+    }
+    console.error("Failed to create ASR batch job:", error);
+    return NextResponse.json(
+      { error: "Failed to create ASR batch job" },
+      { status: 500 },
+    );
+  }
+}
@@ -4,8 +4,10 @@ import { ASR_CONTENT_TAGS, normalizeAsrContentTags } from "@/lib/asr-tags";
 import { downloadFile } from "@/lib/minio";
 import { prisma } from "@/lib/prisma";
 import {
+  type ExtractedAudioSegment,
  type ExtractedVideoFrame,
  extractAudio,
+  extractAudioSegments,
  extractVideoFrames,
 } from "../media";
 import prompt from "./prompt.md";
@@ -16,8 +18,25 @@ type TranscriptAwemeOptions = {
  log?: AsrLogger;
 };

+type TranscriptAudioOptions = {
+  segment?: {
+    index: number;
+    total: number;
+    startMs: number;
+    endMs: number;
+  };
+};
+
 let qwenClient: OpenAI | null = null;

+const AUDIO_DATA_URL_PREFIX = "data:;base64,";
+const DEFAULT_QWEN_MAX_STRING_LENGTH = 28_000_000;
+const DEFAULT_SAFE_AUDIO_DATA_URL_LENGTH = 24_000_000;
+const DEFAULT_AUDIO_BITRATE_KBPS = 128;
+const DEFAULT_AUDIO_CHUNK_MS = 10 * 60 * 1000;
+const MIN_AUDIO_CHUNK_MS = 60 * 1000;
+const CHUNK_REEXTRACT_LIMIT = 5;
+
 function getQwenClient() {
  if (qwenClient) return qwenClient;

@@ -81,11 +100,188 @@ function parseAsrResult(content: string): AsrResult {
  });
 }

+function getPositiveIntEnv(name: string, fallback: number) {
+  const value = Number.parseInt(process.env[name] ?? "", 10);
+  return Number.isFinite(value) && value > 0 ? value : fallback;
+}
+
+function getAudioBitrateKbps() {
+  return getPositiveIntEnv(
+    "ASR_AUDIO_BITRATE_KBPS",
+    DEFAULT_AUDIO_BITRATE_KBPS,
+  );
+}
+
+function getAudioChunkMs() {
+  return Math.max(
+    MIN_AUDIO_CHUNK_MS,
+    getPositiveIntEnv(
+      "ASR_AUDIO_CHUNK_SECONDS",
+      DEFAULT_AUDIO_CHUNK_MS / 1000,
+    ) * 1000,
+  );
+}
+
+function getMaxAudioDataUrlLength() {
+  const providerLimit = getPositiveIntEnv(
+    "QWEN_MAX_STRING_LENGTH",
+    DEFAULT_QWEN_MAX_STRING_LENGTH,
+  );
+  const configuredLimit = getPositiveIntEnv(
+    "ASR_MAX_AUDIO_DATA_URL_LENGTH",
+    DEFAULT_SAFE_AUDIO_DATA_URL_LENGTH,
+  );
+  return Math.max(
+    1_000_000,
+    Math.min(configuredLimit, providerLimit - 1_000_000),
+  );
+}
+
+function getBase64Length(byteLength: number) {
+  return Math.ceil(byteLength / 3) * 4;
+}
+
+function getAudioDataUrlLength(audio: Buffer) {
+  return AUDIO_DATA_URL_PREFIX.length + getBase64Length(audio.length);
+}
+
+function estimateAudioDataUrlLength(durationMs: number, bitrateKbps: number) {
+  const estimatedBytes = Math.ceil((durationMs / 1000) * bitrateKbps * 125);
+  return AUDIO_DATA_URL_PREFIX.length + getBase64Length(estimatedBytes);
+}
+
+function formatTimestamp(ms: number) {
+  const totalSeconds = Math.max(0, Math.floor(ms / 1000));
+  const hours = Math.floor(totalSeconds / 3600);
+  const minutes = Math.floor((totalSeconds % 3600) / 60);
+  const seconds = totalSeconds % 60;
+  if (hours > 0) {
+    return `${hours}:${minutes.toString().padStart(2, "0")}:${seconds
+      .toString()
+      .padStart(2, "0")}`;
+  }
+  return `${minutes}:${seconds.toString().padStart(2, "0")}`;
+}
+
+function getSegmentInstruction(segment?: TranscriptAudioOptions["segment"]) {
+  if (!segment) return "";
+
+  return `\n\n当前音频是长视频的第 ${segment.index + 1}/${segment.total} 段，时间范围 ${formatTimestamp(segment.startMs)}-${formatTimestamp(segment.endMs)}。请只转写这一段音频，不要猜测其他分片未出现的台词；content_tags 仍结合整条视频三帧、视频简介和当前分片内容判断。`;
+}
+
+function shouldChunkBeforeFullExtract(durationMs: number) {
+  if (!durationMs || durationMs <= 0) return false;
+  return (
+    estimateAudioDataUrlLength(durationMs, getAudioBitrateKbps()) >
+    getMaxAudioDataUrlLength()
+  );
+}
+
+function mergeContentTags(results: AsrResult[]) {
+  const counts = new Map<string, { count: number; firstSeen: number }>();
+  let order = 0;
+  for (const result of results) {
+    for (const tag of result.content_tags) {
+      const current = counts.get(tag);
+      if (current) {
+        current.count += 1;
+      } else {
+        counts.set(tag, { count: 1, firstSeen: order });
+        order += 1;
+      }
+    }
+  }
+
+  const sortedTags = [...counts.entries()].sort(
+    (left, right) =>
+      right[1].count - left[1].count || left[1].firstSeen - right[1].firstSeen,
+  );
+  const preferredTags = sortedTags.filter(([tag]) => tag !== "其他");
+  const tags = (preferredTags.length > 0 ? preferredTags : sortedTags)
+    .slice(0, 3)
+    .map(([tag]) => tag);
+
+  return normalizeAsrContentTags(tags);
+}
+
+function mergeLanguages(results: AsrResult[]) {
+  const languages = results
+    .flatMap((result) =>
+      (result.language ?? "")
+        .split(/[,，、\s]+/)
+        .map((item) => item.trim())
+        .filter(Boolean),
+    )
+    .filter((language, index, array) => array.indexOf(language) === index);
+
+  return languages.length > 0 ? languages.join(",") : null;
+}
+
+function getMostCommonAudioType(results: AsrResult[]) {
+  const counts = new Map<string, { count: number; firstSeen: number }>();
+  let order = 0;
+  for (const result of results) {
+    if (!result.audio_type) continue;
+    const current = counts.get(result.audio_type);
+    if (current) {
+      current.count += 1;
+    } else {
+      counts.set(result.audio_type, { count: 1, firstSeen: order });
+      order += 1;
+    }
+  }
+
+  return (
+    [...counts.entries()].sort(
+      (left, right) =>
+        right[1].count - left[1].count ||
+        left[1].firstSeen - right[1].firstSeen,
+    )[0]?.[0] ?? null
+  );
+}
+
+function mergeTranscriptLines(results: AsrResult[]) {
+  const lines: string[] = [];
+  for (const result of results) {
+    for (const line of result.transcript) {
+      const normalizedLine = line.trim();
+      if (!normalizedLine) continue;
+      if (lines.at(-1) === normalizedLine) continue;
+      lines.push(normalizedLine);
+    }
+  }
+  return lines;
+}
+
+function mergeAsrResults(results: AsrResult[]): AsrResult {
+  if (results.length === 0) {
+    throw new Error("No ASR chunk results returned from Qwen Omni");
+  }
+
+  const speechDetected = results.some((result) => result.speech_detected);
+  const nonSpeechSummaries = results
+    .map((result) => result.non_speech_summary?.trim())
+    .filter((summary): summary is string => Boolean(summary));
+
+  return AsrSchema.parse({
+    speech_detected: speechDetected,
+    language: speechDetected ? mergeLanguages(results) : null,
+    audio_type: speechDetected ? null : getMostCommonAudioType(results),
+    transcript: mergeTranscriptLines(results),
+    non_speech_summary:
+      speechDetected || nonSpeechSummaries.length === 0
+        ? null
+        : nonSpeechSummaries.join("；"),
+    content_tags: mergeContentTags(results),
+  });
+}
+
 async function transcriptAudio(
  audio: Buffer,
  frames: ExtractedVideoFrame[],
  videoDesc: string,
  log?: AsrLogger,
+  options: TranscriptAudioOptions = {},
 ) {
  const base64Audio = Buffer.from(audio).toString("base64");
  const client = getQwenClient();
@@ -99,7 +295,7 @@ async function transcriptAudio(
  }));

  log?.(
-    `调用 Qwen Omni: model=${model}, audio=${Math.round(audio.length / 1024)}KB, frames=${frames.length}`,
+    `调用 Qwen Omni: model=${model}, audio=${Math.round(audio.length / 1024)}KB, audioDataUrl=${getAudioDataUrlLength(audio)}, frames=${frames.length}`,
  );

  const completion = await client.chat.completions.create({
@@ -114,11 +310,14 @@ async function transcriptAudio(
          })),
          {
            type: "input_audio",
-            input_audio: { data: `data:;base64,${base64Audio}`, format: "mp3" },
+            input_audio: {
+              data: `${AUDIO_DATA_URL_PREFIX}${base64Audio}`,
+              format: "mp3",
+            },
          },
          {
            type: "text",
-            text: `${prompt}\n\n视频简介：\n${videoDesc || "无"}\n\n已附带视频三帧，时间点约为：${
+            text: `${prompt}${getSegmentInstruction(options.segment)}\n\n视频简介：\n${videoDesc || "无"}\n\n已附带视频三帧，时间点约为：${
              frames
                .map((frame) => `${(frame.timestampMs / 1000).toFixed(1)}s`)
                .join("、") || "无"
@@ -151,6 +350,102 @@ async function transcriptAudio(
  }
 }

+async function extractSafeAudioSegments(
+  videoBuffer: Buffer,
+  durationMs: number,
+  log?: AsrLogger,
+) {
+  if (!durationMs || durationMs <= 0) {
+    throw new Error(
+      "Audio is too large for a single request and video duration is missing",
+    );
+  }
+
+  const maxAudioDataUrlLength = getMaxAudioDataUrlLength();
+  let segmentDurationMs = Math.min(getAudioChunkMs(), durationMs);
+
+  for (let attempt = 1; attempt <= CHUNK_REEXTRACT_LIMIT; attempt++) {
+    log?.(
+      `提取音频分片: chunk=${Math.round(segmentDurationMs / 1000)}s, bitrate=${getAudioBitrateKbps()}kbps`,
+    );
+    const segments = await extractAudioSegments(videoBuffer, {
+      durationMs,
+      segmentDurationMs,
+      format: "mp3",
+      bitrateKbps: getAudioBitrateKbps(),
+    });
+
+    if (segments.length === 0) {
+      throw new Error("Failed to extract audio segments from video");
+    }
+
+    const largestSegmentLength = Math.max(
+      ...segments.map((segment) => getAudioDataUrlLength(segment.buffer)),
+    );
+    log?.(
+      `音频分片提取完成: count=${segments.length}, largestDataUrl=${largestSegmentLength}`,
+    );
+
+    if (largestSegmentLength <= maxAudioDataUrlLength) return segments;
+
+    if (segmentDurationMs <= MIN_AUDIO_CHUNK_MS) {
+      throw new Error(
+        `Audio segment is still too large after splitting: ${largestSegmentLength}`,
+      );
+    }
+
+    const shrinkRatio = Math.max(
+      0.25,
+      Math.min(0.85, (maxAudioDataUrlLength / largestSegmentLength) * 0.8),
+    );
+    segmentDurationMs = Math.max(
+      MIN_AUDIO_CHUNK_MS,
+      Math.floor(segmentDurationMs * shrinkRatio),
+    );
+    log?.(
+      `音频分片仍过大，缩短分片后重试: ${Math.round(segmentDurationMs / 1000)}s`,
+    );
+  }
+
+  throw new Error("Failed to extract safe audio segments from video");
+}
+
+async function transcriptAudioSegments(
+  segments: ExtractedAudioSegment[],
+  frames: ExtractedVideoFrame[],
+  videoDesc: string,
+  log?: AsrLogger,
+) {
+  const results: AsrResult[] = [];
+
+  for (const segment of segments) {
+    log?.(
+      `识别音频分片 ${segment.index + 1}/${segments.length}: ${formatTimestamp(segment.startMs)}-${formatTimestamp(segment.endMs)}`,
+    );
+    const result = await transcriptAudio(
+      segment.buffer,
+      frames,
+      videoDesc,
+      log,
+      {
+        segment: {
+          index: segment.index,
+          total: segments.length,
+          startMs: segment.startMs,
+          endMs: segment.endMs,
+        },
+      },
+    );
+    results.push(result);
+  }
+
+  const mergedResult = mergeAsrResults(results);
+  log?.(
+    `分片结果合并完成: chunks=${results.length}, transcript=${mergedResult.transcript.length} 段, tags=${mergedResult.content_tags.join("、")}`,
+  );
+  return mergedResult;
+}
+
 export async function transcriptAweme(
  awemeId: string,
  options: TranscriptAwemeOptions = {},
@@ -179,24 +474,48 @@ export async function transcriptAweme(
      .join("、")}`,
  );

-  log?.("提取音频");
-  const audioDat = await extractAudio(buffer, {
-    format: "mp3",
-    bitrateKbps: 128,
-  });
+  let result: AsrResult;
+  const maxAudioDataUrlLength = getMaxAudioDataUrlLength();
+  if (shouldChunkBeforeFullExtract(aweme.duration_ms)) {
+    log?.(
+      `预计整段音频会超过安全阈值 ${maxAudioDataUrlLength}，启用长视频分片识别`,
+    );
+    const segments = await extractSafeAudioSegments(
+      buffer,
+      aweme.duration_ms,
+      log,
+    );
+    result = await transcriptAudioSegments(segments, frames, aweme.desc, log);
+  } else {
+    log?.("提取音频");
+    const audioDat = await extractAudio(buffer, {
+      format: "mp3",
+      bitrateKbps: getAudioBitrateKbps(),
+    });

-  if (!audioDat || !audioDat.buffer) {
-    throw new Error("Failed to extract audio from video");
+    if (!audioDat?.buffer) {
+      throw new Error("Failed to extract audio from video");
+    }
+
+    const audioDataUrlLength = getAudioDataUrlLength(audioDat.buffer);
+    log?.(
+      `音频提取完成: ${Math.round(audioDat.buffer.length / 1024)}KB, dataUrl=${audioDataUrlLength}`,
+    );
+
+    if (audioDataUrlLength > maxAudioDataUrlLength) {
+      log?.(
+        `整段音频 data URL 超过安全阈值 ${maxAudioDataUrlLength}，改用分片识别`,
+      );
+      const segments = await extractSafeAudioSegments(
+        buffer,
+        aweme.duration_ms,
+        log,
+      );
+      result = await transcriptAudioSegments(segments, frames, aweme.desc, log);
+    } else {
+      result = await transcriptAudio(audioDat.buffer, frames, aweme.desc, log);
+    }
  }
-  log?.(`音频提取完成: ${Math.round(audioDat.buffer.length / 1024)}KB`);
-
-  // 调用 Qwen Omni 生成转写与固定分类标签
-  const result = await transcriptAudio(
-    audioDat.buffer,
-    frames,
-    aweme.desc,
-    log,
-  );
  log?.(`解析完成: tags=${result.content_tags.join("、")}`);

  // 将转写结果持久化到数据库
@@ -82,6 +82,60 @@ export type ExtractedVideoFrame = {
  timestampMs: number;
 };

+export type ExtractedAudio = {
+  buffer: Buffer;
+  contentType: string;
+  ext: string;
+};
+
+export type ExtractedAudioSegment = ExtractedAudio & {
+  index: number;
+  startMs: number;
+  endMs: number;
+};
+
+type AudioFormat = "mp3" | "aac" | "wav";
+
+function resolveAudioEncoding(format: AudioFormat, bitrateKbps: number) {
+  if (format === "mp3") {
+    return {
+      ext: "mp3",
+      contentType: "audio/mpeg",
+      codecArgs: ["-c:a", "libmp3lame", "-b:a", `${bitrateKbps}k`],
+    };
+  }
+
+  if (format === "aac") {
+    return {
+      ext: "m4a",
+      contentType: "audio/mp4",
+      codecArgs: [
+        "-c:a",
+        "aac",
+        "-b:a",
+        `${bitrateKbps}k`,
+        "-movflags",
+        "+faststart",
+      ],
+    };
+  }
+
+  return {
+    ext: "wav",
+    contentType: "audio/wav",
+    codecArgs: [
+      "-f",
+      "wav",
+      "-acodec",
+      "pcm_s16le",
+      "-ar",
+      "44100",
+      "-ac",
+      "2",
+    ],
+  };
+}
+
 /**
 * 使用 ffprobe 获取视频时长（毫秒）
 */
@@ -234,8 +288,8 @@ export async function extractVideoFrames(
 */
 export async function extractAudio(
  videoBuffer: Buffer,
-  opts?: { format?: "mp3" | "aac" | "wav"; bitrateKbps?: number },
-): Promise<{ buffer: Buffer; contentType: string; ext: string } | null> {
+  opts?: { format?: AudioFormat; bitrateKbps?: number },
+): Promise<ExtractedAudio | null> {
  const ffmpegCmd = process.env.FFMPEG_PATH || "ffmpeg";
  const tmpDir = os.tmpdir();
  const base = `dy_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`;
@@ -243,47 +297,8 @@ export async function extractAudio(

  const format = opts?.format ?? "mp3";
  const bitrate = Math.max(32, Math.min(512, opts?.bitrateKbps ?? 192)); // 安全范围 32~512 kbps
-
-  // 根据目标格式设置输出路径、MIME 与编码参数
-  let outPath = "";
-  let contentType = "";
-  let ext = "";
-  let codecArgs: string[] = [];
-
-  if (format === "mp3") {
-    ext = "mp3";
-    contentType = "audio/mpeg";
-    outPath = path.join(tmpDir, `${base}.${ext}`);
-    codecArgs = ["-c:a", "libmp3lame", "-b:a", `${bitrate}k`];
-  } else if (format === "aac") {
-    // 使用 m4a 容器更通用
-    ext = "m4a";
-    contentType = "audio/mp4";
-    outPath = path.join(tmpDir, `${base}.${ext}`);
-    codecArgs = [
-      "-c:a",
-      "aac",
-      "-b:a",
-      `${bitrate}k`,
-      "-movflags",
-      "+faststart",
-    ];
-  } else {
-    // wav
-    ext = "wav";
-    contentType = "audio/wav";
-    outPath = path.join(tmpDir, `${base}.${ext}`);
-    codecArgs = [
-      "-f",
-      "wav",
-      "-acodec",
-      "pcm_s16le",
-      "-ar",
-      "44100",
-      "-ac",
-      "2",
-    ];
-  }
+  const { ext, contentType, codecArgs } = resolveAudioEncoding(format, bitrate);
+  const outPath = path.join(tmpDir, `${base}.${ext}`);

  try {
    await fs.writeFile(inPath, videoBuffer);
@@ -320,3 +335,88 @@ export async function extractAudio(
    } catch {}
  }
 }
+
+/**
+ * 将视频音轨按时间切成多个音频片段，避免长视频单次请求体过大。
+ */
+export async function extractAudioSegments(
+  videoBuffer: Buffer,
+  opts: {
+    durationMs: number;
+    segmentDurationMs: number;
+    format?: AudioFormat;
+    bitrateKbps?: number;
+  },
+): Promise<ExtractedAudioSegment[]> {
+  const ffmpegCmd = process.env.FFMPEG_PATH || "ffmpeg";
+  const tmpDir = os.tmpdir();
+  const base = `dy_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`;
+  const inPath = path.join(tmpDir, `${base}.mp4`);
+
+  const format = opts.format ?? "mp3";
+  const bitrate = Math.max(32, Math.min(512, opts.bitrateKbps ?? 128));
+  const segmentDurationMs = Math.max(1000, opts.segmentDurationMs);
+  const durationMs = Math.max(0, opts.durationMs);
+  const { ext, contentType, codecArgs } = resolveAudioEncoding(format, bitrate);
+  const outPaths: string[] = [];
+
+  try {
+    await fs.writeFile(inPath, videoBuffer);
+
+    const segments: ExtractedAudioSegment[] = [];
+    let index = 0;
+    for (let startMs = 0; startMs < durationMs; startMs += segmentDurationMs) {
+      const endMs = Math.min(durationMs, startMs + segmentDurationMs);
+      if (endMs - startMs < 500) continue;
+
+      const outPath = path.join(tmpDir, `${base}_${index}.${ext}`);
+      outPaths.push(outPath);
+
+      const args = [
+        "-hide_banner",
+        "-loglevel",
+        "error",
+        "-ss",
+        (startMs / 1000).toFixed(3),
+        "-t",
+        ((endMs - startMs) / 1000).toFixed(3),
+        "-i",
+        inPath,
+        "-vn",
+        ...codecArgs,
+        "-y",
+        outPath,
+      ];
+      await execFileAsync(ffmpegCmd, args, { windowsHide: true });
+      segments.push({
+        buffer: await fs.readFile(outPath),
+        contentType,
+        ext,
+        index,
+        startMs,
+        endMs,
+      });
+      index += 1;
+    }
+
+    return segments;
+  } catch (error) {
+    if (isMissingExecutableError(error)) {
+      console.warn(
+        "系统未检测到 ffmpeg，可安装并配置 PATH 或设置 FFMPEG_PATH 后启用音频分片提取。",
+      );
+      return [];
+    }
+    console.warn(`提取音频分片失败: ${getErrorMessage(error)}`);
+    throw error;
+  } finally {
+    try {
+      await fs.unlink(inPath);
+    } catch {}
+    for (const outPath of outPaths) {
+      try {
+        await fs.unlink(outPath);
+      } catch {}
+    }
+  }
+}