feie9454/douyin-archive
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

允许后端任务

Browse Source
This commit is contained in:
feie9454
2026-06-17 17:58:00 +08:00
parent eac6ad2343
commit b9af979582
6 changed files with 1874 additions and 304 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/api/asr/index.ts
+338 -19
View File
@@ -4,8 +4,10 @@ import { ASR_CONTENT_TAGS, normalizeAsrContentTags } from "@/lib/asr-tags";
import { downloadFile } from "@/lib/minio";
import { prisma } from "@/lib/prisma";
import {
type ExtractedAudioSegment,
type ExtractedVideoFrame,
extractAudio,
extractAudioSegments,
extractVideoFrames,
} from "../media";
import prompt from "./prompt.md";
@@ -16,8 +18,25 @@ type TranscriptAwemeOptions = {
log?: AsrLogger;
};
type TranscriptAudioOptions = {
segment?: {
index: number;
total: number;
startMs: number;
endMs: number;
};
};
let qwenClient: OpenAI | null = null;
const AUDIO_DATA_URL_PREFIX = "data:;base64,";
const DEFAULT_QWEN_MAX_STRING_LENGTH = 28_000_000;
const DEFAULT_SAFE_AUDIO_DATA_URL_LENGTH = 24_000_000;
const DEFAULT_AUDIO_BITRATE_KBPS = 128;
const DEFAULT_AUDIO_CHUNK_MS = 10 * 60 * 1000;
const MIN_AUDIO_CHUNK_MS = 60 * 1000;
const CHUNK_REEXTRACT_LIMIT = 5;
function getQwenClient() {
if (qwenClient) return qwenClient;
@@ -81,11 +100,188 @@ function parseAsrResult(content: string): AsrResult {
});
}
function getPositiveIntEnv(name: string, fallback: number) {
const value = Number.parseInt(process.env[name] ?? "", 10);
return Number.isFinite(value) && value > 0 ? value : fallback;
}
function getAudioBitrateKbps() {
return getPositiveIntEnv(
"ASR_AUDIO_BITRATE_KBPS",
DEFAULT_AUDIO_BITRATE_KBPS,
);
}
function getAudioChunkMs() {
return Math.max(
MIN_AUDIO_CHUNK_MS,
getPositiveIntEnv(
"ASR_AUDIO_CHUNK_SECONDS",
DEFAULT_AUDIO_CHUNK_MS / 1000,
) * 1000,
);
}
function getMaxAudioDataUrlLength() {
const providerLimit = getPositiveIntEnv(
"QWEN_MAX_STRING_LENGTH",
DEFAULT_QWEN_MAX_STRING_LENGTH,
);
const configuredLimit = getPositiveIntEnv(
"ASR_MAX_AUDIO_DATA_URL_LENGTH",
DEFAULT_SAFE_AUDIO_DATA_URL_LENGTH,
);
return Math.max(
1_000_000,
Math.min(configuredLimit, providerLimit - 1_000_000),
);
}
function getBase64Length(byteLength: number) {
return Math.ceil(byteLength / 3) * 4;
}
function getAudioDataUrlLength(audio: Buffer) {
return AUDIO_DATA_URL_PREFIX.length + getBase64Length(audio.length);
}
function estimateAudioDataUrlLength(durationMs: number, bitrateKbps: number) {
const estimatedBytes = Math.ceil((durationMs / 1000) * bitrateKbps * 125);
return AUDIO_DATA_URL_PREFIX.length + getBase64Length(estimatedBytes);
}
function formatTimestamp(ms: number) {
const totalSeconds = Math.max(0, Math.floor(ms / 1000));
const hours = Math.floor(totalSeconds / 3600);
const minutes = Math.floor((totalSeconds % 3600) / 60);
const seconds = totalSeconds % 60;
if (hours > 0) {
return `${hours}:${minutes.toString().padStart(2, "0")}:${seconds
.toString()
.padStart(2, "0")}`;
}
return `${minutes}:${seconds.toString().padStart(2, "0")}`;
}
function getSegmentInstruction(segment?: TranscriptAudioOptions["segment"]) {
if (!segment) return "";
return `\n\n当前音频是长视频的第 ${segment.index + 1}/${segment.total} 段，时间范围 ${formatTimestamp(segment.startMs)}-${formatTimestamp(segment.endMs)}。请只转写这一段音频，不要猜测其他分片未出现的台词；content_tags 仍结合整条视频三帧、视频简介和当前分片内容判断。`;
}
function shouldChunkBeforeFullExtract(durationMs: number) {
if (!durationMs || durationMs <= 0) return false;
return (
estimateAudioDataUrlLength(durationMs, getAudioBitrateKbps()) >
getMaxAudioDataUrlLength()
);
}
function mergeContentTags(results: AsrResult[]) {
const counts = new Map<string, { count: number; firstSeen: number }>();
let order = 0;
for (const result of results) {
for (const tag of result.content_tags) {
const current = counts.get(tag);
if (current) {
current.count += 1;
} else {
counts.set(tag, { count: 1, firstSeen: order });
order += 1;
}
}
}
const sortedTags = [...counts.entries()].sort(
(left, right) =>
right[1].count - left[1].count || left[1].firstSeen - right[1].firstSeen,
);
const preferredTags = sortedTags.filter(([tag]) => tag !== "其他");
const tags = (preferredTags.length > 0 ? preferredTags : sortedTags)
.slice(0, 3)
.map(([tag]) => tag);
return normalizeAsrContentTags(tags);
}
function mergeLanguages(results: AsrResult[]) {
const languages = results
.flatMap((result) =>
(result.language ?? "")
.split(/[,，、\s]+/)
.map((item) => item.trim())
.filter(Boolean),
)
.filter((language, index, array) => array.indexOf(language) === index);
return languages.length > 0 ? languages.join(",") : null;
}
function getMostCommonAudioType(results: AsrResult[]) {
const counts = new Map<string, { count: number; firstSeen: number }>();
let order = 0;
for (const result of results) {
if (!result.audio_type) continue;
const current = counts.get(result.audio_type);
if (current) {
current.count += 1;
} else {
counts.set(result.audio_type, { count: 1, firstSeen: order });
order += 1;
}
}
return (
[...counts.entries()].sort(
(left, right) =>
right[1].count - left[1].count ||
left[1].firstSeen - right[1].firstSeen,
)[0]?.[0] ?? null
);
}
function mergeTranscriptLines(results: AsrResult[]) {
const lines: string[] = [];
for (const result of results) {
for (const line of result.transcript) {
const normalizedLine = line.trim();
if (!normalizedLine) continue;
if (lines.at(-1) === normalizedLine) continue;
lines.push(normalizedLine);
}
}
return lines;
}
function mergeAsrResults(results: AsrResult[]): AsrResult {
if (results.length === 0) {
throw new Error("No ASR chunk results returned from Qwen Omni");
}
const speechDetected = results.some((result) => result.speech_detected);
const nonSpeechSummaries = results
.map((result) => result.non_speech_summary?.trim())
.filter((summary): summary is string => Boolean(summary));
return AsrSchema.parse({
speech_detected: speechDetected,
language: speechDetected ? mergeLanguages(results) : null,
audio_type: speechDetected ? null : getMostCommonAudioType(results),
transcript: mergeTranscriptLines(results),
non_speech_summary:
speechDetected || nonSpeechSummaries.length === 0
? null
: nonSpeechSummaries.join("；"),
content_tags: mergeContentTags(results),
});
}
async function transcriptAudio(
audio: Buffer,
frames: ExtractedVideoFrame[],
videoDesc: string,
log?: AsrLogger,
options: TranscriptAudioOptions = {},
) {
const base64Audio = Buffer.from(audio).toString("base64");
const client = getQwenClient();
@@ -99,7 +295,7 @@ async function transcriptAudio(
}));
log?.(
`调用 Qwen Omni: model=${model}, audio=${Math.round(audio.length / 1024)}KB, frames=${frames.length}`,
`调用 Qwen Omni: model=${model}, audio=${Math.round(audio.length / 1024)}KB, audioDataUrl=${getAudioDataUrlLength(audio)}, frames=${frames.length}`,
);
const completion = await client.chat.completions.create({
@@ -114,11 +310,14 @@ async function transcriptAudio(
})),
{
type: "input_audio",
input_audio: { data: `data:;base64,${base64Audio}`, format: "mp3" },
input_audio: {
data: `${AUDIO_DATA_URL_PREFIX}${base64Audio}`,
format: "mp3",
},
},
{
type: "text",
text: `${prompt}\n\n视频简介：\n${videoDesc || "无"}\n\n已附带视频三帧，时间点约为：${
text: `${prompt}${getSegmentInstruction(options.segment)}\n\n视频简介：\n${videoDesc || "无"}\n\n已附带视频三帧，时间点约为：${
frames
.map((frame) => `${(frame.timestampMs / 1000).toFixed(1)}s`)
.join("、") || "无"
@@ -151,6 +350,102 @@ async function transcriptAudio(
}
}
async function extractSafeAudioSegments(
videoBuffer: Buffer,
durationMs: number,
log?: AsrLogger,
) {
if (!durationMs || durationMs <= 0) {
throw new Error(
"Audio is too large for a single request and video duration is missing",
);
}
const maxAudioDataUrlLength = getMaxAudioDataUrlLength();
let segmentDurationMs = Math.min(getAudioChunkMs(), durationMs);
for (let attempt = 1; attempt <= CHUNK_REEXTRACT_LIMIT; attempt++) {
log?.(
`提取音频分片: chunk=${Math.round(segmentDurationMs / 1000)}s, bitrate=${getAudioBitrateKbps()}kbps`,
);
const segments = await extractAudioSegments(videoBuffer, {
durationMs,
segmentDurationMs,
format: "mp3",
bitrateKbps: getAudioBitrateKbps(),
});
if (segments.length === 0) {
throw new Error("Failed to extract audio segments from video");
}
const largestSegmentLength = Math.max(
...segments.map((segment) => getAudioDataUrlLength(segment.buffer)),
);
log?.(
`音频分片提取完成: count=${segments.length}, largestDataUrl=${largestSegmentLength}`,
);
if (largestSegmentLength <= maxAudioDataUrlLength) return segments;
if (segmentDurationMs <= MIN_AUDIO_CHUNK_MS) {
throw new Error(
`Audio segment is still too large after splitting: ${largestSegmentLength}`,
);
}
const shrinkRatio = Math.max(
0.25,
Math.min(0.85, (maxAudioDataUrlLength / largestSegmentLength) * 0.8),
);
segmentDurationMs = Math.max(
MIN_AUDIO_CHUNK_MS,
Math.floor(segmentDurationMs * shrinkRatio),
);
log?.(
`音频分片仍过大，缩短分片后重试: ${Math.round(segmentDurationMs / 1000)}s`,
);
}
throw new Error("Failed to extract safe audio segments from video");
}
async function transcriptAudioSegments(
segments: ExtractedAudioSegment[],
frames: ExtractedVideoFrame[],
videoDesc: string,
log?: AsrLogger,
) {
const results: AsrResult[] = [];
for (const segment of segments) {
log?.(
`识别音频分片 ${segment.index + 1}/${segments.length}: ${formatTimestamp(segment.startMs)}-${formatTimestamp(segment.endMs)}`,
);
const result = await transcriptAudio(
segment.buffer,
frames,
videoDesc,
log,
{
segment: {
index: segment.index,
total: segments.length,
startMs: segment.startMs,
endMs: segment.endMs,
},
},
);
results.push(result);
}
const mergedResult = mergeAsrResults(results);
log?.(
`分片结果合并完成: chunks=${results.length}, transcript=${mergedResult.transcript.length} 段, tags=${mergedResult.content_tags.join("、")}`,
);
return mergedResult;
}
export async function transcriptAweme(
awemeId: string,
options: TranscriptAwemeOptions = {},
@@ -179,24 +474,48 @@ export async function transcriptAweme(
.join("、")}`,
);
log?.("提取音频");
const audioDat = await extractAudio(buffer, {
format: "mp3",
bitrateKbps: 128,
});
let result: AsrResult;
const maxAudioDataUrlLength = getMaxAudioDataUrlLength();
if (shouldChunkBeforeFullExtract(aweme.duration_ms)) {
log?.(
`预计整段音频会超过安全阈值 ${maxAudioDataUrlLength}，启用长视频分片识别`,
);
const segments = await extractSafeAudioSegments(
buffer,
aweme.duration_ms,
log,
);
result = await transcriptAudioSegments(segments, frames, aweme.desc, log);
} else {
log?.("提取音频");
const audioDat = await extractAudio(buffer, {
format: "mp3",
bitrateKbps: getAudioBitrateKbps(),
});
if (!audioDat || !audioDat.buffer) {
throw new Error("Failed to extract audio from video");
if (!audioDat?.buffer) {
throw new Error("Failed to extract audio from video");
}
const audioDataUrlLength = getAudioDataUrlLength(audioDat.buffer);
log?.(
`音频提取完成: ${Math.round(audioDat.buffer.length / 1024)}KB, dataUrl=${audioDataUrlLength}`,
);
if (audioDataUrlLength > maxAudioDataUrlLength) {
log?.(
`整段音频 data URL 超过安全阈值 ${maxAudioDataUrlLength}，改用分片识别`,
);
const segments = await extractSafeAudioSegments(
buffer,
aweme.duration_ms,
log,
);
result = await transcriptAudioSegments(segments, frames, aweme.desc, log);
} else {
result = await transcriptAudio(audioDat.buffer, frames, aweme.desc, log);
}
}
log?.(`音频提取完成: ${Math.round(audioDat.buffer.length / 1024)}KB`);
// 调用 Qwen Omni 生成转写与固定分类标签
const result = await transcriptAudio(
audioDat.buffer,
frames,
aweme.desc,
log,
);
log?.(`解析完成: tags=${result.content_tags.join("、")}`);
// 将转写结果持久化到数据库