From 6f0e38054b99a33ba2d09777b754b22da8e1548d Mon Sep 17 00:00:00 2001 From: feiezha Date: Sat, 3 Oct 2026 21:58:06 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E5=B0=86=E8=A7=86=E9=A2=91=E8=BD=AC?= =?UTF-8?q?=E5=86=99=E9=87=8D=E6=9E=84=E4=B8=BA=E7=BA=AF=E6=96=87=E5=AD=97?= =?UTF-8?q?=E5=BC=82=E6=AD=A5=E8=AF=AD=E9=9F=B3=E8=AF=86=E5=88=AB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env.example | 18 + .gitignore | 1 + README.md | 70 + app/admin/stt/videos/page.tsx | 582 +++---- app/api/admin/stt/videos/route.ts | 22 +- app/api/fetcher/index.ts | 8 +- app/api/media.ts | 22 +- app/api/stt/batch/route.ts | 22 + app/api/stt/index.ts | 113 -- app/api/stt/prompt.md | 29 - app/api/stt/route.ts | 48 +- app/aweme/[awemeId]/Client.tsx | 2 +- .../[awemeId]/components/TranscriptPanel.tsx | 10 +- app/aweme/[awemeId]/types.ts | 2 - bun.lock | 231 +-- global.d.ts | 9 - lib/minio.ts | 5 + lib/stt/audio.ts | 85 + lib/stt/dashscope.ts | 159 ++ lib/stt/tasks.ts | 141 ++ lib/stt/types.ts | 33 + lib/stt/worker.ts | 112 ++ next.config.ts | 13 - package-lock.json | 1499 ++++++----------- package.json | 11 +- pm2.config.cjs | 14 + .../migration.sql | 47 + prisma/schema.prisma | 83 +- scripts/stt-smoke.ts | 77 + scripts/stt-worker.ts | 34 + tests/stt-media.test.ts | 65 + tests/stt-test-env.ts | 12 + tests/stt.integration.test.ts | 308 ++++ tests/stt.test.ts | 167 ++ 34 files changed, 2283 insertions(+), 1771 deletions(-) create mode 100644 .env.example create mode 100644 app/api/stt/batch/route.ts delete mode 100644 app/api/stt/index.ts delete mode 100644 app/api/stt/prompt.md delete mode 100644 global.d.ts create mode 100644 lib/stt/audio.ts create mode 100644 lib/stt/dashscope.ts create mode 100644 lib/stt/tasks.ts create mode 100644 lib/stt/types.ts create mode 100644 lib/stt/worker.ts create mode 100644 prisma/migrations/20261003000000_plain_transcription/migration.sql create mode 100644 scripts/stt-smoke.ts create mode 100644 scripts/stt-worker.ts create mode 100644 tests/stt-media.test.ts create mode 100644 tests/stt-test-env.ts create mode 100644 tests/stt.integration.test.ts create mode 100644 tests/stt.test.ts diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..29f564f --- /dev/null +++ b/.env.example @@ -0,0 +1,18 @@ +DATABASE_URL=postgresql://postgres:password@localhost:5432/douyin_archive +PORT=3000 +MINIO_ENDPOINT=localhost +MINIO_PORT=9000 +MINIO_USE_SSL=false +MINIO_ACCESS_KEY= +MINIO_SECRET_KEY= +MINIO_BUCKET_NAME=home-page +MINIO_PUBLIC_DOMAIN=https://your-public-storage.example.com + +# Native DashScope ASR, not the OpenAI compatible-mode endpoint. +DASHSCOPE_API_KEY= +DASHSCOPE_BASE_URL=https://dashscope.aliyuncs.com/api/v1 +DASHSCOPE_ASR_MODEL=qwen3-asr-flash-filetrans + +FFMPEG_PATH=ffmpeg +FFPROBE_PATH=ffprobe +CHROMIUM_HEADLESS=true diff --git a/.gitignore b/.gitignore index e5a2413..c2d6f26 100644 --- a/.gitignore +++ b/.gitignore @@ -32,6 +32,7 @@ yarn-error.log* # env files (can opt-in for committing if needed) .env* +!.env.example # vercel .vercel diff --git a/README.md b/README.md index 2c67347..2950c50 100644 --- a/README.md +++ b/README.md @@ -34,3 +34,73 @@ You can check out [the Next.js GitHub repository](https://github.com/vercel/next The easiest way to deploy your Next.js app is to use the [Vercel Platform](https://vercel.com/new?utm_medium=default-template&filter=next.js&utm_source=create-next-app&utm_campaign=create-next-app-readme) from the creators of Next.js. Check out our [Next.js deployment documentation](https://nextjs.org/docs/app/building-your-application/deploying) for more details. + +## 视频文字转写（DashScope ASR） + +仅使用 `qwen3-asr-flash-filetrans` 做文字识别，不使用通用模型理解、分类、总结或润色。新视频保存后入队；历史文字保留，后台可单条重转或批量补转。关闭页面不影响已入队任务。 + +处理流程：从 MinIO 将原视频流式下载到临时文件 → ffprobe 检查音轨及实际时长 → ffmpeg 提取整段 **16 kHz 单声道、64 kbps MP3** → 上传至 MinIO → 将 **音频 URL** 提交到 DashScope → 后台取回文字。没有分块、视频上传到模型或二次模型调用。无音轨直接保存空文字；ASR 空结果显示“未识别到文字”。 + +### 配置与启动 + +参考 `.env.example`。模型配置统一为： + +```dotenv +DASHSCOPE_API_KEY=你的百炼Key +DASHSCOPE_BASE_URL=https://dashscope.aliyuncs.com/api/v1 +DASHSCOPE_ASR_MODEL=qwen3-asr-flash-filetrans +``` + +这是 DashScope **原生 API** 地址，不是 `/compatible-mode/v1`。Key 和端点地域必须匹配。也可使用北京业务空间域名的 `/api/v1` 地址。旧 `OPENAI_*`、`QWEN_*` 和转写模块的全局代理配置不再使用；已有 Omni 的百炼 Key 可用于 ASR，仍需账号开通相应模型权限。 + +需要 ffmpeg、ffprobe（可通过 `FFMPEG_PATH`、`FFPROBE_PATH` 指定）。`MINIO_ENDPOINT`/端口/SSL 必须是运行机器可鉴权读写的 S3 API；`MINIO_PUBLIC_DOMAIN` 必须让 DashScope 能下载所上传的音频。执行器不修改 Bucket 策略。音频存储于 `transcription-audio/<视频ID>/<时间戳>.mp3`；独立对象防止手动重试覆盖仍在使用的音频。可为该前缀配置不少于 7 天的生命周期，视频原文件不受影响。本地视频临时文件在成功或失败后都会清理。 + +```bash +npm ci +npx prisma generate +npm run dev +# 另一个终端（开发与生产均需执行器） +npm run stt:worker +``` + +生产使用 `pm2 startOrReload pm2.config.cjs --env production --update-env`，配置同时包含 `DouyinArchive` 和 `DouyinArchive-ASR`。全局最多两个处理中任务，每 10 秒查一次供应商状态。任务保存在 PostgreSQL，不需要 Redis。未配置 Key 不阻止网站启动，转写会显示配置错误。 + +### 数据库迁移与发布顺序 + +1. 备份现有数据库，保存旧版本用于回退。 +2. 在停止服务前安装依赖、生成 Prisma Client 并运行 `npm run build`。 +3. 停止旧网站及旧后台执行器，运行 `npx prisma migrate deploy`。 +4. 用上述 PM2 命令启动新网站和 ASR 执行器，检查两者日志；确认转写与搜索后再完成发布验收。 + +迁移保留 `VideoTranscript` 的历史文字、关联和时间字段；删除 `speech_detected`、`language`、`audio_type`、`non_speech_summary`，以及部分部署存在的 `content_tags`。不自动补转或重转历史视频。新建 `TranscriptionTask` 保存状态、任务 ID、租约和重试信息。重转成功时原子替换文字，失败保留旧文字。 + +迁移会替换已知的旧全文检索触发器，并维护 `transcript_zh_tsv` 与 GIN 索引。**已有 `public.zhcfg` 中文分词配置会保留**；新库没有该配置时创建 `simple` 回退配置，它不提供中文分词。新部署若需要与旧库相同的中文搜索效果，先安装同样的中文分词扩展和 `zhcfg`；后续调整分词配置后应重新计算现有索引字段。不要在迁移后直接回滚到依赖旧字段的应用；需要回退时恢复数据库备份和旧应用版本。 + +### 接口和任务行为 + +- `POST /api/stt`，JSON `{ "awemeId": "视频ID" }`：首次转写或手动重转；处理中重复点击复用当前任务。 +- `GET /api/stt?awemeId=视频ID`：只读取 `{ status, error, transcript }`，不触发转写。`transcript` 为 `string[] | null`；`null` 表示没有完成记录，`[]` 表示完成但没有文字。 +- `POST /api/stt/batch`：一次将所有没有完成记录、且未在排队或处理中的视频加入数据库队列，失败记录可补转，历史完成记录不重转。 +- 状态为 `PENDING`（尚未入队）、`QUEUED`、`PROCESSING`、`SUCCEEDED`、`FAILED`。后台显示各状态数量、错误和重试按钮。 + +音频限制为 12 小时、2 GB，超限失败而非截断。执行器重启后复用已有供应商任务 ID；租约到期可恢复领取。提交前写入标记，提交阶段崩溃或超时无法确认任务 ID 时，标为失败并要求核对百炼任务后手动重试，避免盲目重复付费。查询和结果下载的网络错误/429/5xx 按 10 秒至 5 分钟退避重试；鉴权和明确的供应商失败不自动重提。超过 24 小时的任务标为失败。 + +### 验证 + +```bash +npm run test:stt +# 必须是独立本地库 douyin_asr_test；此测试会重建该库的 public schema +TEST_DATABASE_URL=postgresql://你的本地用户名@127.0.0.1:5432/douyin_asr_test npm run test:stt:integration +npx tsc --noEmit +npm run build +``` + +部署后可用管理页测试短视频、超过 5 分钟的视频、无发言/无音轨视频。检查新归档自动入队、关页面继续处理、重启续查、失败保留旧文字及文字搜索。 + +也可对已归档的视频独立实测（会上传音频、调用付费 ASR；**不修改数据库里的原转写**）： + +```bash +npx tsx scripts/stt-smoke.ts 7682010237578136858 +``` + +脚本将供应商任务 ID 和结果保存在 `/tmp/douyin-asr-<视频ID>.json`。已有未完成任务时应查询原任务，不要重复提交。 diff --git a/app/admin/stt/videos/page.tsx b/app/admin/stt/videos/page.tsx index f158877..06ce3a5 100644 --- a/app/admin/stt/videos/page.tsx +++ b/app/admin/stt/videos/page.tsx @@ -1,437 +1,257 @@ "use client"; -import { useState, useEffect } from "react"; +import Image from "next/image"; import Link from "next/link"; +import { useCallback, useEffect, useRef, useState } from "react"; import BackButton from "@/app/components/BackButton"; - -type VideoTranscript = { - id: string; - speech_detected: boolean; - language: string | null; - audio_type: string | null; - transcript: string | null; - non_speech_summary: string | null; -}; +import { + isActive, + statusLabels, + type TranscriptionState, + type TranscriptionStatus, +} from "@/lib/stt/types"; type Video = { aweme_id: string; desc: string; cover_url: string; duration_ms: number; - author: { - nickname: string; - }; - transcript: VideoTranscript | null; -}; - -type ApiResponse = { - videos: Video[]; - total: number; + author: { nickname: string }; + transcription: TranscriptionState; }; +const filters: ("all" | TranscriptionStatus)[] = [ + "all", + "PENDING", + "QUEUED", + "PROCESSING", + "SUCCEEDED", + "FAILED", +]; export default function SttVideosPage() { const [videos, setVideos] = useState([]); const [loading, setLoading] = useState(true); const [error, setError] = useState(null); - const [transcribing, setTranscribing] = useState>(new Set()); - const [filter, setFilter] = useState<"all" | "transcribed" | "pending">( - "all", - ); - const [batchTranscribing, setBatchTranscribing] = useState(false); - const [batchProgress, setBatchProgress] = useState({ current: 0, total: 0 }); - - useEffect(() => { - fetchVideos(); - }, []); - - const fetchVideos = async () => { + const [notice, setNotice] = useState(""); + const [submitting, setSubmitting] = useState>(new Set()); + const [batchSubmitting, setBatchSubmitting] = useState(false); + const [filter, setFilter] = useState<"all" | TranscriptionStatus>("all"); + const fetching = useRef(false); + const fetchVideos = useCallback(async () => { + if (fetching.current) return; + fetching.current = true; try { - setLoading(true); - const response = await fetch("/api/admin/stt/videos"); - if (!response.ok) { - throw new Error("Failed to fetch videos"); - } - const data: ApiResponse = await response.json(); + const response = await fetch("/api/admin/stt/videos", { + cache: "no-store", + }); + if (!response.ok) throw new Error("读取转写列表失败"); + const data: { videos: Video[] } = await response.json(); setVideos(data.videos); + setError(null); } catch (err) { - setError(err instanceof Error ? err.message : "Unknown error"); + setError(err instanceof Error ? err.message : "读取失败"); } finally { setLoading(false); + fetching.current = false; } - }; + }, []); + useEffect(() => { + void fetchVideos(); + const timer = setInterval(() => { + void fetchVideos(); + }, 5000); + return () => clearInterval(timer); + }, [fetchVideos]); - // 简单的 sleep 工具 - const sleep = (ms: number) => - new Promise((resolve) => setTimeout(resolve, ms)); - - type PollOptions = { - intervalMs?: number; - maxAttempts?: number; - }; - - // 轮询某个视频的转写状态直至完成或超时 - const pollVideoUntilTranscribed = async ( - awemeId: string, - { intervalMs = 2000, maxAttempts = 30 }: PollOptions = {}, - ): Promise => { - for (let attempt = 0; attempt < maxAttempts; attempt++) { - try { - const resp = await fetch("/api/admin/stt/videos"); - if (resp.ok) { - const data: ApiResponse = await resp.json(); - setVideos(data.videos); - const v = data.videos.find((x) => x.aweme_id === awemeId); - if (v && v.transcript) { - return true; - } - } - } catch (e) { - // 忽略网络错误，继续轮询 - } - await sleep(intervalMs); - } - return false; - }; - - const handleTranscribe = async (awemeId: string) => { - setTranscribing((prev) => new Set(prev).add(awemeId)); + async function handleTranscribe(awemeId: string) { + setSubmitting((prev) => new Set(prev).add(awemeId)); try { - const response = await fetch(`/api/stt?awemeId=${awemeId}`); - if (!response.ok) { - throw new Error("Failed to transcribe video"); - } - // 轮询直到该视频转写完成或超时，避免后端异步导致立即刷新拿不到最新状态 - const done = await pollVideoUntilTranscribed(awemeId); - if (!done) { - // 兜底刷新一次 - await fetchVideos(); - } + const response = await fetch("/api/stt", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ awemeId }), + }); + const data = await response.json(); + if (!response.ok) throw new Error(data.error || "任务入队失败"); + setVideos((prev) => + prev.map((video) => + video.aweme_id === awemeId + ? { ...video, transcription: data } + : video, + ), + ); } catch (err) { - alert(err instanceof Error ? err.message : "Transcription failed"); + setNotice(err instanceof Error ? err.message : "任务入队失败"); } finally { - setTranscribing((prev) => { + setSubmitting((prev) => { const next = new Set(prev); next.delete(awemeId); return next; }); } - }; - - const handleBatchTranscribe = async () => { - const pendingVideos = videos.filter((v) => v.transcript === null); - if (pendingVideos.length === 0) { - alert("没有待转写的视频"); - return; - } - - if ( - !confirm( - `确定要转写 ${pendingVideos.length} 个视频吗？这可能需要较长时间。`, - ) - ) { - return; - } - - setBatchTranscribing(true); - setBatchProgress({ current: 0, total: pendingVideos.length }); - - const pollers: Promise[] = []; - for (let i = 0; i < pendingVideos.length; i++) { - const video = pendingVideos[i]; - setBatchProgress({ current: i + 1, total: pendingVideos.length }); - setTranscribing((prev) => new Set(prev).add(video.aweme_id)); - - try { - const response = await fetch(`/api/stt?awemeId=${video.aweme_id}`); - if (!response.ok) { - console.error(`Failed to transcribe video ${video.aweme_id}`); - } - // 为该视频启动后台轮询，不阻塞批处理的顺序执行 - const p = pollVideoUntilTranscribed(video.aweme_id).finally(() => { - setTranscribing((prev) => { - const next = new Set(prev); - next.delete(video.aweme_id); - return next; - }); - }); - pollers.push(p); - } catch (err) { - console.error(`Error transcribing video ${video.aweme_id}:`, err); - } - } - - // 等待所有轮询结束（完成或超时），再进行最终刷新 - if (pollers.length > 0) { - await Promise.allSettled(pollers); - } - await fetchVideos(); - setBatchTranscribing(false); - setBatchProgress({ current: 0, total: 0 }); - alert("批量转写完成！"); - }; - - const filteredVideos = videos.filter((v) => { - if (filter === "transcribed") return v.transcript !== null; - if (filter === "pending") return v.transcript === null; - return true; - }); - - const stats = { - total: videos.length, - transcribed: videos.filter((v) => v.transcript !== null).length, - pending: videos.filter((v) => v.transcript === null).length, - }; - - if (loading) { - return ( -
-
-
-
-

加载中...

-
-
-
- ); } - - if (error) { - return ( -
-
-
-

错误: {error}

- -
-
-
- ); + async function handleBatch() { + setBatchSubmitting(true); + try { + const response = await fetch("/api/stt/batch", { method: "POST" }); + const data = await response.json(); + if (!response.ok) throw new Error(data.error || "批量入队失败"); + setNotice(`已加入 ${data.queued} 个转写任务；关闭页面后仍会继续处理。`); + await fetchVideos(); + } catch (err) { + setNotice(err instanceof Error ? err.message : "批量入队失败"); + } finally { + setBatchSubmitting(false); + } } + const count = (status: TranscriptionStatus) => + videos.filter((v) => v.transcription.status === status).length; + const eligible = videos.filter( + (v) => + v.transcription.transcript === null && !isActive(v.transcription.status), + ).length; + const shown = videos.filter( + (v) => filter === "all" || v.transcription.status === filter, + ); return ( -
-
- {/* Header */} -
- -

- 视频转写管理 -

-

管理视频的语音转写状态

+
+
+ +

视频文字转写

+

+ 已完成 {count("SUCCEEDED")} · 处理中 {count("PROCESSING")} · 排队中{" "} + {count("QUEUED")} · 失败 {count("FAILED")} · 待转写 {count("PENDING")} +

+ {error && ( +

+ {error} +

+ )} + {notice && ( + {notice} + )} +
+
+ {filters.map((value) => ( + + ))} +
+
- - {/* Stats */} -
-
-
总视频数
-
- {stats.total} -
-
-
-
已转写
-
- {stats.transcribed} -
-
-
-
待转写
-
- {stats.pending} -
-
-
- - {/* Filter */} -
-
-
- - - -
-
- {batchTranscribing && ( -
- 正在转写: {batchProgress.current} / {batchProgress.total} -
- )} - -
-
-
- - {/* Video List */} -
-
- - - - - - - - - - - {filteredVideos.map((video) => ( - - +
- 视频 - - 时长 - - 转写状态 - - 操作 -
-
- -
+
+ + + + + + + + + + + {shown.map((video) => { + const state = video.transcription; + const active = + isActive(state.status) || submitting.has(video.aweme_id); + return ( + + - - - - ))} - -
视频时长状态与文字操作
+
+ {video.cover_url && ( + + )} +
{video.desc || "无描述"} -

+

@{video.author.nickname}

- {Math.floor(video.duration_ms / 1000)}s + + {Math.floor(video.duration_ms / 1000)} 秒 - {video.transcript ? ( -
-
- - 已转写 - - {video.transcript.language && ( - - {video.transcript.language} - - )} -
- {video.transcript.speech_detected ? ( -
- {video.transcript.transcript} -
- ) : ( -
- - {video.transcript.audio_type || "非语音"} - - {video.transcript.non_speech_summary && ( - - - {video.transcript.non_speech_summary} - - )} -
- )} -
- ) : ( - - 待转写 - + 		+

+ {statusLabels[state.status]} +

+ {state.error && ( +

+ {state.error} +

+ )} + {state.transcript !== null && ( +

+ {state.transcript.join("\n") || "未识别到文字"} +

)} 		- {video.transcript ? ( - - ) : ( - - )} + +
-
- - {filteredVideos.length === 0 && ( -
没有找到视频
+ ); + })} +
+ {!shown.length && ( +

+ {loading ? "加载中…" : "没有找到视频"} +

)}
-
+
); } diff --git a/app/api/admin/stt/videos/route.ts b/app/api/admin/stt/videos/route.ts index 26b104c..cff2223 100644 --- a/app/api/admin/stt/videos/route.ts +++ b/app/api/admin/stt/videos/route.ts @@ -1,26 +1,23 @@ import { NextResponse } from "next/server"; -import { prisma } from "@/lib/prisma"; import { getFileUrl } from "@/lib/minio"; +import { prisma } from "@/lib/prisma"; +import { transcriptionState } from "@/lib/stt/types"; export async function GET() { try { const videos = await prisma.video.findMany({ orderBy: { created_at: "desc" }, - take: 1000, include: { author: { select: { nickname: true, }, }, + transcriptionTask: true, transcript: { select: { id: true, - speech_detected: true, - language: true, - audio_type: true, transcript: true, - non_speech_summary: true, }, }, }, @@ -29,21 +26,12 @@ export async function GET() { const formattedVideos = videos.map((v) => ({ aweme_id: v.aweme_id, desc: v.desc, - cover_url: getFileUrl(v.cover_url ?? ""), + cover_url: v.cover_url ? getFileUrl(v.cover_url) : "", duration_ms: v.duration_ms, author: { nickname: v.author.nickname, }, - transcript: v.transcript - ? { - id: v.transcript.id, - speech_detected: v.transcript.speech_detected, - language: v.transcript.language, - audio_type: v.transcript.audio_type, - transcript: v.transcript.transcript, - non_speech_summary: v.transcript.non_speech_summary, - } - : null, + transcription: transcriptionState(v.transcript, v.transcriptionTask), })); return NextResponse.json({ diff --git a/app/api/fetcher/index.ts b/app/api/fetcher/index.ts index 069cf79..6650f64 100644 --- a/app/api/fetcher/index.ts +++ b/app/api/fetcher/index.ts @@ -21,7 +21,7 @@ import { releaseIsolatedContext, } from "@/app/api/fetcher/browser"; import { extractFirstFrame } from "@/app/api/media"; -import { transcriptAweme } from "../stt"; +import { enqueueTranscription } from "@/lib/stt/tasks"; const DETAIL_PATH = "/aweme/v1/web/aweme/detail/"; const COMMENT_PATH = "/aweme/v1/web/comment/list/"; @@ -429,7 +429,11 @@ export async function scrapeDouyin(url: string) { fps ?? undefined, ); console.log(chalk.green.bold("✓ 视频作品保存成功")); - transcriptAweme(detail.aweme_detail.aweme_id).catch((e) => {}); // 异步转写，不阻塞主流程 + try { + await enqueueTranscription(detail.aweme_detail.aweme_id); + } catch { + console.error("视频已保存，但转写任务入队失败，可在后台批量补转"); + } return { type: "video", ...saved }; } else { throw new ScrapeError( diff --git a/app/api/media.ts b/app/api/media.ts index a8786c9..0d85cb1 100644 --- a/app/api/media.ts +++ b/app/api/media.ts @@ -111,17 +111,17 @@ export async function getVideoDuration( } /** - * 使用 ffmpeg 将视频二进制转换为音频，返回音频 Buffer + * 使用 ffmpeg 将视频 Buffer 或本地文件转换为音频，返回音频 Buffer * * 默认输出 mp3，可通过 opts.format 指定 'mp3' | 'aac' | 'wav' - * - mp3: 使用 libmp3lame，contentType 为 'audio/mpeg' + * - mp3: 使用 libmp3lame，16 kHz 单声道，contentType 为 'audio/mpeg' * - aac: 输出 m4a 容器（AAC 编码），contentType 为 'audio/mp4' * - wav: PCM 16-bit，44100Hz，双声道，contentType 为 'audio/wav' * * 若系统未安装 ffmpeg 或未在 PATH 中，返回 null 并给出提示。 */ export async function extractAudio( - videoBuffer: Buffer, + videoBuffer: Buffer | string, opts?: { format?: "mp3" | "aac" | "wav"; bitrateKbps?: number }, ): Promise<{ buffer: Buffer; contentType: string; ext: string } | null> { const ffmpegCmd = process.env.FFMPEG_PATH || "ffmpeg"; @@ -129,6 +129,7 @@ export async function extractAudio( const base = `dy_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`; const inPath = path.join(tmpDir, `${base}.mp4`); + const inputPath = typeof videoBuffer === "string" ? videoBuffer : inPath; const format = opts?.format ?? "mp3"; const bitrate = Math.max(32, Math.min(512, opts?.bitrateKbps ?? 192)); // 安全范围 32~512 kbps @@ -174,19 +175,28 @@ export async function extractAudio( } try { - await fs.writeFile(inPath, videoBuffer); + if (typeof videoBuffer !== "string") await fs.writeFile(inPath, videoBuffer); const args = [ "-hide_banner", "-loglevel", "error", "-i", - inPath, + inputPath, + "-map", + "0:a:0", + "-ac", + "1", + "-ar", + "16000", "-vn", // 丢弃视频流 ...codecArgs, "-y", outPath, ]; - await execFileAsync(ffmpegCmd, args, { windowsHide: true }); + await execFileAsync(ffmpegCmd, args, { + windowsHide: true, + timeout: 60 * 60 * 1000, + }); const audio = await fs.readFile(outPath); return { buffer: audio, contentType, ext }; } catch (e: any) { diff --git a/app/api/stt/batch/route.ts b/app/api/stt/batch/route.ts new file mode 100644 index 0000000..91994b2 --- /dev/null +++ b/app/api/stt/batch/route.ts @@ -0,0 +1,22 @@ +import { NextResponse } from "next/server"; +import { getAsrConfig } from "@/lib/stt/dashscope"; +import { enqueueMissingTranscriptions } from "@/lib/stt/tasks"; +export const runtime = "nodejs"; +export async function POST() { + try { + getAsrConfig(); + } catch (error) { + return NextResponse.json( + { error: (error as Error).message }, + { status: 503 }, + ); + } + try { + return NextResponse.json( + { queued: await enqueueMissingTranscriptions() }, + { status: 202 }, + ); + } catch { + return NextResponse.json({ error: "批量任务入队失败" }, { status: 500 }); + } +} diff --git a/app/api/stt/index.ts b/app/api/stt/index.ts deleted file mode 100644 index 4c05c7f..0000000 --- a/app/api/stt/index.ts +++ /dev/null @@ -1,113 +0,0 @@ -import fs from "fs"; -import OpenAI from "openai"; -import prompt from "./prompt.md"; -import { prisma } from "@/lib/prisma"; -import { downloadFile } from "@/lib/minio"; -import { extractAudio } from "../media"; -import { z } from "zod"; -import { zodResponseFormat } from "openai/helpers/zod"; -import { ProxyAgent, setGlobalDispatcher } from "undici"; - -const proxy = process.env.HTTPS_PROXY || process.env.HTTP_PROXY; -if (proxy) { - setGlobalDispatcher(new ProxyAgent(proxy)); -} - -const client = new OpenAI({ - apiKey: process.env.OPENAI_API_KEY, - baseURL: process.env.OPENAI_API_BASE_URL, -}); - -// Zod schema aligned with prompt.md -export const SttSchema = z - .object({ - speech_detected: z.boolean(), - language: z.string().min(1).nullable(), - audio_type: z.string().nullable(), - transcript: z.array(z.string()), - non_speech_summary: z.string().nullable(), - }) - .strict(); - -export type SttResult = z.infer; - -async function transcriptAudio(audio: Buffer | string) { - if (typeof audio === "string") { - audio = fs.readFileSync(audio); - } - const base64Audio = Buffer.from(audio).toString("base64"); - - const completion = await client.chat.completions.create({ - model: "gemini-2.5-flash", - messages: [ - { - role: "user", - content: [ - { type: "text", text: prompt }, - { - type: "input_audio", - input_audio: { data: base64Audio, format: "mp3" }, - }, - ], - }, - ], - response_format: zodResponseFormat(SttSchema, "stt_result"), - }); - - const data = completion.choices?.[0]?.message; - console.log("转写结果", data.content); - - if (!data || !data.content) { - throw new Error("No STT result returned from model"); - } - try { - return JSON.parse(data.content) as SttResult; - } catch (e) { - throw new Error("Failed to parse STT result JSON"); - } -} - -export async function transcriptAweme(awemeId: string): Promise { - const aweme = await prisma.video.findUnique({ - where: { aweme_id: awemeId }, - }); - if (!aweme) { - throw new Error("Aweme not found or aweme is not a video post"); - } - const vPath = aweme.video_url; - const buffer = await downloadFile(vPath); - - const audioDat = await extractAudio(buffer, { - format: "mp3", - bitrateKbps: 128, - }); - - if (!audioDat || !audioDat.buffer) { - throw new Error("Failed to extract audio from video"); - } - - // 调用大模型生成转写结果 - const result = await transcriptAudio(audioDat.buffer); - - // 将转写结果持久化到数据库 - await prisma.videoTranscript.upsert({ - where: { videoId: awemeId }, - update: { - speech_detected: result.speech_detected, - language: result.language, - audio_type: result.audio_type, - transcript: result.transcript, - non_speech_summary: result.non_speech_summary, - }, - create: { - videoId: awemeId, - speech_detected: result.speech_detected, - language: result.language, - audio_type: result.audio_type, - transcript: result.transcript, - non_speech_summary: result.non_speech_summary, - }, - }); - - return result; -} diff --git a/app/api/stt/prompt.md b/app/api/stt/prompt.md deleted file mode 100644 index ef32b13..0000000 --- a/app/api/stt/prompt.md +++ /dev/null @@ -1,29 +0,0 @@ -你将接收一段音频。请完成：你将接收一段音频。语音活动检测（VAD）与声源分类。 - -**输出 JSON Schema（示例）** - -```json -{ - "speech_detected": true, - "language": "zh-CN", - "audio_type": null, - "transcript": ["大家好，我是xxx", "欢迎来到今天的视频", "今天我们来聊聊AI的未来"], - "non_speech_summary": null, -} -``` - -**当无发言时返回：** - -```json -{ - "speech_detected": false, - "language": null, - "audio_type": "music | ambience | animal | mechanical | other", - "transcript": [], - "non_speech_summary": "示例：纯音乐-钢琴独奏，节奏舒缓；或 环境声-雨声伴随雷鸣。", -} -``` - -** 规则补充 ** - -* 允许`language` 为多标签（如 "zh-CN,en"）或为 `null`（无发言时）。 diff --git a/app/api/stt/route.ts b/app/api/stt/route.ts index 40ca475..9350233 100644 --- a/app/api/stt/route.ts +++ b/app/api/stt/route.ts @@ -1,23 +1,33 @@ +import { type NextRequest, NextResponse } from "next/server"; +import { enqueueTranscription, readTranscription } from "@/lib/stt/tasks"; +import { isActive } from "@/lib/stt/types"; export const runtime = "nodejs"; -import { NextRequest, NextResponse } from "next/server"; -import { prisma } from "@/lib/prisma"; -import type { FeedItem, FeedResponse } from "@/app/types/feed"; -import { getFileUrl } from "@/lib/minio"; -import { transcriptAweme } from "."; - -// Contract -// Inputs: search params { before?: ISOString, limit?: number } -// Output: { items: FeedItem[], nextCursor: ISOString | null } - +export const dynamic = "force-dynamic"; +const validId = (id: unknown): id is string => + typeof id === "string" && /^[A-Za-z0-9_-]{1,128}$/.test(id); export async function GET(req: NextRequest) { - const { searchParams } = new URL(req.url); - const awemeId = searchParams.get("awemeId"); - if (!awemeId) { - return NextResponse.json( - { error: "Missing awemeId parameter" }, - { status: 400 }, - ); + const id = req.nextUrl.searchParams.get("awemeId"); + if (!validId(id)) + return NextResponse.json({ error: "需要有效的 awemeId" }, { status: 400 }); + try { + const state = await readTranscription(id); + return state + ? NextResponse.json(state) + : NextResponse.json({ error: "视频不存在" }, { status: 404 }); + } catch { + return NextResponse.json({ error: "读取转写状态失败" }, { status: 500 }); + } +} +export async function POST(req: NextRequest) { + const body = await req.json().catch(() => null); + if (!validId(body?.awemeId)) + return NextResponse.json({ error: "需要有效的 awemeId" }, { status: 400 }); + try { + const state = await enqueueTranscription(body.awemeId, true); + return state + ? NextResponse.json(state, { status: isActive(state.status) ? 202 : 200 }) + : NextResponse.json({ error: "视频不存在" }, { status: 404 }); + } catch { + return NextResponse.json({ error: "转写任务入队失败" }, { status: 500 }); } - const script = await transcriptAweme(awemeId); - return NextResponse.json(script); } diff --git a/app/aweme/[awemeId]/Client.tsx b/app/aweme/[awemeId]/Client.tsx index 901b1d7..904f68f 100644 --- a/app/aweme/[awemeId]/Client.tsx +++ b/app/aweme/[awemeId]/Client.tsx @@ -556,7 +556,7 @@ export default function AwemeDetailClient({ segmentProgress={imageCarouselState.segProgress} musicUrl={isVideo ? undefined : (data as ImageData).music_url} audioRef={audioRef} - hasTranscript={isVideo && !!transcript?.speech_detected} + hasTranscript={isVideo && !!transcript} onShowTranscript={() => setTranscriptOpen(true)} onTogglePlay={togglePlay} onSeek={handleControlSeek} diff --git a/app/aweme/[awemeId]/components/TranscriptPanel.tsx b/app/aweme/[awemeId]/components/TranscriptPanel.tsx index e2cb36b..bdd30b9 100644 --- a/app/aweme/[awemeId]/components/TranscriptPanel.tsx +++ b/app/aweme/[awemeId]/components/TranscriptPanel.tsx @@ -36,7 +36,7 @@ export function TranscriptPanel({ }); }; - if (!transcript?.speech_detected || !transcript.transcript?.length) { + if (!transcript?.transcript?.length) { return (
-

该视频暂无语音转录内容

+

{transcript ? "未识别到文字" : "该视频尚未转写"}

@@ -80,11 +80,7 @@ export function TranscriptPanel({

语音转录

- {transcript.language && ( -

- 语言: {transcript.language} -

- )} +