完成转录功能，中文脚本搜索，播放界面显示脚本
This commit is contained in:
@@ -0,0 +1,60 @@
|
||||
import { NextResponse } from 'next/server';
|
||||
import { prisma } from '@/lib/prisma';
|
||||
import { getFileUrl } from '@/lib/minio';
|
||||
|
||||
export async function GET() {
|
||||
try {
|
||||
const videos = await prisma.video.findMany({
|
||||
orderBy: { created_at: 'desc' },
|
||||
take: 1000,
|
||||
include: {
|
||||
author: {
|
||||
select: {
|
||||
nickname: true,
|
||||
},
|
||||
},
|
||||
transcript: {
|
||||
select: {
|
||||
id: true,
|
||||
speech_detected: true,
|
||||
language: true,
|
||||
audio_type: true,
|
||||
transcript: true,
|
||||
non_speech_summary: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const formattedVideos = videos.map((v) => ({
|
||||
aweme_id: v.aweme_id,
|
||||
desc: v.desc,
|
||||
cover_url: getFileUrl(v.cover_url ?? ''),
|
||||
duration_ms: v.duration_ms,
|
||||
author: {
|
||||
nickname: v.author.nickname,
|
||||
},
|
||||
transcript: v.transcript
|
||||
? {
|
||||
id: v.transcript.id,
|
||||
speech_detected: v.transcript.speech_detected,
|
||||
language: v.transcript.language,
|
||||
audio_type: v.transcript.audio_type,
|
||||
transcript: v.transcript.transcript,
|
||||
non_speech_summary: v.transcript.non_speech_summary,
|
||||
}
|
||||
: null,
|
||||
}));
|
||||
|
||||
return NextResponse.json({
|
||||
videos: formattedVideos,
|
||||
total: videos.length,
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('Failed to fetch videos:', error);
|
||||
return NextResponse.json(
|
||||
{ error: 'Failed to fetch videos' },
|
||||
{ status: 500 }
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -12,6 +12,7 @@ import { saveToDB, saveImagePostToDB } from '@/app/api/fetcher/persist';
|
||||
import chalk from 'chalk';
|
||||
import { acquireIsolatedContext, releaseIsolatedContext } from '@/app/api/fetcher/browser';
|
||||
import { extractFirstFrame } from '@/app/api/media';
|
||||
import { transcriptAweme } from '../stt';
|
||||
|
||||
const DETAIL_PATH = '/aweme/v1/web/aweme/detail/';
|
||||
const COMMENT_PATH = '/aweme/v1/web/comment/list/';
|
||||
@@ -308,6 +309,7 @@ export async function scrapeDouyin(url: string) {
|
||||
|
||||
const saved = await saveToDB(context, detail, comments, uploadedUrl, bestPlayAddr?.width, bestPlayAddr?.height, coverUrl, fps ?? undefined);
|
||||
console.log(chalk.green.bold('✓ 视频作品保存成功'));
|
||||
transcriptAweme(detail.aweme_detail.aweme_id).catch((e) => {}); // 异步转写，不阻塞主流程
|
||||
return { type: "video", ...saved };
|
||||
} else {
|
||||
throw new ScrapeError('无法判定作品类型,接口响应异常', 500, 'UNKNOWN_TYPE');
|
||||
|
||||
@@ -0,0 +1,164 @@
|
||||
import { json } from '@/lib/json';
|
||||
import { getFileUrl } from '@/lib/minio';
|
||||
import { prisma } from '@/lib/prisma'; // 你的 Prisma 客户端实例
|
||||
import { NextResponse } from 'next/server';
|
||||
|
||||
export async function GET(req: Request) {
|
||||
const { searchParams } = new URL(req.url);
|
||||
const q = (searchParams.get('q') || '').trim();
|
||||
const page = Math.max(1, Number(searchParams.get('page') || 1));
|
||||
const limit = Math.min(50, Math.max(1, Number(searchParams.get('limit') || 20)));
|
||||
const offset = (page - 1) * limit;
|
||||
|
||||
if (!q) {
|
||||
return NextResponse.json({ results: [], total: 0 });
|
||||
}
|
||||
|
||||
try {
|
||||
// 核心查询：Postgres 中文全文检索（视频转写 + 视频desc + 图文desc）+ 相关度排序 + 摘要高亮
|
||||
// 说明：
|
||||
// - 合并搜索来源：VideoTranscript.transcript_zh_tsv、Video.desc 与 ImagePost.desc
|
||||
// - 使用 UNION ALL 合并视频和图文的搜索结果
|
||||
// - rank 取两者匹配得分的最大值，snippet 优先展示转写匹配，否则展示 desc 匹配
|
||||
const rows = await prisma.$queryRaw<
|
||||
{
|
||||
id: string;
|
||||
awemeId: string;
|
||||
type: 'video' | 'image';
|
||||
rank: number;
|
||||
snippet: string;
|
||||
}[]
|
||||
>`
|
||||
WITH tsq AS (
|
||||
SELECT websearch_to_tsquery('zhcfg', ${q}) AS query
|
||||
)
|
||||
-- 视频搜索结果
|
||||
SELECT
|
||||
COALESCE(vt.id, 'desc:' || v.aweme_id) AS id,
|
||||
v.aweme_id AS "awemeId",
|
||||
'video'::text AS type,
|
||||
GREATEST(
|
||||
COALESCE(ts_rank(vt.transcript_zh_tsv, tsq.query), 0),
|
||||
COALESCE(ts_rank(to_tsvector('zhcfg', v.desc), tsq.query), 0)
|
||||
) AS rank,
|
||||
CASE
|
||||
WHEN vt.transcript_zh_tsv @@ tsq.query THEN
|
||||
ts_headline(
|
||||
'zhcfg',
|
||||
array_to_string(vt.transcript, ' '),
|
||||
tsq.query,
|
||||
'StartSel=<mark>,StopSel=</mark>,MaxFragments=2,MinWords=2,MaxWords=20'
|
||||
)
|
||||
ELSE
|
||||
ts_headline(
|
||||
'zhcfg',
|
||||
v.desc,
|
||||
tsq.query,
|
||||
'StartSel=<mark>,StopSel=</mark>,MaxFragments=2,MinWords=2,MaxWords=20'
|
||||
)
|
||||
END AS snippet
|
||||
FROM "Video" v
|
||||
LEFT JOIN "VideoTranscript" vt ON vt."videoId" = v.aweme_id,
|
||||
tsq
|
||||
WHERE
|
||||
(vt.transcript_zh_tsv @@ tsq.query OR to_tsvector('zhcfg', v.desc) @@ tsq.query)
|
||||
|
||||
UNION ALL
|
||||
|
||||
-- 图文搜索结果
|
||||
SELECT
|
||||
'img_desc:' || ip.aweme_id AS id,
|
||||
ip.aweme_id AS "awemeId",
|
||||
'image'::text AS type,
|
||||
ts_rank(to_tsvector('zhcfg', ip.desc), tsq.query) AS rank,
|
||||
ts_headline(
|
||||
'zhcfg',
|
||||
ip.desc,
|
||||
tsq.query,
|
||||
'StartSel=<mark>,StopSel=</mark>,MaxFragments=2,MinWords=2,MaxWords=20'
|
||||
) AS snippet
|
||||
FROM "ImagePost" ip, tsq
|
||||
WHERE to_tsvector('zhcfg', ip.desc) @@ tsq.query
|
||||
|
||||
ORDER BY rank DESC
|
||||
OFFSET ${offset}
|
||||
LIMIT ${limit};
|
||||
`;
|
||||
|
||||
// 查询总数（参数化，避免注入）
|
||||
const totalRows = await prisma.$queryRaw<{
|
||||
count: number;
|
||||
}[]>`
|
||||
WITH tsq AS (
|
||||
SELECT websearch_to_tsquery('zhcfg', ${q}) AS query
|
||||
)
|
||||
SELECT (
|
||||
(SELECT COUNT(*)::int FROM "Video" v
|
||||
LEFT JOIN "VideoTranscript" vt ON vt."videoId" = v.aweme_id
|
||||
WHERE (vt.transcript_zh_tsv @@ tsq.query OR to_tsvector('zhcfg', v.desc) @@ tsq.query))
|
||||
+
|
||||
(SELECT COUNT(*)::int FROM "ImagePost" ip
|
||||
WHERE to_tsvector('zhcfg', ip.desc) @@ tsq.query)
|
||||
) AS count
|
||||
FROM tsq;
|
||||
`;
|
||||
|
||||
// 分离视频和图文ID
|
||||
const videoIds = rows.filter(r => r.type === 'video').map(r => r.awemeId);
|
||||
const imagePostIds = rows.filter(r => r.type === 'image').map(r => r.awemeId);
|
||||
|
||||
// 批量查询视频元信息
|
||||
const videos = videoIds.length > 0 ? (await prisma.video.findMany({
|
||||
where: { aweme_id: { in: videoIds } },
|
||||
select: {
|
||||
aweme_id: true,
|
||||
desc: true,
|
||||
cover_url: true,
|
||||
video_url: true,
|
||||
duration_ms: true,
|
||||
author: true
|
||||
},
|
||||
})).map(v => (
|
||||
{ ...v, cover_url: getFileUrl(v.cover_url || ''),
|
||||
author: { ...v.author, avatar_url: getFileUrl(v.author.avatar_url || '') },
|
||||
video_url: getFileUrl(v.video_url || '') })
|
||||
) : [];
|
||||
|
||||
// 批量查询图文元信息
|
||||
const imagePosts = imagePostIds.length > 0 ? (await prisma.imagePost.findMany({
|
||||
where: { aweme_id: { in: imagePostIds } },
|
||||
select: {
|
||||
aweme_id: true,
|
||||
desc: true,
|
||||
author: true,
|
||||
images: {
|
||||
orderBy: { order: 'asc' },
|
||||
take: 1,
|
||||
select: {
|
||||
url: true,
|
||||
width: true,
|
||||
height: true,
|
||||
}
|
||||
}
|
||||
},
|
||||
})).map(ip => ({
|
||||
...ip,
|
||||
author: { ...ip.author, avatar_url: getFileUrl(ip.author.avatar_url || '') },
|
||||
cover_url: ip.images[0] ? getFileUrl(ip.images[0].url) : null,
|
||||
})) : [];
|
||||
|
||||
return json({
|
||||
results: rows.map(r => ({
|
||||
...r,
|
||||
video: r.type === 'video' ? videos.find(v => v.aweme_id === r.awemeId) : undefined,
|
||||
imagePost: r.type === 'image' ? imagePosts.find(ip => ip.aweme_id === r.awemeId) : undefined,
|
||||
})),
|
||||
total: totalRows?.[0]?.count ?? 0,
|
||||
page,
|
||||
limit,
|
||||
});
|
||||
} catch (err) {
|
||||
console.error('Search error:', err);
|
||||
return NextResponse.json({ error: 'Search failed' }, { status: 500 });
|
||||
}
|
||||
}
|
||||
+55
-15
@@ -4,45 +4,62 @@ import prompt from "./prompt.md";
|
||||
import { prisma } from "@/lib/prisma";
|
||||
import { downloadFile } from "@/lib/minio";
|
||||
import { extractAudio } from "../media";
|
||||
import { z } from "zod";
|
||||
import { zodResponseFormat } from "openai/helpers/zod";
|
||||
|
||||
const client = new OpenAI({
|
||||
apiKey: process.env.OPENAI_API_KEY,
|
||||
baseURL: process.env.OPENAI_API_BASE_URL,
|
||||
});
|
||||
|
||||
// Zod schema aligned with prompt.md
|
||||
export const SttSchema = z
|
||||
.object({
|
||||
speech_detected: z.boolean(),
|
||||
language: z.string().min(1).nullable(),
|
||||
audio_type: z.string().nullable(),
|
||||
transcript: z.array(z.string()),
|
||||
non_speech_summary: z.string().nullable(),
|
||||
})
|
||||
.strict();
|
||||
|
||||
export type SttResult = z.infer<typeof SttSchema>;
|
||||
|
||||
async function transcriptAudio(audio: Buffer | string) {
|
||||
if (typeof audio === "string") {
|
||||
audio = fs.readFileSync(audio);
|
||||
}
|
||||
const base64Audio = Buffer.from(audio).toString("base64");
|
||||
|
||||
const response = await client.chat.completions.create({
|
||||
model: "gemini-2.5-flash-lite",
|
||||
const completion = await client.chat.completions.create({
|
||||
model: "gemini-2.5-flash",
|
||||
messages: [
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
{
|
||||
type: "text",
|
||||
text: prompt,
|
||||
},
|
||||
{ type: "text", text: prompt },
|
||||
{
|
||||
type: "input_audio",
|
||||
input_audio: {
|
||||
data: base64Audio,
|
||||
format: "mp3",
|
||||
},
|
||||
input_audio: { data: base64Audio, format: "mp3" },
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
response_format: zodResponseFormat(SttSchema, "stt_result"),
|
||||
});
|
||||
|
||||
console.log(response.choices[0].message.content);
|
||||
return response.choices[0].message.content;
|
||||
const data = completion.choices?.[0]?.message
|
||||
if (!data) {
|
||||
throw new Error("No STT result returned from model");
|
||||
}
|
||||
const parsed = SttSchema.safeParse(data?.content).data;
|
||||
if (!parsed) {
|
||||
throw new Error("Failed to parse STT result with zod");
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
export async function transcriptAweme(awemeId: string) {
|
||||
export async function transcriptAweme(awemeId: string): Promise<SttResult> {
|
||||
const aweme = await prisma.video.findUnique({
|
||||
where: { aweme_id: awemeId },
|
||||
});
|
||||
@@ -52,11 +69,34 @@ export async function transcriptAweme(awemeId: string) {
|
||||
const vPath = aweme.video_url
|
||||
const buffer = await downloadFile(vPath);
|
||||
|
||||
const audioDat = await extractAudio(buffer, { format: "mp3", bitrateKbps: 96 });
|
||||
const audioDat = await extractAudio(buffer, { format: "mp3", bitrateKbps: 128 });
|
||||
|
||||
if (!audioDat || !audioDat.buffer) {
|
||||
throw new Error("Failed to extract audio from video");
|
||||
}
|
||||
|
||||
return await transcriptAudio(audioDat.buffer);
|
||||
// 调用大模型生成转写结果
|
||||
const result = await transcriptAudio(audioDat.buffer);
|
||||
|
||||
// 将转写结果持久化到数据库
|
||||
await prisma.videoTranscript.upsert({
|
||||
where: { videoId: awemeId },
|
||||
update: {
|
||||
speech_detected: result.speech_detected,
|
||||
language: result.language,
|
||||
audio_type: result.audio_type,
|
||||
transcript: result.transcript,
|
||||
non_speech_summary: result.non_speech_summary,
|
||||
},
|
||||
create: {
|
||||
videoId: awemeId,
|
||||
speech_detected: result.speech_detected,
|
||||
language: result.language,
|
||||
audio_type: result.audio_type,
|
||||
transcript: result.transcript,
|
||||
non_speech_summary: result.non_speech_summary,
|
||||
},
|
||||
});
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
+5
-27
@@ -1,42 +1,24 @@
|
||||
你将接收一段音频。请完成：
|
||||
A.语音活动检测（VAD）与声源分类；
|
||||
B.条件式处理：
|
||||
- 若包含可辨识的人类发言：** 进行转录 **（保留原语言，不翻译），并尽可能给出说话人分离与时间戳；
|
||||
- 若不包含人类发言：** 不转录 **，仅返回音频类型与简要描述。
|
||||
C.严格输出为下方 JSON，字段不得缺失或额外编造。听不清处用“[听不清]”。
|
||||
你将接收一段音频。请完成：你将接收一段音频。语音活动检测（VAD）与声源分类。
|
||||
|
||||
** 输出 JSON Schema（示例）**
|
||||
**输出 JSON Schema（示例）**
|
||||
|
||||
```json
|
||||
{
|
||||
"speech_detected": true,
|
||||
"language": "zh-CN",
|
||||
"audio_type": null,
|
||||
"background": "music | ambience | none | unknown",
|
||||
"transcript": [
|
||||
{
|
||||
"start": 0.00,
|
||||
"end": 3.42,
|
||||
"text": "大家好，我是……"
|
||||
},
|
||||
{
|
||||
"start": 3.50,
|
||||
"end": 6.10,
|
||||
"text": "欢迎来到今天的节目。"
|
||||
}
|
||||
],
|
||||
"transcript": ["大家好，我是xxx", "欢迎来到今天的视频", "今天我们来聊聊AI的未来"],
|
||||
"non_speech_summary": null,
|
||||
}
|
||||
```
|
||||
>
|
||||
** 当无发言时返回：**
|
||||
|
||||
**当无发言时返回：**
|
||||
|
||||
```json
|
||||
{
|
||||
"speech_detected": false,
|
||||
"language": null,
|
||||
"audio_type": "music | ambience | animal | mechanical | other",
|
||||
"background": "none",
|
||||
"transcript": [],
|
||||
"non_speech_summary": "示例：纯音乐-钢琴独奏，节奏舒缓；或 环境声-雨声伴随雷鸣。",
|
||||
}
|
||||
@@ -44,8 +26,4 @@ C.严格输出为下方 JSON，字段不得缺失或额外编造。听不清处
|
||||
|
||||
** 规则补充 **
|
||||
|
||||
* 只要存在可理解的人类发言（即便有音乐 / 噪声），就执行转录，并在 `background` 标注“music / ambience”。
|
||||
* 不要将唱词 / 哼唱视为“发言”；若仅有人声演唱且无口语发言，视为 ** 音乐 **。
|
||||
* 不要臆测未听清内容；不要添加与音频无关的信息。
|
||||
* 时间单位统一为秒，保留两位小数。
|
||||
* 允许`language` 为多标签（如 "zh-CN,en"）或为 `null`（无发言时）。
|
||||
|
||||
Reference in New Issue
Block a user