Copilot
@@ -1,6 +1,6 @@
|
||||
import { NextResponse } from "next/server";
|
||||
import { prisma } from "@/lib/prisma";
|
||||
import { getFileUrl } from "@/lib/minio";
|
||||
import { prisma } from "@/lib/prisma";
|
||||
|
||||
export async function GET() {
|
||||
try {
|
||||
@@ -21,6 +21,7 @@ export async function GET() {
|
||||
audio_type: true,
|
||||
transcript: true,
|
||||
non_speech_summary: true,
|
||||
content_tags: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -42,6 +43,7 @@ export async function GET() {
|
||||
audio_type: v.transcript.audio_type,
|
||||
transcript: v.transcript.transcript,
|
||||
non_speech_summary: v.transcript.non_speech_summary,
|
||||
content_tags: v.transcript.content_tags,
|
||||
}
|
||||
: null,
|
||||
}));
|
||||
@@ -0,0 +1,227 @@
|
||||
import OpenAI from "openai";
|
||||
import { z } from "zod";
|
||||
import { ASR_CONTENT_TAGS, normalizeAsrContentTags } from "@/lib/asr-tags";
|
||||
import { downloadFile } from "@/lib/minio";
|
||||
import { prisma } from "@/lib/prisma";
|
||||
import {
|
||||
type ExtractedVideoFrame,
|
||||
extractAudio,
|
||||
extractVideoFrames,
|
||||
} from "../media";
|
||||
import prompt from "./prompt.md";
|
||||
|
||||
export type AsrLogger = (message: string) => void;
|
||||
|
||||
type TranscriptAwemeOptions = {
|
||||
log?: AsrLogger;
|
||||
};
|
||||
|
||||
let qwenClient: OpenAI | null = null;
|
||||
|
||||
function getQwenClient() {
|
||||
if (qwenClient) return qwenClient;
|
||||
|
||||
const apiKey = process.env.QWEN_API_KEY || process.env.DASHSCOPE_API_KEY;
|
||||
if (!apiKey) {
|
||||
throw new Error("Missing QWEN_API_KEY environment variable");
|
||||
}
|
||||
|
||||
qwenClient = new OpenAI({
|
||||
apiKey,
|
||||
baseURL:
|
||||
process.env.QWEN_API_BASE_URL ||
|
||||
"https://dashscope.aliyuncs.com/compatible-mode/v1",
|
||||
});
|
||||
|
||||
return qwenClient;
|
||||
}
|
||||
|
||||
// Zod schema aligned with prompt.md
|
||||
export const AsrSchema = z
|
||||
.object({
|
||||
speech_detected: z.boolean(),
|
||||
language: z.string().min(1).nullable(),
|
||||
audio_type: z.string().nullable(),
|
||||
transcript: z.array(z.string()),
|
||||
non_speech_summary: z.string().nullable(),
|
||||
content_tags: z.array(z.enum(ASR_CONTENT_TAGS)).min(1).max(3),
|
||||
})
|
||||
.strict();
|
||||
|
||||
const RawAsrSchema = z.object({
|
||||
speech_detected: z.boolean(),
|
||||
language: z.string().min(1).nullable(),
|
||||
audio_type: z.string().nullable(),
|
||||
transcript: z.array(z.string()),
|
||||
non_speech_summary: z.string().nullable(),
|
||||
content_tags: z.unknown().optional(),
|
||||
});
|
||||
|
||||
export type AsrResult = z.infer<typeof AsrSchema>;
|
||||
|
||||
function extractJsonObject(text: string) {
|
||||
const trimmed = text.trim();
|
||||
const fenced = trimmed.match(/```(?:json)?\s*([\s\S]*?)\s*```/i);
|
||||
if (fenced?.[1]) return fenced[1].trim();
|
||||
|
||||
const start = trimmed.indexOf("{");
|
||||
const end = trimmed.lastIndexOf("}");
|
||||
if (start >= 0 && end > start) {
|
||||
return trimmed.slice(start, end + 1);
|
||||
}
|
||||
|
||||
return trimmed;
|
||||
}
|
||||
|
||||
function parseAsrResult(content: string): AsrResult {
|
||||
const parsed = RawAsrSchema.parse(JSON.parse(extractJsonObject(content)));
|
||||
return AsrSchema.parse({
|
||||
...parsed,
|
||||
content_tags: normalizeAsrContentTags(parsed.content_tags),
|
||||
});
|
||||
}
|
||||
|
||||
async function transcriptAudio(
|
||||
audio: Buffer,
|
||||
frames: ExtractedVideoFrame[],
|
||||
videoDesc: string,
|
||||
log?: AsrLogger,
|
||||
) {
|
||||
const base64Audio = Buffer.from(audio).toString("base64");
|
||||
const client = getQwenClient();
|
||||
const model = process.env.QWEN_OMNI_MODEL || "qwen3.5-omni-plus";
|
||||
const frameContentParts = frames.map((frame, index) => ({
|
||||
type: "image_url" as const,
|
||||
image_url: {
|
||||
url: `data:${frame.contentType};base64,${frame.buffer.toString("base64")}`,
|
||||
},
|
||||
index,
|
||||
}));
|
||||
|
||||
log?.(
|
||||
`调用 Qwen Omni: model=${model}, audio=${Math.round(audio.length / 1024)}KB, frames=${frames.length}`,
|
||||
);
|
||||
|
||||
const completion = await client.chat.completions.create({
|
||||
model,
|
||||
messages: [
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
...frameContentParts.map(({ type, image_url }) => ({
|
||||
type,
|
||||
image_url,
|
||||
})),
|
||||
{
|
||||
type: "input_audio",
|
||||
input_audio: { data: `data:;base64,${base64Audio}`, format: "mp3" },
|
||||
},
|
||||
{
|
||||
type: "text",
|
||||
text: `${prompt}\n\n视频简介：\n${videoDesc || "无"}\n\n已附带视频三帧，时间点约为：${
|
||||
frames
|
||||
.map((frame) => `${(frame.timestampMs / 1000).toFixed(1)}s`)
|
||||
.join("、") || "无"
|
||||
}。请结合画面和音频判断内容标签。`,
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
modalities: ["text"],
|
||||
stream: true,
|
||||
stream_options: { include_usage: true },
|
||||
});
|
||||
|
||||
let content = "";
|
||||
for await (const chunk of completion) {
|
||||
const deltaContent = chunk.choices?.[0]?.delta?.content;
|
||||
if (typeof deltaContent === "string") content += deltaContent;
|
||||
}
|
||||
|
||||
log?.(`Qwen Omni 原始输出: ${content}`);
|
||||
|
||||
if (!content.trim()) {
|
||||
throw new Error("No ASR result returned from Qwen Omni");
|
||||
}
|
||||
|
||||
try {
|
||||
return parseAsrResult(content);
|
||||
} catch {
|
||||
throw new Error("Failed to parse ASR result JSON");
|
||||
}
|
||||
}
|
||||
|
||||
export async function transcriptAweme(
|
||||
awemeId: string,
|
||||
options: TranscriptAwemeOptions = {},
|
||||
): Promise<AsrResult> {
|
||||
const log = options.log;
|
||||
log?.(`查询视频: ${awemeId}`);
|
||||
const aweme = await prisma.video.findUnique({
|
||||
where: { aweme_id: awemeId },
|
||||
});
|
||||
if (!aweme) {
|
||||
throw new Error("Aweme not found or aweme is not a video post");
|
||||
}
|
||||
const vPath = aweme.video_url;
|
||||
log?.(`下载视频文件: ${vPath}`);
|
||||
const buffer = await downloadFile(vPath);
|
||||
log?.(`视频下载完成: ${Math.round(buffer.length / 1024)}KB`);
|
||||
|
||||
log?.("提取视频三帧");
|
||||
const frames = await extractVideoFrames(buffer, {
|
||||
count: 3,
|
||||
durationMs: aweme.duration_ms,
|
||||
});
|
||||
log?.(
|
||||
`视频帧提取完成: ${frames
|
||||
.map((frame) => `${(frame.timestampMs / 1000).toFixed(1)}s`)
|
||||
.join("、")}`,
|
||||
);
|
||||
|
||||
log?.("提取音频");
|
||||
const audioDat = await extractAudio(buffer, {
|
||||
format: "mp3",
|
||||
bitrateKbps: 128,
|
||||
});
|
||||
|
||||
if (!audioDat || !audioDat.buffer) {
|
||||
throw new Error("Failed to extract audio from video");
|
||||
}
|
||||
log?.(`音频提取完成: ${Math.round(audioDat.buffer.length / 1024)}KB`);
|
||||
|
||||
// 调用 Qwen Omni 生成转写与固定分类标签
|
||||
const result = await transcriptAudio(
|
||||
audioDat.buffer,
|
||||
frames,
|
||||
aweme.desc,
|
||||
log,
|
||||
);
|
||||
log?.(`解析完成: tags=${result.content_tags.join("、")}`);
|
||||
|
||||
// 将转写结果持久化到数据库
|
||||
log?.("写入数据库 VideoTranscript");
|
||||
await prisma.videoTranscript.upsert({
|
||||
where: { videoId: awemeId },
|
||||
update: {
|
||||
speech_detected: result.speech_detected,
|
||||
language: result.language,
|
||||
audio_type: result.audio_type,
|
||||
transcript: result.transcript,
|
||||
non_speech_summary: result.non_speech_summary,
|
||||
content_tags: result.content_tags,
|
||||
},
|
||||
create: {
|
||||
videoId: awemeId,
|
||||
speech_detected: result.speech_detected,
|
||||
language: result.language,
|
||||
audio_type: result.audio_type,
|
||||
transcript: result.transcript,
|
||||
non_speech_summary: result.non_speech_summary,
|
||||
content_tags: result.content_tags,
|
||||
},
|
||||
});
|
||||
log?.("数据库写入完成");
|
||||
|
||||
return result;
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,39 @@
|
||||
你将接收一段音频、视频中固定抽取的三帧画面，并可能收到一段视频描述作为辅助信息。请完成：语音活动检测（VAD）、声源分类、语音转文字，以及内容标签分类。
|
||||
|
||||
只能输出一个合法 JSON 对象，不要输出 Markdown、解释文字或代码块。
|
||||
|
||||
`content_tags` 必须从以下固定标签中选择 1 到 3 个，不能自造标签：
|
||||
|
||||
美女、游戏、时政、美食、旅行、体育、科技、教育、宠物、音乐、影视、财经、生活、其他
|
||||
|
||||
**输出 JSON Schema（示例）**
|
||||
|
||||
```json
|
||||
{
|
||||
"speech_detected": true,
|
||||
"language": "zh-CN",
|
||||
"audio_type": null,
|
||||
"transcript": ["大家好，我是xxx", "欢迎来到今天的视频", "今天我们来聊聊AI的未来"],
|
||||
"non_speech_summary": null,
|
||||
"content_tags": ["科技", "教育"]
|
||||
}
|
||||
```
|
||||
|
||||
**当无发言时返回：**
|
||||
|
||||
```json
|
||||
{
|
||||
"speech_detected": false,
|
||||
"language": null,
|
||||
"audio_type": "music | ambience | animal | mechanical | other",
|
||||
"transcript": [],
|
||||
"non_speech_summary": "示例：纯音乐-钢琴独奏，节奏舒缓；或 环境声-雨声伴随雷鸣。",
|
||||
"content_tags": ["音乐"]
|
||||
}
|
||||
```
|
||||
|
||||
** 规则补充 **
|
||||
|
||||
* 允许`language` 为多标签（如 "zh-CN,en"）或为 `null`（无发言时）。
|
||||
* `transcript` 按自然语义分句，每段尽量保持完整短句。
|
||||
* 必须结合三帧画面、音频内容和辅助视频描述判断标签；信息不足时使用 `其他`。
|
||||
@@ -0,0 +1,22 @@
|
||||
export const runtime = "nodejs";
|
||||
|
||||
import { type NextRequest, NextResponse } from "next/server";
|
||||
import { transcriptAweme } from ".";
|
||||
|
||||
export async function GET(req: NextRequest) {
|
||||
const { searchParams } = new URL(req.url);
|
||||
const awemeId = searchParams.get("awemeId");
|
||||
if (!awemeId) {
|
||||
return NextResponse.json(
|
||||
{ error: "Missing awemeId parameter" },
|
||||
{ status: 400 },
|
||||
);
|
||||
}
|
||||
try {
|
||||
const result = await transcriptAweme(awemeId, /* {log: (msg) => console.log(`[ASR] ${msg}`)} */);
|
||||
return NextResponse.json(result);
|
||||
} catch (error) {
|
||||
console.error("ASR failed:", error);
|
||||
return NextResponse.json({ error: "ASR failed" }, { status: 500 });
|
||||
}
|
||||
}
|
||||
+51
-26
@@ -1,7 +1,8 @@
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { prisma } from "@/lib/prisma";
|
||||
import { type NextRequest, NextResponse } from "next/server";
|
||||
import type { FeedItem, FeedResponse } from "@/app/types/feed";
|
||||
import { isAsrContentTag } from "@/lib/asr-tags";
|
||||
import { getFileUrl } from "@/lib/minio";
|
||||
import { prisma } from "@/lib/prisma";
|
||||
|
||||
// Contract
|
||||
// Inputs: search params { before?: ISOString, limit?: number }
|
||||
@@ -11,24 +12,44 @@ export async function GET(req: NextRequest) {
|
||||
const { searchParams } = new URL(req.url);
|
||||
const limitParam = searchParams.get("limit");
|
||||
const beforeParam = searchParams.get("before");
|
||||
const rawTag = (searchParams.get("tag") || "").trim();
|
||||
const selectedTag = rawTag && isAsrContentTag(rawTag) ? rawTag : null;
|
||||
|
||||
if (rawTag && !selectedTag) {
|
||||
return NextResponse.json({ error: "Invalid tag" }, { status: 400 });
|
||||
}
|
||||
|
||||
const limit = Math.min(Math.max(Number(limitParam ?? "24"), 1), 60); // 1..60
|
||||
const before = beforeParam ? new Date(beforeParam) : null;
|
||||
const videoWhere = {
|
||||
...(before ? { created_at: { lt: before } } : {}),
|
||||
...(selectedTag
|
||||
? { transcript: { is: { content_tags: { has: selectedTag } } } }
|
||||
: {}),
|
||||
};
|
||||
|
||||
// fetch chunk from both tables
|
||||
const [videos, posts] = await Promise.all([
|
||||
prisma.video.findMany({
|
||||
where: before ? { created_at: { lt: before } } : undefined,
|
||||
where: videoWhere,
|
||||
orderBy: { created_at: "desc" },
|
||||
take: limit,
|
||||
include: { author: true },
|
||||
}),
|
||||
prisma.imagePost.findMany({
|
||||
where: before ? { created_at: { lt: before } } : undefined,
|
||||
orderBy: { created_at: "desc" },
|
||||
take: limit,
|
||||
include: { author: true, images: { orderBy: { order: "asc" }, take: 1 } },
|
||||
include: {
|
||||
author: true,
|
||||
transcript: { select: { content_tags: true } },
|
||||
},
|
||||
}),
|
||||
selectedTag
|
||||
? Promise.resolve([])
|
||||
: prisma.imagePost.findMany({
|
||||
where: before ? { created_at: { lt: before } } : undefined,
|
||||
orderBy: { created_at: "desc" },
|
||||
take: limit,
|
||||
include: {
|
||||
author: true,
|
||||
images: { orderBy: { order: "asc" }, take: 1 },
|
||||
},
|
||||
}),
|
||||
]);
|
||||
|
||||
const merged: FeedItem[] = [
|
||||
@@ -47,29 +68,33 @@ export async function GET(req: NextRequest) {
|
||||
sec_uid: v.author.sec_uid,
|
||||
},
|
||||
likes: Number(v.digg_count),
|
||||
content_tags: v.transcript?.content_tags ?? [],
|
||||
})),
|
||||
...posts.map((p) => ({
|
||||
type: "image" as const,
|
||||
aweme_id: p.aweme_id,
|
||||
created_at: p.created_at,
|
||||
desc: p.desc,
|
||||
cover_url: getFileUrl(p.images?.[0]?.url ?? null),
|
||||
width: p.images?.[0]?.width ?? null,
|
||||
height: p.images?.[0]?.height ?? null,
|
||||
author: {
|
||||
nickname: p.author.nickname,
|
||||
avatar_url: getFileUrl(p.author.avatar_url ?? ""),
|
||||
sec_uid: p.author.sec_uid,
|
||||
},
|
||||
likes: Number(p.digg_count),
|
||||
})),
|
||||
...posts.map((p) => {
|
||||
const coverPath = p.images?.[0]?.url;
|
||||
return {
|
||||
type: "image" as const,
|
||||
aweme_id: p.aweme_id,
|
||||
created_at: p.created_at,
|
||||
desc: p.desc,
|
||||
cover_url: coverPath ? getFileUrl(coverPath) : null,
|
||||
width: p.images?.[0]?.width ?? null,
|
||||
height: p.images?.[0]?.height ?? null,
|
||||
author: {
|
||||
nickname: p.author.nickname,
|
||||
avatar_url: getFileUrl(p.author.avatar_url ?? ""),
|
||||
sec_uid: p.author.sec_uid,
|
||||
},
|
||||
likes: Number(p.digg_count),
|
||||
};
|
||||
}),
|
||||
]
|
||||
.sort((a, b) => +new Date(b.created_at) - +new Date(a.created_at))
|
||||
.slice(0, limit);
|
||||
|
||||
const nextCursor =
|
||||
merged.length > 0
|
||||
? new Date(merged[merged.length - 1].created_at as any).toISOString()
|
||||
? new Date(merged[merged.length - 1].created_at).toISOString()
|
||||
: null;
|
||||
const payload: FeedResponse = { items: merged, nextCursor };
|
||||
return NextResponse.json(payload);
|
||||
|
||||
@@ -21,7 +21,7 @@ import {
|
||||
releaseIsolatedContext,
|
||||
} from "@/app/api/fetcher/browser";
|
||||
import { extractFirstFrame } from "@/app/api/media";
|
||||
import { transcriptAweme } from "../stt";
|
||||
import { transcriptAweme } from "../asr";
|
||||
|
||||
const DETAIL_PATH = "/aweme/v1/web/aweme/detail/";
|
||||
const COMMENT_PATH = "/aweme/v1/web/comment/list/";
|
||||
|
||||
+136
-28
@@ -1,11 +1,28 @@
|
||||
import { execFile } from "child_process";
|
||||
import { promises as fs } from "fs";
|
||||
import os from "os";
|
||||
import path from "path";
|
||||
import { promisify } from "util";
|
||||
import { execFile } from "node:child_process";
|
||||
import { promises as fs } from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { promisify } from "node:util";
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
|
||||
function getErrorMessage(error: unknown) {
|
||||
return error instanceof Error ? error.message : String(error);
|
||||
}
|
||||
|
||||
function getErrorCode(error: unknown) {
|
||||
return typeof error === "object" && error !== null && "code" in error
|
||||
? String((error as { code?: unknown }).code)
|
||||
: null;
|
||||
}
|
||||
|
||||
function isMissingExecutableError(error: unknown) {
|
||||
return (
|
||||
getErrorCode(error) === "ENOENT" ||
|
||||
/not found|is not recognized/i.test(getErrorMessage(error))
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* 使用 ffmpeg 从视频二进制中提取第一帧，返回 JPEG buffer
|
||||
*/
|
||||
@@ -40,18 +57,14 @@ export async function extractFirstFrame(
|
||||
await execFileAsync(ffmpegCmd, args, { windowsHide: true });
|
||||
const img = await fs.readFile(outPath);
|
||||
return { buffer: img, contentType: "image/jpeg", ext: "jpg" };
|
||||
} catch (e: any) {
|
||||
if (
|
||||
e &&
|
||||
(e.code === "ENOENT" ||
|
||||
/not found|is not recognized/i.test(String(e.message)))
|
||||
) {
|
||||
} catch (error) {
|
||||
if (isMissingExecutableError(error)) {
|
||||
console.warn(
|
||||
"系统未检测到 ffmpeg，可安装并配置 PATH 或设置 FFMPEG_PATH 后启用封面提取。",
|
||||
);
|
||||
return null;
|
||||
}
|
||||
throw e;
|
||||
throw error;
|
||||
} finally {
|
||||
try {
|
||||
await fs.unlink(inPath);
|
||||
@@ -62,6 +75,13 @@ export async function extractFirstFrame(
|
||||
}
|
||||
}
|
||||
|
||||
export type ExtractedVideoFrame = {
|
||||
buffer: Buffer;
|
||||
contentType: string;
|
||||
ext: string;
|
||||
timestampMs: number;
|
||||
};
|
||||
|
||||
/**
|
||||
* 使用 ffprobe 获取视频时长（毫秒）
|
||||
*/
|
||||
@@ -88,20 +108,16 @@ export async function getVideoDuration(
|
||||
windowsHide: true,
|
||||
});
|
||||
const durationSeconds = parseFloat(stdout.trim());
|
||||
if (isNaN(durationSeconds)) return null;
|
||||
if (Number.isNaN(durationSeconds)) return null;
|
||||
return Math.round(durationSeconds * 1000); // 转换为毫秒
|
||||
} catch (e: any) {
|
||||
if (
|
||||
e &&
|
||||
(e.code === "ENOENT" ||
|
||||
/not found|is not recognized/i.test(String(e.message)))
|
||||
) {
|
||||
} catch (error) {
|
||||
if (isMissingExecutableError(error)) {
|
||||
console.warn(
|
||||
"系统未检测到 ffprobe，可安装并配置 PATH 或设置 FFPROBE_PATH 后启用时长提取。",
|
||||
);
|
||||
return null;
|
||||
}
|
||||
console.warn(`获取视频时长失败: ${e?.message || e}`);
|
||||
console.warn(`获取视频时长失败: ${getErrorMessage(error)}`);
|
||||
return null;
|
||||
} finally {
|
||||
try {
|
||||
@@ -110,6 +126,102 @@ export async function getVideoDuration(
|
||||
}
|
||||
}
|
||||
|
||||
function getFrameTimestamps(durationMs: number | null, count: number) {
|
||||
if (!durationMs || durationMs <= 0) {
|
||||
return Array.from({ length: count }, (_, index) => index * 1000);
|
||||
}
|
||||
|
||||
if (count === 1) return [0];
|
||||
|
||||
const maxTimestamp = Math.max(0, durationMs - 200);
|
||||
if (count === 3) {
|
||||
return [
|
||||
Math.min(maxTimestamp, Math.max(0, Math.round(durationMs * 0.1))),
|
||||
Math.min(maxTimestamp, Math.max(0, Math.round(durationMs * 0.5))),
|
||||
Math.min(maxTimestamp, Math.max(0, Math.round(durationMs * 0.9))),
|
||||
];
|
||||
}
|
||||
|
||||
return Array.from({ length: count }, (_, index) => {
|
||||
const ratio = count === 1 ? 0 : index / (count - 1);
|
||||
return Math.min(maxTimestamp, Math.max(0, Math.round(durationMs * ratio)));
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* 使用 ffmpeg 从视频中提取固定数量的 JPEG 帧。
|
||||
*/
|
||||
export async function extractVideoFrames(
|
||||
videoBuffer: Buffer,
|
||||
opts?: { count?: number; durationMs?: number | null },
|
||||
): Promise<ExtractedVideoFrame[]> {
|
||||
const ffmpegCmd = process.env.FFMPEG_PATH || "ffmpeg";
|
||||
const tmpDir = os.tmpdir();
|
||||
const base = `dy_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`;
|
||||
const inPath = path.join(tmpDir, `${base}.mp4`);
|
||||
const count = Math.max(1, opts?.count ?? 3);
|
||||
const outPaths = Array.from({ length: count }, (_, index) =>
|
||||
path.join(tmpDir, `${base}_${index}.jpg`),
|
||||
);
|
||||
|
||||
try {
|
||||
await fs.writeFile(inPath, videoBuffer);
|
||||
const durationMs =
|
||||
opts?.durationMs ?? (await getVideoDuration(videoBuffer));
|
||||
const timestamps = getFrameTimestamps(durationMs, count);
|
||||
|
||||
const frames: ExtractedVideoFrame[] = [];
|
||||
for (let index = 0; index < count; index++) {
|
||||
const timestampSeconds = Math.max(0, timestamps[index] / 1000).toFixed(3);
|
||||
const args = [
|
||||
"-hide_banner",
|
||||
"-loglevel",
|
||||
"error",
|
||||
"-ss",
|
||||
timestampSeconds,
|
||||
"-i",
|
||||
inPath,
|
||||
"-frames:v",
|
||||
"1",
|
||||
"-q:v",
|
||||
"3",
|
||||
"-vf",
|
||||
"scale=768:-2:force_original_aspect_ratio=decrease",
|
||||
"-f",
|
||||
"image2",
|
||||
"-y",
|
||||
outPaths[index],
|
||||
];
|
||||
await execFileAsync(ffmpegCmd, args, { windowsHide: true });
|
||||
frames.push({
|
||||
buffer: await fs.readFile(outPaths[index]),
|
||||
contentType: "image/jpeg",
|
||||
ext: "jpg",
|
||||
timestampMs: timestamps[index],
|
||||
});
|
||||
}
|
||||
|
||||
return frames;
|
||||
} catch (error) {
|
||||
if (isMissingExecutableError(error)) {
|
||||
console.warn(
|
||||
"系统未检测到 ffmpeg，可安装并配置 PATH 或设置 FFMPEG_PATH 后启用视频帧提取。",
|
||||
);
|
||||
return [];
|
||||
}
|
||||
throw error;
|
||||
} finally {
|
||||
try {
|
||||
await fs.unlink(inPath);
|
||||
} catch {}
|
||||
for (const outPath of outPaths) {
|
||||
try {
|
||||
await fs.unlink(outPath);
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 使用 ffmpeg 将视频二进制转换为音频，返回音频 Buffer
|
||||
*
|
||||
@@ -189,20 +301,16 @@ export async function extractAudio(
|
||||
await execFileAsync(ffmpegCmd, args, { windowsHide: true });
|
||||
const audio = await fs.readFile(outPath);
|
||||
return { buffer: audio, contentType, ext };
|
||||
} catch (e: any) {
|
||||
if (
|
||||
e &&
|
||||
(e.code === "ENOENT" ||
|
||||
/not found|is not recognized/i.test(String(e.message)))
|
||||
) {
|
||||
} catch (error) {
|
||||
if (isMissingExecutableError(error)) {
|
||||
console.warn(
|
||||
"系统未检测到 ffmpeg，可安装并配置 PATH 或设置 FFMPEG_PATH 后启用音频提取。",
|
||||
);
|
||||
return null;
|
||||
}
|
||||
// 一些环境可能缺少特定编码器（如 libmp3lame），提示并抛出原始错误
|
||||
console.warn(`提取音频失败: ${e?.message || e}`);
|
||||
throw e;
|
||||
console.warn(`提取音频失败: ${getErrorMessage(error)}`);
|
||||
throw error;
|
||||
} finally {
|
||||
try {
|
||||
await fs.unlink(inPath);
|
||||
|
||||
+133
-80
@@ -1,11 +1,32 @@
|
||||
import { NextResponse } from "next/server";
|
||||
import { isAsrContentTag } from "@/lib/asr-tags";
|
||||
import { json } from "@/lib/json";
|
||||
import { getFileUrl } from "@/lib/minio";
|
||||
import { prisma } from "@/lib/prisma"; // 你的 Prisma 客户端实例
|
||||
import { NextResponse } from "next/server";
|
||||
import { prisma } from "@/lib/prisma";
|
||||
|
||||
type SearchRow = {
|
||||
id: string;
|
||||
awemeId: string;
|
||||
type: "video" | "image";
|
||||
rank: number;
|
||||
snippet: string;
|
||||
contentTags: string[];
|
||||
};
|
||||
|
||||
function sanitizeSnippetHtml(snippet: string) {
|
||||
return snippet
|
||||
.replaceAll("&", "&")
|
||||
.replaceAll("<", "<")
|
||||
.replaceAll(">", ">")
|
||||
.replaceAll("<mark>", "<mark>")
|
||||
.replaceAll("</mark>", "</mark>");
|
||||
}
|
||||
|
||||
export async function GET(req: Request) {
|
||||
const { searchParams } = new URL(req.url);
|
||||
const q = (searchParams.get("q") || "").trim();
|
||||
const rawTag = (searchParams.get("tag") || "").trim();
|
||||
const contentTag = rawTag && isAsrContentTag(rawTag) ? rawTag : null;
|
||||
const page = Math.max(1, Number(searchParams.get("page") || 1));
|
||||
const limit = Math.min(
|
||||
50,
|
||||
@@ -13,7 +34,11 @@ export async function GET(req: Request) {
|
||||
);
|
||||
const offset = (page - 1) * limit;
|
||||
|
||||
if (!q) {
|
||||
if (rawTag && !contentTag) {
|
||||
return NextResponse.json({ error: "Invalid tag" }, { status: 400 });
|
||||
}
|
||||
|
||||
if (!q && !contentTag) {
|
||||
return NextResponse.json({ results: [], total: 0 });
|
||||
}
|
||||
|
||||
@@ -23,90 +48,117 @@ export async function GET(req: Request) {
|
||||
// - 合并搜索来源：VideoTranscript.transcript_zh_tsv、Video.desc 与 ImagePost.desc
|
||||
// - 使用 UNION ALL 合并视频和图文的搜索结果
|
||||
// - rank 取两者匹配得分的最大值，snippet 优先展示转写匹配，否则展示 desc 匹配
|
||||
const rows = await prisma.$queryRaw<
|
||||
{
|
||||
id: string;
|
||||
awemeId: string;
|
||||
type: "video" | "image";
|
||||
rank: number;
|
||||
snippet: string;
|
||||
}[]
|
||||
>`
|
||||
WITH tsq AS (
|
||||
SELECT websearch_to_tsquery('zhcfg', ${q}) AS query
|
||||
)
|
||||
-- 视频搜索结果
|
||||
SELECT
|
||||
COALESCE(vt.id, 'desc:' || v.aweme_id) AS id,
|
||||
v.aweme_id AS "awemeId",
|
||||
'video'::text AS type,
|
||||
GREATEST(
|
||||
COALESCE(ts_rank(vt.transcript_zh_tsv, tsq.query), 0),
|
||||
COALESCE(ts_rank(to_tsvector('zhcfg', v.desc), tsq.query), 0)
|
||||
) AS rank,
|
||||
CASE
|
||||
WHEN vt.transcript_zh_tsv @@ tsq.query THEN
|
||||
const rows = q
|
||||
? await prisma.$queryRaw<SearchRow[]>`
|
||||
WITH tsq AS (
|
||||
SELECT websearch_to_tsquery('zhcfg', ${q}) AS query
|
||||
)
|
||||
-- 视频搜索结果
|
||||
SELECT
|
||||
COALESCE(vt.id, 'desc:' || v.aweme_id) AS id,
|
||||
v.aweme_id AS "awemeId",
|
||||
'video'::text AS type,
|
||||
GREATEST(
|
||||
COALESCE(ts_rank(vt.transcript_zh_tsv, tsq.query), 0),
|
||||
COALESCE(ts_rank(to_tsvector('zhcfg', v.desc), tsq.query), 0)
|
||||
) AS rank,
|
||||
CASE
|
||||
WHEN vt.transcript_zh_tsv @@ tsq.query THEN
|
||||
ts_headline(
|
||||
'zhcfg',
|
||||
array_to_string(vt.transcript, ' '),
|
||||
tsq.query,
|
||||
'StartSel=<mark>,StopSel=</mark>,MaxFragments=2,MinWords=2,MaxWords=20'
|
||||
)
|
||||
ELSE
|
||||
ts_headline(
|
||||
'zhcfg',
|
||||
v.desc,
|
||||
tsq.query,
|
||||
'StartSel=<mark>,StopSel=</mark>,MaxFragments=2,MinWords=2,MaxWords=20'
|
||||
)
|
||||
END AS snippet,
|
||||
COALESCE(vt.content_tags, ARRAY[]::text[]) AS "contentTags"
|
||||
FROM "Video" v
|
||||
LEFT JOIN "VideoTranscript" vt ON vt."videoId" = v.aweme_id,
|
||||
tsq
|
||||
WHERE
|
||||
(vt.transcript_zh_tsv @@ tsq.query OR to_tsvector('zhcfg', v.desc) @@ tsq.query)
|
||||
AND CASE
|
||||
WHEN ${contentTag}::text IS NULL THEN TRUE
|
||||
ELSE vt.content_tags @> ARRAY[${contentTag}]::text[]
|
||||
END
|
||||
|
||||
UNION ALL
|
||||
|
||||
-- 图文搜索结果
|
||||
SELECT
|
||||
'img_desc:' || ip.aweme_id AS id,
|
||||
ip.aweme_id AS "awemeId",
|
||||
'image'::text AS type,
|
||||
ts_rank(to_tsvector('zhcfg', ip.desc), tsq.query) AS rank,
|
||||
ts_headline(
|
||||
'zhcfg',
|
||||
array_to_string(vt.transcript, ' '),
|
||||
ip.desc,
|
||||
tsq.query,
|
||||
'StartSel=<mark>,StopSel=</mark>,MaxFragments=2,MinWords=2,MaxWords=20'
|
||||
)
|
||||
ELSE
|
||||
ts_headline(
|
||||
'zhcfg',
|
||||
) AS snippet,
|
||||
ARRAY[]::text[] AS "contentTags"
|
||||
FROM "ImagePost" ip, tsq
|
||||
WHERE to_tsvector('zhcfg', ip.desc) @@ tsq.query
|
||||
AND ${contentTag}::text IS NULL
|
||||
|
||||
ORDER BY rank DESC
|
||||
OFFSET ${offset}
|
||||
LIMIT ${limit};
|
||||
`
|
||||
: await prisma.$queryRaw<SearchRow[]>`
|
||||
SELECT
|
||||
vt.id AS id,
|
||||
v.aweme_id AS "awemeId",
|
||||
'video'::text AS type,
|
||||
1.0::double precision AS rank,
|
||||
COALESCE(
|
||||
NULLIF(array_to_string(vt.transcript[1:2], ' '), ''),
|
||||
vt.non_speech_summary,
|
||||
v.desc,
|
||||
tsq.query,
|
||||
'StartSel=<mark>,StopSel=</mark>,MaxFragments=2,MinWords=2,MaxWords=20'
|
||||
)
|
||||
END AS snippet
|
||||
FROM "Video" v
|
||||
LEFT JOIN "VideoTranscript" vt ON vt."videoId" = v.aweme_id,
|
||||
tsq
|
||||
WHERE
|
||||
(vt.transcript_zh_tsv @@ tsq.query OR to_tsvector('zhcfg', v.desc) @@ tsq.query)
|
||||
|
||||
UNION ALL
|
||||
|
||||
-- 图文搜索结果
|
||||
SELECT
|
||||
'img_desc:' || ip.aweme_id AS id,
|
||||
ip.aweme_id AS "awemeId",
|
||||
'image'::text AS type,
|
||||
ts_rank(to_tsvector('zhcfg', ip.desc), tsq.query) AS rank,
|
||||
ts_headline(
|
||||
'zhcfg',
|
||||
ip.desc,
|
||||
tsq.query,
|
||||
'StartSel=<mark>,StopSel=</mark>,MaxFragments=2,MinWords=2,MaxWords=20'
|
||||
) AS snippet
|
||||
FROM "ImagePost" ip, tsq
|
||||
WHERE to_tsvector('zhcfg', ip.desc) @@ tsq.query
|
||||
|
||||
ORDER BY rank DESC
|
||||
OFFSET ${offset}
|
||||
LIMIT ${limit};
|
||||
`;
|
||||
'标签匹配'
|
||||
) AS snippet,
|
||||
vt.content_tags AS "contentTags"
|
||||
FROM "VideoTranscript" vt
|
||||
JOIN "Video" v ON v.aweme_id = vt."videoId"
|
||||
WHERE vt.content_tags @> ARRAY[${contentTag}]::text[]
|
||||
ORDER BY v.created_at DESC
|
||||
OFFSET ${offset}
|
||||
LIMIT ${limit};
|
||||
`;
|
||||
|
||||
// 查询总数（参数化，避免注入）
|
||||
const totalRows = await prisma.$queryRaw<
|
||||
{
|
||||
count: number;
|
||||
}[]
|
||||
>`
|
||||
WITH tsq AS (
|
||||
SELECT websearch_to_tsquery('zhcfg', ${q}) AS query
|
||||
)
|
||||
SELECT (
|
||||
(SELECT COUNT(*)::int FROM "Video" v
|
||||
LEFT JOIN "VideoTranscript" vt ON vt."videoId" = v.aweme_id
|
||||
WHERE (vt.transcript_zh_tsv @@ tsq.query OR to_tsvector('zhcfg', v.desc) @@ tsq.query))
|
||||
+
|
||||
(SELECT COUNT(*)::int FROM "ImagePost" ip
|
||||
WHERE to_tsvector('zhcfg', ip.desc) @@ tsq.query)
|
||||
) AS count
|
||||
FROM tsq;
|
||||
`;
|
||||
const totalRows = q
|
||||
? await prisma.$queryRaw<{ count: number }[]>`
|
||||
WITH tsq AS (
|
||||
SELECT websearch_to_tsquery('zhcfg', ${q}) AS query
|
||||
)
|
||||
SELECT (
|
||||
(SELECT COUNT(*)::int FROM "Video" v
|
||||
LEFT JOIN "VideoTranscript" vt ON vt."videoId" = v.aweme_id
|
||||
WHERE (vt.transcript_zh_tsv @@ tsq.query OR to_tsvector('zhcfg', v.desc) @@ tsq.query)
|
||||
AND CASE
|
||||
WHEN ${contentTag}::text IS NULL THEN TRUE
|
||||
ELSE vt.content_tags @> ARRAY[${contentTag}]::text[]
|
||||
END)
|
||||
+
|
||||
(SELECT COUNT(*)::int FROM "ImagePost" ip
|
||||
WHERE to_tsvector('zhcfg', ip.desc) @@ tsq.query
|
||||
AND ${contentTag}::text IS NULL)
|
||||
) AS count
|
||||
FROM tsq;
|
||||
`
|
||||
: await prisma.$queryRaw<{ count: number }[]>`
|
||||
SELECT COUNT(*)::int AS count
|
||||
FROM "VideoTranscript" vt
|
||||
WHERE vt.content_tags @> ARRAY[${contentTag}]::text[];
|
||||
`;
|
||||
|
||||
// 分离视频和图文ID
|
||||
const videoIds = rows
|
||||
@@ -176,6 +228,7 @@ export async function GET(req: Request) {
|
||||
return json({
|
||||
results: rows.map((r) => ({
|
||||
...r,
|
||||
snippet: sanitizeSnippetHtml(r.snippet),
|
||||
video:
|
||||
r.type === "video"
|
||||
? videos.find((v) => v.aweme_id === r.awemeId)
|
||||
|
||||
@@ -1,113 +0,0 @@
|
||||
import fs from "fs";
|
||||
import OpenAI from "openai";
|
||||
import prompt from "./prompt.md";
|
||||
import { prisma } from "@/lib/prisma";
|
||||
import { downloadFile } from "@/lib/minio";
|
||||
import { extractAudio } from "../media";
|
||||
import { z } from "zod";
|
||||
import { zodResponseFormat } from "openai/helpers/zod";
|
||||
import { ProxyAgent, setGlobalDispatcher } from "undici";
|
||||
|
||||
const proxy = process.env.HTTPS_PROXY || process.env.HTTP_PROXY;
|
||||
if (proxy) {
|
||||
setGlobalDispatcher(new ProxyAgent(proxy));
|
||||
}
|
||||
|
||||
const client = new OpenAI({
|
||||
apiKey: process.env.OPENAI_API_KEY,
|
||||
baseURL: process.env.OPENAI_API_BASE_URL,
|
||||
});
|
||||
|
||||
// Zod schema aligned with prompt.md
|
||||
export const SttSchema = z
|
||||
.object({
|
||||
speech_detected: z.boolean(),
|
||||
language: z.string().min(1).nullable(),
|
||||
audio_type: z.string().nullable(),
|
||||
transcript: z.array(z.string()),
|
||||
non_speech_summary: z.string().nullable(),
|
||||
})
|
||||
.strict();
|
||||
|
||||
export type SttResult = z.infer<typeof SttSchema>;
|
||||
|
||||
async function transcriptAudio(audio: Buffer | string) {
|
||||
if (typeof audio === "string") {
|
||||
audio = fs.readFileSync(audio);
|
||||
}
|
||||
const base64Audio = Buffer.from(audio).toString("base64");
|
||||
|
||||
const completion = await client.chat.completions.create({
|
||||
model: "gemini-2.5-flash",
|
||||
messages: [
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
{ type: "text", text: prompt },
|
||||
{
|
||||
type: "input_audio",
|
||||
input_audio: { data: base64Audio, format: "mp3" },
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
response_format: zodResponseFormat(SttSchema, "stt_result"),
|
||||
});
|
||||
|
||||
const data = completion.choices?.[0]?.message;
|
||||
console.log("转写结果", data.content);
|
||||
|
||||
if (!data || !data.content) {
|
||||
throw new Error("No STT result returned from model");
|
||||
}
|
||||
try {
|
||||
return JSON.parse(data.content) as SttResult;
|
||||
} catch (e) {
|
||||
throw new Error("Failed to parse STT result JSON");
|
||||
}
|
||||
}
|
||||
|
||||
export async function transcriptAweme(awemeId: string): Promise<SttResult> {
|
||||
const aweme = await prisma.video.findUnique({
|
||||
where: { aweme_id: awemeId },
|
||||
});
|
||||
if (!aweme) {
|
||||
throw new Error("Aweme not found or aweme is not a video post");
|
||||
}
|
||||
const vPath = aweme.video_url;
|
||||
const buffer = await downloadFile(vPath);
|
||||
|
||||
const audioDat = await extractAudio(buffer, {
|
||||
format: "mp3",
|
||||
bitrateKbps: 128,
|
||||
});
|
||||
|
||||
if (!audioDat || !audioDat.buffer) {
|
||||
throw new Error("Failed to extract audio from video");
|
||||
}
|
||||
|
||||
// 调用大模型生成转写结果
|
||||
const result = await transcriptAudio(audioDat.buffer);
|
||||
|
||||
// 将转写结果持久化到数据库
|
||||
await prisma.videoTranscript.upsert({
|
||||
where: { videoId: awemeId },
|
||||
update: {
|
||||
speech_detected: result.speech_detected,
|
||||
language: result.language,
|
||||
audio_type: result.audio_type,
|
||||
transcript: result.transcript,
|
||||
non_speech_summary: result.non_speech_summary,
|
||||
},
|
||||
create: {
|
||||
videoId: awemeId,
|
||||
speech_detected: result.speech_detected,
|
||||
language: result.language,
|
||||
audio_type: result.audio_type,
|
||||
transcript: result.transcript,
|
||||
non_speech_summary: result.non_speech_summary,
|
||||
},
|
||||
});
|
||||
|
||||
return result;
|
||||
}
|
||||
@@ -1,29 +0,0 @@
|
||||
你将接收一段音频。请完成：你将接收一段音频。语音活动检测（VAD）与声源分类。
|
||||
|
||||
**输出 JSON Schema（示例）**
|
||||
|
||||
```json
|
||||
{
|
||||
"speech_detected": true,
|
||||
"language": "zh-CN",
|
||||
"audio_type": null,
|
||||
"transcript": ["大家好，我是xxx", "欢迎来到今天的视频", "今天我们来聊聊AI的未来"],
|
||||
"non_speech_summary": null,
|
||||
}
|
||||
```
|
||||
|
||||
**当无发言时返回：**
|
||||
|
||||
```json
|
||||
{
|
||||
"speech_detected": false,
|
||||
"language": null,
|
||||
"audio_type": "music | ambience | animal | mechanical | other",
|
||||
"transcript": [],
|
||||
"non_speech_summary": "示例：纯音乐-钢琴独奏，节奏舒缓；或 环境声-雨声伴随雷鸣。",
|
||||
}
|
||||
```
|
||||
|
||||
** 规则补充 **
|
||||
|
||||
* 允许`language` 为多标签（如 "zh-CN,en"）或为 `null`（无发言时）。
|
||||
@@ -1,23 +0,0 @@
|
||||
export const runtime = "nodejs";
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { prisma } from "@/lib/prisma";
|
||||
import type { FeedItem, FeedResponse } from "@/app/types/feed";
|
||||
import { getFileUrl } from "@/lib/minio";
|
||||
import { transcriptAweme } from ".";
|
||||
|
||||
// Contract
|
||||
// Inputs: search params { before?: ISOString, limit?: number }
|
||||
// Output: { items: FeedItem[], nextCursor: ISOString | null }
|
||||
|
||||
export async function GET(req: NextRequest) {
|
||||
const { searchParams } = new URL(req.url);
|
||||
const awemeId = searchParams.get("awemeId");
|
||||
if (!awemeId) {
|
||||
return NextResponse.json(
|
||||
{ error: "Missing awemeId parameter" },
|
||||
{ status: 400 },
|
||||
);
|
||||
}
|
||||
const script = await transcriptAweme(awemeId);
|
||||
return NextResponse.json(script);
|
||||
}
|
||||
Reference in New Issue
Block a user