feie9454/douyin-archive
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add manual video tagging

Browse Source
This commit is contained in:
feie9454
2026-04-29 21:20:21 +08:00
parent 02d37f3953
commit eac6ad2343
17 changed files with 1066 additions and 418 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/api/admin/asr/videos/route.ts
+29 -20
View File
@@ -27,26 +27,35 @@ export async function GET() {
},
});
const formattedVideos = videos.map((v) => ({
aweme_id: v.aweme_id,
desc: v.desc,
cover_url: getFileUrl(v.cover_url ?? ""),
duration_ms: v.duration_ms,
author: {
nickname: v.author.nickname,
},
transcript: v.transcript
? {
id: v.transcript.id,
speech_detected: v.transcript.speech_detected,
language: v.transcript.language,
audio_type: v.transcript.audio_type,
transcript: v.transcript.transcript,
non_speech_summary: v.transcript.non_speech_summary,
content_tags: v.transcript.content_tags,
}
: null,
}));
const formattedVideos = videos.map((v) => {
const contentTags =
v.manual_content_tags.length > 0
? v.manual_content_tags
: (v.transcript?.content_tags ?? []);
return {
aweme_id: v.aweme_id,
desc: v.desc,
cover_url: getFileUrl(v.cover_url ?? ""),
duration_ms: v.duration_ms,
manual_content_tags: v.manual_content_tags,
content_tags: contentTags,
author: {
nickname: v.author.nickname,
},
transcript: v.transcript
? {
id: v.transcript.id,
speech_detected: v.transcript.speech_detected,
language: v.transcript.language,
audio_type: v.transcript.audio_type,
transcript: v.transcript.transcript,
non_speech_summary: v.transcript.non_speech_summary,
content_tags: v.transcript.content_tags,
}
: null,
};
});
return NextResponse.json({
videos: formattedVideos,
app/api/asr/route.ts
+36 -1
View File
@@ -3,6 +3,30 @@ export const runtime = "nodejs";
import { type NextRequest, NextResponse } from "next/server";
import { transcriptAweme } from ".";
const runningJobs = new Map<string, Promise<void>>();
function startBackgroundAsr(awemeId: string) {
if (runningJobs.has(awemeId)) return false;
const job = transcriptAweme(awemeId, {
log: (message) => console.log(`[ASR ${awemeId}] ${message}`),
})
.then((result) => {
console.log(
`[ASR ${awemeId}] done tags=${result.content_tags.join("、")}`,
);
})
.catch((error) => {
console.error(`[ASR ${awemeId}] failed:`, error);
})
.finally(() => {
runningJobs.delete(awemeId);
});
runningJobs.set(awemeId, job);
return true;
}
export async function GET(req: NextRequest) {
const { searchParams } = new URL(req.url);
const awemeId = searchParams.get("awemeId");
@@ -12,8 +36,19 @@ export async function GET(req: NextRequest) {
{ status: 400 },
);
}
const asyncMode =
searchParams.get("async") === "1" || searchParams.get("mode") === "async";
if (asyncMode) {
const started = startBackgroundAsr(awemeId);
return NextResponse.json(
{ awemeId, status: started ? "started" : "running" },
{ status: 202 },
);
}
try {
const result = await transcriptAweme(awemeId, /* {log: (msg) => console.log(`[ASR] ${msg}`)} */);
const result = await transcriptAweme(awemeId);
return NextResponse.json(result);
} catch (error) {
console.error("ASR failed:", error);
app/api/asr/tags/route.ts
+127
View File
@@ -0,0 +1,127 @@
export const runtime = "nodejs";
import { type NextRequest, NextResponse } from "next/server";
import { isAsrContentTag } from "@/lib/asr-tags";
import { prisma } from "@/lib/prisma";
type UpdateMode = "set" | "add";
function uniqueValidTags(tags: unknown[]) {
const validTags: string[] = [];
for (const tag of tags) {
if (typeof tag !== "string" || !isAsrContentTag(tag)) {
throw new Error("Invalid tag");
}
if (!validTags.includes(tag)) validTags.push(tag);
}
return validTags;
}
function resolveContentTags(manualTags: string[], asrTags: string[]) {
return manualTags.length > 0 ? manualTags : asrTags;
}
export async function POST(req: NextRequest) {
let body: unknown;
try {
body = await req.json();
} catch {
return NextResponse.json({ error: "Invalid JSON body" }, { status: 400 });
}
if (!body || typeof body !== "object") {
return NextResponse.json(
{ error: "Invalid request body" },
{ status: 400 },
);
}
const raw = body as {
awemeId?: unknown;
awemeIds?: unknown;
tag?: unknown;
contentTags?: unknown;
mode?: unknown;
};
const awemeIds = Array.isArray(raw.awemeIds)
? raw.awemeIds
: typeof raw.awemeId === "string"
? [raw.awemeId]
: [];
const normalizedAwemeIds = [...new Set(awemeIds)]
.filter(
(id): id is string => typeof id === "string" && id.trim().length > 0,
)
.map((id) => id.trim());
if (normalizedAwemeIds.length === 0 || normalizedAwemeIds.length > 200) {
return NextResponse.json(
{ error: "awemeIds must contain 1 to 200 ids" },
{ status: 400 },
);
}
const mode: UpdateMode = raw.mode === "add" ? "add" : "set";
const rawTags = Array.isArray(raw.contentTags)
? raw.contentTags
: typeof raw.tag === "string"
? [raw.tag]
: [];
let contentTags: string[];
try {
contentTags = uniqueValidTags(rawTags);
} catch {
return NextResponse.json({ error: "Invalid tag" }, { status: 400 });
}
if (mode === "add" && contentTags.length === 0) {
return NextResponse.json(
{ error: "add mode requires at least one tag" },
{ status: 400 },
);
}
const videos = await prisma.video.findMany({
where: { aweme_id: { in: normalizedAwemeIds } },
select: {
aweme_id: true,
manual_content_tags: true,
transcript: { select: { content_tags: true } },
},
});
if (videos.length === 0) {
return NextResponse.json({ error: "No videos found" }, { status: 404 });
}
const updates = videos.map((video) => {
const asrTags = video.transcript?.content_tags ?? [];
const baseTags = resolveContentTags(video.manual_content_tags, asrTags);
const manualTags =
mode === "add"
? [...new Set([...baseTags, ...contentTags])]
: contentTags;
return { awemeId: video.aweme_id, manualTags, asrTags };
});
await prisma.$transaction(
updates.map((update) =>
prisma.video.update({
where: { aweme_id: update.awemeId },
data: { manual_content_tags: update.manualTags },
}),
),
);
return NextResponse.json({
updated: updates.map((update) => ({
aweme_id: update.awemeId,
manual_content_tags: update.manualTags,
content_tags: resolveContentTags(update.manualTags, update.asrTags),
})),
missing: normalizedAwemeIds.filter(
(id) => !videos.some((video) => video.aweme_id === id),
),
});
}
app/api/feed/route.ts
+14 -3
View File
@@ -1,6 +1,6 @@
import { type NextRequest, NextResponse } from "next/server";
import type { FeedItem, FeedResponse } from "@/app/types/feed";
import { isAsrContentTag } from "@/lib/asr-tags";
import { isAsrContentTag, resolveEffectiveContentTags } from "@/lib/asr-tags";
import { getFileUrl } from "@/lib/minio";
import { prisma } from "@/lib/prisma";
@@ -24,7 +24,15 @@ export async function GET(req: NextRequest) {
const videoWhere = {
...(before ? { created_at: { lt: before } } : {}),
...(selectedTag
? { transcript: { is: { content_tags: { has: selectedTag } } } }
? {
OR: [
{ manual_content_tags: { has: selectedTag } },
{
manual_content_tags: { isEmpty: true },
transcript: { is: { content_tags: { has: selectedTag } } },
},
],
}
: {}),
};
@@ -68,7 +76,10 @@ export async function GET(req: NextRequest) {
sec_uid: v.author.sec_uid,
},
likes: Number(v.digg_count),
content_tags: v.transcript?.content_tags ?? [],
content_tags: resolveEffectiveContentTags(
v.manual_content_tags,
v.transcript?.content_tags,
),
})),
...posts.map((p) => {
const coverPath = p.images?.[0]?.url;
app/api/search/route.ts
+25 -10
View File
@@ -78,7 +78,10 @@ export async function GET(req: Request) {
'StartSel=<mark>,StopSel=</mark>,MaxFragments=2,MinWords=2,MaxWords=20'
)
END AS snippet,
COALESCE(vt.content_tags, ARRAY[]::text[]) AS "contentTags"
CASE
WHEN cardinality(v.manual_content_tags) > 0 THEN v.manual_content_tags
ELSE COALESCE(vt.content_tags, ARRAY[]::text[])
END AS "contentTags"
FROM "Video" v
LEFT JOIN "VideoTranscript" vt ON vt."videoId" = v.aweme_id,
tsq
@@ -86,7 +89,8 @@ export async function GET(req: Request) {
(vt.transcript_zh_tsv @@ tsq.query OR to_tsvector('zhcfg', v.desc) @@ tsq.query)
AND CASE
WHEN ${contentTag}::text IS NULL THEN TRUE
ELSE vt.content_tags @> ARRAY[${contentTag}]::text[]
WHEN cardinality(v.manual_content_tags) > 0 THEN v.manual_content_tags @> ARRAY[${contentTag}]::text[]
ELSE COALESCE(vt.content_tags, ARRAY[]::text[]) @> ARRAY[${contentTag}]::text[]
END
UNION ALL
@@ -114,7 +118,7 @@ export async function GET(req: Request) {
`
: await prisma.$queryRaw<SearchRow[]>`
SELECT
vt.id AS id,
COALESCE(vt.id, 'tag:' || v.aweme_id) AS id,
v.aweme_id AS "awemeId",
'video'::text AS type,
1.0::double precision AS rank,
@@ -124,10 +128,16 @@ export async function GET(req: Request) {
v.desc,
'标签匹配'
) AS snippet,
vt.content_tags AS "contentTags"
FROM "VideoTranscript" vt
JOIN "Video" v ON v.aweme_id = vt."videoId"
WHERE vt.content_tags @> ARRAY[${contentTag}]::text[]
CASE
WHEN cardinality(v.manual_content_tags) > 0 THEN v.manual_content_tags
ELSE COALESCE(vt.content_tags, ARRAY[]::text[])
END AS "contentTags"
FROM "Video" v
LEFT JOIN "VideoTranscript" vt ON vt."videoId" = v.aweme_id
WHERE CASE
WHEN cardinality(v.manual_content_tags) > 0 THEN v.manual_content_tags @> ARRAY[${contentTag}]::text[]
ELSE COALESCE(vt.content_tags, ARRAY[]::text[]) @> ARRAY[${contentTag}]::text[]
END
ORDER BY v.created_at DESC
OFFSET ${offset}
LIMIT ${limit};
@@ -145,7 +155,8 @@ export async function GET(req: Request) {
WHERE (vt.transcript_zh_tsv @@ tsq.query OR to_tsvector('zhcfg', v.desc) @@ tsq.query)
AND CASE
WHEN ${contentTag}::text IS NULL THEN TRUE
ELSE vt.content_tags @> ARRAY[${contentTag}]::text[]
WHEN cardinality(v.manual_content_tags) > 0 THEN v.manual_content_tags @> ARRAY[${contentTag}]::text[]
ELSE COALESCE(vt.content_tags, ARRAY[]::text[]) @> ARRAY[${contentTag}]::text[]
END)
+
(SELECT COUNT(*)::int FROM "ImagePost" ip
@@ -156,8 +167,12 @@ export async function GET(req: Request) {
`
: await prisma.$queryRaw<{ count: number }[]>`
SELECT COUNT(*)::int AS count
FROM "VideoTranscript" vt
WHERE vt.content_tags @> ARRAY[${contentTag}]::text[];
FROM "Video" v
LEFT JOIN "VideoTranscript" vt ON vt."videoId" = v.aweme_id
WHERE CASE
WHEN cardinality(v.manual_content_tags) > 0 THEN v.manual_content_tags @> ARRAY[${contentTag}]::text[]
ELSE COALESCE(vt.content_tags, ARRAY[]::text[]) @> ARRAY[${contentTag}]::text[]
END;
`;
// 分离视频和图文ID