优化图文页
This commit is contained in:
@@ -1,11 +1,11 @@
|
||||
import { NextResponse } from 'next/server';
|
||||
import { prisma } from '@/lib/prisma';
|
||||
import { getFileUrl } from '@/lib/minio';
|
||||
import { NextResponse } from "next/server";
|
||||
import { prisma } from "@/lib/prisma";
|
||||
import { getFileUrl } from "@/lib/minio";
|
||||
|
||||
export async function GET() {
|
||||
try {
|
||||
const videos = await prisma.video.findMany({
|
||||
orderBy: { created_at: 'desc' },
|
||||
orderBy: { created_at: "desc" },
|
||||
take: 1000,
|
||||
include: {
|
||||
author: {
|
||||
@@ -29,7 +29,7 @@ export async function GET() {
|
||||
const formattedVideos = videos.map((v) => ({
|
||||
aweme_id: v.aweme_id,
|
||||
desc: v.desc,
|
||||
cover_url: getFileUrl(v.cover_url ?? ''),
|
||||
cover_url: getFileUrl(v.cover_url ?? ""),
|
||||
duration_ms: v.duration_ms,
|
||||
author: {
|
||||
nickname: v.author.nickname,
|
||||
@@ -51,10 +51,10 @@ export async function GET() {
|
||||
total: videos.length,
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('Failed to fetch videos:', error);
|
||||
console.error("Failed to fetch videos:", error);
|
||||
return NextResponse.json(
|
||||
{ error: 'Failed to fetch videos' },
|
||||
{ status: 500 }
|
||||
{ error: "Failed to fetch videos" },
|
||||
{ status: 500 },
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,15 +1,18 @@
|
||||
import { NextRequest, NextResponse } from 'next/server';
|
||||
import { prisma } from '@/lib/prisma';
|
||||
import type { FeedItem, FeedResponse } from '@/app/types/feed';
|
||||
import { getFileUrl } from '@/lib/minio';
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { prisma } from "@/lib/prisma";
|
||||
import type { FeedItem, FeedResponse } from "@/app/types/feed";
|
||||
import { getFileUrl } from "@/lib/minio";
|
||||
|
||||
export async function GET(req: NextRequest, { params }: { params: Promise<{ secUid: string }> }) {
|
||||
export async function GET(
|
||||
req: NextRequest,
|
||||
{ params }: { params: Promise<{ secUid: string }> },
|
||||
) {
|
||||
const secUid = (await params).secUid;
|
||||
const { searchParams } = new URL(req.url);
|
||||
const limitParam = searchParams.get('limit');
|
||||
const beforeParam = searchParams.get('before');
|
||||
const limitParam = searchParams.get("limit");
|
||||
const beforeParam = searchParams.get("before");
|
||||
|
||||
const limit = Math.min(Math.max(Number(limitParam ?? '24'), 1), 60); // 1..60
|
||||
const limit = Math.min(Math.max(Number(limitParam ?? "24"), 1), 60); // 1..60
|
||||
const before = beforeParam ? new Date(beforeParam) : null;
|
||||
|
||||
// fetch chunk from both tables
|
||||
@@ -17,20 +20,20 @@ export async function GET(req: NextRequest, { params }: { params: Promise<{ secU
|
||||
prisma.video.findMany({
|
||||
where: {
|
||||
authorId: secUid,
|
||||
...(before ? { created_at: { lt: before } } : {})
|
||||
...(before ? { created_at: { lt: before } } : {}),
|
||||
},
|
||||
orderBy: { created_at: 'desc' },
|
||||
orderBy: { created_at: "desc" },
|
||||
take: limit,
|
||||
include: { author: true },
|
||||
}),
|
||||
prisma.imagePost.findMany({
|
||||
where: {
|
||||
authorId: secUid,
|
||||
...(before ? { created_at: { lt: before } } : {})
|
||||
...(before ? { created_at: { lt: before } } : {}),
|
||||
},
|
||||
orderBy: { created_at: 'desc' },
|
||||
orderBy: { created_at: "desc" },
|
||||
take: limit,
|
||||
include: { author: true, images: { orderBy: { order: 'asc' }, take: 1 } },
|
||||
include: { author: true, images: { orderBy: { order: "asc" }, take: 1 } },
|
||||
}),
|
||||
]);
|
||||
|
||||
@@ -41,11 +44,15 @@ export async function GET(req: NextRequest, { params }: { params: Promise<{ secU
|
||||
created_at: v.created_at,
|
||||
desc: v.desc,
|
||||
video_url: getFileUrl(v.video_url),
|
||||
cover_url: getFileUrl(v.cover_url ?? 'default_cover.png'),
|
||||
cover_url: getFileUrl(v.cover_url ?? "default_cover.png"),
|
||||
width: v.width ?? null,
|
||||
height: v.height ?? null,
|
||||
author: { nickname: v.author.nickname, avatar_url: getFileUrl(v.author.avatar_url ?? ''), sec_uid: v.author.sec_uid },
|
||||
likes: Number(v.digg_count)
|
||||
author: {
|
||||
nickname: v.author.nickname,
|
||||
avatar_url: getFileUrl(v.author.avatar_url ?? ""),
|
||||
sec_uid: v.author.sec_uid,
|
||||
},
|
||||
likes: Number(v.digg_count),
|
||||
})),
|
||||
...posts.map((p) => ({
|
||||
type: "image" as const,
|
||||
@@ -55,13 +62,21 @@ export async function GET(req: NextRequest, { params }: { params: Promise<{ secU
|
||||
cover_url: getFileUrl(p.images?.[0]?.url ?? null),
|
||||
width: p.images?.[0]?.width ?? null,
|
||||
height: p.images?.[0]?.height ?? null,
|
||||
author: { nickname: p.author.nickname, avatar_url: getFileUrl(p.author.avatar_url ?? ''), sec_uid: p.author.sec_uid },
|
||||
likes: Number(p.digg_count)
|
||||
author: {
|
||||
nickname: p.author.nickname,
|
||||
avatar_url: getFileUrl(p.author.avatar_url ?? ""),
|
||||
sec_uid: p.author.sec_uid,
|
||||
},
|
||||
likes: Number(p.digg_count),
|
||||
})),
|
||||
].sort((a, b) => +new Date(b.created_at) - +new Date(a.created_at))
|
||||
]
|
||||
.sort((a, b) => +new Date(b.created_at) - +new Date(a.created_at))
|
||||
.slice(0, limit);
|
||||
|
||||
const nextCursor = merged.length > 0 ? new Date(merged[merged.length - 1].created_at as any).toISOString() : null;
|
||||
const nextCursor =
|
||||
merged.length > 0
|
||||
? new Date(merged[merged.length - 1].created_at as any).toISOString()
|
||||
: null;
|
||||
const payload: FeedResponse = { items: merged, nextCursor };
|
||||
return NextResponse.json(payload);
|
||||
}
|
||||
|
||||
@@ -12,8 +12,14 @@ export async function GET(req: NextRequest) {
|
||||
|
||||
// Find current item timestamp from either table
|
||||
const [video, post] = await Promise.all([
|
||||
prisma.video.findUnique({ where: { aweme_id: awemeId }, select: { aweme_id: true, created_at: true } }),
|
||||
prisma.imagePost.findUnique({ where: { aweme_id: awemeId }, select: { aweme_id: true, created_at: true } }),
|
||||
prisma.video.findUnique({
|
||||
where: { aweme_id: awemeId },
|
||||
select: { aweme_id: true, created_at: true },
|
||||
}),
|
||||
prisma.imagePost.findUnique({
|
||||
where: { aweme_id: awemeId },
|
||||
select: { aweme_id: true, created_at: true },
|
||||
}),
|
||||
]);
|
||||
|
||||
const current = video ?? post;
|
||||
@@ -52,25 +58,61 @@ export async function GET(req: NextRequest) {
|
||||
]);
|
||||
|
||||
const pickPrev = (() => {
|
||||
const cands: { type: "video" | "image"; aweme_id: string; created_at: Date }[] = [];
|
||||
if (newerVideo) cands.push({ type: "video", aweme_id: newerVideo.aweme_id, created_at: newerVideo.created_at as unknown as Date });
|
||||
if (newerPost) cands.push({ type: "image", aweme_id: newerPost.aweme_id, created_at: newerPost.created_at as unknown as Date });
|
||||
const cands: {
|
||||
type: "video" | "image";
|
||||
aweme_id: string;
|
||||
created_at: Date;
|
||||
}[] = [];
|
||||
if (newerVideo)
|
||||
cands.push({
|
||||
type: "video",
|
||||
aweme_id: newerVideo.aweme_id,
|
||||
created_at: newerVideo.created_at as unknown as Date,
|
||||
});
|
||||
if (newerPost)
|
||||
cands.push({
|
||||
type: "image",
|
||||
aweme_id: newerPost.aweme_id,
|
||||
created_at: newerPost.created_at as unknown as Date,
|
||||
});
|
||||
if (cands.length === 0) return undefined;
|
||||
// nearest newer -> minimal created_at
|
||||
cands.sort((a, b) => +a.created_at - +b.created_at);
|
||||
const h = cands[0];
|
||||
return { type: h.type, aweme_id: h.aweme_id, created_at: h.created_at.toISOString() };
|
||||
return {
|
||||
type: h.type,
|
||||
aweme_id: h.aweme_id,
|
||||
created_at: h.created_at.toISOString(),
|
||||
};
|
||||
})();
|
||||
|
||||
const pickNext = (() => {
|
||||
const cands: { type: "video" | "image"; aweme_id: string; created_at: Date }[] = [];
|
||||
if (olderVideo) cands.push({ type: "video", aweme_id: olderVideo.aweme_id, created_at: olderVideo.created_at as unknown as Date });
|
||||
if (olderPost) cands.push({ type: "image", aweme_id: olderPost.aweme_id, created_at: olderPost.created_at as unknown as Date });
|
||||
const cands: {
|
||||
type: "video" | "image";
|
||||
aweme_id: string;
|
||||
created_at: Date;
|
||||
}[] = [];
|
||||
if (olderVideo)
|
||||
cands.push({
|
||||
type: "video",
|
||||
aweme_id: olderVideo.aweme_id,
|
||||
created_at: olderVideo.created_at as unknown as Date,
|
||||
});
|
||||
if (olderPost)
|
||||
cands.push({
|
||||
type: "image",
|
||||
aweme_id: olderPost.aweme_id,
|
||||
created_at: olderPost.created_at as unknown as Date,
|
||||
});
|
||||
if (cands.length === 0) return undefined;
|
||||
// nearest older -> maximal created_at among older
|
||||
cands.sort((a, b) => +b.created_at - +a.created_at);
|
||||
const h = cands[0];
|
||||
return { type: h.type, aweme_id: h.aweme_id, created_at: h.created_at.toISOString() };
|
||||
return {
|
||||
type: h.type,
|
||||
aweme_id: h.aweme_id,
|
||||
created_at: h.created_at.toISOString(),
|
||||
};
|
||||
})();
|
||||
|
||||
return NextResponse.json({ prev: pickPrev ?? null, next: pickNext ?? null });
|
||||
|
||||
@@ -5,7 +5,7 @@ import { NextRequest, NextResponse } from "next/server";
|
||||
|
||||
export async function GET(
|
||||
request: NextRequest,
|
||||
{ params }: { params: Promise<{ awemeId: string }> }
|
||||
{ params }: { params: Promise<{ awemeId: string }> },
|
||||
) {
|
||||
const awemeId = (await params).awemeId;
|
||||
const searchParams = request.nextUrl.searchParams;
|
||||
@@ -13,7 +13,8 @@ export async function GET(
|
||||
const take = parseInt(searchParams.get("take") || "20", 10);
|
||||
|
||||
// ranked 模式参数（均为可选，提供合理默认值）
|
||||
const seed = searchParams.get("seed") || new Date().toISOString().slice(0, 10); // 默认按日期稳定
|
||||
const seed =
|
||||
searchParams.get("seed") || new Date().toISOString().slice(0, 10); // 默认按日期稳定
|
||||
const snapshotIso = searchParams.get("snapshot");
|
||||
const snapshot = snapshotIso ? new Date(snapshotIso) : new Date(); // 用于时间衰减的基准时间，确保单次会话稳定
|
||||
const halfLifeHours = parseFloat(searchParams.get("halfLifeHours") || "24");
|
||||
@@ -21,8 +22,6 @@ export async function GET(
|
||||
const wTime = parseFloat(searchParams.get("wTime") || "2"); // 时间衰减权重
|
||||
const wJit = parseFloat(searchParams.get("wJit") || "10"); // 随机扰动权重（稳定随机）
|
||||
|
||||
|
||||
|
||||
try {
|
||||
// 查找是视频还是图文
|
||||
const [video, post] = await Promise.all([
|
||||
@@ -41,9 +40,7 @@ export async function GET(
|
||||
}
|
||||
|
||||
// 构建查询条件
|
||||
const where = video
|
||||
? { videoId: awemeId }
|
||||
: { imagePostId: awemeId };
|
||||
const where = video ? { videoId: awemeId } : { imagePostId: awemeId };
|
||||
|
||||
// 按「热度 + 时间衰减 + 稳定随机扰动」打分排序；否则使用稳定的时间排序
|
||||
const total = await prisma.comment.count({ where });
|
||||
@@ -68,7 +65,9 @@ export async function GET(
|
||||
${wJit} * ${jitterExpr}
|
||||
)`;
|
||||
|
||||
const whereField = (await video) ? Prisma.sql`c."videoId"` : Prisma.sql`c."imagePostId"`;
|
||||
const whereField = (await video)
|
||||
? Prisma.sql`c."videoId"`
|
||||
: Prisma.sql`c."imagePostId"`;
|
||||
|
||||
const rows: Array<{
|
||||
cid: string;
|
||||
@@ -92,22 +91,41 @@ export async function GET(
|
||||
ORDER BY ${scoreExpr} DESC, c."created_at" DESC, c."cid" ASC
|
||||
OFFSET ${skip}
|
||||
LIMIT ${take}
|
||||
`
|
||||
`,
|
||||
);
|
||||
|
||||
// 批量查询每条评论的配图/贴纸
|
||||
const cids = rows.map(r => r.cid);
|
||||
const cids = rows.map((r) => r.cid);
|
||||
const images = cids.length
|
||||
? await prisma.commentImage.findMany({
|
||||
where: { commentId: { in: cids } },
|
||||
orderBy: { order: 'asc' },
|
||||
select: { commentId: true, url: true, width: true, height: true, order: true },
|
||||
})
|
||||
where: { commentId: { in: cids } },
|
||||
orderBy: { order: "asc" },
|
||||
select: {
|
||||
commentId: true,
|
||||
url: true,
|
||||
width: true,
|
||||
height: true,
|
||||
order: true,
|
||||
},
|
||||
})
|
||||
: [];
|
||||
const group = new Map<string, { url: string; width?: number | null; height?: number | null; order: number }[]>();
|
||||
const group = new Map<
|
||||
string,
|
||||
{
|
||||
url: string;
|
||||
width?: number | null;
|
||||
height?: number | null;
|
||||
order: number;
|
||||
}[]
|
||||
>();
|
||||
for (const img of images) {
|
||||
const arr = group.get(img.commentId) || [];
|
||||
arr.push({ url: img.url, width: img.width, height: img.height, order: img.order });
|
||||
arr.push({
|
||||
url: img.url,
|
||||
width: img.width,
|
||||
height: img.height,
|
||||
order: img.order,
|
||||
});
|
||||
group.set(img.commentId, arr);
|
||||
}
|
||||
const formattedComments = rows.map((c) => ({
|
||||
@@ -117,9 +135,15 @@ export async function GET(
|
||||
digg_count: Number(c.digg_count),
|
||||
user: {
|
||||
nickname: c.nickname,
|
||||
avatar_url: getFileUrl(c.avatar_url || 'default_avatar.png'),
|
||||
avatar_url: getFileUrl(c.avatar_url || "default_avatar.png"),
|
||||
},
|
||||
images: (group.get(c.cid) || []).sort((a, b) => a.order - b.order).map(i => ({ url: getFileUrl(i.url), width: i.width ?? undefined, height: i.height ?? undefined })),
|
||||
images: (group.get(c.cid) || [])
|
||||
.sort((a, b) => a.order - b.order)
|
||||
.map((i) => ({
|
||||
url: getFileUrl(i.url),
|
||||
width: i.width ?? undefined,
|
||||
height: i.height ?? undefined,
|
||||
})),
|
||||
}));
|
||||
|
||||
return NextResponse.json({
|
||||
@@ -134,9 +158,6 @@ export async function GET(
|
||||
});
|
||||
} catch (error) {
|
||||
console.error("获取评论失败:", error);
|
||||
return NextResponse.json(
|
||||
{ error: "获取评论失败" },
|
||||
{ status: 500 }
|
||||
);
|
||||
return NextResponse.json({ error: "获取评论失败" }, { status: 500 });
|
||||
}
|
||||
}
|
||||
|
||||
+29
-17
@@ -1,7 +1,7 @@
|
||||
import { NextRequest, NextResponse } from 'next/server';
|
||||
import { prisma } from '@/lib/prisma';
|
||||
import type { FeedItem, FeedResponse } from '@/app/types/feed';
|
||||
import { getFileUrl } from '@/lib/minio';
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { prisma } from "@/lib/prisma";
|
||||
import type { FeedItem, FeedResponse } from "@/app/types/feed";
|
||||
import { getFileUrl } from "@/lib/minio";
|
||||
|
||||
// Contract
|
||||
// Inputs: search params { before?: ISOString, limit?: number }
|
||||
@@ -9,25 +9,25 @@ import { getFileUrl } from '@/lib/minio';
|
||||
|
||||
export async function GET(req: NextRequest) {
|
||||
const { searchParams } = new URL(req.url);
|
||||
const limitParam = searchParams.get('limit');
|
||||
const beforeParam = searchParams.get('before');
|
||||
const limitParam = searchParams.get("limit");
|
||||
const beforeParam = searchParams.get("before");
|
||||
|
||||
const limit = Math.min(Math.max(Number(limitParam ?? '24'), 1), 60); // 1..60
|
||||
const limit = Math.min(Math.max(Number(limitParam ?? "24"), 1), 60); // 1..60
|
||||
const before = beforeParam ? new Date(beforeParam) : null;
|
||||
|
||||
// fetch chunk from both tables
|
||||
const [videos, posts] = await Promise.all([
|
||||
prisma.video.findMany({
|
||||
where: before ? { created_at: { lt: before } } : undefined,
|
||||
orderBy: { created_at: 'desc' },
|
||||
orderBy: { created_at: "desc" },
|
||||
take: limit,
|
||||
include: { author: true },
|
||||
}),
|
||||
prisma.imagePost.findMany({
|
||||
where: before ? { created_at: { lt: before } } : undefined,
|
||||
orderBy: { created_at: 'desc' },
|
||||
orderBy: { created_at: "desc" },
|
||||
take: limit,
|
||||
include: { author: true, images: { orderBy: { order: 'asc' }, take: 1 } },
|
||||
include: { author: true, images: { orderBy: { order: "asc" }, take: 1 } },
|
||||
}),
|
||||
]);
|
||||
|
||||
@@ -38,11 +38,15 @@ export async function GET(req: NextRequest) {
|
||||
created_at: v.created_at,
|
||||
desc: v.desc,
|
||||
video_url: getFileUrl(v.video_url),
|
||||
cover_url: getFileUrl(v.cover_url ?? 'default_cover.png'),
|
||||
cover_url: getFileUrl(v.cover_url ?? "default_cover.png"),
|
||||
width: v.width ?? null,
|
||||
height: v.height ?? null,
|
||||
author: { nickname: v.author.nickname, avatar_url: getFileUrl(v.author.avatar_url ?? ''), sec_uid: v.author.sec_uid },
|
||||
likes: Number(v.digg_count)
|
||||
author: {
|
||||
nickname: v.author.nickname,
|
||||
avatar_url: getFileUrl(v.author.avatar_url ?? ""),
|
||||
sec_uid: v.author.sec_uid,
|
||||
},
|
||||
likes: Number(v.digg_count),
|
||||
})),
|
||||
...posts.map((p) => ({
|
||||
type: "image" as const,
|
||||
@@ -52,13 +56,21 @@ export async function GET(req: NextRequest) {
|
||||
cover_url: getFileUrl(p.images?.[0]?.url ?? null),
|
||||
width: p.images?.[0]?.width ?? null,
|
||||
height: p.images?.[0]?.height ?? null,
|
||||
author: { nickname: p.author.nickname, avatar_url: getFileUrl(p.author.avatar_url ?? ''), sec_uid: p.author.sec_uid },
|
||||
likes: Number(p.digg_count)
|
||||
author: {
|
||||
nickname: p.author.nickname,
|
||||
avatar_url: getFileUrl(p.author.avatar_url ?? ""),
|
||||
sec_uid: p.author.sec_uid,
|
||||
},
|
||||
likes: Number(p.digg_count),
|
||||
})),
|
||||
].sort((a, b) => +new Date(b.created_at) - +new Date(a.created_at))
|
||||
]
|
||||
.sort((a, b) => +new Date(b.created_at) - +new Date(a.created_at))
|
||||
.slice(0, limit);
|
||||
|
||||
const nextCursor = merged.length > 0 ? new Date(merged[merged.length - 1].created_at as any).toISOString() : null;
|
||||
const nextCursor =
|
||||
merged.length > 0
|
||||
? new Date(merged[merged.length - 1].created_at as any).toISOString()
|
||||
: null;
|
||||
const payload: FeedResponse = { items: merged, nextCursor };
|
||||
return NextResponse.json(payload);
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
// index.js
|
||||
import { chromium } from 'playwright-extra';
|
||||
import stealth from 'puppeteer-extra-plugin-stealth'
|
||||
import { chromium } from "playwright-extra";
|
||||
import stealth from "puppeteer-extra-plugin-stealth";
|
||||
|
||||
chromium.use(stealth());
|
||||
|
||||
@@ -8,9 +8,9 @@ chromium.use(stealth());
|
||||
const browser = await chromium.launch({ headless: false }); // 需要可视化就 false
|
||||
const context = await browser.newContext();
|
||||
const page = await context.newPage();
|
||||
await page.goto('https://bot.sannysoft.com/'); // 常用自测页
|
||||
console.log('Title:', await page.title());
|
||||
await page.screenshot({ path: 'stealth.png', fullPage: true });
|
||||
await page.goto("https://bot.sannysoft.com/"); // 常用自测页
|
||||
console.log("Title:", await page.title());
|
||||
await page.screenshot({ path: "stealth.png", fullPage: true });
|
||||
setTimeout(async () => {
|
||||
await browser.close();
|
||||
}, 1000_000);
|
||||
|
||||
+72
-71
@@ -1,99 +1,98 @@
|
||||
export const runtime = 'nodejs'
|
||||
export const runtime = "nodejs";
|
||||
|
||||
import { type Browser, type BrowserContext } from 'playwright'
|
||||
import { chromium } from 'playwright-extra' // A simple singleton manager for a persistent Chromium context with ref-counting. // Prevents concurrent tasks from closing the shared window prematurely.
|
||||
import stealth from 'puppeteer-extra-plugin-stealth'
|
||||
import { type Browser, type BrowserContext } from "playwright";
|
||||
import { chromium } from "playwright-extra"; // A simple singleton manager for a persistent Chromium context with ref-counting. // Prevents concurrent tasks from closing the shared window prematurely.
|
||||
import stealth from "puppeteer-extra-plugin-stealth";
|
||||
chromium.use(stealth());
|
||||
|
||||
let contextPromise: Promise<BrowserContext> | null = null
|
||||
let context: BrowserContext | null = null
|
||||
let refCount = 0
|
||||
let idleCloseTimer: NodeJS.Timeout | null = null
|
||||
let contextPromise: Promise<BrowserContext> | null = null;
|
||||
let context: BrowserContext | null = null;
|
||||
let refCount = 0;
|
||||
let idleCloseTimer: NodeJS.Timeout | null = null;
|
||||
|
||||
const USER_DATA_DIR = process.env.USER_DATA_DIR ?? 'chrome-profile/douyin'
|
||||
const USER_DATA_DIR = process.env.USER_DATA_DIR ?? "chrome-profile/douyin";
|
||||
|
||||
async function launchContext(): Promise<BrowserContext> {
|
||||
const ctx = await chromium.launchPersistentContext(
|
||||
USER_DATA_DIR,
|
||||
{
|
||||
headless: process.env.CHROMIUM_HEADLESS === 'true',
|
||||
viewport: {
|
||||
width: Number(process.env.CHROMIUM_VIEWPORT_WIDTH ?? 1280),
|
||||
height: Number(process.env.CHROMIUM_VIEWPORT_HEIGHT ?? 1080)
|
||||
}
|
||||
}
|
||||
)
|
||||
const ctx = await chromium.launchPersistentContext(USER_DATA_DIR, {
|
||||
headless: process.env.CHROMIUM_HEADLESS === "true",
|
||||
viewport: {
|
||||
width: Number(process.env.CHROMIUM_VIEWPORT_WIDTH ?? 1280),
|
||||
height: Number(process.env.CHROMIUM_VIEWPORT_HEIGHT ?? 1080),
|
||||
},
|
||||
});
|
||||
// When the context is closed externally, reset manager state
|
||||
ctx.on('close', () => {
|
||||
context = null
|
||||
contextPromise = null
|
||||
refCount = 0
|
||||
ctx.on("close", () => {
|
||||
context = null;
|
||||
contextPromise = null;
|
||||
refCount = 0;
|
||||
if (idleCloseTimer) {
|
||||
clearTimeout(idleCloseTimer)
|
||||
idleCloseTimer = null
|
||||
clearTimeout(idleCloseTimer);
|
||||
idleCloseTimer = null;
|
||||
}
|
||||
})
|
||||
return ctx
|
||||
});
|
||||
return ctx;
|
||||
}
|
||||
|
||||
export async function acquireBrowserContext(): Promise<BrowserContext> {
|
||||
// Cancel any pending idle close if a new consumer arrives
|
||||
if (idleCloseTimer) {
|
||||
clearTimeout(idleCloseTimer)
|
||||
idleCloseTimer = null
|
||||
clearTimeout(idleCloseTimer);
|
||||
idleCloseTimer = null;
|
||||
}
|
||||
|
||||
if (context) {
|
||||
refCount += 1
|
||||
return context
|
||||
refCount += 1;
|
||||
return context;
|
||||
}
|
||||
|
||||
if (!contextPromise) {
|
||||
contextPromise = launchContext()
|
||||
contextPromise = launchContext();
|
||||
}
|
||||
context = await contextPromise
|
||||
refCount += 1
|
||||
return context
|
||||
context = await contextPromise;
|
||||
refCount += 1;
|
||||
return context;
|
||||
}
|
||||
|
||||
export async function releaseBrowserContext(options?: { idleMillis?: number }): Promise<void> {
|
||||
const idleMillis = options?.idleMillis ?? 15_000
|
||||
refCount = Math.max(0, refCount - 1)
|
||||
export async function releaseBrowserContext(options?: {
|
||||
idleMillis?: number;
|
||||
}): Promise<void> {
|
||||
const idleMillis = options?.idleMillis ?? 15_000;
|
||||
refCount = Math.max(0, refCount - 1);
|
||||
|
||||
if (refCount > 0 || !context) return
|
||||
if (refCount > 0 || !context) return;
|
||||
|
||||
// Delay the close to allow bursty workloads to reuse the context
|
||||
if (idleCloseTimer) {
|
||||
clearTimeout(idleCloseTimer)
|
||||
idleCloseTimer = null
|
||||
clearTimeout(idleCloseTimer);
|
||||
idleCloseTimer = null;
|
||||
}
|
||||
idleCloseTimer = setTimeout(async () => {
|
||||
try {
|
||||
if (context && refCount === 0) {
|
||||
await context.close()
|
||||
await context.close();
|
||||
}
|
||||
} finally {
|
||||
context = null
|
||||
contextPromise = null
|
||||
idleCloseTimer = null
|
||||
context = null;
|
||||
contextPromise = null;
|
||||
idleCloseTimer = null;
|
||||
}
|
||||
}, idleMillis)
|
||||
}, idleMillis);
|
||||
}
|
||||
|
||||
// --- Isolated context support for per-scrape independence ---
|
||||
|
||||
const isolatedMap = new WeakMap<BrowserContext, Browser>()
|
||||
const isolatedMap = new WeakMap<BrowserContext, Browser>();
|
||||
|
||||
async function getSharedStorageState(): Promise<any | undefined> {
|
||||
try {
|
||||
const shared = await acquireBrowserContext()
|
||||
const state = await shared.storageState()
|
||||
const shared = await acquireBrowserContext();
|
||||
const state = await shared.storageState();
|
||||
// Do not force-close immediately; keep ref-counting behavior
|
||||
await releaseBrowserContext()
|
||||
return state
|
||||
await releaseBrowserContext();
|
||||
return state;
|
||||
} catch {
|
||||
// If shared context not available, proceed without storageState
|
||||
return undefined
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -103,37 +102,39 @@ async function getSharedStorageState(): Promise<any | undefined> {
|
||||
* so you remain logged-in, but isolates network events, cache and listeners.
|
||||
*/
|
||||
export async function acquireIsolatedContext(): Promise<BrowserContext> {
|
||||
const storageState = await getSharedStorageState()
|
||||
const storageState = await getSharedStorageState();
|
||||
const browser = await chromium.launch({
|
||||
headless: process.env.CHROMIUM_HEADLESS === 'true'
|
||||
})
|
||||
headless: process.env.CHROMIUM_HEADLESS === "true",
|
||||
});
|
||||
const ctx = await browser.newContext({
|
||||
storageState,
|
||||
viewport: {
|
||||
width: Number(process.env.CHROMIUM_VIEWPORT_WIDTH ?? 1280),
|
||||
height: Number(process.env.CHROMIUM_VIEWPORT_HEIGHT ?? 1080)
|
||||
}
|
||||
})
|
||||
isolatedMap.set(ctx, browser)
|
||||
ctx.on('close', () => {
|
||||
const b = isolatedMap.get(ctx)
|
||||
height: Number(process.env.CHROMIUM_VIEWPORT_HEIGHT ?? 1080),
|
||||
},
|
||||
});
|
||||
isolatedMap.set(ctx, browser);
|
||||
ctx.on("close", () => {
|
||||
const b = isolatedMap.get(ctx);
|
||||
if (b) {
|
||||
b.close().catch(() => {})
|
||||
isolatedMap.delete(ctx)
|
||||
b.close().catch(() => {});
|
||||
isolatedMap.delete(ctx);
|
||||
}
|
||||
})
|
||||
return ctx
|
||||
});
|
||||
return ctx;
|
||||
}
|
||||
|
||||
export async function releaseIsolatedContext(ctx: BrowserContext | null | undefined): Promise<void> {
|
||||
if (!ctx) return
|
||||
export async function releaseIsolatedContext(
|
||||
ctx: BrowserContext | null | undefined,
|
||||
): Promise<void> {
|
||||
if (!ctx) return;
|
||||
try {
|
||||
await ctx.close()
|
||||
await ctx.close();
|
||||
} finally {
|
||||
const b = isolatedMap.get(ctx)
|
||||
const b = isolatedMap.get(ctx);
|
||||
if (b) {
|
||||
await b.close().catch(() => {})
|
||||
isolatedMap.delete(ctx)
|
||||
await b.close().catch(() => {});
|
||||
isolatedMap.delete(ctx);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+416
-287
@@ -1,22 +1,31 @@
|
||||
export const runtime = 'nodejs'
|
||||
export const runtime = "nodejs";
|
||||
// src/scrapeDouyin.ts
|
||||
import { BrowserContext, Page, type Response } from 'playwright';
|
||||
import { chromium } from 'playwright-extra';
|
||||
import { prisma } from '@/lib/prisma';
|
||||
import { uploadFile, generateUniqueFileName } from '@/lib/minio';
|
||||
import { createCamelCompatibleProxy } from '@/app/api/fetcher/utils';
|
||||
import { waitForFirstResponse, waitForResponseWithTimeout, safeJson, downloadBinary, collectResponsesWithinTime } from '@/app/api/fetcher/network';
|
||||
import { pickBestPlayAddr } from '@/app/api/fetcher/media';
|
||||
import { handleImagePost } from '@/app/api/fetcher/uploader';
|
||||
import { saveToDB, saveImagePostToDB } from '@/app/api/fetcher/persist';
|
||||
import chalk from 'chalk';
|
||||
import { acquireIsolatedContext, releaseIsolatedContext } from '@/app/api/fetcher/browser';
|
||||
import { extractFirstFrame } from '@/app/api/media';
|
||||
import { transcriptAweme } from '../stt';
|
||||
import { BrowserContext, Page, type Response } from "playwright";
|
||||
import { chromium } from "playwright-extra";
|
||||
import { prisma } from "@/lib/prisma";
|
||||
import { uploadFile, generateUniqueFileName } from "@/lib/minio";
|
||||
import { createCamelCompatibleProxy } from "@/app/api/fetcher/utils";
|
||||
import {
|
||||
waitForFirstResponse,
|
||||
waitForResponseWithTimeout,
|
||||
safeJson,
|
||||
downloadBinary,
|
||||
collectResponsesWithinTime,
|
||||
} from "@/app/api/fetcher/network";
|
||||
import { pickBestPlayAddr } from "@/app/api/fetcher/media";
|
||||
import { handleImagePost } from "@/app/api/fetcher/uploader";
|
||||
import { saveToDB, saveImagePostToDB } from "@/app/api/fetcher/persist";
|
||||
import chalk from "chalk";
|
||||
import {
|
||||
acquireIsolatedContext,
|
||||
releaseIsolatedContext,
|
||||
} from "@/app/api/fetcher/browser";
|
||||
import { extractFirstFrame } from "@/app/api/media";
|
||||
import { transcriptAweme } from "../stt";
|
||||
|
||||
const DETAIL_PATH = '/aweme/v1/web/aweme/detail/';
|
||||
const COMMENT_PATH = '/aweme/v1/web/comment/list/';
|
||||
const POST_PATH = '/aweme/v1/web/aweme/post/'
|
||||
const DETAIL_PATH = "/aweme/v1/web/aweme/detail/";
|
||||
const COMMENT_PATH = "/aweme/v1/web/comment/list/";
|
||||
const POST_PATH = "/aweme/v1/web/aweme/post/";
|
||||
|
||||
/**
|
||||
* 滚动页面并收集评论
|
||||
@@ -26,324 +35,444 @@ const POST_PATH = '/aweme/v1/web/aweme/post/'
|
||||
* @returns 收集到的所有评论响应
|
||||
*/
|
||||
async function scrollAndCollectComments(
|
||||
context: BrowserContext,
|
||||
page: Page,
|
||||
durationMs: number = 10_000
|
||||
context: BrowserContext,
|
||||
page: Page,
|
||||
durationMs: number = 10_000,
|
||||
): Promise<Response[]> {
|
||||
console.log(chalk.blue(`📜 开始滚动页面收集评论（持续 ${durationMs / 1000} 秒）...`));
|
||||
console.log(
|
||||
chalk.blue(`📜 开始滚动页面收集评论（持续 ${durationMs / 1000} 秒）...`),
|
||||
);
|
||||
|
||||
// 启动评论响应收集器
|
||||
const commentResponsesPromise = collectResponsesWithinTime(
|
||||
context,
|
||||
(r: Response) => r.url().includes(COMMENT_PATH) && r.status() === 200 && r.request().frame()?.page() === page,
|
||||
durationMs
|
||||
);
|
||||
// 启动评论响应收集器
|
||||
const commentResponsesPromise = collectResponsesWithinTime(
|
||||
context,
|
||||
(r: Response) =>
|
||||
r.url().includes(COMMENT_PATH) &&
|
||||
r.status() === 200 &&
|
||||
r.request().frame()?.page() === page,
|
||||
durationMs,
|
||||
);
|
||||
|
||||
// 在指定时间内持续滚动页面
|
||||
const startTime = Date.now();
|
||||
const scrollInterval = 500;
|
||||
let scrollCount = 0;
|
||||
const selector = "div[data-e2e='comment-list']";
|
||||
// 在指定时间内持续滚动页面
|
||||
const startTime = Date.now();
|
||||
const scrollInterval = 500;
|
||||
let scrollCount = 0;
|
||||
const selector = "div[data-e2e='comment-list']";
|
||||
|
||||
// 1) 等元素出现并可见
|
||||
await page.waitForSelector(selector, { state: 'visible', timeout: 5000 });
|
||||
// 1) 等元素出现并可见
|
||||
await page.waitForSelector(selector, { state: "visible", timeout: 5000 });
|
||||
|
||||
// 2) 确保滚动到可见区域
|
||||
const list = page.locator(selector);
|
||||
await list.scrollIntoViewIfNeeded();
|
||||
// 2) 确保滚动到可见区域
|
||||
const list = page.locator(selector);
|
||||
await list.scrollIntoViewIfNeeded();
|
||||
|
||||
// 3) 执行 hover（推荐用 locator 的 hover）
|
||||
list.hover({ timeout: 5000 }).catch(() => { });
|
||||
while (Date.now() - startTime < durationMs - 500) { // 留 500ms 缓冲
|
||||
try {
|
||||
list.hover({ timeout: 2000 }).catch(() => { });
|
||||
// 使用 Playwright 的 mouse.wheel 方法滚动
|
||||
// 每次滚动一大段距离
|
||||
// await list.hover();
|
||||
const scrollAmount = 1500;
|
||||
await page.mouse.wheel(0, scrollAmount);
|
||||
// 3) 执行 hover（推荐用 locator 的 hover）
|
||||
list.hover({ timeout: 5000 }).catch(() => {});
|
||||
while (Date.now() - startTime < durationMs - 500) {
|
||||
// 留 500ms 缓冲
|
||||
try {
|
||||
list.hover({ timeout: 2000 }).catch(() => {});
|
||||
// 使用 Playwright 的 mouse.wheel 方法滚动
|
||||
// 每次滚动一大段距离
|
||||
// await list.hover();
|
||||
const scrollAmount = 1500;
|
||||
await page.mouse.wheel(0, scrollAmount);
|
||||
|
||||
scrollCount++;
|
||||
console.log(chalk.gray(` ↓ 第 ${scrollCount} 次滚动`));
|
||||
scrollCount++;
|
||||
console.log(chalk.gray(` ↓ 第 ${scrollCount} 次滚动`));
|
||||
|
||||
// 等待一段时间，让评论加载
|
||||
await page.waitForTimeout(scrollInterval);
|
||||
|
||||
} catch (e) {
|
||||
console.warn(chalk.yellow(` ⚠ 滚动时出现警告: ${(e as Error)?.message}`));
|
||||
}
|
||||
// 等待一段时间，让评论加载
|
||||
await page.waitForTimeout(scrollInterval);
|
||||
} catch (e) {
|
||||
console.warn(
|
||||
chalk.yellow(` ⚠ 滚动时出现警告: ${(e as Error)?.message}`),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// 等待收集器完成
|
||||
const commentResponses = await commentResponsesPromise;
|
||||
console.log(chalk.green(`✓ 评论收集完成，共收集到 ${commentResponses.length} 个评论响应`));
|
||||
// 等待收集器完成
|
||||
const commentResponses = await commentResponsesPromise;
|
||||
console.log(
|
||||
chalk.green(
|
||||
`✓ 评论收集完成，共收集到 ${commentResponses.length} 个评论响应`,
|
||||
),
|
||||
);
|
||||
|
||||
return commentResponses;
|
||||
return commentResponses;
|
||||
}
|
||||
|
||||
async function readPostMem(context: BrowserContext, page: Page) {
|
||||
const md = await page.evaluate(() => {
|
||||
// @ts-ignore
|
||||
let data = window.__pace_captured__.find(i => i[1] && i[1].includes(`"awemeId":`))[1]
|
||||
return JSON.parse(data.slice(data.indexOf("{")).replaceAll("]\n", ''))
|
||||
// return {aweme: { detail: {} } };
|
||||
}).catch(() => null);
|
||||
const md = await page
|
||||
.evaluate(() => {
|
||||
const captured = (window as any).__pace_captured__ as Array<unknown[]>;
|
||||
let data = captured.find(
|
||||
(item) => typeof item[1] === "string" && item[1].includes(`"awemeId":`),
|
||||
)?.[1] as string | undefined;
|
||||
if (!data) return null;
|
||||
return JSON.parse(data.slice(data.indexOf("{")).replaceAll("]\n", ""));
|
||||
// return {aweme: { detail: {} } };
|
||||
})
|
||||
.catch(() => null);
|
||||
|
||||
// await new Promise((res) => setTimeout(res, 1000000));
|
||||
// await new Promise((res) => setTimeout(res, 1000000));
|
||||
|
||||
let aweme_mem = md?.aweme?.detail as DouyinImageAweme;
|
||||
if (!aweme_mem) throw new Error('页面内存数据中未找到作品详情');
|
||||
let aweme_mem = md?.aweme?.detail as DouyinImageAweme;
|
||||
if (!aweme_mem) throw new Error("页面内存数据中未找到作品详情");
|
||||
|
||||
// @ts-ignore
|
||||
aweme_mem.author = aweme_mem.authorInfo
|
||||
// @ts-ignore
|
||||
aweme_mem.statistics = aweme_mem.stats
|
||||
// @ts-ignore
|
||||
aweme_mem.author = aweme_mem.authorInfo;
|
||||
// @ts-ignore
|
||||
aweme_mem.statistics = aweme_mem.stats;
|
||||
|
||||
const comments = md.comment ? createCamelCompatibleProxy<DouyinCommentResponse>(md.comment) : null;
|
||||
const aweme = createCamelCompatibleProxy(aweme_mem);
|
||||
const comments = md.comment
|
||||
? createCamelCompatibleProxy<DouyinCommentResponse>(md.comment)
|
||||
: null;
|
||||
const aweme = createCamelCompatibleProxy(aweme_mem);
|
||||
|
||||
return { aweme, comments }
|
||||
return { aweme, comments };
|
||||
}
|
||||
|
||||
|
||||
export class ScrapeError extends Error {
|
||||
constructor(
|
||||
message: string,
|
||||
public statusCode: number = 500,
|
||||
public code?: string
|
||||
) {
|
||||
super(message);
|
||||
this.name = 'ScrapeError';
|
||||
}
|
||||
constructor(
|
||||
message: string,
|
||||
public statusCode: number = 500,
|
||||
public code?: string,
|
||||
) {
|
||||
super(message);
|
||||
this.name = "ScrapeError";
|
||||
}
|
||||
}
|
||||
|
||||
export async function scrapeDouyin(url: string) {
|
||||
console.log(chalk.blue('🚀 启动共享 Chromium 浏览器...'));
|
||||
let context: BrowserContext | null = await acquireIsolatedContext();
|
||||
const page = await context.newPage();
|
||||
console.log(chalk.cyan(`📄 正在访问: ${chalk.underline(url)}`));
|
||||
console.log(chalk.blue("🚀 启动共享 Chromium 浏览器..."));
|
||||
let context: BrowserContext | null = await acquireIsolatedContext();
|
||||
const page = await context.newPage();
|
||||
console.log(chalk.cyan(`📄 正在访问: ${chalk.underline(url)}`));
|
||||
|
||||
await page.addInitScript(() => {
|
||||
// 建一个全局容器存捕获的数据
|
||||
(window as any).__pace_captured__ = [];
|
||||
await page.addInitScript(() => {
|
||||
// 建一个全局容器存捕获的数据
|
||||
(window as any).__pace_captured__ = [];
|
||||
|
||||
// 用 Proxy 包装一个数组，拦截 push
|
||||
const captured = (window as any).__pace_captured__;
|
||||
const proxyArr = new Proxy([] as any[], {
|
||||
get(target, prop, receiver) {
|
||||
if (prop === 'push') {
|
||||
return (...items: any[]) => {
|
||||
try { captured.push(...items); } catch { }
|
||||
return Array.prototype.push.apply(target, items);
|
||||
};
|
||||
}
|
||||
return Reflect.get(target, prop, receiver);
|
||||
},
|
||||
set(target, prop, value, receiver) {
|
||||
// 兼容站点可能直接赋初始数组： self.__pace_f = [a,b]
|
||||
if (prop === 'length') return Reflect.set(target, prop, value, receiver);
|
||||
return Reflect.set(target, prop, value, receiver);
|
||||
}
|
||||
});
|
||||
|
||||
(self as any).__pace_f = proxyArr;
|
||||
(window as any).__pace_f = proxyArr;
|
||||
// 用 Proxy 包装一个数组，拦截 push
|
||||
const captured = (window as any).__pace_captured__;
|
||||
const proxyArr = new Proxy([] as any[], {
|
||||
get(target, prop, receiver) {
|
||||
if (prop === "push") {
|
||||
return (...items: any[]) => {
|
||||
try {
|
||||
captured.push(...items);
|
||||
} catch {}
|
||||
return Array.prototype.push.apply(target, items);
|
||||
};
|
||||
}
|
||||
return Reflect.get(target, prop, receiver);
|
||||
},
|
||||
set(target, prop, value, receiver) {
|
||||
// 兼容站点可能直接赋初始数组： self.__pace_f = [a,b]
|
||||
if (prop === "length")
|
||||
return Reflect.set(target, prop, value, receiver);
|
||||
return Reflect.set(target, prop, value, receiver);
|
||||
},
|
||||
});
|
||||
|
||||
(self as any).__pace_f = proxyArr;
|
||||
(window as any).__pace_f = proxyArr;
|
||||
});
|
||||
|
||||
try {
|
||||
// 先注册“先到先得”的监听，再导航，避免漏包
|
||||
const firstTypePromise = waitForFirstResponse(
|
||||
context,
|
||||
[
|
||||
{
|
||||
key: "detail",
|
||||
test: (r: Response) =>
|
||||
r.url().includes(DETAIL_PATH) &&
|
||||
r.status() === 200 &&
|
||||
r.request().frame()?.page() === page,
|
||||
},
|
||||
{
|
||||
key: "post",
|
||||
test: (r: Response) =>
|
||||
r.url().includes(POST_PATH) &&
|
||||
r.status() === 200 &&
|
||||
r.request().frame()?.page() === page,
|
||||
},
|
||||
],
|
||||
10_000,
|
||||
);
|
||||
|
||||
await page.goto(url, { waitUntil: "domcontentloaded", timeout: 20_000 });
|
||||
|
||||
// 查找页面中是否存在 "视频不存在" 的提示
|
||||
const isNotFound = await page
|
||||
.locator("text=视频不存在")
|
||||
.count()
|
||||
.then((count) => count > 0)
|
||||
.catch(() => false);
|
||||
if (isNotFound) {
|
||||
console.error(chalk.red("✗ 视频不存在或已被删除"));
|
||||
throw new ScrapeError("视频不存在或已被删除", 404, "VIDEO_NOT_FOUND");
|
||||
}
|
||||
|
||||
// 等待作品类型判定
|
||||
const firstType = await firstTypePromise;
|
||||
|
||||
// 尝试从内存读取图文数据（如果是图文作品）
|
||||
let memoryData: {
|
||||
aweme: any;
|
||||
comments: DouyinCommentResponse | null;
|
||||
} | null = null;
|
||||
try {
|
||||
// 先注册“先到先得”的监听，再导航，避免漏包
|
||||
const firstTypePromise = waitForFirstResponse(context, [
|
||||
{ key: 'detail', test: (r: Response) => r.url().includes(DETAIL_PATH) && r.status() === 200 && r.request().frame()?.page() === page },
|
||||
{ key: 'post', test: (r: Response) => r.url().includes(POST_PATH) && r.status() === 200 && r.request().frame()?.page() === page },
|
||||
], 10_000);
|
||||
memoryData = await readPostMem(context, page);
|
||||
console.log(chalk.green("✓ 从内存读取图文数据成功"));
|
||||
} catch {
|
||||
// 内存读取失败，稍后通过网络获取
|
||||
}
|
||||
|
||||
await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 20_000 });
|
||||
if (!firstType && !memoryData) {
|
||||
console.error(chalk.red("✗ 既无法从内存读取数据,也无法从网络获得数据"));
|
||||
throw new ScrapeError(
|
||||
"无法获取作品数据,可能是网络问题或作品已下架",
|
||||
404,
|
||||
"NO_DATA",
|
||||
);
|
||||
}
|
||||
|
||||
// 查找页面中是否存在 "视频不存在" 的提示
|
||||
const isNotFound = await page.locator('text=视频不存在').count().then(count => count > 0).catch(() => false);
|
||||
if (isNotFound) {
|
||||
console.error(chalk.red('✗ 视频不存在或已被删除'));
|
||||
throw new ScrapeError('视频不存在或已被删除', 404, 'VIDEO_NOT_FOUND');
|
||||
}
|
||||
console.log(
|
||||
chalk.cyan(
|
||||
`📡 检测到作品类型: ${chalk.bold(firstType?.key === "post" || memoryData ? "图文" : "视频")}`,
|
||||
),
|
||||
);
|
||||
|
||||
// 等待作品类型判定
|
||||
const firstType = await firstTypePromise;
|
||||
let allComments: DouyinComment[] = [];
|
||||
try {
|
||||
// 开始滚动并收集评论
|
||||
const commentResponses = await scrollAndCollectComments(context, page);
|
||||
|
||||
// 尝试从内存读取图文数据（如果是图文作品）
|
||||
let memoryData: { aweme: any; comments: DouyinCommentResponse | null } | null = null;
|
||||
// 解析所有收集到的评论响应
|
||||
for (const commentRes of commentResponses) {
|
||||
try {
|
||||
memoryData = await readPostMem(context, page);
|
||||
console.log(chalk.green('✓ 从内存读取图文数据成功'));
|
||||
} catch {
|
||||
// 内存读取失败，稍后通过网络获取
|
||||
const commentData = await safeJson<DouyinCommentResponse>(commentRes);
|
||||
if (commentData?.comments?.length) {
|
||||
allComments.push(...commentData.comments);
|
||||
}
|
||||
} catch (e) {
|
||||
console.warn(
|
||||
chalk.yellow(`⚠ 解析评论响应失败: ${(e as Error)?.message}`),
|
||||
);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.warn(
|
||||
chalk.yellow(`⚠ 评论收集失败: ${(error as Error)?.message}`),
|
||||
);
|
||||
}
|
||||
|
||||
if (!firstType && !memoryData) {
|
||||
console.error(chalk.red('✗ 既无法从内存读取数据,也无法从网络获得数据'));
|
||||
throw new ScrapeError('无法获取作品数据,可能是网络问题或作品已下架', 404, 'NO_DATA');
|
||||
// 去重评论（根据 cid）
|
||||
const uniqueComments = Array.from(
|
||||
new Map(allComments.map((c) => [c.cid, c])).values(),
|
||||
);
|
||||
|
||||
console.log(
|
||||
chalk.green(
|
||||
`✓ 共收集到 ${uniqueComments.length} 条独立评论（去重前: ${allComments.length}）`,
|
||||
),
|
||||
);
|
||||
|
||||
// 如果从内存读取到了评论，合并进来作为兜底
|
||||
let comments: DouyinCommentResponse;
|
||||
if (memoryData?.comments?.comments?.length) {
|
||||
console.log(
|
||||
chalk.blue(
|
||||
`📝 合并内存中的 ${memoryData.comments.comments.length} 条评论`,
|
||||
),
|
||||
);
|
||||
const memComments = memoryData.comments.comments;
|
||||
const mergedMap = new Map(uniqueComments.map((c) => [c.cid, c]));
|
||||
for (const c of memComments) {
|
||||
if (!mergedMap.has(c.cid)) {
|
||||
mergedMap.set(c.cid, c);
|
||||
}
|
||||
}
|
||||
comments = {
|
||||
comments: Array.from(mergedMap.values()),
|
||||
total: mergedMap.size,
|
||||
status_code: 0,
|
||||
};
|
||||
console.log(chalk.green(`✓ 合并后共 ${comments.comments.length} 条评论`));
|
||||
} else {
|
||||
comments = {
|
||||
comments: uniqueComments,
|
||||
total: uniqueComments.length,
|
||||
status_code: 0,
|
||||
};
|
||||
}
|
||||
|
||||
console.log(chalk.cyan(`📡 检测到作品类型: ${chalk.bold(firstType?.key === 'post' || memoryData ? '图文' : '视频')}`));
|
||||
// 分支：视频 or 图文（两者只会有一个命中，先到先得）
|
||||
// 优先处理内存数据（图文）
|
||||
if (memoryData) {
|
||||
const aweme = memoryData.aweme;
|
||||
const uploads = await handleImagePost(context, aweme);
|
||||
const saved = await saveImagePostToDB(context, aweme, comments, uploads); // 传递完整 JSON
|
||||
console.log(chalk.green.bold("✓ 图文作品保存成功"));
|
||||
return { type: "image", ...saved };
|
||||
} else if (firstType?.key === "post") {
|
||||
// 图文作品（网络）
|
||||
const postJson = await safeJson<DouyinPostListResponse>(
|
||||
firstType.response,
|
||||
);
|
||||
if (!postJson?.aweme_list?.length)
|
||||
throw new ScrapeError("图文作品响应为空", 404, "EMPTY_POST_RESPONSE");
|
||||
|
||||
let allComments: DouyinComment[] = [];
|
||||
try {
|
||||
// 开始滚动并收集评论
|
||||
const commentResponses = await scrollAndCollectComments(context, page);
|
||||
const currentURL = page.url();
|
||||
const target_aweme_id = currentURL.split("/").at(-1);
|
||||
const awemeList = postJson.aweme_list as unknown as DouyinImageAweme[];
|
||||
let aweme = awemeList.find(
|
||||
(pt: DouyinImageAweme) => pt.aweme_id === target_aweme_id,
|
||||
);
|
||||
if (!aweme) {
|
||||
throw new ScrapeError(
|
||||
"无法找到目标作品,可能已被删除",
|
||||
404,
|
||||
"POST_NOT_FOUND",
|
||||
);
|
||||
}
|
||||
|
||||
// 解析所有收集到的评论响应
|
||||
for (const commentRes of commentResponses) {
|
||||
try {
|
||||
const commentData = await safeJson<DouyinCommentResponse>(commentRes);
|
||||
if (commentData?.comments?.length) {
|
||||
allComments.push(...commentData.comments);
|
||||
}
|
||||
} catch (e) {
|
||||
console.warn(chalk.yellow(`⚠ 解析评论响应失败: ${(e as Error)?.message}`));
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.warn(chalk.yellow(`⚠ 评论收集失败: ${(error as Error)?.message}`));
|
||||
}
|
||||
const uploads = await handleImagePost(context, aweme);
|
||||
const saved = await saveImagePostToDB(
|
||||
context,
|
||||
aweme,
|
||||
comments,
|
||||
uploads,
|
||||
postJson,
|
||||
); // 传递完整 JSON
|
||||
console.log(chalk.green.bold("✓ 图文作品保存成功"));
|
||||
return { type: "image", ...saved };
|
||||
} else if (firstType?.key === "detail") {
|
||||
// 视频作品
|
||||
const detail = (await safeJson<DouyinVideoDetailResponse>(
|
||||
firstType.response,
|
||||
))!;
|
||||
|
||||
// 找到比特率最高的 url
|
||||
const bestPlayAddr = pickBestPlayAddr(
|
||||
detail?.aweme_detail?.video.bit_rate,
|
||||
);
|
||||
const bestVUrl = bestPlayAddr?.url_list?.[0];
|
||||
const fps = bestPlayAddr?.FPS ?? null; // 提取 FPS
|
||||
|
||||
// 去重评论（根据 cid）
|
||||
const uniqueComments = Array.from(
|
||||
new Map(allComments.map(c => [c.cid, c])).values()
|
||||
console.log(chalk.cyan(`📹 最佳视频 URL: ${chalk.dim(bestVUrl)}`));
|
||||
console.log(chalk.cyan(`🎞️ 视频帧率: ${chalk.bold(fps || "N/A")} FPS`));
|
||||
if (bestPlayAddr?.width && bestPlayAddr?.height) {
|
||||
console.log(
|
||||
chalk.cyan(
|
||||
`📐 视频分辨率: ${chalk.bold(`${bestPlayAddr.width}x${bestPlayAddr.height}`)}`,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
// 下载视频并上传至 MinIO，获取外链
|
||||
let uploadedUrl: string | undefined;
|
||||
let coverUrl: string | undefined;
|
||||
if (bestVUrl && detail?.aweme_detail) {
|
||||
console.log(chalk.blue("⬇️ 正在下载视频..."));
|
||||
const { buffer, contentType, ext } = await downloadBinary(
|
||||
context,
|
||||
bestVUrl,
|
||||
);
|
||||
const awemeId = detail.aweme_detail.aweme_id;
|
||||
const fileName = generateUniqueFileName(
|
||||
`${awemeId}.${ext}`,
|
||||
"douyin/videos",
|
||||
);
|
||||
|
||||
console.log(chalk.green(`✓ 共收集到 ${uniqueComments.length} 条独立评论（去重前: ${allComments.length}）`));
|
||||
console.log(chalk.blue("⬆️ 正在上传视频到 MinIO..."));
|
||||
uploadedUrl = await uploadFile(buffer, fileName, {
|
||||
"Content-Type": contentType,
|
||||
});
|
||||
console.log(
|
||||
chalk.green(`✓ 视频上传成功: ${chalk.underline(uploadedUrl)}`),
|
||||
);
|
||||
|
||||
// 如果从内存读取到了评论，合并进来作为兜底
|
||||
let comments: DouyinCommentResponse;
|
||||
if (memoryData?.comments?.comments?.length) {
|
||||
console.log(chalk.blue(`📝 合并内存中的 ${memoryData.comments.comments.length} 条评论`));
|
||||
const memComments = memoryData.comments.comments;
|
||||
const mergedMap = new Map(uniqueComments.map(c => [c.cid, c]));
|
||||
for (const c of memComments) {
|
||||
if (!mergedMap.has(c.cid)) {
|
||||
mergedMap.set(c.cid, c);
|
||||
}
|
||||
}
|
||||
comments = {
|
||||
comments: Array.from(mergedMap.values()),
|
||||
total: mergedMap.size,
|
||||
status_code: 0
|
||||
};
|
||||
console.log(chalk.green(`✓ 合并后共 ${comments.comments.length} 条评论`));
|
||||
} else {
|
||||
comments = {
|
||||
comments: uniqueComments,
|
||||
total: uniqueComments.length,
|
||||
status_code: 0
|
||||
};
|
||||
}
|
||||
|
||||
// 分支：视频 or 图文（两者只会有一个命中，先到先得）
|
||||
// 优先处理内存数据（图文）
|
||||
if (memoryData) {
|
||||
const aweme = memoryData.aweme;
|
||||
const uploads = await handleImagePost(context, aweme);
|
||||
const saved = await saveImagePostToDB(context, aweme, comments, uploads); // 传递完整 JSON
|
||||
console.log(chalk.green.bold('✓ 图文作品保存成功'));
|
||||
return { type: "image", ...saved };
|
||||
} else if (firstType?.key === 'post') {
|
||||
// 图文作品（网络）
|
||||
const postJson = await safeJson<DouyinPostListResponse>(firstType.response);
|
||||
if (!postJson?.aweme_list?.length) throw new ScrapeError('图文作品响应为空', 404, 'EMPTY_POST_RESPONSE');
|
||||
|
||||
const currentURL = page.url();
|
||||
const target_aweme_id = currentURL.split('/').at(-1);
|
||||
const awemeList = postJson.aweme_list as unknown as DouyinImageAweme[];
|
||||
let aweme = awemeList.find((pt: DouyinImageAweme) => pt.aweme_id === target_aweme_id);
|
||||
if (!aweme) {
|
||||
throw new ScrapeError('无法找到目标作品,可能已被删除', 404, 'POST_NOT_FOUND');
|
||||
}
|
||||
|
||||
const uploads = await handleImagePost(context, aweme);
|
||||
const saved = await saveImagePostToDB(context, aweme, comments, uploads, postJson); // 传递完整 JSON
|
||||
console.log(chalk.green.bold('✓ 图文作品保存成功'));
|
||||
return { type: "image", ...saved };
|
||||
} else if (firstType?.key === 'detail') {
|
||||
// 视频作品
|
||||
const detail = (await safeJson<DouyinVideoDetailResponse>(firstType.response))!;
|
||||
|
||||
// 找到比特率最高的 url
|
||||
const bestPlayAddr = pickBestPlayAddr(
|
||||
detail?.aweme_detail?.video.bit_rate
|
||||
// 提取首帧作为封面并上传
|
||||
try {
|
||||
console.log(chalk.blue("🖼️ 正在提取视频封面..."));
|
||||
const cover = await extractFirstFrame(buffer);
|
||||
if (cover) {
|
||||
const coverName = generateUniqueFileName(
|
||||
`${awemeId}.jpg`,
|
||||
"douyin/covers",
|
||||
);
|
||||
const bestVUrl = bestPlayAddr?.url_list?.[0];
|
||||
const fps = bestPlayAddr?.FPS ?? null; // 提取 FPS
|
||||
|
||||
console.log(chalk.cyan(`📹 最佳视频 URL: ${chalk.dim(bestVUrl)}`));
|
||||
console.log(chalk.cyan(`🎞️ 视频帧率: ${chalk.bold(fps || 'N/A')} FPS`));
|
||||
if (bestPlayAddr?.width && bestPlayAddr?.height) {
|
||||
console.log(chalk.cyan(`📐 视频分辨率: ${chalk.bold(`${bestPlayAddr.width}x${bestPlayAddr.height}`)}`));
|
||||
}
|
||||
|
||||
// 下载视频并上传至 MinIO，获取外链
|
||||
let uploadedUrl: string | undefined;
|
||||
let coverUrl: string | undefined;
|
||||
if (bestVUrl && detail?.aweme_detail) {
|
||||
console.log(chalk.blue('⬇️ 正在下载视频...'));
|
||||
const { buffer, contentType, ext } = await downloadBinary(context, bestVUrl);
|
||||
const awemeId = detail.aweme_detail.aweme_id;
|
||||
const fileName = generateUniqueFileName(`${awemeId}.${ext}`, 'douyin/videos');
|
||||
|
||||
console.log(chalk.blue('⬆️ 正在上传视频到 MinIO...'));
|
||||
uploadedUrl = await uploadFile(buffer, fileName, { 'Content-Type': contentType });
|
||||
console.log(chalk.green(`✓ 视频上传成功: ${chalk.underline(uploadedUrl)}`));
|
||||
|
||||
// 提取首帧作为封面并上传
|
||||
try {
|
||||
console.log(chalk.blue('🖼️ 正在提取视频封面...'));
|
||||
const cover = await extractFirstFrame(buffer);
|
||||
if (cover) {
|
||||
const coverName = generateUniqueFileName(`${awemeId}.jpg`, 'douyin/covers');
|
||||
coverUrl = await uploadFile(cover.buffer, coverName, { 'Content-Type': cover.contentType });
|
||||
console.log(chalk.green(`✓ 封面上传成功: ${chalk.underline(coverUrl)}`));
|
||||
}
|
||||
} catch (e) {
|
||||
console.warn(chalk.yellow(`⚠ 提取封面失败，跳过: ${(e as Error)?.message || e}`));
|
||||
}
|
||||
}
|
||||
|
||||
const saved = await saveToDB(context, detail, comments, uploadedUrl, bestPlayAddr?.width, bestPlayAddr?.height, coverUrl, fps ?? undefined);
|
||||
console.log(chalk.green.bold('✓ 视频作品保存成功'));
|
||||
transcriptAweme(detail.aweme_detail.aweme_id).catch((e) => {}); // 异步转写，不阻塞主流程
|
||||
return { type: "video", ...saved };
|
||||
} else {
|
||||
throw new ScrapeError('无法判定作品类型,接口响应异常', 500, 'UNKNOWN_TYPE');
|
||||
}
|
||||
} catch (error) {
|
||||
// 如果是我们自定义的错误,直接抛出
|
||||
if (error instanceof ScrapeError) {
|
||||
throw error;
|
||||
coverUrl = await uploadFile(cover.buffer, coverName, {
|
||||
"Content-Type": cover.contentType,
|
||||
});
|
||||
console.log(
|
||||
chalk.green(`✓ 封面上传成功: ${chalk.underline(coverUrl)}`),
|
||||
);
|
||||
}
|
||||
} catch (e) {
|
||||
console.warn(
|
||||
chalk.yellow(`⚠ 提取封面失败，跳过: ${(e as Error)?.message || e}`),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// 处理其他类型的错误
|
||||
const errMsg = (error as Error)?.message || String(error);
|
||||
console.error(chalk.red(`✗ 爬取失败: ${errMsg}`));
|
||||
|
||||
// 根据错误类型返回不同的状态码
|
||||
if (errMsg.includes('timeout') || errMsg.includes('超时')) {
|
||||
throw new ScrapeError('请求超时,请稍后重试', 408, 'TIMEOUT');
|
||||
}
|
||||
if (errMsg.includes('页面内存数据中未找到作品详情')) {
|
||||
throw new ScrapeError('作品数据加载失败', 404, 'DATA_NOT_LOADED');
|
||||
}
|
||||
if (errMsg.includes('net::')) {
|
||||
throw new ScrapeError('网络连接失败', 503, 'NETWORK_ERROR');
|
||||
}
|
||||
|
||||
// 默认服务器错误
|
||||
throw new ScrapeError(errMsg || '爬取过程中发生未知错误', 500, 'UNKNOWN_ERROR');
|
||||
} finally {
|
||||
console.log(chalk.gray('🧹 清理资源...'));
|
||||
try { await page.close({ runBeforeUnload: true }); } catch { }
|
||||
// 关闭本次任务的隔离上下文与浏览器
|
||||
await releaseIsolatedContext(context);
|
||||
await prisma.$disconnect();
|
||||
console.log(chalk.gray('✓ 资源清理完成'));
|
||||
const saved = await saveToDB(
|
||||
context,
|
||||
detail,
|
||||
comments,
|
||||
uploadedUrl,
|
||||
bestPlayAddr?.width,
|
||||
bestPlayAddr?.height,
|
||||
coverUrl,
|
||||
fps ?? undefined,
|
||||
);
|
||||
console.log(chalk.green.bold("✓ 视频作品保存成功"));
|
||||
transcriptAweme(detail.aweme_detail.aweme_id).catch((e) => {}); // 异步转写，不阻塞主流程
|
||||
return { type: "video", ...saved };
|
||||
} else {
|
||||
throw new ScrapeError(
|
||||
"无法判定作品类型,接口响应异常",
|
||||
500,
|
||||
"UNKNOWN_TYPE",
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
// 如果是我们自定义的错误,直接抛出
|
||||
if (error instanceof ScrapeError) {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
// 处理其他类型的错误
|
||||
const errMsg = (error as Error)?.message || String(error);
|
||||
console.error(chalk.red(`✗ 爬取失败: ${errMsg}`));
|
||||
|
||||
// 根据错误类型返回不同的状态码
|
||||
if (errMsg.includes("timeout") || errMsg.includes("超时")) {
|
||||
throw new ScrapeError("请求超时,请稍后重试", 408, "TIMEOUT");
|
||||
}
|
||||
if (errMsg.includes("页面内存数据中未找到作品详情")) {
|
||||
throw new ScrapeError("作品数据加载失败", 404, "DATA_NOT_LOADED");
|
||||
}
|
||||
if (errMsg.includes("net::")) {
|
||||
throw new ScrapeError("网络连接失败", 503, "NETWORK_ERROR");
|
||||
}
|
||||
|
||||
// 默认服务器错误
|
||||
throw new ScrapeError(
|
||||
errMsg || "爬取过程中发生未知错误",
|
||||
500,
|
||||
"UNKNOWN_ERROR",
|
||||
);
|
||||
} finally {
|
||||
console.log(chalk.gray("🧹 清理资源..."));
|
||||
try {
|
||||
await page.close({ runBeforeUnload: true });
|
||||
} catch {}
|
||||
// 关闭本次任务的隔离上下文与浏览器
|
||||
await releaseIsolatedContext(context);
|
||||
await prisma.$disconnect();
|
||||
console.log(chalk.gray("✓ 资源清理完成"));
|
||||
}
|
||||
}
|
||||
|
||||
+10
-11
@@ -1,17 +1,16 @@
|
||||
export const runtime = 'nodejs'
|
||||
|
||||
import { execFile } from 'child_process';
|
||||
import { promisify } from 'util';
|
||||
export const runtime = "nodejs";
|
||||
|
||||
import { execFile } from "child_process";
|
||||
import { promisify } from "util";
|
||||
|
||||
export function pickBestPlayAddr(variants: PlayVariant[] | undefined | null) {
|
||||
if (!variants?.length) return null;
|
||||
if (!variants?.length) return null;
|
||||
|
||||
const best = variants.reduce((best, cur) => {
|
||||
const b1 = best?.bit_rate ?? -1;
|
||||
const b2 = cur?.bit_rate ?? -1;
|
||||
return b2 > b1 ? cur : best;
|
||||
});
|
||||
const best = variants.reduce((best, cur) => {
|
||||
const b1 = best?.bit_rate ?? -1;
|
||||
const b2 = cur?.bit_rate ?? -1;
|
||||
return b2 > b1 ? cur : best;
|
||||
});
|
||||
|
||||
return best?.play_addr ?? null;
|
||||
return best?.play_addr ?? null;
|
||||
}
|
||||
|
||||
+128
-127
@@ -1,18 +1,18 @@
|
||||
export const runtime = 'nodejs'
|
||||
export const runtime = "nodejs";
|
||||
|
||||
import type { BrowserContext, Response } from 'playwright';
|
||||
import type { BrowserContext, Response } from "playwright";
|
||||
|
||||
export async function safeJson<T>(res: Response): Promise<T | null> {
|
||||
const ctype = res.headers()['content-type'] || '';
|
||||
if (ctype.includes('application/json')) {
|
||||
return (await res.json()) as T;
|
||||
}
|
||||
const t = await res.text();
|
||||
try {
|
||||
return JSON.parse(t) as T;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
const ctype = res.headers()["content-type"] || "";
|
||||
if (ctype.includes("application/json")) {
|
||||
return (await res.json()) as T;
|
||||
}
|
||||
const t = await res.text();
|
||||
try {
|
||||
return JSON.parse(t) as T;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -21,30 +21,31 @@ export async function safeJson<T>(res: Response): Promise<T | null> {
|
||||
* - referrer 使用链接本身
|
||||
*/
|
||||
export async function downloadBinary(
|
||||
context: BrowserContext,
|
||||
url: string,
|
||||
context: BrowserContext,
|
||||
url: string,
|
||||
): Promise<{ buffer: Buffer; contentType: string; ext: string }> {
|
||||
console.log('下载：', url);
|
||||
console.log("下载：", url);
|
||||
|
||||
const headers = {
|
||||
referer: 'https://www.douyin.com/',
|
||||
} as Record<string, string>;
|
||||
const headers = {
|
||||
referer: "https://www.douyin.com/",
|
||||
} as Record<string, string>;
|
||||
|
||||
const res = await context.request.get(url, {
|
||||
headers,
|
||||
maxRedirects: 3,
|
||||
timeout: 240_000,
|
||||
failOnStatusCode: true,
|
||||
});
|
||||
const res = await context.request.get(url, {
|
||||
headers,
|
||||
maxRedirects: 3,
|
||||
timeout: 240_000,
|
||||
failOnStatusCode: true,
|
||||
});
|
||||
|
||||
if (!res.ok()) {
|
||||
throw new Error(`下载内容失败: ${res.status()} ${res.statusText()}`);
|
||||
}
|
||||
if (!res.ok()) {
|
||||
throw new Error(`下载内容失败: ${res.status()} ${res.statusText()}`);
|
||||
}
|
||||
|
||||
const buffer = await res.body();
|
||||
const contentType = res.headers()['content-type'] || 'application/octet-stream';
|
||||
const ext = (contentType.split('/')[1] || 'bin').split(';')[0] || 'bin';
|
||||
return { buffer, contentType, ext };
|
||||
const buffer = await res.body();
|
||||
const contentType =
|
||||
res.headers()["content-type"] || "application/octet-stream";
|
||||
const ext = (contentType.split("/")[1] || "bin").split(";")[0] || "bin";
|
||||
return { buffer, contentType, ext };
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -52,46 +53,46 @@ export async function downloadBinary(
|
||||
* - 不为每个候选单独设长超时，改用整体兜底超时，避免无意义等待。
|
||||
*/
|
||||
export function waitForFirstResponse(
|
||||
context: BrowserContext,
|
||||
candidates: { key: string; test: (r: Response) => boolean }[],
|
||||
timeoutMs = 20_000
|
||||
context: BrowserContext,
|
||||
candidates: { key: string; test: (r: Response) => boolean }[],
|
||||
timeoutMs = 20_000,
|
||||
): Promise<{ key: string; response: Response } | null> {
|
||||
return new Promise((resolve) => {
|
||||
let resolved = false;
|
||||
let timer: NodeJS.Timeout | undefined;
|
||||
return new Promise((resolve) => {
|
||||
let resolved = false;
|
||||
let timer: NodeJS.Timeout | undefined;
|
||||
|
||||
const handler = (res: Response) => {
|
||||
if (resolved) return;
|
||||
for (const c of candidates) {
|
||||
try {
|
||||
if (c.test(res)) {
|
||||
resolved = true;
|
||||
cleanup();
|
||||
resolve({ key: c.key, response: res });
|
||||
return;
|
||||
}
|
||||
} catch {
|
||||
// ignore predicate errors
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const cleanup = () => {
|
||||
context.off('response', handler);
|
||||
if (timer) clearTimeout(timer);
|
||||
};
|
||||
|
||||
context.on('response', handler);
|
||||
if (timeoutMs > 0) {
|
||||
timer = setTimeout(() => {
|
||||
if (!resolved) {
|
||||
resolved = true;
|
||||
cleanup();
|
||||
resolve(null);
|
||||
}
|
||||
}, timeoutMs);
|
||||
const handler = (res: Response) => {
|
||||
if (resolved) return;
|
||||
for (const c of candidates) {
|
||||
try {
|
||||
if (c.test(res)) {
|
||||
resolved = true;
|
||||
cleanup();
|
||||
resolve({ key: c.key, response: res });
|
||||
return;
|
||||
}
|
||||
} catch {
|
||||
// ignore predicate errors
|
||||
}
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
const cleanup = () => {
|
||||
context.off("response", handler);
|
||||
if (timer) clearTimeout(timer);
|
||||
};
|
||||
|
||||
context.on("response", handler);
|
||||
if (timeoutMs > 0) {
|
||||
timer = setTimeout(() => {
|
||||
if (!resolved) {
|
||||
resolved = true;
|
||||
cleanup();
|
||||
resolve(null);
|
||||
}
|
||||
}, timeoutMs);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -99,76 +100,76 @@ export function waitForFirstResponse(
|
||||
* 用于评论等需要滚动加载的数据
|
||||
*/
|
||||
export function collectResponsesWithinTime(
|
||||
context: BrowserContext,
|
||||
predicate: (r: Response) => boolean,
|
||||
durationMs: number
|
||||
context: BrowserContext,
|
||||
predicate: (r: Response) => boolean,
|
||||
durationMs: number,
|
||||
): Promise<Response[]> {
|
||||
return new Promise((resolve) => {
|
||||
const collected: Response[] = [];
|
||||
const seenUrls = new Set<string>();
|
||||
let timer: NodeJS.Timeout | undefined;
|
||||
return new Promise((resolve) => {
|
||||
const collected: Response[] = [];
|
||||
const seenUrls = new Set<string>();
|
||||
let timer: NodeJS.Timeout | undefined;
|
||||
|
||||
const handler = (res: Response) => {
|
||||
try {
|
||||
if (predicate(res)) {
|
||||
// 使用 URL 去重，避免重复收集同一个请求
|
||||
const url = res.url();
|
||||
if (!seenUrls.has(url)) {
|
||||
seenUrls.add(url);
|
||||
collected.push(res);
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// ignore predicate errors
|
||||
}
|
||||
};
|
||||
const handler = (res: Response) => {
|
||||
try {
|
||||
if (predicate(res)) {
|
||||
// 使用 URL 去重，避免重复收集同一个请求
|
||||
const url = res.url();
|
||||
if (!seenUrls.has(url)) {
|
||||
seenUrls.add(url);
|
||||
collected.push(res);
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// ignore predicate errors
|
||||
}
|
||||
};
|
||||
|
||||
const cleanup = () => {
|
||||
context.off('response', handler);
|
||||
if (timer) clearTimeout(timer);
|
||||
};
|
||||
const cleanup = () => {
|
||||
context.off("response", handler);
|
||||
if (timer) clearTimeout(timer);
|
||||
};
|
||||
|
||||
context.on('response', handler);
|
||||
timer = setTimeout(() => {
|
||||
cleanup();
|
||||
resolve(collected);
|
||||
}, durationMs);
|
||||
});
|
||||
context.on("response", handler);
|
||||
timer = setTimeout(() => {
|
||||
cleanup();
|
||||
resolve(collected);
|
||||
}, durationMs);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* 等待符合条件的单个 Response，带短超时；用于评论等"可有可无"的数据。
|
||||
*/
|
||||
export function waitForResponseWithTimeout(
|
||||
context: BrowserContext,
|
||||
predicate: (r: Response) => boolean,
|
||||
timeoutMs = 5_000
|
||||
context: BrowserContext,
|
||||
predicate: (r: Response) => boolean,
|
||||
timeoutMs = 5_000,
|
||||
): Promise<Response> {
|
||||
return new Promise<Response>((resolve, reject) => {
|
||||
let timer: NodeJS.Timeout | undefined;
|
||||
return new Promise<Response>((resolve, reject) => {
|
||||
let timer: NodeJS.Timeout | undefined;
|
||||
|
||||
const handler = (res: Response) => {
|
||||
try {
|
||||
if (predicate(res)) {
|
||||
cleanup();
|
||||
resolve(res);
|
||||
}
|
||||
} catch {
|
||||
// ignore predicate errors
|
||||
}
|
||||
};
|
||||
|
||||
const cleanup = () => {
|
||||
context.off('response', handler);
|
||||
if (timer) clearTimeout(timer);
|
||||
};
|
||||
|
||||
context.on('response', handler);
|
||||
if (timeoutMs > 0) {
|
||||
timer = setTimeout(() => {
|
||||
cleanup();
|
||||
reject(new Error('timeout'));
|
||||
}, timeoutMs);
|
||||
const handler = (res: Response) => {
|
||||
try {
|
||||
if (predicate(res)) {
|
||||
cleanup();
|
||||
resolve(res);
|
||||
}
|
||||
});
|
||||
} catch {
|
||||
// ignore predicate errors
|
||||
}
|
||||
};
|
||||
|
||||
const cleanup = () => {
|
||||
context.off("response", handler);
|
||||
if (timer) clearTimeout(timer);
|
||||
};
|
||||
|
||||
context.on("response", handler);
|
||||
if (timeoutMs > 0) {
|
||||
timer = setTimeout(() => {
|
||||
cleanup();
|
||||
reject(new Error("timeout"));
|
||||
}, timeoutMs);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
+399
-331
@@ -1,359 +1,427 @@
|
||||
import type { BrowserContext } from 'playwright';
|
||||
import { prisma } from '@/lib/prisma';
|
||||
import { uploadAvatarFromUrl, uploadImageFromUrl } from './uploader';
|
||||
import { firstUrl } from './utils';
|
||||
import type { BrowserContext } from "playwright";
|
||||
import { prisma } from "@/lib/prisma";
|
||||
import { uploadAvatarFromUrl, uploadImageFromUrl } from "./uploader";
|
||||
import { firstUrl } from "./utils";
|
||||
|
||||
export async function saveToDB(
|
||||
context: BrowserContext,
|
||||
detailResp: DouyinVideoDetailResponse,
|
||||
commentResp: DouyinCommentResponse,
|
||||
videoUrl?: string,
|
||||
width?: number,
|
||||
height?: number,
|
||||
coverUrl?: string,
|
||||
fps?: number
|
||||
context: BrowserContext,
|
||||
detailResp: DouyinVideoDetailResponse,
|
||||
commentResp: DouyinCommentResponse,
|
||||
videoUrl?: string,
|
||||
width?: number,
|
||||
height?: number,
|
||||
coverUrl?: string,
|
||||
fps?: number,
|
||||
) {
|
||||
if (!detailResp?.aweme_detail) throw new Error('视频详情为空');
|
||||
const d = detailResp.aweme_detail;
|
||||
if (!detailResp?.aweme_detail) throw new Error("视频详情为空");
|
||||
const d = detailResp.aweme_detail;
|
||||
|
||||
// 1) Upsert Author
|
||||
const authorAvatarSrc = firstUrl(d.author.avatar_thumb?.url_list);
|
||||
const authorAvatarUploaded = await uploadAvatarFromUrl(context, authorAvatarSrc, `authors/${d.author.sec_uid}`);
|
||||
const author = await prisma.author.upsert({
|
||||
where: { sec_uid: d.author.sec_uid },
|
||||
create: {
|
||||
sec_uid: d.author.sec_uid,
|
||||
uid: d.author.uid,
|
||||
nickname: d.author.nickname,
|
||||
signature: d.author.signature ?? null,
|
||||
avatar_url: authorAvatarUploaded ?? null,
|
||||
follower_count: BigInt(d.author.follower_count || 0),
|
||||
total_favorited: BigInt(d.author.total_favorited || 0),
|
||||
unique_id: d.author.unique_id ?? null,
|
||||
short_id: d.author.short_id ?? null,
|
||||
},
|
||||
update: {
|
||||
uid: d.author.uid,
|
||||
nickname: d.author.nickname,
|
||||
signature: d.author.signature ?? null,
|
||||
avatar_url: authorAvatarUploaded ?? null,
|
||||
follower_count: BigInt(d.author.follower_count || 0),
|
||||
total_favorited: BigInt(d.author.total_favorited || 0),
|
||||
unique_id: d.author.unique_id ?? null,
|
||||
short_id: d.author.short_id ?? null,
|
||||
// 1) Upsert Author
|
||||
const authorAvatarSrc = firstUrl(d.author.avatar_thumb?.url_list);
|
||||
const authorAvatarUploaded = await uploadAvatarFromUrl(
|
||||
context,
|
||||
authorAvatarSrc,
|
||||
`authors/${d.author.sec_uid}`,
|
||||
);
|
||||
const author = await prisma.author.upsert({
|
||||
where: { sec_uid: d.author.sec_uid },
|
||||
create: {
|
||||
sec_uid: d.author.sec_uid,
|
||||
uid: d.author.uid,
|
||||
nickname: d.author.nickname,
|
||||
signature: d.author.signature ?? null,
|
||||
avatar_url: authorAvatarUploaded ?? null,
|
||||
follower_count: BigInt(d.author.follower_count || 0),
|
||||
total_favorited: BigInt(d.author.total_favorited || 0),
|
||||
unique_id: d.author.unique_id ?? null,
|
||||
short_id: d.author.short_id ?? null,
|
||||
},
|
||||
update: {
|
||||
uid: d.author.uid,
|
||||
nickname: d.author.nickname,
|
||||
signature: d.author.signature ?? null,
|
||||
avatar_url: authorAvatarUploaded ?? null,
|
||||
follower_count: BigInt(d.author.follower_count || 0),
|
||||
total_favorited: BigInt(d.author.total_favorited || 0),
|
||||
unique_id: d.author.unique_id ?? null,
|
||||
short_id: d.author.short_id ?? null,
|
||||
},
|
||||
});
|
||||
|
||||
// 2) Upsert Video
|
||||
const video = await prisma.video.upsert({
|
||||
where: { aweme_id: d.aweme_id },
|
||||
create: {
|
||||
aweme_id: d.aweme_id,
|
||||
desc: d.desc,
|
||||
preview_title: d.preview_title ?? null,
|
||||
duration_ms: d.duration,
|
||||
created_at: new Date((d.create_time || 0) * 1000),
|
||||
share_url: d.share_url,
|
||||
digg_count: BigInt(d.statistics?.digg_count || 0),
|
||||
comment_count: BigInt(d.statistics?.comment_count || 0),
|
||||
share_count: BigInt(d.statistics?.share_count || 0),
|
||||
collect_count: BigInt(d.statistics?.collect_count || 0),
|
||||
authorId: author.sec_uid,
|
||||
tags: d.tags?.map((t) => t.tag_name) ?? [],
|
||||
video_url: videoUrl ?? "",
|
||||
width: width ?? null,
|
||||
height: height ?? null,
|
||||
cover_url: coverUrl ?? null,
|
||||
fps: fps ?? null,
|
||||
raw_json: detailResp as any, // 保存完整接口 JSON
|
||||
},
|
||||
update: {
|
||||
desc: d.desc,
|
||||
preview_title: d.preview_title ?? null,
|
||||
duration_ms: d.duration,
|
||||
created_at: new Date((d.create_time || 0) * 1000),
|
||||
share_url: d.share_url,
|
||||
digg_count: BigInt(d.statistics?.digg_count || 0),
|
||||
comment_count: BigInt(d.statistics?.comment_count || 0),
|
||||
share_count: BigInt(d.statistics?.share_count || 0),
|
||||
collect_count: BigInt(d.statistics?.collect_count || 0),
|
||||
authorId: author.sec_uid,
|
||||
...(videoUrl ? { video_url: videoUrl } : {}),
|
||||
...(width ? { width } : {}),
|
||||
...(height ? { height } : {}),
|
||||
...(coverUrl ? { cover_url: coverUrl } : {}),
|
||||
...(fps ? { fps } : {}),
|
||||
raw_json: detailResp as any, // 更新完整接口 JSON
|
||||
},
|
||||
});
|
||||
|
||||
// 3) Upsert Comments + CommentUser
|
||||
const comments = commentResp?.comments ?? [];
|
||||
for (const c of comments) {
|
||||
const origAvatar: string | null =
|
||||
firstUrl(c.user?.avatar_thumb?.url_list) ?? null;
|
||||
const nameHint = `comment-users/${(c.user?.nickname || "unknown").replace(/\s+/g, "_")}-${c.cid}`;
|
||||
const uploadedAvatar = await uploadAvatarFromUrl(
|
||||
context,
|
||||
origAvatar ?? undefined,
|
||||
nameHint,
|
||||
);
|
||||
const finalAvatar = uploadedAvatar ?? origAvatar; // string | null
|
||||
const finalAvatarKey = finalAvatar ?? "";
|
||||
const cu = await prisma.commentUser.upsert({
|
||||
where: {
|
||||
nickname_avatar_url: {
|
||||
nickname: c.user?.nickname || "未知用户",
|
||||
avatar_url: finalAvatarKey,
|
||||
},
|
||||
},
|
||||
create: {
|
||||
nickname: c.user?.nickname || "未知用户",
|
||||
avatar_url: finalAvatar ?? null,
|
||||
},
|
||||
update: {
|
||||
avatar_url: finalAvatar ?? null,
|
||||
},
|
||||
});
|
||||
|
||||
// 2) Upsert Video
|
||||
const video = await prisma.video.upsert({
|
||||
where: { aweme_id: d.aweme_id },
|
||||
create: {
|
||||
aweme_id: d.aweme_id,
|
||||
desc: d.desc,
|
||||
preview_title: d.preview_title ?? null,
|
||||
duration_ms: d.duration,
|
||||
created_at: new Date((d.create_time || 0) * 1000),
|
||||
share_url: d.share_url,
|
||||
digg_count: BigInt(d.statistics?.digg_count || 0),
|
||||
comment_count: BigInt(d.statistics?.comment_count || 0),
|
||||
share_count: BigInt(d.statistics?.share_count || 0),
|
||||
collect_count: BigInt(d.statistics?.collect_count || 0),
|
||||
authorId: author.sec_uid,
|
||||
tags: (d.tags?.map(t => t.tag_name) ?? []),
|
||||
video_url: videoUrl ?? '',
|
||||
width: width ?? null,
|
||||
height: height ?? null,
|
||||
cover_url: coverUrl ?? null,
|
||||
fps: fps ?? null,
|
||||
raw_json: detailResp as any, // 保存完整接口 JSON
|
||||
},
|
||||
update: {
|
||||
desc: d.desc,
|
||||
preview_title: d.preview_title ?? null,
|
||||
duration_ms: d.duration,
|
||||
created_at: new Date((d.create_time || 0) * 1000),
|
||||
share_url: d.share_url,
|
||||
digg_count: BigInt(d.statistics?.digg_count || 0),
|
||||
comment_count: BigInt(d.statistics?.comment_count || 0),
|
||||
share_count: BigInt(d.statistics?.share_count || 0),
|
||||
collect_count: BigInt(d.statistics?.collect_count || 0),
|
||||
authorId: author.sec_uid,
|
||||
...(videoUrl ? { video_url: videoUrl } : {}),
|
||||
...(width ? { width } : {}),
|
||||
...(height ? { height } : {}),
|
||||
...(coverUrl ? { cover_url: coverUrl } : {}),
|
||||
...(fps ? { fps } : {}),
|
||||
raw_json: detailResp as any, // 更新完整接口 JSON
|
||||
},
|
||||
const savedComment = await prisma.comment.upsert({
|
||||
where: { cid: c.cid },
|
||||
create: {
|
||||
cid: c.cid,
|
||||
text: c.text,
|
||||
digg_count: BigInt(c.digg_count || 0),
|
||||
created_at: new Date((c.create_time || 0) * 1000),
|
||||
videoId: video.aweme_id,
|
||||
userId: cu.id,
|
||||
},
|
||||
update: {
|
||||
text: c.text,
|
||||
digg_count: BigInt(c.digg_count || 0),
|
||||
created_at: new Date((c.create_time || 0) * 1000),
|
||||
videoId: video.aweme_id,
|
||||
userId: cu.id,
|
||||
},
|
||||
});
|
||||
|
||||
// 3) Upsert Comments + CommentUser
|
||||
const comments = commentResp?.comments ?? [];
|
||||
for (const c of comments) {
|
||||
const origAvatar: string | null = firstUrl(c.user?.avatar_thumb?.url_list) ?? null;
|
||||
const nameHint = `comment-users/${(c.user?.nickname || 'unknown').replace(/\s+/g, '_')}-${c.cid}`;
|
||||
const uploadedAvatar = await uploadAvatarFromUrl(context, origAvatar ?? undefined, nameHint);
|
||||
const finalAvatar = uploadedAvatar ?? origAvatar; // string | null
|
||||
const finalAvatarKey = finalAvatar ?? '';
|
||||
const cu = await prisma.commentUser.upsert({
|
||||
where: {
|
||||
nickname_avatar_url: {
|
||||
nickname: c.user?.nickname || '未知用户',
|
||||
avatar_url: finalAvatarKey,
|
||||
},
|
||||
},
|
||||
create: {
|
||||
nickname: c.user?.nickname || '未知用户',
|
||||
avatar_url: finalAvatar ?? null,
|
||||
},
|
||||
update: {
|
||||
avatar_url: finalAvatar ?? null,
|
||||
},
|
||||
// 处理评论贴纸/配图上传与入库
|
||||
try {
|
||||
const sources: {
|
||||
url?: string | null;
|
||||
width?: number;
|
||||
height?: number;
|
||||
}[] = [];
|
||||
// 贴纸（当作第一张）
|
||||
const stickerUrl = firstUrl(c.sticker?.animate_url?.url_list);
|
||||
if (stickerUrl) {
|
||||
sources.push({
|
||||
url: stickerUrl,
|
||||
width: c.sticker?.animate_url?.width,
|
||||
height: c.sticker?.animate_url?.height,
|
||||
});
|
||||
}
|
||||
// 配图列表
|
||||
const imgs = c.image_list || [];
|
||||
for (const it of imgs) {
|
||||
const u = firstUrl(it?.origin_url?.url_list);
|
||||
if (u)
|
||||
sources.push({
|
||||
url: u,
|
||||
width: it?.origin_url?.width as any,
|
||||
height: it?.origin_url?.height as any,
|
||||
});
|
||||
}
|
||||
|
||||
const savedComment = await prisma.comment.upsert({
|
||||
where: { cid: c.cid },
|
||||
create: {
|
||||
cid: c.cid,
|
||||
text: c.text,
|
||||
digg_count: BigInt(c.digg_count || 0),
|
||||
created_at: new Date((c.create_time || 0) * 1000),
|
||||
videoId: video.aweme_id,
|
||||
userId: cu.id,
|
||||
},
|
||||
update: {
|
||||
text: c.text,
|
||||
digg_count: BigInt(c.digg_count || 0),
|
||||
created_at: new Date((c.create_time || 0) * 1000),
|
||||
videoId: video.aweme_id,
|
||||
userId: cu.id,
|
||||
},
|
||||
for (let i = 0; i < sources.length; i++) {
|
||||
const s = sources[i];
|
||||
const uploaded = await uploadImageFromUrl(
|
||||
context,
|
||||
s.url ?? undefined,
|
||||
`comments/${c.cid}/${i}`,
|
||||
);
|
||||
if (!uploaded) continue;
|
||||
await prisma.commentImage.upsert({
|
||||
where: { commentId_order: { commentId: savedComment.cid, order: i } },
|
||||
create: {
|
||||
commentId: savedComment.cid,
|
||||
order: i,
|
||||
url: uploaded,
|
||||
width: typeof s.width === "number" ? s.width : null,
|
||||
height: typeof s.height === "number" ? s.height : null,
|
||||
},
|
||||
update: {
|
||||
url: uploaded,
|
||||
width: typeof s.width === "number" ? s.width : null,
|
||||
height: typeof s.height === "number" ? s.height : null,
|
||||
},
|
||||
});
|
||||
|
||||
// 处理评论贴纸/配图上传与入库
|
||||
try {
|
||||
const sources: { url?: string | null; width?: number; height?: number }[] = [];
|
||||
// 贴纸（当作第一张）
|
||||
const stickerUrl = firstUrl(c.sticker?.animate_url?.url_list);
|
||||
if (stickerUrl) {
|
||||
sources.push({ url: stickerUrl, width: c.sticker?.animate_url?.width, height: c.sticker?.animate_url?.height });
|
||||
}
|
||||
// 配图列表
|
||||
const imgs = c.image_list || [];
|
||||
for (const it of imgs) {
|
||||
const u = firstUrl(it?.origin_url?.url_list);
|
||||
if (u) sources.push({ url: u, width: it?.origin_url?.width as any, height: it?.origin_url?.height as any });
|
||||
}
|
||||
|
||||
for (let i = 0; i < sources.length; i++) {
|
||||
const s = sources[i];
|
||||
const uploaded = await uploadImageFromUrl(
|
||||
context,
|
||||
s.url ?? undefined,
|
||||
`comments/${c.cid}/${i}`,
|
||||
);
|
||||
if (!uploaded) continue;
|
||||
await prisma.commentImage.upsert({
|
||||
where: { commentId_order: { commentId: savedComment.cid, order: i } },
|
||||
create: {
|
||||
commentId: savedComment.cid,
|
||||
order: i,
|
||||
url: uploaded,
|
||||
width: typeof s.width === 'number' ? s.width : null,
|
||||
height: typeof s.height === 'number' ? s.height : null,
|
||||
},
|
||||
update: {
|
||||
url: uploaded,
|
||||
width: typeof s.width === 'number' ? s.width : null,
|
||||
height: typeof s.height === 'number' ? s.height : null,
|
||||
},
|
||||
});
|
||||
}
|
||||
} catch (e) {
|
||||
console.warn('[comment-images] 保存失败：', (e as Error)?.message || e);
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
console.warn("[comment-images] 保存失败：", (e as Error)?.message || e);
|
||||
}
|
||||
}
|
||||
|
||||
return { aweme_id: video.aweme_id, author_sec_uid: author.sec_uid, comment_count: comments.length };
|
||||
return {
|
||||
aweme_id: video.aweme_id,
|
||||
author_sec_uid: author.sec_uid,
|
||||
comment_count: comments.length,
|
||||
};
|
||||
}
|
||||
|
||||
export async function saveImagePostToDB(
|
||||
context: BrowserContext,
|
||||
aweme: DouyinImageAweme,
|
||||
commentResp: DouyinCommentResponse,
|
||||
uploads: { images: { url: string; width?: number; height?: number, video?: string }[]; musicUrl?: string },
|
||||
rawJson?: any
|
||||
context: BrowserContext,
|
||||
aweme: DouyinImageAweme,
|
||||
commentResp: DouyinCommentResponse,
|
||||
uploads: {
|
||||
images: {
|
||||
url: string;
|
||||
width?: number;
|
||||
height?: number;
|
||||
video?: string;
|
||||
duration?: number;
|
||||
}[];
|
||||
musicUrl?: string;
|
||||
},
|
||||
rawJson?: any,
|
||||
) {
|
||||
if (!aweme?.author?.sec_uid) throw new Error('作者 sec_uid 缺失');
|
||||
if (!aweme?.author?.sec_uid) throw new Error("作者 sec_uid 缺失");
|
||||
|
||||
// Upsert Author（与视频一致）
|
||||
const authorAvatarSrc = firstUrl(aweme.author.avatar_thumb?.url_list);
|
||||
const authorAvatarUploaded = await uploadAvatarFromUrl(context, authorAvatarSrc, `authors/${aweme.author.sec_uid}`);
|
||||
const author = await prisma.author.upsert({
|
||||
where: { sec_uid: aweme.author.sec_uid },
|
||||
create: {
|
||||
sec_uid: aweme.author.sec_uid,
|
||||
uid: aweme.author.uid,
|
||||
nickname: aweme.author.nickname,
|
||||
signature: aweme.author.signature ?? null,
|
||||
avatar_url: authorAvatarUploaded ?? null,
|
||||
follower_count: BigInt((aweme.author as any).follower_count || 0),
|
||||
total_favorited: BigInt((aweme.author as any).total_favorited || 0),
|
||||
unique_id: (aweme.author as any).unique_id ?? null,
|
||||
short_id: (aweme.author as any).short_id ?? null,
|
||||
},
|
||||
update: {
|
||||
uid: aweme.author.uid,
|
||||
nickname: aweme.author.nickname,
|
||||
signature: aweme.author.signature ?? null,
|
||||
avatar_url: authorAvatarUploaded ?? null,
|
||||
follower_count: BigInt((aweme.author as any).follower_count || 0),
|
||||
total_favorited: BigInt((aweme.author as any).total_favorited || 0),
|
||||
unique_id: (aweme.author as any).unique_id ?? null,
|
||||
short_id: (aweme.author as any).short_id ?? null,
|
||||
// Upsert Author（与视频一致）
|
||||
const authorAvatarSrc = firstUrl(aweme.author.avatar_thumb?.url_list);
|
||||
const authorAvatarUploaded = await uploadAvatarFromUrl(
|
||||
context,
|
||||
authorAvatarSrc,
|
||||
`authors/${aweme.author.sec_uid}`,
|
||||
);
|
||||
const author = await prisma.author.upsert({
|
||||
where: { sec_uid: aweme.author.sec_uid },
|
||||
create: {
|
||||
sec_uid: aweme.author.sec_uid,
|
||||
uid: aweme.author.uid,
|
||||
nickname: aweme.author.nickname,
|
||||
signature: aweme.author.signature ?? null,
|
||||
avatar_url: authorAvatarUploaded ?? null,
|
||||
follower_count: BigInt((aweme.author as any).follower_count || 0),
|
||||
total_favorited: BigInt((aweme.author as any).total_favorited || 0),
|
||||
unique_id: (aweme.author as any).unique_id ?? null,
|
||||
short_id: (aweme.author as any).short_id ?? null,
|
||||
},
|
||||
update: {
|
||||
uid: aweme.author.uid,
|
||||
nickname: aweme.author.nickname,
|
||||
signature: aweme.author.signature ?? null,
|
||||
avatar_url: authorAvatarUploaded ?? null,
|
||||
follower_count: BigInt((aweme.author as any).follower_count || 0),
|
||||
total_favorited: BigInt((aweme.author as any).total_favorited || 0),
|
||||
unique_id: (aweme.author as any).unique_id ?? null,
|
||||
short_id: (aweme.author as any).short_id ?? null,
|
||||
},
|
||||
});
|
||||
|
||||
// Upsert ImagePost
|
||||
const imagePost = await prisma.imagePost.upsert({
|
||||
where: { aweme_id: aweme.aweme_id },
|
||||
create: {
|
||||
aweme_id: aweme.aweme_id,
|
||||
desc: aweme.desc,
|
||||
created_at: new Date((aweme.create_time || 0) * 1000),
|
||||
share_url: aweme.share_url || "",
|
||||
digg_count: BigInt(aweme.statistics?.digg_count || 0),
|
||||
comment_count: BigInt(aweme.statistics?.comment_count || 0),
|
||||
share_count: BigInt(aweme.statistics?.share_count || 0),
|
||||
collect_count: BigInt(aweme.statistics?.collect_count || 0),
|
||||
authorId: author.sec_uid,
|
||||
tags: aweme.video_tag?.map((t) => t.tag_name) ?? [],
|
||||
music_url: uploads.musicUrl ?? null,
|
||||
raw_json: rawJson ?? null, // 保存完整接口 JSON
|
||||
},
|
||||
update: {
|
||||
desc: aweme.desc,
|
||||
created_at: new Date((aweme.create_time || 0) * 1000),
|
||||
share_url: aweme.share_url,
|
||||
digg_count: BigInt(aweme.statistics?.digg_count || 0),
|
||||
comment_count: BigInt(aweme.statistics?.comment_count || 0),
|
||||
share_count: BigInt(aweme.statistics?.share_count || 0),
|
||||
collect_count: BigInt(aweme.statistics?.collect_count || 0),
|
||||
authorId: author.sec_uid,
|
||||
tags: aweme.video_tag?.map((t) => t.tag_name) ?? [],
|
||||
music_url: uploads.musicUrl ?? undefined,
|
||||
raw_json: rawJson ?? undefined, // 更新完整接口 JSON
|
||||
},
|
||||
});
|
||||
|
||||
// Upsert ImageFiles（按顺序）
|
||||
for (let i = 0; i < uploads.images.length; i++) {
|
||||
const { url, width, height, video, duration } = uploads.images[i];
|
||||
const durationMs =
|
||||
typeof duration === "number" && Number.isFinite(duration)
|
||||
? Math.max(1, Math.round(duration))
|
||||
: null;
|
||||
await prisma.imageFile.upsert({
|
||||
where: { postId_order: { postId: imagePost.aweme_id, order: i } },
|
||||
create: {
|
||||
postId: imagePost.aweme_id,
|
||||
order: i,
|
||||
url,
|
||||
width: typeof width === "number" ? width : null,
|
||||
height: typeof height === "number" ? height : null,
|
||||
animated: video || null,
|
||||
duration: durationMs,
|
||||
},
|
||||
update: {
|
||||
url,
|
||||
width: typeof width === "number" ? width : null,
|
||||
height: typeof height === "number" ? height : null,
|
||||
animated: video || null,
|
||||
duration: durationMs,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
// 评论入库：关联到 ImagePost
|
||||
const comments = commentResp?.comments ?? [];
|
||||
for (const c of comments) {
|
||||
const origAvatar: string | null =
|
||||
firstUrl(c.user?.avatar_thumb?.url_list) ?? null;
|
||||
const nameHint = `comment-users/${(c.user?.nickname || "unknown").replace(/\s+/g, "_")}-${c.cid}`;
|
||||
const uploadedAvatar = await uploadAvatarFromUrl(
|
||||
context,
|
||||
origAvatar ?? undefined,
|
||||
nameHint,
|
||||
);
|
||||
const finalAvatar = uploadedAvatar ?? origAvatar; // string | null
|
||||
const finalAvatarKey = finalAvatar ?? "";
|
||||
const cu = await prisma.commentUser.upsert({
|
||||
where: {
|
||||
nickname_avatar_url: {
|
||||
nickname: c.user?.nickname || "未知用户",
|
||||
avatar_url: finalAvatarKey,
|
||||
},
|
||||
},
|
||||
create: {
|
||||
nickname: c.user?.nickname || "未知用户",
|
||||
avatar_url: finalAvatar ?? null,
|
||||
},
|
||||
update: {
|
||||
avatar_url: finalAvatar ?? null,
|
||||
},
|
||||
});
|
||||
|
||||
// Upsert ImagePost
|
||||
const imagePost = await prisma.imagePost.upsert({
|
||||
where: { aweme_id: aweme.aweme_id },
|
||||
create: {
|
||||
aweme_id: aweme.aweme_id,
|
||||
desc: aweme.desc,
|
||||
created_at: new Date((aweme.create_time || 0) * 1000),
|
||||
share_url: aweme.share_url || '',
|
||||
digg_count: BigInt(aweme.statistics?.digg_count || 0),
|
||||
comment_count: BigInt(aweme.statistics?.comment_count || 0),
|
||||
share_count: BigInt(aweme.statistics?.share_count || 0),
|
||||
collect_count: BigInt(aweme.statistics?.collect_count || 0),
|
||||
authorId: author.sec_uid,
|
||||
tags: (aweme.video_tag?.map(t => t.tag_name) ?? []),
|
||||
music_url: uploads.musicUrl ?? null,
|
||||
raw_json: rawJson ?? null, // 保存完整接口 JSON
|
||||
},
|
||||
update: {
|
||||
desc: aweme.desc,
|
||||
created_at: new Date((aweme.create_time || 0) * 1000),
|
||||
share_url: aweme.share_url,
|
||||
digg_count: BigInt(aweme.statistics?.digg_count || 0),
|
||||
comment_count: BigInt(aweme.statistics?.comment_count || 0),
|
||||
share_count: BigInt(aweme.statistics?.share_count || 0),
|
||||
collect_count: BigInt(aweme.statistics?.collect_count || 0),
|
||||
authorId: author.sec_uid,
|
||||
tags: (aweme.video_tag?.map(t => t.tag_name) ?? []),
|
||||
music_url: uploads.musicUrl ?? undefined,
|
||||
raw_json: rawJson ?? undefined, // 更新完整接口 JSON
|
||||
},
|
||||
const savedComment = await prisma.comment.upsert({
|
||||
where: { cid: c.cid },
|
||||
create: {
|
||||
cid: c.cid,
|
||||
text: c.text,
|
||||
digg_count: BigInt(c.digg_count || 0),
|
||||
created_at: new Date((c.create_time || 0) * 1000),
|
||||
imagePostId: imagePost.aweme_id,
|
||||
userId: cu.id,
|
||||
},
|
||||
update: {
|
||||
text: c.text,
|
||||
digg_count: BigInt(c.digg_count || 0),
|
||||
created_at: new Date((c.create_time || 0) * 1000),
|
||||
imagePostId: imagePost.aweme_id,
|
||||
userId: cu.id,
|
||||
},
|
||||
});
|
||||
|
||||
// Upsert ImageFiles（按顺序）
|
||||
for (let i = 0; i < uploads.images.length; i++) {
|
||||
const { url, width, height, video } = uploads.images[i];
|
||||
await prisma.imageFile.upsert({
|
||||
where: { postId_order: { postId: imagePost.aweme_id, order: i } },
|
||||
create: {
|
||||
postId: imagePost.aweme_id,
|
||||
order: i,
|
||||
url,
|
||||
width: typeof width === 'number' ? width : null,
|
||||
height: typeof height === 'number' ? height : null,
|
||||
animated: video || null,
|
||||
},
|
||||
update: {
|
||||
url,
|
||||
width: typeof width === 'number' ? width : null,
|
||||
height: typeof height === 'number' ? height : null,
|
||||
animated: video || null,
|
||||
},
|
||||
// 处理评论贴纸/配图上传与入库
|
||||
try {
|
||||
const sources: {
|
||||
url?: string | null;
|
||||
width?: number;
|
||||
height?: number;
|
||||
}[] = [];
|
||||
// 贴纸（当作第一张）
|
||||
const stickerUrl = firstUrl(c.sticker?.animate_url?.url_list);
|
||||
if (stickerUrl) {
|
||||
sources.push({
|
||||
url: stickerUrl,
|
||||
width: c.sticker?.animate_url?.width,
|
||||
height: c.sticker?.animate_url?.height,
|
||||
});
|
||||
}
|
||||
// 配图列表
|
||||
const imgs = c.image_list || [];
|
||||
for (const it of imgs) {
|
||||
const u = firstUrl(it?.origin_url?.url_list);
|
||||
if (u)
|
||||
sources.push({
|
||||
url: u,
|
||||
width: it?.origin_url?.width as any,
|
||||
height: it?.origin_url?.height as any,
|
||||
});
|
||||
}
|
||||
|
||||
for (let i = 0; i < sources.length; i++) {
|
||||
const s = sources[i];
|
||||
const uploaded = await uploadImageFromUrl(
|
||||
context,
|
||||
s.url ?? undefined,
|
||||
`comments/${c.cid}/${i}`,
|
||||
);
|
||||
if (!uploaded) continue;
|
||||
await prisma.commentImage.upsert({
|
||||
where: { commentId_order: { commentId: savedComment.cid, order: i } },
|
||||
create: {
|
||||
commentId: savedComment.cid,
|
||||
order: i,
|
||||
url: uploaded,
|
||||
width: typeof s.width === "number" ? s.width : null,
|
||||
height: typeof s.height === "number" ? s.height : null,
|
||||
},
|
||||
update: {
|
||||
url: uploaded,
|
||||
width: typeof s.width === "number" ? s.width : null,
|
||||
height: typeof s.height === "number" ? s.height : null,
|
||||
},
|
||||
});
|
||||
}
|
||||
} catch (e) {
|
||||
console.warn("[comment-images] 保存失败：", (e as Error)?.message || e);
|
||||
}
|
||||
}
|
||||
|
||||
// 评论入库：关联到 ImagePost
|
||||
const comments = commentResp?.comments ?? [];
|
||||
for (const c of comments) {
|
||||
const origAvatar: string | null = firstUrl(c.user?.avatar_thumb?.url_list) ?? null;
|
||||
const nameHint = `comment-users/${(c.user?.nickname || 'unknown').replace(/\s+/g, '_')}-${c.cid}`;
|
||||
const uploadedAvatar = await uploadAvatarFromUrl(context, origAvatar ?? undefined, nameHint);
|
||||
const finalAvatar = uploadedAvatar ?? origAvatar; // string | null
|
||||
const finalAvatarKey = finalAvatar ?? '';
|
||||
const cu = await prisma.commentUser.upsert({
|
||||
where: {
|
||||
nickname_avatar_url: {
|
||||
nickname: c.user?.nickname || '未知用户',
|
||||
avatar_url: finalAvatarKey,
|
||||
},
|
||||
},
|
||||
create: {
|
||||
nickname: c.user?.nickname || '未知用户',
|
||||
avatar_url: finalAvatar ?? null,
|
||||
},
|
||||
update: {
|
||||
avatar_url: finalAvatar ?? null,
|
||||
},
|
||||
});
|
||||
|
||||
const savedComment = await prisma.comment.upsert({
|
||||
where: { cid: c.cid },
|
||||
create: {
|
||||
cid: c.cid,
|
||||
text: c.text,
|
||||
digg_count: BigInt(c.digg_count || 0),
|
||||
created_at: new Date((c.create_time || 0) * 1000),
|
||||
imagePostId: imagePost.aweme_id,
|
||||
userId: cu.id,
|
||||
},
|
||||
update: {
|
||||
text: c.text,
|
||||
digg_count: BigInt(c.digg_count || 0),
|
||||
created_at: new Date((c.create_time || 0) * 1000),
|
||||
imagePostId: imagePost.aweme_id,
|
||||
userId: cu.id,
|
||||
},
|
||||
});
|
||||
|
||||
// 处理评论贴纸/配图上传与入库
|
||||
try {
|
||||
const sources: { url?: string | null; width?: number; height?: number }[] = [];
|
||||
// 贴纸（当作第一张）
|
||||
const stickerUrl = firstUrl(c.sticker?.animate_url?.url_list);
|
||||
if (stickerUrl) {
|
||||
sources.push({ url: stickerUrl, width: c.sticker?.animate_url?.width, height: c.sticker?.animate_url?.height });
|
||||
}
|
||||
// 配图列表
|
||||
const imgs = c.image_list || [];
|
||||
for (const it of imgs) {
|
||||
const u = firstUrl(it?.origin_url?.url_list);
|
||||
if (u) sources.push({ url: u, width: it?.origin_url?.width as any, height: it?.origin_url?.height as any });
|
||||
}
|
||||
|
||||
for (let i = 0; i < sources.length; i++) {
|
||||
const s = sources[i];
|
||||
const uploaded = await uploadImageFromUrl(
|
||||
context,
|
||||
s.url ?? undefined,
|
||||
`comments/${c.cid}/${i}`,
|
||||
);
|
||||
if (!uploaded) continue;
|
||||
await prisma.commentImage.upsert({
|
||||
where: { commentId_order: { commentId: savedComment.cid, order: i } },
|
||||
create: {
|
||||
commentId: savedComment.cid,
|
||||
order: i,
|
||||
url: uploaded,
|
||||
width: typeof s.width === 'number' ? s.width : null,
|
||||
height: typeof s.height === 'number' ? s.height : null,
|
||||
},
|
||||
update: {
|
||||
url: uploaded,
|
||||
width: typeof s.width === 'number' ? s.width : null,
|
||||
height: typeof s.height === 'number' ? s.height : null,
|
||||
},
|
||||
});
|
||||
}
|
||||
} catch (e) {
|
||||
console.warn('[comment-images] 保存失败：', (e as Error)?.message || e);
|
||||
}
|
||||
}
|
||||
|
||||
return { aweme_id: imagePost.aweme_id, author_sec_uid: author.sec_uid, image_count: uploads.images.length, comment_count: comments.length };
|
||||
return {
|
||||
aweme_id: imagePost.aweme_id,
|
||||
author_sec_uid: author.sec_uid,
|
||||
image_count: uploads.images.length,
|
||||
comment_count: comments.length,
|
||||
};
|
||||
}
|
||||
|
||||
+44
-44
@@ -1,51 +1,51 @@
|
||||
export const runtime = 'nodejs'
|
||||
export const runtime = "nodejs";
|
||||
|
||||
import { NextRequest, NextResponse } from 'next/server'
|
||||
import { prisma } from '@/lib/prisma'
|
||||
import { scrapeDouyin, ScrapeError } from '.';
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { prisma } from "@/lib/prisma";
|
||||
import { scrapeDouyin, ScrapeError } from ".";
|
||||
|
||||
async function handleDouyinScrape(req: NextRequest) {
|
||||
const { searchParams } = new URL(req.url);
|
||||
const videoUrl = searchParams.get('url');
|
||||
|
||||
if (!videoUrl) {
|
||||
return NextResponse.json(
|
||||
{ error: '缺少视频URL', code: 'MISSING_URL' },
|
||||
{ status: 400 }
|
||||
);
|
||||
const { searchParams } = new URL(req.url);
|
||||
const videoUrl = searchParams.get("url");
|
||||
|
||||
if (!videoUrl) {
|
||||
return NextResponse.json(
|
||||
{ error: "缺少视频URL", code: "MISSING_URL" },
|
||||
{ status: 400 },
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
// 调用爬虫函数
|
||||
const result = await scrapeDouyin(videoUrl);
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
data: result,
|
||||
});
|
||||
} catch (error) {
|
||||
// 处理自定义的 ScrapeError
|
||||
if (error instanceof ScrapeError) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: error.message,
|
||||
code: error.code,
|
||||
},
|
||||
{ status: error.statusCode },
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
// 调用爬虫函数
|
||||
const result = await scrapeDouyin(videoUrl);
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
data: result
|
||||
});
|
||||
} catch (error) {
|
||||
// 处理自定义的 ScrapeError
|
||||
if (error instanceof ScrapeError) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: error.message,
|
||||
code: error.code
|
||||
},
|
||||
{ status: error.statusCode }
|
||||
);
|
||||
}
|
||||
|
||||
// 处理未知错误
|
||||
console.error('未捕获的错误:', error);
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: '服务器内部错误',
|
||||
code: 'INTERNAL_ERROR'
|
||||
},
|
||||
{ status: 500 }
|
||||
);
|
||||
}
|
||||
// 处理未知错误
|
||||
console.error("未捕获的错误:", error);
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: "服务器内部错误",
|
||||
code: "INTERNAL_ERROR",
|
||||
},
|
||||
{ status: 500 },
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export const GET = handleDouyinScrape
|
||||
export const GET = handleDouyinScrape;
|
||||
|
||||
Vendored
+33
-33
@@ -17,17 +17,17 @@ interface DouyinComment {
|
||||
animate_url: {
|
||||
width: number;
|
||||
height: number;
|
||||
url_list: string[]
|
||||
}
|
||||
},
|
||||
url_list: string[];
|
||||
};
|
||||
};
|
||||
|
||||
image_list?: {
|
||||
origin_url:{
|
||||
origin_url: {
|
||||
width: number;
|
||||
height: number;
|
||||
url_list: string[]
|
||||
}
|
||||
}[]
|
||||
url_list: string[];
|
||||
};
|
||||
}[];
|
||||
}
|
||||
|
||||
/** 用户信息（精简版） */
|
||||
@@ -45,35 +45,35 @@ interface DouyinVideoDetailResponse {
|
||||
}
|
||||
/** 作者信息（精简版） */
|
||||
interface DouyinAuthor {
|
||||
uid: string; // 用户ID
|
||||
sec_uid: string; // 安全UID
|
||||
nickname: string; // 用户昵称
|
||||
signature: string; // 个性签名
|
||||
uid: string; // 用户ID
|
||||
sec_uid: string; // 安全UID
|
||||
nickname: string; // 用户昵称
|
||||
signature: string; // 个性签名
|
||||
avatar_thumb: {
|
||||
url_list: string[]; // 头像URL（可取第一个）
|
||||
url_list: string[]; // 头像URL（可取第一个）
|
||||
};
|
||||
follower_count: number; // 粉丝数
|
||||
total_favorited: number; // 获赞总数
|
||||
unique_id: string; // 抖音号
|
||||
short_id: string; // 短ID
|
||||
follower_count: number; // 粉丝数
|
||||
total_favorited: number; // 获赞总数
|
||||
unique_id: string; // 抖音号
|
||||
short_id: string; // 短ID
|
||||
}
|
||||
/** 视频详情 */
|
||||
interface DouyinVideoDetail {
|
||||
aweme_id: string; // 视频ID
|
||||
desc: string; // 视频描述
|
||||
preview_title?: string; // 视频标题（有些字段中叫 preview_title）
|
||||
duration: number; // 视频时长（毫秒）
|
||||
create_time: number; // 创建时间（时间戳）
|
||||
share_url: string; // 视频分享链接
|
||||
aweme_id: string; // 视频ID
|
||||
desc: string; // 视频描述
|
||||
preview_title?: string; // 视频标题（有些字段中叫 preview_title）
|
||||
duration: number; // 视频时长（毫秒）
|
||||
create_time: number; // 创建时间（时间戳）
|
||||
share_url: string; // 视频分享链接
|
||||
|
||||
statistics: {
|
||||
digg_count: number; // 点赞数
|
||||
comment_count: number; // 评论数
|
||||
share_count: number; // 分享数
|
||||
collect_count: number; // 收藏数
|
||||
digg_count: number; // 点赞数
|
||||
comment_count: number; // 评论数
|
||||
share_count: number; // 分享数
|
||||
collect_count: number; // 收藏数
|
||||
};
|
||||
|
||||
author: DouyinAuthor; // 作者信息
|
||||
author: DouyinAuthor; // 作者信息
|
||||
video: VideoPlayBasic;
|
||||
tags: VideoTagBasic[];
|
||||
}
|
||||
@@ -88,9 +88,9 @@ interface VideoPlayBasic {
|
||||
|
||||
/** 单个清晰度变体（来自 bit_rate[*] + play_addr） */
|
||||
interface PlayVariant {
|
||||
format: string; // mp4 等
|
||||
format: string; // mp4 等
|
||||
FPS: number;
|
||||
bit_rate: number; // bit_rate.bit_rate
|
||||
bit_rate: number; // bit_rate.bit_rate
|
||||
|
||||
/** 直连播放地址（最关键） */
|
||||
play_addr: {
|
||||
@@ -101,7 +101,7 @@ interface PlayVariant {
|
||||
data_size: number;
|
||||
FPS: number;
|
||||
is_bytevc1: number; // 0 or 1
|
||||
is_h265: number; // 0 or 1
|
||||
is_h265: number; // 0 or 1
|
||||
};
|
||||
}
|
||||
|
||||
@@ -132,7 +132,7 @@ interface DouyinImageAweme {
|
||||
};
|
||||
author: DouyinAuthor; // 复用视频作者类型（需包含 sec_uid）
|
||||
images: DouyinImageInfo[]; // 图片列表
|
||||
music?: DouyinMusicBasic; // 背景音乐（可选）
|
||||
music?: DouyinMusicBasic; // 背景音乐（可选）
|
||||
video_tag?: VideoTagBasic[]; // 标签
|
||||
}
|
||||
|
||||
@@ -143,7 +143,7 @@ interface DouyinImageInfo {
|
||||
width: number;
|
||||
height: number;
|
||||
video: {
|
||||
play_addr: { src: string }[]
|
||||
play_addr: { src: string }[];
|
||||
} | null; // 如果是动图，会有 video 信息
|
||||
}
|
||||
|
||||
@@ -157,4 +157,4 @@ interface DouyinMusicBasic {
|
||||
uri?: string;
|
||||
url_list: string[]; // 真实可下载地址
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
+150
-92
@@ -1,117 +1,175 @@
|
||||
export const runtime = 'nodejs'
|
||||
export const runtime = "nodejs";
|
||||
|
||||
import type { BrowserContext } from 'playwright';
|
||||
import { uploadFile, generateUniqueFileName } from '@/lib/minio';
|
||||
import { downloadBinary } from './network';
|
||||
import { pickFirstUrl } from './utils';
|
||||
import { getVideoDuration } from '@/app/api/media';
|
||||
import type { BrowserContext } from "playwright";
|
||||
import { uploadFile, generateUniqueFileName } from "@/lib/minio";
|
||||
import { downloadBinary } from "./network";
|
||||
import { pickFirstUrl } from "./utils";
|
||||
import { getVideoDuration } from "@/app/api/media";
|
||||
|
||||
/**
|
||||
* 下载头像并上传到 MinIO，返回外链；失败时回退为原始链接。
|
||||
*/
|
||||
export async function uploadAvatarFromUrl(
|
||||
context: BrowserContext,
|
||||
srcUrl?: string | null,
|
||||
nameHint?: string,
|
||||
context: BrowserContext,
|
||||
srcUrl?: string | null,
|
||||
nameHint?: string,
|
||||
): Promise<string | undefined> {
|
||||
if (!srcUrl) return undefined;
|
||||
try {
|
||||
const { buffer, contentType, ext } = await downloadBinary(context, srcUrl);
|
||||
const safeExt = ext || 'jpg';
|
||||
const baseName = nameHint ? `${nameHint}.${safeExt}` : `avatar.${safeExt}`;
|
||||
const fileName = generateUniqueFileName(baseName, 'douyin/avatars');
|
||||
const uploaded = await uploadFile(buffer, fileName, { 'Content-Type': contentType });
|
||||
return uploaded;
|
||||
} catch (e) {
|
||||
console.warn('[avatar] 上传失败，使用原始链接：', (e as Error)?.message || e);
|
||||
return srcUrl || undefined;
|
||||
}
|
||||
if (!srcUrl) return undefined;
|
||||
try {
|
||||
const { buffer, contentType, ext } = await downloadBinary(context, srcUrl);
|
||||
const safeExt = ext || "jpg";
|
||||
const baseName = nameHint ? `${nameHint}.${safeExt}` : `avatar.${safeExt}`;
|
||||
const fileName = generateUniqueFileName(baseName, "douyin/avatars");
|
||||
const uploaded = await uploadFile(buffer, fileName, {
|
||||
"Content-Type": contentType,
|
||||
});
|
||||
return uploaded;
|
||||
} catch (e) {
|
||||
console.warn(
|
||||
"[avatar] 上传失败，使用原始链接：",
|
||||
(e as Error)?.message || e,
|
||||
);
|
||||
return srcUrl || undefined;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 下载任意图片并上传到 MinIO，返回外链；失败时回退为原始链接。
|
||||
*/
|
||||
export async function uploadImageFromUrl(
|
||||
context: BrowserContext,
|
||||
srcUrl?: string | null,
|
||||
nameHint?: string,
|
||||
context: BrowserContext,
|
||||
srcUrl?: string | null,
|
||||
nameHint?: string,
|
||||
): Promise<string | undefined> {
|
||||
if (!srcUrl) return undefined;
|
||||
try {
|
||||
const { buffer, contentType, ext } = await downloadBinary(context, srcUrl);
|
||||
const safeExt = ext || 'jpg';
|
||||
const baseName = nameHint ? `${nameHint}.${safeExt}` : `image.${safeExt}`;
|
||||
const fileName = generateUniqueFileName(baseName, 'douyin/comment-images');
|
||||
const uploaded = await uploadFile(buffer, fileName, { 'Content-Type': contentType });
|
||||
return uploaded;
|
||||
} catch (e) {
|
||||
console.warn('[image] 上传失败，使用原始链接：', (e as Error)?.message || e);
|
||||
return srcUrl || undefined;
|
||||
}
|
||||
if (!srcUrl) return undefined;
|
||||
try {
|
||||
const { buffer, contentType, ext } = await downloadBinary(context, srcUrl);
|
||||
const safeExt = ext || "jpg";
|
||||
const baseName = nameHint ? `${nameHint}.${safeExt}` : `image.${safeExt}`;
|
||||
const fileName = generateUniqueFileName(baseName, "douyin/comment-images");
|
||||
const uploaded = await uploadFile(buffer, fileName, {
|
||||
"Content-Type": contentType,
|
||||
});
|
||||
return uploaded;
|
||||
} catch (e) {
|
||||
console.warn(
|
||||
"[image] 上传失败，使用原始链接：",
|
||||
(e as Error)?.message || e,
|
||||
);
|
||||
return srcUrl || undefined;
|
||||
}
|
||||
}
|
||||
|
||||
/** 下载图文作品的图片和音乐并上传到 MinIO */
|
||||
export async function handleImagePost(
|
||||
context: BrowserContext,
|
||||
aweme: DouyinImageAweme
|
||||
): Promise<{ images: { url: string; width?: number; height?: number; video?: string; duration?: number }[]; musicUrl?: string }> {
|
||||
const awemeId = aweme.aweme_id;
|
||||
const uploadedImages: { url: string; width?: number; height?: number; video?: string; duration?: number }[] = [];
|
||||
context: BrowserContext,
|
||||
aweme: DouyinImageAweme,
|
||||
): Promise<{
|
||||
images: {
|
||||
url: string;
|
||||
width?: number;
|
||||
height?: number;
|
||||
video?: string;
|
||||
duration?: number;
|
||||
}[];
|
||||
musicUrl?: string;
|
||||
}> {
|
||||
const awemeId = aweme.aweme_id;
|
||||
const uploadedImages: {
|
||||
url: string;
|
||||
width?: number;
|
||||
height?: number;
|
||||
video?: string;
|
||||
duration?: number;
|
||||
}[] = [];
|
||||
|
||||
// 下载图片（顺序保持）
|
||||
for (let i = 0; i < (aweme.images?.length || 0); i++) {
|
||||
const img = aweme.images[i];
|
||||
const url = pickFirstUrl(img?.url_list);
|
||||
if (!url) continue;
|
||||
const { buffer, contentType, ext } = await downloadBinary(context, url);
|
||||
const safeExt = ext || 'jpg';
|
||||
const fileName = generateUniqueFileName(`${awemeId}/${i}.${safeExt}`, 'douyin/images');
|
||||
const uploaded = await uploadFile(buffer, fileName, { 'Content-Type': contentType });
|
||||
// 下载图片（顺序保持）
|
||||
for (let i = 0; i < (aweme.images?.length || 0); i++) {
|
||||
const img = aweme.images[i];
|
||||
const url = pickFirstUrl(img?.url_list);
|
||||
if (!url) continue;
|
||||
const { buffer, contentType, ext } = await downloadBinary(context, url);
|
||||
const safeExt = ext || "jpg";
|
||||
const fileName = generateUniqueFileName(
|
||||
`${awemeId}/${i}.${safeExt}`,
|
||||
"douyin/images",
|
||||
);
|
||||
const uploaded = await uploadFile(buffer, fileName, {
|
||||
"Content-Type": contentType,
|
||||
});
|
||||
|
||||
if (img.video?.play_addr) {
|
||||
// 如果是动图，下载 video 并上传
|
||||
const videoUrl = img.video.play_addr[0]?.src;
|
||||
if (videoUrl) {
|
||||
try {
|
||||
const { buffer: videoBuffer, contentType: videoContentType, ext: videoExt } = await downloadBinary(context, videoUrl);
|
||||
const safeVideoExt = videoExt || 'mp4';
|
||||
const videoFileName = generateUniqueFileName(`${awemeId}/${i}_animated.${safeVideoExt}`, 'douyin/images');
|
||||
const uploadedVideo = await uploadFile(videoBuffer, videoFileName, { 'Content-Type': videoContentType });
|
||||
|
||||
// 获取动图时长
|
||||
const duration = await getVideoDuration(videoBuffer);
|
||||
|
||||
// 将动图的 video URL 和 duration 也存储起来
|
||||
uploadedImages.push({
|
||||
url: uploaded,
|
||||
width: img?.width,
|
||||
height: img?.height,
|
||||
video: uploadedVideo,
|
||||
duration: duration ?? undefined
|
||||
});
|
||||
|
||||
if (duration) {
|
||||
console.log(`[image] 动图 ${i} 时长: ${duration}ms`);
|
||||
}
|
||||
} catch (e) {
|
||||
console.warn(`[image] 动图视频上传失败，跳过：`, (e as Error)?.message || e);
|
||||
uploadedImages.push({ url: uploaded, width: img?.width, height: img?.height });
|
||||
}
|
||||
}
|
||||
} else {
|
||||
uploadedImages.push({ url: uploaded, width: img?.width, height: img?.height });
|
||||
if (img.video?.play_addr) {
|
||||
// 如果是动图，下载 video 并上传
|
||||
const videoUrl = img.video.play_addr[0]?.src;
|
||||
if (videoUrl) {
|
||||
try {
|
||||
const {
|
||||
buffer: videoBuffer,
|
||||
contentType: videoContentType,
|
||||
ext: videoExt,
|
||||
} = await downloadBinary(context, videoUrl);
|
||||
const safeVideoExt = videoExt || "mp4";
|
||||
const videoFileName = generateUniqueFileName(
|
||||
`${awemeId}/${i}_animated.${safeVideoExt}`,
|
||||
"douyin/images",
|
||||
);
|
||||
const uploadedVideo = await uploadFile(videoBuffer, videoFileName, {
|
||||
"Content-Type": videoContentType,
|
||||
});
|
||||
|
||||
// 获取动图时长
|
||||
const duration = await getVideoDuration(videoBuffer);
|
||||
|
||||
// 将动图的 video URL 和 duration 也存储起来
|
||||
uploadedImages.push({
|
||||
url: uploaded,
|
||||
width: img?.width,
|
||||
height: img?.height,
|
||||
video: uploadedVideo,
|
||||
duration: duration ?? undefined,
|
||||
});
|
||||
|
||||
if (duration) {
|
||||
console.log(`[image] 动图 ${i} 时长: ${duration}ms`);
|
||||
}
|
||||
} catch (e) {
|
||||
console.warn(
|
||||
`[image] 动图视频上传失败，跳过：`,
|
||||
(e as Error)?.message || e,
|
||||
);
|
||||
uploadedImages.push({
|
||||
url: uploaded,
|
||||
width: img?.width,
|
||||
height: img?.height,
|
||||
});
|
||||
}
|
||||
}
|
||||
} else {
|
||||
uploadedImages.push({
|
||||
url: uploaded,
|
||||
width: img?.width,
|
||||
height: img?.height,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// 下载音乐（可选）
|
||||
let musicUrl: string | undefined;
|
||||
const audioSrc = pickFirstUrl(aweme.music?.play_url?.url_list);
|
||||
if (audioSrc) {
|
||||
const { buffer, contentType, ext } = await downloadBinary(context, audioSrc);
|
||||
const safeExt = ext || 'mp3';
|
||||
const fileName = generateUniqueFileName(`${awemeId}.${safeExt}`, 'douyin/audios');
|
||||
musicUrl = await uploadFile(buffer, fileName, { 'Content-Type': contentType });
|
||||
}
|
||||
// 下载音乐（可选）
|
||||
let musicUrl: string | undefined;
|
||||
const audioSrc = pickFirstUrl(aweme.music?.play_url?.url_list);
|
||||
if (audioSrc) {
|
||||
const { buffer, contentType, ext } = await downloadBinary(
|
||||
context,
|
||||
audioSrc,
|
||||
);
|
||||
const safeExt = ext || "mp3";
|
||||
const fileName = generateUniqueFileName(
|
||||
`${awemeId}.${safeExt}`,
|
||||
"douyin/audios",
|
||||
);
|
||||
musicUrl = await uploadFile(buffer, fileName, {
|
||||
"Content-Type": contentType,
|
||||
});
|
||||
}
|
||||
|
||||
return { images: uploadedImages, musicUrl };
|
||||
return { images: uploadedImages, musicUrl };
|
||||
}
|
||||
|
||||
+38
-37
@@ -1,11 +1,11 @@
|
||||
export const runtime = 'nodejs'
|
||||
export const runtime = "nodejs";
|
||||
|
||||
export function toCamelCaseKey(key: string): string {
|
||||
return key.replace(/_([a-zA-Z])/g, (_, c: string) => c.toUpperCase());
|
||||
return key.replace(/_([a-zA-Z])/g, (_, c: string) => c.toUpperCase());
|
||||
}
|
||||
|
||||
export function toSnakeCaseKey(key: string): string {
|
||||
return key.replace(/[A-Z]/g, (m) => `_${m.toLowerCase()}`);
|
||||
return key.replace(/[A-Z]/g, (m) => `_${m.toLowerCase()}`);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -13,52 +13,53 @@ export function toSnakeCaseKey(key: string): string {
|
||||
* 访问顺序：原名 -> camelCase -> snake_case
|
||||
*/
|
||||
export function createCamelCompatibleProxy<T extends object>(root: T): T {
|
||||
const seen = new WeakMap<object, any>();
|
||||
const seen = new WeakMap<object, any>();
|
||||
|
||||
const wrap = (value: any): any => {
|
||||
if (value === null || typeof value !== 'object') return value;
|
||||
if (seen.has(value)) return seen.get(value);
|
||||
const proxied = new Proxy(value, handler);
|
||||
seen.set(value, proxied);
|
||||
return proxied;
|
||||
};
|
||||
const wrap = (value: any): any => {
|
||||
if (value === null || typeof value !== "object") return value;
|
||||
if (seen.has(value)) return seen.get(value);
|
||||
const proxied = new Proxy(value, handler);
|
||||
seen.set(value, proxied);
|
||||
return proxied;
|
||||
};
|
||||
|
||||
const handler: ProxyHandler<any> = {
|
||||
get(target, prop, receiver) {
|
||||
// 非字符串属性（如 Symbol、数字索引）直接透传
|
||||
if (typeof prop !== 'string') {
|
||||
return wrap(Reflect.get(target, prop, receiver));
|
||||
}
|
||||
const handler: ProxyHandler<any> = {
|
||||
get(target, prop, receiver) {
|
||||
// 非字符串属性（如 Symbol、数字索引）直接透传
|
||||
if (typeof prop !== "string") {
|
||||
return wrap(Reflect.get(target, prop, receiver));
|
||||
}
|
||||
|
||||
const primary = prop;
|
||||
const primary = prop;
|
||||
|
||||
if (primary in target) return wrap(Reflect.get(target, primary, receiver));
|
||||
if (primary in target)
|
||||
return wrap(Reflect.get(target, primary, receiver));
|
||||
|
||||
const camel = toCamelCaseKey(primary);
|
||||
if (camel in target) return wrap(Reflect.get(target, camel, receiver));
|
||||
const camel = toCamelCaseKey(primary);
|
||||
if (camel in target) return wrap(Reflect.get(target, camel, receiver));
|
||||
|
||||
const snake = toSnakeCaseKey(primary);
|
||||
if (snake in target) return wrap(Reflect.get(target, snake, receiver));
|
||||
const snake = toSnakeCaseKey(primary);
|
||||
if (snake in target) return wrap(Reflect.get(target, snake, receiver));
|
||||
|
||||
return wrap(Reflect.get(target, prop, receiver));
|
||||
},
|
||||
has(target, prop) {
|
||||
if (typeof prop !== 'string') return prop in target;
|
||||
const primary = prop === 'auther' ? 'autherInfo' : prop;
|
||||
return (
|
||||
primary in target ||
|
||||
toCamelCaseKey(primary) in target ||
|
||||
toSnakeCaseKey(primary) in target
|
||||
);
|
||||
}
|
||||
};
|
||||
return wrap(Reflect.get(target, prop, receiver));
|
||||
},
|
||||
has(target, prop) {
|
||||
if (typeof prop !== "string") return prop in target;
|
||||
const primary = prop === "auther" ? "autherInfo" : prop;
|
||||
return (
|
||||
primary in target ||
|
||||
toCamelCaseKey(primary) in target ||
|
||||
toSnakeCaseKey(primary) in target
|
||||
);
|
||||
},
|
||||
};
|
||||
|
||||
return wrap(root);
|
||||
return wrap(root);
|
||||
}
|
||||
|
||||
/** 选择首个可用 URL */
|
||||
export function pickFirstUrl(list?: string[]) {
|
||||
return Array.isArray(list) && list.length ? list[0] : undefined;
|
||||
return Array.isArray(list) && list.length ? list[0] : undefined;
|
||||
}
|
||||
|
||||
// 别名，兼容旧命名
|
||||
|
||||
+179
-120
@@ -1,80 +1,113 @@
|
||||
import { execFile } from 'child_process';
|
||||
import { promises as fs } from 'fs';
|
||||
import os from 'os';
|
||||
import path from 'path';
|
||||
import { promisify } from 'util';
|
||||
import { execFile } from "child_process";
|
||||
import { promises as fs } from "fs";
|
||||
import os from "os";
|
||||
import path from "path";
|
||||
import { promisify } from "util";
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
|
||||
/**
|
||||
* 使用 ffmpeg 从视频二进制中提取第一帧，返回 JPEG buffer
|
||||
*/
|
||||
export async function extractFirstFrame(videoBuffer: Buffer): Promise<{ buffer: Buffer; contentType: string; ext: string } | null> {
|
||||
const ffmpegCmd = process.env.FFMPEG_PATH || 'ffmpeg';
|
||||
const tmpDir = os.tmpdir();
|
||||
const base = `dy_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`;
|
||||
const inPath = path.join(tmpDir, `${base}.mp4`);
|
||||
const outPath = path.join(tmpDir, `${base}.jpg`);
|
||||
export async function extractFirstFrame(
|
||||
videoBuffer: Buffer,
|
||||
): Promise<{ buffer: Buffer; contentType: string; ext: string } | null> {
|
||||
const ffmpegCmd = process.env.FFMPEG_PATH || "ffmpeg";
|
||||
const tmpDir = os.tmpdir();
|
||||
const base = `dy_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`;
|
||||
const inPath = path.join(tmpDir, `${base}.mp4`);
|
||||
const outPath = path.join(tmpDir, `${base}.jpg`);
|
||||
|
||||
try {
|
||||
await fs.writeFile(inPath, videoBuffer);
|
||||
const args = [
|
||||
'-hide_banner',
|
||||
'-loglevel', 'error',
|
||||
'-ss', '0',
|
||||
'-i', inPath,
|
||||
'-frames:v', '1',
|
||||
'-q:v', '2',
|
||||
'-f', 'image2',
|
||||
'-y',
|
||||
outPath,
|
||||
];
|
||||
await execFileAsync(ffmpegCmd, args, { windowsHide: true });
|
||||
const img = await fs.readFile(outPath);
|
||||
return { buffer: img, contentType: 'image/jpeg', ext: 'jpg' };
|
||||
} catch (e: any) {
|
||||
if (e && (e.code === 'ENOENT' || /not found|is not recognized/i.test(String(e.message)))) {
|
||||
console.warn('系统未检测到 ffmpeg，可安装并配置 PATH 或设置 FFMPEG_PATH 后启用封面提取。');
|
||||
return null;
|
||||
}
|
||||
throw e;
|
||||
} finally {
|
||||
try { await fs.unlink(inPath); } catch { }
|
||||
try { await fs.unlink(outPath); } catch { }
|
||||
try {
|
||||
await fs.writeFile(inPath, videoBuffer);
|
||||
const args = [
|
||||
"-hide_banner",
|
||||
"-loglevel",
|
||||
"error",
|
||||
"-ss",
|
||||
"0",
|
||||
"-i",
|
||||
inPath,
|
||||
"-frames:v",
|
||||
"1",
|
||||
"-q:v",
|
||||
"2",
|
||||
"-f",
|
||||
"image2",
|
||||
"-y",
|
||||
outPath,
|
||||
];
|
||||
await execFileAsync(ffmpegCmd, args, { windowsHide: true });
|
||||
const img = await fs.readFile(outPath);
|
||||
return { buffer: img, contentType: "image/jpeg", ext: "jpg" };
|
||||
} catch (e: any) {
|
||||
if (
|
||||
e &&
|
||||
(e.code === "ENOENT" ||
|
||||
/not found|is not recognized/i.test(String(e.message)))
|
||||
) {
|
||||
console.warn(
|
||||
"系统未检测到 ffmpeg，可安装并配置 PATH 或设置 FFMPEG_PATH 后启用封面提取。",
|
||||
);
|
||||
return null;
|
||||
}
|
||||
throw e;
|
||||
} finally {
|
||||
try {
|
||||
await fs.unlink(inPath);
|
||||
} catch {}
|
||||
try {
|
||||
await fs.unlink(outPath);
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 使用 ffprobe 获取视频时长（毫秒）
|
||||
*/
|
||||
export async function getVideoDuration(videoBuffer: Buffer): Promise<number | null> {
|
||||
const ffprobeCmd = process.env.FFPROBE_PATH || 'ffprobe';
|
||||
const tmpDir = os.tmpdir();
|
||||
const base = `dy_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`;
|
||||
const inPath = path.join(tmpDir, `${base}.mp4`);
|
||||
export async function getVideoDuration(
|
||||
videoBuffer: Buffer,
|
||||
): Promise<number | null> {
|
||||
const ffprobeCmd = process.env.FFPROBE_PATH || "ffprobe";
|
||||
const tmpDir = os.tmpdir();
|
||||
const base = `dy_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`;
|
||||
const inPath = path.join(tmpDir, `${base}.mp4`);
|
||||
|
||||
try {
|
||||
await fs.writeFile(inPath, videoBuffer);
|
||||
const args = [
|
||||
'-v', 'error',
|
||||
'-show_entries', 'format=duration',
|
||||
'-of', 'default=noprint_wrappers=1:nokey=1',
|
||||
inPath,
|
||||
];
|
||||
const { stdout } = await execFileAsync(ffprobeCmd, args, { windowsHide: true });
|
||||
const durationSeconds = parseFloat(stdout.trim());
|
||||
if (isNaN(durationSeconds)) return null;
|
||||
return Math.round(durationSeconds * 1000); // 转换为毫秒
|
||||
} catch (e: any) {
|
||||
if (e && (e.code === 'ENOENT' || /not found|is not recognized/i.test(String(e.message)))) {
|
||||
console.warn('系统未检测到 ffprobe，可安装并配置 PATH 或设置 FFPROBE_PATH 后启用时长提取。');
|
||||
return null;
|
||||
}
|
||||
console.warn(`获取视频时长失败: ${e?.message || e}`);
|
||||
return null;
|
||||
} finally {
|
||||
try { await fs.unlink(inPath); } catch { }
|
||||
try {
|
||||
await fs.writeFile(inPath, videoBuffer);
|
||||
const args = [
|
||||
"-v",
|
||||
"error",
|
||||
"-show_entries",
|
||||
"format=duration",
|
||||
"-of",
|
||||
"default=noprint_wrappers=1:nokey=1",
|
||||
inPath,
|
||||
];
|
||||
const { stdout } = await execFileAsync(ffprobeCmd, args, {
|
||||
windowsHide: true,
|
||||
});
|
||||
const durationSeconds = parseFloat(stdout.trim());
|
||||
if (isNaN(durationSeconds)) return null;
|
||||
return Math.round(durationSeconds * 1000); // 转换为毫秒
|
||||
} catch (e: any) {
|
||||
if (
|
||||
e &&
|
||||
(e.code === "ENOENT" ||
|
||||
/not found|is not recognized/i.test(String(e.message)))
|
||||
) {
|
||||
console.warn(
|
||||
"系统未检测到 ffprobe，可安装并配置 PATH 或设置 FFPROBE_PATH 后启用时长提取。",
|
||||
);
|
||||
return null;
|
||||
}
|
||||
console.warn(`获取视频时长失败: ${e?.message || e}`);
|
||||
return null;
|
||||
} finally {
|
||||
try {
|
||||
await fs.unlink(inPath);
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -88,68 +121,94 @@ export async function getVideoDuration(videoBuffer: Buffer): Promise<number | nu
|
||||
* 若系统未安装 ffmpeg 或未在 PATH 中，返回 null 并给出提示。
|
||||
*/
|
||||
export async function extractAudio(
|
||||
videoBuffer: Buffer,
|
||||
opts?: { format?: 'mp3' | 'aac' | 'wav'; bitrateKbps?: number }
|
||||
videoBuffer: Buffer,
|
||||
opts?: { format?: "mp3" | "aac" | "wav"; bitrateKbps?: number },
|
||||
): Promise<{ buffer: Buffer; contentType: string; ext: string } | null> {
|
||||
const ffmpegCmd = process.env.FFMPEG_PATH || 'ffmpeg';
|
||||
const tmpDir = os.tmpdir();
|
||||
const base = `dy_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`;
|
||||
const inPath = path.join(tmpDir, `${base}.mp4`);
|
||||
const ffmpegCmd = process.env.FFMPEG_PATH || "ffmpeg";
|
||||
const tmpDir = os.tmpdir();
|
||||
const base = `dy_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`;
|
||||
const inPath = path.join(tmpDir, `${base}.mp4`);
|
||||
|
||||
const format = opts?.format ?? 'mp3';
|
||||
const bitrate = Math.max(32, Math.min(512, opts?.bitrateKbps ?? 192)); // 安全范围 32~512 kbps
|
||||
const format = opts?.format ?? "mp3";
|
||||
const bitrate = Math.max(32, Math.min(512, opts?.bitrateKbps ?? 192)); // 安全范围 32~512 kbps
|
||||
|
||||
// 根据目标格式设置输出路径、MIME 与编码参数
|
||||
let outPath = '';
|
||||
let contentType = '';
|
||||
let ext = '';
|
||||
let codecArgs: string[] = [];
|
||||
// 根据目标格式设置输出路径、MIME 与编码参数
|
||||
let outPath = "";
|
||||
let contentType = "";
|
||||
let ext = "";
|
||||
let codecArgs: string[] = [];
|
||||
|
||||
if (format === 'mp3') {
|
||||
ext = 'mp3';
|
||||
contentType = 'audio/mpeg';
|
||||
outPath = path.join(tmpDir, `${base}.${ext}`);
|
||||
codecArgs = ['-c:a', 'libmp3lame', '-b:a', `${bitrate}k`];
|
||||
} else if (format === 'aac') {
|
||||
// 使用 m4a 容器更通用
|
||||
ext = 'm4a';
|
||||
contentType = 'audio/mp4';
|
||||
outPath = path.join(tmpDir, `${base}.${ext}`);
|
||||
codecArgs = ['-c:a', 'aac', '-b:a', `${bitrate}k`, '-movflags', '+faststart'];
|
||||
} else {
|
||||
// wav
|
||||
ext = 'wav';
|
||||
contentType = 'audio/wav';
|
||||
outPath = path.join(tmpDir, `${base}.${ext}`);
|
||||
codecArgs = ['-f', 'wav', '-acodec', 'pcm_s16le', '-ar', '44100', '-ac', '2'];
|
||||
if (format === "mp3") {
|
||||
ext = "mp3";
|
||||
contentType = "audio/mpeg";
|
||||
outPath = path.join(tmpDir, `${base}.${ext}`);
|
||||
codecArgs = ["-c:a", "libmp3lame", "-b:a", `${bitrate}k`];
|
||||
} else if (format === "aac") {
|
||||
// 使用 m4a 容器更通用
|
||||
ext = "m4a";
|
||||
contentType = "audio/mp4";
|
||||
outPath = path.join(tmpDir, `${base}.${ext}`);
|
||||
codecArgs = [
|
||||
"-c:a",
|
||||
"aac",
|
||||
"-b:a",
|
||||
`${bitrate}k`,
|
||||
"-movflags",
|
||||
"+faststart",
|
||||
];
|
||||
} else {
|
||||
// wav
|
||||
ext = "wav";
|
||||
contentType = "audio/wav";
|
||||
outPath = path.join(tmpDir, `${base}.${ext}`);
|
||||
codecArgs = [
|
||||
"-f",
|
||||
"wav",
|
||||
"-acodec",
|
||||
"pcm_s16le",
|
||||
"-ar",
|
||||
"44100",
|
||||
"-ac",
|
||||
"2",
|
||||
];
|
||||
}
|
||||
|
||||
try {
|
||||
await fs.writeFile(inPath, videoBuffer);
|
||||
const args = [
|
||||
"-hide_banner",
|
||||
"-loglevel",
|
||||
"error",
|
||||
"-i",
|
||||
inPath,
|
||||
"-vn", // 丢弃视频流
|
||||
...codecArgs,
|
||||
"-y",
|
||||
outPath,
|
||||
];
|
||||
await execFileAsync(ffmpegCmd, args, { windowsHide: true });
|
||||
const audio = await fs.readFile(outPath);
|
||||
return { buffer: audio, contentType, ext };
|
||||
} catch (e: any) {
|
||||
if (
|
||||
e &&
|
||||
(e.code === "ENOENT" ||
|
||||
/not found|is not recognized/i.test(String(e.message)))
|
||||
) {
|
||||
console.warn(
|
||||
"系统未检测到 ffmpeg，可安装并配置 PATH 或设置 FFMPEG_PATH 后启用音频提取。",
|
||||
);
|
||||
return null;
|
||||
}
|
||||
|
||||
// 一些环境可能缺少特定编码器（如 libmp3lame），提示并抛出原始错误
|
||||
console.warn(`提取音频失败: ${e?.message || e}`);
|
||||
throw e;
|
||||
} finally {
|
||||
try {
|
||||
await fs.writeFile(inPath, videoBuffer);
|
||||
const args = [
|
||||
'-hide_banner',
|
||||
'-loglevel', 'error',
|
||||
'-i', inPath,
|
||||
'-vn', // 丢弃视频流
|
||||
...codecArgs,
|
||||
'-y',
|
||||
outPath,
|
||||
];
|
||||
await execFileAsync(ffmpegCmd, args, { windowsHide: true });
|
||||
const audio = await fs.readFile(outPath);
|
||||
return { buffer: audio, contentType, ext };
|
||||
} catch (e: any) {
|
||||
if (e && (e.code === 'ENOENT' || /not found|is not recognized/i.test(String(e.message)))) {
|
||||
console.warn('系统未检测到 ffmpeg，可安装并配置 PATH 或设置 FFMPEG_PATH 后启用音频提取。');
|
||||
return null;
|
||||
}
|
||||
// 一些环境可能缺少特定编码器（如 libmp3lame），提示并抛出原始错误
|
||||
console.warn(`提取音频失败: ${e?.message || e}`);
|
||||
throw e;
|
||||
} finally {
|
||||
try { await fs.unlink(inPath); } catch { }
|
||||
try {
|
||||
if (outPath) await fs.unlink(outPath);
|
||||
} catch { }
|
||||
}
|
||||
await fs.unlink(inPath);
|
||||
} catch {}
|
||||
try {
|
||||
if (outPath) await fs.unlink(outPath);
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
|
||||
+86
-54
@@ -1,13 +1,16 @@
|
||||
import { json } from '@/lib/json';
|
||||
import { getFileUrl } from '@/lib/minio';
|
||||
import { prisma } from '@/lib/prisma'; // 你的 Prisma 客户端实例
|
||||
import { NextResponse } from 'next/server';
|
||||
import { json } from "@/lib/json";
|
||||
import { getFileUrl } from "@/lib/minio";
|
||||
import { prisma } from "@/lib/prisma"; // 你的 Prisma 客户端实例
|
||||
import { NextResponse } from "next/server";
|
||||
|
||||
export async function GET(req: Request) {
|
||||
const { searchParams } = new URL(req.url);
|
||||
const q = (searchParams.get('q') || '').trim();
|
||||
const page = Math.max(1, Number(searchParams.get('page') || 1));
|
||||
const limit = Math.min(50, Math.max(1, Number(searchParams.get('limit') || 20)));
|
||||
const q = (searchParams.get("q") || "").trim();
|
||||
const page = Math.max(1, Number(searchParams.get("page") || 1));
|
||||
const limit = Math.min(
|
||||
50,
|
||||
Math.max(1, Number(searchParams.get("limit") || 20)),
|
||||
);
|
||||
const offset = (page - 1) * limit;
|
||||
|
||||
if (!q) {
|
||||
@@ -24,7 +27,7 @@ export async function GET(req: Request) {
|
||||
{
|
||||
id: string;
|
||||
awemeId: string;
|
||||
type: 'video' | 'image';
|
||||
type: "video" | "image";
|
||||
rank: number;
|
||||
snippet: string;
|
||||
}[]
|
||||
@@ -86,9 +89,11 @@ export async function GET(req: Request) {
|
||||
`;
|
||||
|
||||
// 查询总数（参数化，避免注入）
|
||||
const totalRows = await prisma.$queryRaw<{
|
||||
count: number;
|
||||
}[]>`
|
||||
const totalRows = await prisma.$queryRaw<
|
||||
{
|
||||
count: number;
|
||||
}[]
|
||||
>`
|
||||
WITH tsq AS (
|
||||
SELECT websearch_to_tsquery('zhcfg', ${q}) AS query
|
||||
)
|
||||
@@ -104,61 +109,88 @@ export async function GET(req: Request) {
|
||||
`;
|
||||
|
||||
// 分离视频和图文ID
|
||||
const videoIds = rows.filter(r => r.type === 'video').map(r => r.awemeId);
|
||||
const imagePostIds = rows.filter(r => r.type === 'image').map(r => r.awemeId);
|
||||
const videoIds = rows
|
||||
.filter((r) => r.type === "video")
|
||||
.map((r) => r.awemeId);
|
||||
const imagePostIds = rows
|
||||
.filter((r) => r.type === "image")
|
||||
.map((r) => r.awemeId);
|
||||
|
||||
// 批量查询视频元信息
|
||||
const videos = videoIds.length > 0 ? (await prisma.video.findMany({
|
||||
where: { aweme_id: { in: videoIds } },
|
||||
select: {
|
||||
aweme_id: true,
|
||||
desc: true,
|
||||
cover_url: true,
|
||||
video_url: true,
|
||||
duration_ms: true,
|
||||
author: true
|
||||
},
|
||||
})).map(v => (
|
||||
{ ...v, cover_url: getFileUrl(v.cover_url || ''),
|
||||
author: { ...v.author, avatar_url: getFileUrl(v.author.avatar_url || '') },
|
||||
video_url: getFileUrl(v.video_url || '') })
|
||||
) : [];
|
||||
const videos =
|
||||
videoIds.length > 0
|
||||
? (
|
||||
await prisma.video.findMany({
|
||||
where: { aweme_id: { in: videoIds } },
|
||||
select: {
|
||||
aweme_id: true,
|
||||
desc: true,
|
||||
cover_url: true,
|
||||
video_url: true,
|
||||
duration_ms: true,
|
||||
author: true,
|
||||
},
|
||||
})
|
||||
).map((v) => ({
|
||||
...v,
|
||||
cover_url: getFileUrl(v.cover_url || ""),
|
||||
author: {
|
||||
...v.author,
|
||||
avatar_url: getFileUrl(v.author.avatar_url || ""),
|
||||
},
|
||||
video_url: getFileUrl(v.video_url || ""),
|
||||
}))
|
||||
: [];
|
||||
|
||||
// 批量查询图文元信息
|
||||
const imagePosts = imagePostIds.length > 0 ? (await prisma.imagePost.findMany({
|
||||
where: { aweme_id: { in: imagePostIds } },
|
||||
select: {
|
||||
aweme_id: true,
|
||||
desc: true,
|
||||
author: true,
|
||||
images: {
|
||||
orderBy: { order: 'asc' },
|
||||
take: 1,
|
||||
select: {
|
||||
url: true,
|
||||
width: true,
|
||||
height: true,
|
||||
}
|
||||
}
|
||||
},
|
||||
})).map(ip => ({
|
||||
...ip,
|
||||
author: { ...ip.author, avatar_url: getFileUrl(ip.author.avatar_url || '') },
|
||||
cover_url: ip.images[0] ? getFileUrl(ip.images[0].url) : null,
|
||||
})) : [];
|
||||
const imagePosts =
|
||||
imagePostIds.length > 0
|
||||
? (
|
||||
await prisma.imagePost.findMany({
|
||||
where: { aweme_id: { in: imagePostIds } },
|
||||
select: {
|
||||
aweme_id: true,
|
||||
desc: true,
|
||||
author: true,
|
||||
images: {
|
||||
orderBy: { order: "asc" },
|
||||
take: 1,
|
||||
select: {
|
||||
url: true,
|
||||
width: true,
|
||||
height: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
).map((ip) => ({
|
||||
...ip,
|
||||
author: {
|
||||
...ip.author,
|
||||
avatar_url: getFileUrl(ip.author.avatar_url || ""),
|
||||
},
|
||||
cover_url: ip.images[0] ? getFileUrl(ip.images[0].url) : null,
|
||||
}))
|
||||
: [];
|
||||
|
||||
return json({
|
||||
results: rows.map(r => ({
|
||||
results: rows.map((r) => ({
|
||||
...r,
|
||||
video: r.type === 'video' ? videos.find(v => v.aweme_id === r.awemeId) : undefined,
|
||||
imagePost: r.type === 'image' ? imagePosts.find(ip => ip.aweme_id === r.awemeId) : undefined,
|
||||
video:
|
||||
r.type === "video"
|
||||
? videos.find((v) => v.aweme_id === r.awemeId)
|
||||
: undefined,
|
||||
imagePost:
|
||||
r.type === "image"
|
||||
? imagePosts.find((ip) => ip.aweme_id === r.awemeId)
|
||||
: undefined,
|
||||
})),
|
||||
total: totalRows?.[0]?.count ?? 0,
|
||||
page,
|
||||
limit,
|
||||
});
|
||||
} catch (err) {
|
||||
console.error('Search error:', err);
|
||||
return NextResponse.json({ error: 'Search failed' }, { status: 500 });
|
||||
console.error("Search error:", err);
|
||||
return NextResponse.json({ error: "Search failed" }, { status: 500 });
|
||||
}
|
||||
}
|
||||
|
||||
@@ -54,7 +54,7 @@ async function transcriptAudio(audio: Buffer | string) {
|
||||
response_format: zodResponseFormat(SttSchema, "stt_result"),
|
||||
});
|
||||
|
||||
const data = completion.choices?.[0]?.message
|
||||
const data = completion.choices?.[0]?.message;
|
||||
console.log("转写结果", data.content);
|
||||
|
||||
if (!data || !data.content) {
|
||||
@@ -74,10 +74,13 @@ export async function transcriptAweme(awemeId: string): Promise<SttResult> {
|
||||
if (!aweme) {
|
||||
throw new Error("Aweme not found or aweme is not a video post");
|
||||
}
|
||||
const vPath = aweme.video_url
|
||||
const vPath = aweme.video_url;
|
||||
const buffer = await downloadFile(vPath);
|
||||
|
||||
const audioDat = await extractAudio(buffer, { format: "mp3", bitrateKbps: 128 });
|
||||
const audioDat = await extractAudio(buffer, {
|
||||
format: "mp3",
|
||||
bitrateKbps: 128,
|
||||
});
|
||||
|
||||
if (!audioDat || !audioDat.buffer) {
|
||||
throw new Error("Failed to extract audio from video");
|
||||
|
||||
+12
-9
@@ -1,20 +1,23 @@
|
||||
export const runtime = "nodejs";
|
||||
import { NextRequest, NextResponse } from 'next/server';
|
||||
import { prisma } from '@/lib/prisma';
|
||||
import type { FeedItem, FeedResponse } from '@/app/types/feed';
|
||||
import { getFileUrl } from '@/lib/minio';
|
||||
import { transcriptAweme } from '.';
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { prisma } from "@/lib/prisma";
|
||||
import type { FeedItem, FeedResponse } from "@/app/types/feed";
|
||||
import { getFileUrl } from "@/lib/minio";
|
||||
import { transcriptAweme } from ".";
|
||||
|
||||
// Contract
|
||||
// Inputs: search params { before?: ISOString, limit?: number }
|
||||
// Output: { items: FeedItem[], nextCursor: ISOString | null }
|
||||
|
||||
export async function GET(req: NextRequest) {
|
||||
const { searchParams } = new URL(req.url)
|
||||
const awemeId = searchParams.get('awemeId');
|
||||
const { searchParams } = new URL(req.url);
|
||||
const awemeId = searchParams.get("awemeId");
|
||||
if (!awemeId) {
|
||||
return NextResponse.json({ error: "Missing awemeId parameter" }, { status: 400 });
|
||||
return NextResponse.json(
|
||||
{ error: "Missing awemeId parameter" },
|
||||
{ status: 400 },
|
||||
);
|
||||
}
|
||||
const script = await transcriptAweme(awemeId);
|
||||
return NextResponse.json(script);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user