feie9454/douyin-archive
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

反爬虫优化，评论图片支持

Browse Source
This commit is contained in:
feie9454
2025-10-23 13:25:51 +08:00
parent 590b26c420
commit 5a5eba19e4
25 changed files with 1082 additions and 112 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/api/comments/[awemeId]/route.ts
+175
View File
@@ -0,0 +1,175 @@
import { prisma } from "@/lib/prisma";
import { Prisma } from "@prisma/client";
import { NextRequest, NextResponse } from "next/server";
export async function GET(
request: NextRequest,
{ params }: { params: Promise<{ awemeId: string }> }
) {
const awemeId = (await params).awemeId;
const searchParams = request.nextUrl.searchParams;
const skip = parseInt(searchParams.get("skip") || "0", 10);
const take = parseInt(searchParams.get("take") || "20", 10);
const mode = (searchParams.get("mode") || "").toLowerCase();
// ranked 模式参数（均为可选，提供合理默认值）
const seed = searchParams.get("seed") || new Date().toISOString().slice(0, 10); // 默认按日期稳定
const snapshotIso = searchParams.get("snapshot");
const snapshot = snapshotIso ? new Date(snapshotIso) : new Date(); // 用于时间衰减的基准时间，确保单次会话稳定
const halfLifeHours = parseFloat(searchParams.get("halfLifeHours") || "24");
const wPop = parseFloat(searchParams.get("wPop") || "1"); // 热度权重
const wTime = parseFloat(searchParams.get("wTime") || "2"); // 时间衰减权重
const wJit = parseFloat(searchParams.get("wJit") || "10"); // 随机扰动权重（稳定随机）
try {
// 查找是视频还是图文
const [video, post] = await Promise.all([
prisma.video.findUnique({
where: { aweme_id: awemeId },
select: { aweme_id: true },
}),
prisma.imagePost.findUnique({
where: { aweme_id: awemeId },
select: { aweme_id: true },
}),
]);
if (!video && !post) {
return NextResponse.json({ error: "作品不存在" }, { status: 404 });
}
// 构建查询条件
const where = video
? { videoId: awemeId }
: { imagePostId: awemeId };
// ranked 模式：按「热度 + 时间衰减 + 稳定随机扰动」打分排序；否则使用稳定的时间排序
if (mode === "ranked") {
const total = await prisma.comment.count({ where });
// 计算稳定随机扰动：基于 (cid, seed) 的 md5 -> 取前4字节 -> 映射到 [0,1)
const jitterExpr = Prisma.sql`(
(
get_byte(decode(md5(c."cid" || ':' || ${seed}), 'hex'), 0)::double precision * 16777216.0 +
get_byte(decode(md5(c."cid" || ':' || ${seed}), 'hex'), 1)::double precision * 65536.0 +
get_byte(decode(md5(c."cid" || ':' || ${seed}), 'hex'), 2)::double precision * 256.0 +
get_byte(decode(md5(c."cid" || ':' || ${seed}), 'hex'), 3)::double precision
) / 4294967295.0
)`;
const snapshotTs = snapshot.toISOString();
// popularity: ln(1 + digg_count)
// time decay: exp(- age_hours / halfLifeHours)，使用固定 snapshot 基准保证会话内稳定
const scoreExpr = Prisma.sql`(
${wPop} * ln(1 + c."digg_count"::numeric) +
${wTime} * exp(- (extract(epoch from (${snapshotTs}::timestamptz - c."created_at")) / 3600.0) / ${halfLifeHours}) +
${wJit} * ${jitterExpr}
)`;
const whereField = (await video) ? Prisma.sql`c."videoId"` : Prisma.sql`c."imagePostId"`;
const rows: Array<{
cid: string;
text: string;
created_at: Date;
digg_count: bigint;
nickname: string;
avatar_url: string | null;
}> = await prisma.$queryRaw(
Prisma.sql`
SELECT
c."cid",
c."text",
c."created_at",
c."digg_count",
u."nickname",
u."avatar_url"
FROM "Comment" c
JOIN "CommentUser" u ON u."id" = c."userId"
WHERE ${whereField} = ${awemeId}
ORDER BY ${scoreExpr} DESC, c."created_at" DESC, c."cid" ASC
OFFSET ${skip}
LIMIT ${take}
`
);
// 批量查询每条评论的配图/贴纸
const cids = rows.map(r => r.cid);
const images = cids.length
? await prisma.commentImage.findMany({
where: { commentId: { in: cids } },
orderBy: { order: 'asc' },
select: { commentId: true, url: true, width: true, height: true, order: true },
})
: [];
const group = new Map<string, { url: string; width?: number | null; height?: number | null; order: number }[]>();
for (const img of images) {
const arr = group.get(img.commentId) || [];
arr.push({ url: img.url, width: img.width, height: img.height, order: img.order });
group.set(img.commentId, arr);
}
const formattedComments = rows.map((c) => ({
cid: c.cid,
text: c.text,
created_at: c.created_at,
digg_count: Number(c.digg_count),
user: {
nickname: c.nickname,
avatar_url: c.avatar_url || undefined,
},
images: (group.get(c.cid) || []).sort((a,b)=>a.order-b.order).map(i=>({ url: i.url, width: i.width ?? undefined, height: i.height ?? undefined })),
}));
return NextResponse.json({
comments: formattedComments,
total,
hasMore: skip + take < total,
// 回传用于稳定排序的参数，前端可在后续请求中透传
seed,
snapshot: snapshotTs,
weights: { wPop, wTime, wJit, halfLifeHours },
mode: "ranked",
});
}
// 默认模式：获取评论总数和分页数据（使用稳定且不可变的排序，避免因 digg_count 变化导致翻页重复/遗漏）
const [total, comments] = await Promise.all([
prisma.comment.count({ where }),
prisma.comment.findMany({
where,
orderBy: [
{ created_at: "desc" },
{ cid: "asc" },
],
include: { user: true, images: { orderBy: { order: 'asc' }, select: { url: true, width: true, height: true } } },
skip,
take,
}),
]);
const formattedComments = comments.map((c) => ({
cid: c.cid,
text: c.text,
created_at: c.created_at,
digg_count: Number(c.digg_count),
user: {
nickname: c.user.nickname,
avatar_url: c.user.avatar_url,
},
images: (c.images || []).map(i=>({ url: i.url, width: i.width ?? undefined, height: i.height ?? undefined })),
}));
return NextResponse.json({
comments: formattedComments,
total,
hasMore: skip + take < total,
});
} catch (error) {
console.error("获取评论失败:", error);
return NextResponse.json(
{ error: "获取评论失败" },
{ status: 500 }
);
}
}
app/api/fetcher/anti-anti-detector.ts
+17
View File
@@ -0,0 +1,17 @@
// index.js
import { chromium } from 'playwright-extra';
import stealth from 'puppeteer-extra-plugin-stealth'
chromium.use(stealth());
(async () => {
const browser = await chromium.launch({ headless: false }); // 需要可视化就 false
const context = await browser.newContext();
const page = await context.newPage();
await page.goto('https://bot.sannysoft.com/'); // 常用自测页
console.log('Title:', await page.title());
await page.screenshot({ path: 'stealth.png', fullPage: true });
setTimeout(async () => {
await browser.close();
}, 1000_000);
})();
app/api/fetcher/browser.ts
+65 -5
View File
@@ -1,20 +1,22 @@
import { chromium, type BrowserContext } from 'playwright'
export const runtime = 'nodejs'
// A simple singleton manager for a persistent Chromium context with ref-counting.
// Prevents concurrent tasks from closing the shared window prematurely.
import { type Browser, type BrowserContext } from 'playwright'
import { chromium } from 'playwright-extra' // A simple singleton manager for a persistent Chromium context with ref-counting. // Prevents concurrent tasks from closing the shared window prematurely.
import stealth from 'puppeteer-extra-plugin-stealth'
chromium.use(stealth());
let contextPromise: Promise<BrowserContext> | null = null
let context: BrowserContext | null = null
let refCount = 0
let idleCloseTimer: NodeJS.Timeout | null = null
const USER_DATA_DIR = 'chrome-profile/douyin'
const USER_DATA_DIR = process.env.USER_DATA_DIR ?? 'chrome-profile/douyin'
async function launchContext(): Promise<BrowserContext> {
const ctx = await chromium.launchPersistentContext(
USER_DATA_DIR,
{
headless: Boolean(process.env.CHROMIUM_HEADLESS ?? 'false'),
headless: process.env.CHROMIUM_HEADLESS === 'true',
viewport: {
width: Number(process.env.CHROMIUM_VIEWPORT_WIDTH ?? 1280),
height: Number(process.env.CHROMIUM_VIEWPORT_HEIGHT ?? 1080)
@@ -77,3 +79,61 @@ export async function releaseBrowserContext(options?: { idleMillis?: number }):
}
}, idleMillis)
}
// --- Isolated context support for per-scrape independence ---
const isolatedMap = new WeakMap<BrowserContext, Browser>()
async function getSharedStorageState(): Promise<any | undefined> {
try {
const shared = await acquireBrowserContext()
const state = await shared.storageState()
// Do not force-close immediately; keep ref-counting behavior
await releaseBrowserContext()
return state
} catch {
// If shared context not available, proceed without storageState
return undefined
}
}
/**
* Create a brand-new, independent Browser + Context for a single scrape.
* Attempts to import cookies/localStorage from the shared persistent profile
* so you remain logged-in, but isolates network events, cache and listeners.
*/
export async function acquireIsolatedContext(): Promise<BrowserContext> {
const storageState = await getSharedStorageState()
const browser = await chromium.launch({
headless: process.env.CHROMIUM_HEADLESS === 'true'
})
const ctx = await browser.newContext({
storageState,
viewport: {
width: Number(process.env.CHROMIUM_VIEWPORT_WIDTH ?? 1280),
height: Number(process.env.CHROMIUM_VIEWPORT_HEIGHT ?? 1080)
}
})
isolatedMap.set(ctx, browser)
ctx.on('close', () => {
const b = isolatedMap.get(ctx)
if (b) {
b.close().catch(() => {})
isolatedMap.delete(ctx)
}
})
return ctx
}
export async function releaseIsolatedContext(ctx: BrowserContext | null | undefined): Promise<void> {
if (!ctx) return
try {
await ctx.close()
} finally {
const b = isolatedMap.get(ctx)
if (b) {
await b.close().catch(() => {})
isolatedMap.delete(ctx)
}
}
}
app/api/fetcher/index.ts
+152 -57
View File
@@ -1,19 +1,84 @@
export const runtime = 'nodejs'
// src/scrapeDouyin.ts
import { BrowserContext, Page, chromium, type Response } from 'playwright';
import { BrowserContext, Page, type Response } from 'playwright';
import { chromium } from 'playwright-extra';
import { prisma } from '@/lib/prisma';
import { uploadFile, generateUniqueFileName } from '@/lib/minio';
import { createCamelCompatibleProxy } from '@/app/api/fetcher/utils';
import { waitForFirstResponse, waitForResponseWithTimeout, safeJson, downloadBinary } from '@/app/api/fetcher/network';
import { waitForFirstResponse, waitForResponseWithTimeout, safeJson, downloadBinary, collectResponsesWithinTime } from '@/app/api/fetcher/network';
import { pickBestPlayAddr, extractFirstFrame } from '@/app/api/fetcher/media';
import { handleImagePost } from '@/app/api/fetcher/uploader';
import { saveToDB, saveImagePostToDB } from '@/app/api/fetcher/persist';
import chalk from 'chalk';
import { acquireBrowserContext, releaseBrowserContext } from '@/app/api/fetcher/browser';
import { acquireIsolatedContext, releaseIsolatedContext } from '@/app/api/fetcher/browser';
const DETAIL_PATH = '/aweme/v1/web/aweme/detail/';
const COMMENT_PATH = '/aweme/v1/web/comment/list/';
const POST_PATH = '/aweme/v1/web/aweme/post/'
/**
* 滚动页面并收集评论
* @param context 浏览器上下文
* @param page 页面对象
* @param durationMs 持续时间（毫秒）
* @returns 收集到的所有评论响应
*/
async function scrollAndCollectComments(
context: BrowserContext,
page: Page,
durationMs: number = 10_000
): Promise<Response[]> {
console.log(chalk.blue(`📜 开始滚动页面收集评论（持续 ${durationMs / 1000} 秒）...`));
// 启动评论响应收集器
const commentResponsesPromise = collectResponsesWithinTime(
context,
(r: Response) => r.url().includes(COMMENT_PATH) && r.status() === 200 && r.request().frame()?.page() === page,
durationMs
);
// 在指定时间内持续滚动页面
const startTime = Date.now();
const scrollInterval = 500;
let scrollCount = 0;
const selector = "div[data-e2e='comment-list']";
// 1) 等元素出现并可见
await page.waitForSelector(selector, { state: 'visible', timeout: 5000 });
// 2) 确保滚动到可见区域
const list = page.locator(selector);
await list.scrollIntoViewIfNeeded();
// 3) 执行 hover（推荐用 locator 的 hover）
list.hover({ timeout: 5000 }).catch(() => { });
while (Date.now() - startTime < durationMs - 500) { // 留 500ms 缓冲
try {
list.hover({ timeout: 2000 }).catch(() => { });
// 使用 Playwright 的 mouse.wheel 方法滚动
// 每次滚动一大段距离
// await list.hover();
const scrollAmount = 1500;
await page.mouse.wheel(0, scrollAmount);
scrollCount++;
console.log(chalk.gray(` ↓ 第 ${scrollCount} 次滚动`));
// 等待一段时间，让评论加载
await page.waitForTimeout(scrollInterval);
} catch (e) {
console.warn(chalk.yellow(` ⚠ 滚动时出现警告: ${(e as Error)?.message}`));
}
}
// 等待收集器完成
const commentResponses = await commentResponsesPromise;
console.log(chalk.green(`✓ 评论收集完成，共收集到 ${commentResponses.length} 个评论响应`));
return commentResponses;
}
async function readPostMem(context: BrowserContext, page: Page) {
const md = await page.evaluate(() => {
// @ts-ignore
@@ -47,8 +112,7 @@ export class ScrapeError extends Error {
export async function scrapeDouyin(url: string) {
console.log(chalk.blue('🚀 启动共享 Chromium 浏览器...'));
const context = await acquireBrowserContext();
let context: BrowserContext | null = await acquireIsolatedContext();
const page = await context.newPage();
console.log(chalk.cyan(`📄 正在访问: ${chalk.underline(url)}`));
@@ -82,16 +146,11 @@ export async function scrapeDouyin(url: string) {
try {
// 先注册“先到先得”的监听，再导航，避免漏包
const firstTypePromise = waitForFirstResponse(context, [
{ key: 'detail', test: (r: Response) => r.url().includes(DETAIL_PATH) && r.status() === 200 },
{ key: 'post', test: (r: Response) => r.url().includes(POST_PATH) && r.status() === 200 },
], 40_000);
{ key: 'detail', test: (r: Response) => r.url().includes(DETAIL_PATH) && r.status() === 200 && r.request().frame()?.page() === page },
{ key: 'post', test: (r: Response) => r.url().includes(POST_PATH) && r.status() === 200 && r.request().frame()?.page() === page },
], 10_000);
// 评论只做短时“有就用、没有不等”的监听
const commentPromise = waitForResponseWithTimeout(
context, (r: Response) => r.url().includes(COMMENT_PATH) && r.status() === 200, 40_000
).catch(() => null);
await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 60_000 });
await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 20_000 });
// 查找页面中是否存在 "视频不存在" 的提示
const isNotFound = await page.locator('text=视频不存在').count().then(count => count > 0).catch(() => false);
@@ -100,52 +159,88 @@ export async function scrapeDouyin(url: string) {
throw new ScrapeError('视频不存在或已被删除', 404, 'VIDEO_NOT_FOUND');
}
try {
// 优先尝试从内存读取图文数据
let { aweme, comments } = await readPostMem(context, page);
console.log(chalk.green('✓ 从内存读取图文数据成功'));
const uploads = await handleImagePost(context, aweme);
if (!comments) {
console.warn(chalk.yellow('⚠ 从内存读取评论数据失败，尝试从网络请求获取评论数据'));
const commentRes = await commentPromise;
comments = commentRes && await safeJson<DouyinCommentResponse>(commentRes);
if (!comments) {
console.warn(chalk.yellow('⚠ 无法从内存读取评论数据，且网络请求也未返回评论数据'));
comments = { comments: [], total: 0, status_code: 0 };
} else {
console.log(chalk.green('✓ 从网络请求获取评论数据成功'));
}
} else {
console.log(chalk.green('✓ 从内存读取评论数据成功'));
}
const saved = await saveImagePostToDB(context, aweme, comments, uploads); // 传递完整 JSON
console.log(chalk.green.bold('✓ 图文作品保存成功'));
return { type: "image", ...saved };
} catch {
}
const commentRes = await commentPromise;
// 等待作品类型判定
const firstType = await firstTypePromise;
if (!firstType) {
// 尝试从内存读取图文数据（如果是图文作品）
let memoryData: { aweme: any; comments: DouyinCommentResponse | null } | null = null;
try {
memoryData = await readPostMem(context, page);
console.log(chalk.green('✓ 从内存读取图文数据成功'));
} catch {
// 内存读取失败，稍后通过网络获取
}
if (!firstType && !memoryData) {
console.error(chalk.red('✗ 既无法从内存读取数据,也无法从网络获得数据'));
throw new ScrapeError('无法获取作品数据,可能是网络问题或作品已下架', 404, 'NO_DATA');
}
console.log(chalk.cyan(`📡 检测到作品类型: ${chalk.bold(firstType.key === 'post' ? '图文' : '视频')}`));
console.log(chalk.cyan(`📡 检测到作品类型: ${chalk.bold(firstType?.key === 'post' || memoryData ? '图文' : '视频')}`));
let comments = commentRes && await safeJson<DouyinCommentResponse>(commentRes);
if (!comments) {
console.warn(chalk.yellow('⚠ 无法从内存读取评论数据，且网络请求也未返回评论数据'));
comments = { comments: [], total: 0, status_code: 0 };
let allComments: DouyinComment[] = [];
try {
// 开始滚动并收集评论
const commentResponses = await scrollAndCollectComments(context, page);
// 解析所有收集到的评论响应
for (const commentRes of commentResponses) {
try {
const commentData = await safeJson<DouyinCommentResponse>(commentRes);
if (commentData?.comments?.length) {
allComments.push(...commentData.comments);
}
} catch (e) {
console.warn(chalk.yellow(`⚠ 解析评论响应失败: ${(e as Error)?.message}`));
}
}
} catch (error) {
console.warn(chalk.yellow(`⚠ 评论收集失败: ${(error as Error)?.message}`));
}
// 去重评论（根据 cid）
const uniqueComments = Array.from(
new Map(allComments.map(c => [c.cid, c])).values()
);
console.log(chalk.green(`✓ 共收集到 ${uniqueComments.length} 条独立评论（去重前: ${allComments.length}）`));
// 如果从内存读取到了评论，合并进来作为兜底
let comments: DouyinCommentResponse;
if (memoryData?.comments?.comments?.length) {
console.log(chalk.blue(`📝 合并内存中的 ${memoryData.comments.comments.length} 条评论`));
const memComments = memoryData.comments.comments;
const mergedMap = new Map(uniqueComments.map(c => [c.cid, c]));
for (const c of memComments) {
if (!mergedMap.has(c.cid)) {
mergedMap.set(c.cid, c);
}
}
comments = {
comments: Array.from(mergedMap.values()),
total: mergedMap.size,
status_code: 0
};
console.log(chalk.green(`✓ 合并后共 ${comments.comments.length} 条评论`));
} else {
comments = {
comments: uniqueComments,
total: uniqueComments.length,
status_code: 0
};
}
// 分支：视频 or 图文（两者只会有一个命中，先到先得）
if (firstType.key === 'post') {
// 图文作品
// 优先处理内存数据（图文）
if (memoryData) {
const aweme = memoryData.aweme;
const uploads = await handleImagePost(context, aweme);
const saved = await saveImagePostToDB(context, aweme, comments, uploads); // 传递完整 JSON
console.log(chalk.green.bold('✓ 图文作品保存成功'));
return { type: "image", ...saved };
} else if (firstType?.key === 'post') {
// 图文作品（网络）
const postJson = await safeJson<DouyinPostListResponse>(firstType.response);
if (!postJson?.aweme_list?.length) throw new ScrapeError('图文作品响应为空', 404, 'EMPTY_POST_RESPONSE');
@@ -161,7 +256,7 @@ export async function scrapeDouyin(url: string) {
const saved = await saveImagePostToDB(context, aweme, comments, uploads, postJson); // 传递完整 JSON
console.log(chalk.green.bold('✓ 图文作品保存成功'));
return { type: "image", ...saved };
} else if (firstType.key === 'detail') {
} else if (firstType?.key === 'detail') {
// 视频作品
const detail = (await safeJson<DouyinVideoDetailResponse>(firstType.response))!;
@@ -216,11 +311,11 @@ export async function scrapeDouyin(url: string) {
if (error instanceof ScrapeError) {
throw error;
}
// 处理其他类型的错误
const errMsg = (error as Error)?.message || String(error);
console.error(chalk.red(`✗ 爬取失败: ${errMsg}`));
// 根据错误类型返回不同的状态码
if (errMsg.includes('timeout') || errMsg.includes('超时')) {
throw new ScrapeError('请求超时,请稍后重试', 408, 'TIMEOUT');
@@ -231,14 +326,14 @@ export async function scrapeDouyin(url: string) {
if (errMsg.includes('net::')) {
throw new ScrapeError('网络连接失败', 503, 'NETWORK_ERROR');
}
// 默认服务器错误
throw new ScrapeError(errMsg || '爬取过程中发生未知错误', 500, 'UNKNOWN_ERROR');
} finally {
console.log(chalk.gray('🧹 清理资源...'));
try { await page.close({ runBeforeUnload: true }); } catch {}
// 仅释放共享上下文的引用，不直接关闭窗口
await releaseBrowserContext();
try { await page.close({ runBeforeUnload: true }); } catch { }
// 关闭本次任务的隔离上下文与浏览器
await releaseIsolatedContext(context);
await prisma.$disconnect();
console.log(chalk.gray('✓ 资源清理完成'));
}
app/api/fetcher/media.ts
+2
View File
@@ -1,3 +1,5 @@
export const runtime = 'nodejs'
import { promises as fs } from 'fs';
import os from 'os';
import path from 'path';
app/api/fetcher/network.ts
+47 -3
View File
@@ -1,3 +1,5 @@
export const runtime = 'nodejs'
import type { BrowserContext, Response } from 'playwright';
export async function safeJson<T>(res: Response): Promise<T | null> {
@@ -20,12 +22,12 @@ export async function safeJson<T>(res: Response): Promise<T | null> {
*/
export async function downloadBinary(
context: BrowserContext,
url: string
url: string,
): Promise<{ buffer: Buffer; contentType: string; ext: string }> {
console.log('下载：', url);
const headers = {
referer: url,
referer: 'https://www.douyin.com/',
} as Record<string, string>;
const res = await context.request.get(url, {
@@ -93,7 +95,49 @@ export function waitForFirstResponse(
}
/**
* 等待符合条件的单个 Response，带短超时；用于评论等“可有可无”的数据。
* 在指定时间内持续收集所有符合条件的 Response
* 用于评论等需要滚动加载的数据
*/
export function collectResponsesWithinTime(
context: BrowserContext,
predicate: (r: Response) => boolean,
durationMs: number
): Promise<Response[]> {
return new Promise((resolve) => {
const collected: Response[] = [];
const seenUrls = new Set<string>();
let timer: NodeJS.Timeout | undefined;
const handler = (res: Response) => {
try {
if (predicate(res)) {
// 使用 URL 去重，避免重复收集同一个请求
const url = res.url();
if (!seenUrls.has(url)) {
seenUrls.add(url);
collected.push(res);
}
}
} catch {
// ignore predicate errors
}
};
const cleanup = () => {
context.off('response', handler);
if (timer) clearTimeout(timer);
};
context.on('response', handler);
timer = setTimeout(() => {
cleanup();
resolve(collected);
}, durationMs);
});
}
/**
* 等待符合条件的单个 Response，带短超时；用于评论等"可有可无"的数据。
*/
export function waitForResponseWithTimeout(
context: BrowserContext,
app/api/fetcher/persist.ts
+91 -7
View File
@@ -1,6 +1,6 @@
import type { BrowserContext } from 'playwright';
import { prisma } from '@/lib/prisma';
import { uploadAvatarFromUrl } from './uploader';
import { uploadAvatarFromUrl, uploadImageFromUrl } from './uploader';
import { firstUrl } from './utils';
export async function saveToDB(
@@ -111,7 +111,7 @@ export async function saveToDB(
},
});
await prisma.comment.upsert({
const savedComment = await prisma.comment.upsert({
where: { cid: c.cid },
create: {
cid: c.cid,
@@ -129,6 +129,49 @@ export async function saveToDB(
userId: cu.id,
},
});
// 处理评论贴纸/配图上传与入库
try {
const sources: { url?: string | null; width?: number; height?: number }[] = [];
// 贴纸（当作第一张）
const stickerUrl = firstUrl(c.sticker?.animate_url?.url_list);
if (stickerUrl) {
sources.push({ url: stickerUrl, width: c.sticker?.animate_url?.width, height: c.sticker?.animate_url?.height });
}
// 配图列表
const imgs = c.image_list || [];
for (const it of imgs) {
const u = firstUrl(it?.origin_url?.url_list);
if (u) sources.push({ url: u, width: it?.origin_url?.width as any, height: it?.origin_url?.height as any });
}
for (let i = 0; i < sources.length; i++) {
const s = sources[i];
const uploaded = await uploadImageFromUrl(
context,
s.url ?? undefined,
`comments/${c.cid}/${i}`,
);
if (!uploaded) continue;
await prisma.commentImage.upsert({
where: { commentId_order: { commentId: savedComment.cid, order: i } },
create: {
commentId: savedComment.cid,
order: i,
url: uploaded,
width: typeof s.width === 'number' ? s.width : null,
height: typeof s.height === 'number' ? s.height : null,
},
update: {
url: uploaded,
width: typeof s.width === 'number' ? s.width : null,
height: typeof s.height === 'number' ? s.height : null,
},
});
}
} catch (e) {
console.warn('[comment-images] 保存失败：', (e as Error)?.message || e);
}
}
return { aweme_id: video.aweme_id, author_sec_uid: author.sec_uid, comment_count: comments.length };
@@ -138,7 +181,7 @@ export async function saveImagePostToDB(
context: BrowserContext,
aweme: DouyinImageAweme,
commentResp: DouyinCommentResponse,
uploads: { images: { url: string; width?: number; height?: number; video?: string; duration?: number }[]; musicUrl?: string },
uploads: { images: { url: string; width?: number; height?: number, video?: string }[]; musicUrl?: string },
rawJson?: any
) {
if (!aweme?.author?.sec_uid) throw new Error('作者 sec_uid 缺失');
@@ -205,7 +248,7 @@ export async function saveImagePostToDB(
// Upsert ImageFiles（按顺序）
for (let i = 0; i < uploads.images.length; i++) {
const { url, width, height, video, duration } = uploads.images[i];
const { url, width, height, video } = uploads.images[i];
await prisma.imageFile.upsert({
where: { postId_order: { postId: imagePost.aweme_id, order: i } },
create: {
@@ -215,14 +258,12 @@ export async function saveImagePostToDB(
width: typeof width === 'number' ? width : null,
height: typeof height === 'number' ? height : null,
animated: video || null,
duration: typeof duration === 'number' ? duration : null,
},
update: {
url,
width: typeof width === 'number' ? width : null,
height: typeof height === 'number' ? height : null,
animated: video || null,
duration: typeof duration === 'number' ? duration : null,
},
});
}
@@ -251,7 +292,7 @@ export async function saveImagePostToDB(
},
});
await prisma.comment.upsert({
const savedComment = await prisma.comment.upsert({
where: { cid: c.cid },
create: {
cid: c.cid,
@@ -269,6 +310,49 @@ export async function saveImagePostToDB(
userId: cu.id,
},
});
// 处理评论贴纸/配图上传与入库
try {
const sources: { url?: string | null; width?: number; height?: number }[] = [];
// 贴纸（当作第一张）
const stickerUrl = firstUrl(c.sticker?.animate_url?.url_list);
if (stickerUrl) {
sources.push({ url: stickerUrl, width: c.sticker?.animate_url?.width, height: c.sticker?.animate_url?.height });
}
// 配图列表
const imgs = c.image_list || [];
for (const it of imgs) {
const u = firstUrl(it?.origin_url?.url_list);
if (u) sources.push({ url: u, width: it?.origin_url?.width as any, height: it?.origin_url?.height as any });
}
for (let i = 0; i < sources.length; i++) {
const s = sources[i];
const uploaded = await uploadImageFromUrl(
context,
s.url ?? undefined,
`comments/${c.cid}/${i}`,
);
if (!uploaded) continue;
await prisma.commentImage.upsert({
where: { commentId_order: { commentId: savedComment.cid, order: i } },
create: {
commentId: savedComment.cid,
order: i,
url: uploaded,
width: typeof s.width === 'number' ? s.width : null,
height: typeof s.height === 'number' ? s.height : null,
},
update: {
url: uploaded,
width: typeof s.width === 'number' ? s.width : null,
height: typeof s.height === 'number' ? s.height : null,
},
});
}
} catch (e) {
console.warn('[comment-images] 保存失败：', (e as Error)?.message || e);
}
}
return { aweme_id: imagePost.aweme_id, author_sec_uid: author.sec_uid, image_count: uploads.images.length, comment_count: comments.length };
app/api/fetcher/route.ts
+2
View File
@@ -1,3 +1,5 @@
export const runtime = 'nodejs'
import { NextRequest, NextResponse } from 'next/server'
import { prisma } from '@/lib/prisma'
import { scrapeDouyin, ScrapeError } from '.';
app/api/fetcher/types.d.ts
Vendored
+16
View File
@@ -12,6 +12,22 @@ interface DouyinComment {
digg_count: number; // 点赞数
create_time: number; // 创建时间（时间戳）
user: DouyinUser; // 评论用户
sticker?: {
id: number;
animate_url: {
width: number;
height: number;
url_list: string[]
}
},
image_list?: {
origin_url:{
width: number;
height: number;
url_list: string[]
}
}[]
}
/** 用户信息（精简版） */
app/api/fetcher/uploader.ts
+25 -1
View File
@@ -1,3 +1,5 @@
export const runtime = 'nodejs'
import type { BrowserContext } from 'playwright';
import { uploadFile, generateUniqueFileName } from '@/lib/minio';
import { downloadBinary } from './network';
@@ -10,7 +12,7 @@ import { getVideoDuration } from './media';
export async function uploadAvatarFromUrl(
context: BrowserContext,
srcUrl?: string | null,
nameHint?: string
nameHint?: string,
): Promise<string | undefined> {
if (!srcUrl) return undefined;
try {
@@ -26,6 +28,28 @@ export async function uploadAvatarFromUrl(
}
}
/**
* 下载任意图片并上传到 MinIO，返回外链；失败时回退为原始链接。
*/
export async function uploadImageFromUrl(
context: BrowserContext,
srcUrl?: string | null,
nameHint?: string,
): Promise<string | undefined> {
if (!srcUrl) return undefined;
try {
const { buffer, contentType, ext } = await downloadBinary(context, srcUrl);
const safeExt = ext || 'jpg';
const baseName = nameHint ? `${nameHint}.${safeExt}` : `image.${safeExt}`;
const fileName = generateUniqueFileName(baseName, 'douyin/comment-images');
const uploaded = await uploadFile(buffer, fileName, { 'Content-Type': contentType });
return uploaded;
} catch (e) {
console.warn('[image] 上传失败，使用原始链接：', (e as Error)?.message || e);
return srcUrl || undefined;
}
}
/** 下载图文作品的图片和音乐并上传到 MinIO */
export async function handleImagePost(
context: BrowserContext,
app/api/fetcher/utils.ts
+2
View File
@@ -1,3 +1,5 @@
export const runtime = 'nodejs'
export function toCamelCaseKey(key: string): string {
return key.replace(/_([a-zA-Z])/g, (_, c: string) => c.toUpperCase());
}