反爬虫优化,评论图片支持

This commit is contained in:
2025-10-23 13:25:51 +08:00
parent 590b26c420
commit 5a5eba19e4
25 changed files with 1082 additions and 112 deletions
+175
View File
@@ -0,0 +1,175 @@
import { prisma } from "@/lib/prisma";
import { Prisma } from "@prisma/client";
import { NextRequest, NextResponse } from "next/server";
export async function GET(
request: NextRequest,
{ params }: { params: Promise<{ awemeId: string }> }
) {
const awemeId = (await params).awemeId;
const searchParams = request.nextUrl.searchParams;
const skip = parseInt(searchParams.get("skip") || "0", 10);
const take = parseInt(searchParams.get("take") || "20", 10);
const mode = (searchParams.get("mode") || "").toLowerCase();
// ranked 模式参数(均为可选,提供合理默认值)
const seed = searchParams.get("seed") || new Date().toISOString().slice(0, 10); // 默认按日期稳定
const snapshotIso = searchParams.get("snapshot");
const snapshot = snapshotIso ? new Date(snapshotIso) : new Date(); // 用于时间衰减的基准时间,确保单次会话稳定
const halfLifeHours = parseFloat(searchParams.get("halfLifeHours") || "24");
const wPop = parseFloat(searchParams.get("wPop") || "1"); // 热度权重
const wTime = parseFloat(searchParams.get("wTime") || "2"); // 时间衰减权重
const wJit = parseFloat(searchParams.get("wJit") || "10"); // 随机扰动权重(稳定随机)
try {
// 查找是视频还是图文
const [video, post] = await Promise.all([
prisma.video.findUnique({
where: { aweme_id: awemeId },
select: { aweme_id: true },
}),
prisma.imagePost.findUnique({
where: { aweme_id: awemeId },
select: { aweme_id: true },
}),
]);
if (!video && !post) {
return NextResponse.json({ error: "作品不存在" }, { status: 404 });
}
// 构建查询条件
const where = video
? { videoId: awemeId }
: { imagePostId: awemeId };
// ranked 模式:按「热度 + 时间衰减 + 稳定随机扰动」打分排序;否则使用稳定的时间排序
if (mode === "ranked") {
const total = await prisma.comment.count({ where });
// 计算稳定随机扰动:基于 (cid, seed) 的 md5 -> 取前4字节 -> 映射到 [0,1)
const jitterExpr = Prisma.sql`(
(
get_byte(decode(md5(c."cid" || ':' || ${seed}), 'hex'), 0)::double precision * 16777216.0 +
get_byte(decode(md5(c."cid" || ':' || ${seed}), 'hex'), 1)::double precision * 65536.0 +
get_byte(decode(md5(c."cid" || ':' || ${seed}), 'hex'), 2)::double precision * 256.0 +
get_byte(decode(md5(c."cid" || ':' || ${seed}), 'hex'), 3)::double precision
) / 4294967295.0
)`;
const snapshotTs = snapshot.toISOString();
// popularity: ln(1 + digg_count)
// time decay: exp(- age_hours / halfLifeHours),使用固定 snapshot 基准保证会话内稳定
const scoreExpr = Prisma.sql`(
${wPop} * ln(1 + c."digg_count"::numeric) +
${wTime} * exp(- (extract(epoch from (${snapshotTs}::timestamptz - c."created_at")) / 3600.0) / ${halfLifeHours}) +
${wJit} * ${jitterExpr}
)`;
const whereField = (await video) ? Prisma.sql`c."videoId"` : Prisma.sql`c."imagePostId"`;
const rows: Array<{
cid: string;
text: string;
created_at: Date;
digg_count: bigint;
nickname: string;
avatar_url: string | null;
}> = await prisma.$queryRaw(
Prisma.sql`
SELECT
c."cid",
c."text",
c."created_at",
c."digg_count",
u."nickname",
u."avatar_url"
FROM "Comment" c
JOIN "CommentUser" u ON u."id" = c."userId"
WHERE ${whereField} = ${awemeId}
ORDER BY ${scoreExpr} DESC, c."created_at" DESC, c."cid" ASC
OFFSET ${skip}
LIMIT ${take}
`
);
// 批量查询每条评论的配图/贴纸
const cids = rows.map(r => r.cid);
const images = cids.length
? await prisma.commentImage.findMany({
where: { commentId: { in: cids } },
orderBy: { order: 'asc' },
select: { commentId: true, url: true, width: true, height: true, order: true },
})
: [];
const group = new Map<string, { url: string; width?: number | null; height?: number | null; order: number }[]>();
for (const img of images) {
const arr = group.get(img.commentId) || [];
arr.push({ url: img.url, width: img.width, height: img.height, order: img.order });
group.set(img.commentId, arr);
}
const formattedComments = rows.map((c) => ({
cid: c.cid,
text: c.text,
created_at: c.created_at,
digg_count: Number(c.digg_count),
user: {
nickname: c.nickname,
avatar_url: c.avatar_url || undefined,
},
images: (group.get(c.cid) || []).sort((a,b)=>a.order-b.order).map(i=>({ url: i.url, width: i.width ?? undefined, height: i.height ?? undefined })),
}));
return NextResponse.json({
comments: formattedComments,
total,
hasMore: skip + take < total,
// 回传用于稳定排序的参数,前端可在后续请求中透传
seed,
snapshot: snapshotTs,
weights: { wPop, wTime, wJit, halfLifeHours },
mode: "ranked",
});
}
// 默认模式:获取评论总数和分页数据(使用稳定且不可变的排序,避免因 digg_count 变化导致翻页重复/遗漏)
const [total, comments] = await Promise.all([
prisma.comment.count({ where }),
prisma.comment.findMany({
where,
orderBy: [
{ created_at: "desc" },
{ cid: "asc" },
],
include: { user: true, images: { orderBy: { order: 'asc' }, select: { url: true, width: true, height: true } } },
skip,
take,
}),
]);
const formattedComments = comments.map((c) => ({
cid: c.cid,
text: c.text,
created_at: c.created_at,
digg_count: Number(c.digg_count),
user: {
nickname: c.user.nickname,
avatar_url: c.user.avatar_url,
},
images: (c.images || []).map(i=>({ url: i.url, width: i.width ?? undefined, height: i.height ?? undefined })),
}));
return NextResponse.json({
comments: formattedComments,
total,
hasMore: skip + take < total,
});
} catch (error) {
console.error("获取评论失败:", error);
return NextResponse.json(
{ error: "获取评论失败" },
{ status: 500 }
);
}
}
+17
View File
@@ -0,0 +1,17 @@
// index.js
import { chromium } from 'playwright-extra';
import stealth from 'puppeteer-extra-plugin-stealth'
chromium.use(stealth());
(async () => {
const browser = await chromium.launch({ headless: false }); // 需要可视化就 false
const context = await browser.newContext();
const page = await context.newPage();
await page.goto('https://bot.sannysoft.com/'); // 常用自测页
console.log('Title:', await page.title());
await page.screenshot({ path: 'stealth.png', fullPage: true });
setTimeout(async () => {
await browser.close();
}, 1000_000);
})();
+65 -5
View File
@@ -1,20 +1,22 @@
import { chromium, type BrowserContext } from 'playwright'
export const runtime = 'nodejs'
// A simple singleton manager for a persistent Chromium context with ref-counting.
// Prevents concurrent tasks from closing the shared window prematurely.
import { type Browser, type BrowserContext } from 'playwright'
import { chromium } from 'playwright-extra' // A simple singleton manager for a persistent Chromium context with ref-counting. // Prevents concurrent tasks from closing the shared window prematurely.
import stealth from 'puppeteer-extra-plugin-stealth'
chromium.use(stealth());
let contextPromise: Promise<BrowserContext> | null = null
let context: BrowserContext | null = null
let refCount = 0
let idleCloseTimer: NodeJS.Timeout | null = null
const USER_DATA_DIR = 'chrome-profile/douyin'
const USER_DATA_DIR = process.env.USER_DATA_DIR ?? 'chrome-profile/douyin'
async function launchContext(): Promise<BrowserContext> {
const ctx = await chromium.launchPersistentContext(
USER_DATA_DIR,
{
headless: Boolean(process.env.CHROMIUM_HEADLESS ?? 'false'),
headless: process.env.CHROMIUM_HEADLESS === 'true',
viewport: {
width: Number(process.env.CHROMIUM_VIEWPORT_WIDTH ?? 1280),
height: Number(process.env.CHROMIUM_VIEWPORT_HEIGHT ?? 1080)
@@ -77,3 +79,61 @@ export async function releaseBrowserContext(options?: { idleMillis?: number }):
}
}, idleMillis)
}
// --- Isolated context support for per-scrape independence ---
const isolatedMap = new WeakMap<BrowserContext, Browser>()
async function getSharedStorageState(): Promise<any | undefined> {
try {
const shared = await acquireBrowserContext()
const state = await shared.storageState()
// Do not force-close immediately; keep ref-counting behavior
await releaseBrowserContext()
return state
} catch {
// If shared context not available, proceed without storageState
return undefined
}
}
/**
* Create a brand-new, independent Browser + Context for a single scrape.
* Attempts to import cookies/localStorage from the shared persistent profile
* so you remain logged-in, but isolates network events, cache and listeners.
*/
export async function acquireIsolatedContext(): Promise<BrowserContext> {
const storageState = await getSharedStorageState()
const browser = await chromium.launch({
headless: process.env.CHROMIUM_HEADLESS === 'true'
})
const ctx = await browser.newContext({
storageState,
viewport: {
width: Number(process.env.CHROMIUM_VIEWPORT_WIDTH ?? 1280),
height: Number(process.env.CHROMIUM_VIEWPORT_HEIGHT ?? 1080)
}
})
isolatedMap.set(ctx, browser)
ctx.on('close', () => {
const b = isolatedMap.get(ctx)
if (b) {
b.close().catch(() => {})
isolatedMap.delete(ctx)
}
})
return ctx
}
export async function releaseIsolatedContext(ctx: BrowserContext | null | undefined): Promise<void> {
if (!ctx) return
try {
await ctx.close()
} finally {
const b = isolatedMap.get(ctx)
if (b) {
await b.close().catch(() => {})
isolatedMap.delete(ctx)
}
}
}
+152 -57
View File
@@ -1,19 +1,84 @@
export const runtime = 'nodejs'
// src/scrapeDouyin.ts
import { BrowserContext, Page, chromium, type Response } from 'playwright';
import { BrowserContext, Page, type Response } from 'playwright';
import { chromium } from 'playwright-extra';
import { prisma } from '@/lib/prisma';
import { uploadFile, generateUniqueFileName } from '@/lib/minio';
import { createCamelCompatibleProxy } from '@/app/api/fetcher/utils';
import { waitForFirstResponse, waitForResponseWithTimeout, safeJson, downloadBinary } from '@/app/api/fetcher/network';
import { waitForFirstResponse, waitForResponseWithTimeout, safeJson, downloadBinary, collectResponsesWithinTime } from '@/app/api/fetcher/network';
import { pickBestPlayAddr, extractFirstFrame } from '@/app/api/fetcher/media';
import { handleImagePost } from '@/app/api/fetcher/uploader';
import { saveToDB, saveImagePostToDB } from '@/app/api/fetcher/persist';
import chalk from 'chalk';
import { acquireBrowserContext, releaseBrowserContext } from '@/app/api/fetcher/browser';
import { acquireIsolatedContext, releaseIsolatedContext } from '@/app/api/fetcher/browser';
const DETAIL_PATH = '/aweme/v1/web/aweme/detail/';
const COMMENT_PATH = '/aweme/v1/web/comment/list/';
const POST_PATH = '/aweme/v1/web/aweme/post/'
/**
* 滚动页面并收集评论
* @param context 浏览器上下文
* @param page 页面对象
* @param durationMs 持续时间(毫秒)
* @returns 收集到的所有评论响应
*/
async function scrollAndCollectComments(
context: BrowserContext,
page: Page,
durationMs: number = 10_000
): Promise<Response[]> {
console.log(chalk.blue(`📜 开始滚动页面收集评论(持续 ${durationMs / 1000} 秒)...`));
// 启动评论响应收集器
const commentResponsesPromise = collectResponsesWithinTime(
context,
(r: Response) => r.url().includes(COMMENT_PATH) && r.status() === 200 && r.request().frame()?.page() === page,
durationMs
);
// 在指定时间内持续滚动页面
const startTime = Date.now();
const scrollInterval = 500;
let scrollCount = 0;
const selector = "div[data-e2e='comment-list']";
// 1) 等元素出现并可见
await page.waitForSelector(selector, { state: 'visible', timeout: 5000 });
// 2) 确保滚动到可见区域
const list = page.locator(selector);
await list.scrollIntoViewIfNeeded();
// 3) 执行 hover(推荐用 locator 的 hover)
list.hover({ timeout: 5000 }).catch(() => { });
while (Date.now() - startTime < durationMs - 500) { // 留 500ms 缓冲
try {
list.hover({ timeout: 2000 }).catch(() => { });
// 使用 Playwright 的 mouse.wheel 方法滚动
// 每次滚动一大段距离
// await list.hover();
const scrollAmount = 1500;
await page.mouse.wheel(0, scrollAmount);
scrollCount++;
console.log(chalk.gray(` ↓ 第 ${scrollCount} 次滚动`));
// 等待一段时间,让评论加载
await page.waitForTimeout(scrollInterval);
} catch (e) {
console.warn(chalk.yellow(` ⚠ 滚动时出现警告: ${(e as Error)?.message}`));
}
}
// 等待收集器完成
const commentResponses = await commentResponsesPromise;
console.log(chalk.green(`✓ 评论收集完成,共收集到 ${commentResponses.length} 个评论响应`));
return commentResponses;
}
async function readPostMem(context: BrowserContext, page: Page) {
const md = await page.evaluate(() => {
// @ts-ignore
@@ -47,8 +112,7 @@ export class ScrapeError extends Error {
export async function scrapeDouyin(url: string) {
console.log(chalk.blue('🚀 启动共享 Chromium 浏览器...'));
const context = await acquireBrowserContext();
let context: BrowserContext | null = await acquireIsolatedContext();
const page = await context.newPage();
console.log(chalk.cyan(`📄 正在访问: ${chalk.underline(url)}`));
@@ -82,16 +146,11 @@ export async function scrapeDouyin(url: string) {
try {
// 先注册“先到先得”的监听,再导航,避免漏包
const firstTypePromise = waitForFirstResponse(context, [
{ key: 'detail', test: (r: Response) => r.url().includes(DETAIL_PATH) && r.status() === 200 },
{ key: 'post', test: (r: Response) => r.url().includes(POST_PATH) && r.status() === 200 },
], 40_000);
{ key: 'detail', test: (r: Response) => r.url().includes(DETAIL_PATH) && r.status() === 200 && r.request().frame()?.page() === page },
{ key: 'post', test: (r: Response) => r.url().includes(POST_PATH) && r.status() === 200 && r.request().frame()?.page() === page },
], 10_000);
// 评论只做短时“有就用、没有不等”的监听
const commentPromise = waitForResponseWithTimeout(
context, (r: Response) => r.url().includes(COMMENT_PATH) && r.status() === 200, 40_000
).catch(() => null);
await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 60_000 });
await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 20_000 });
// 查找页面中是否存在 "视频不存在" 的提示
const isNotFound = await page.locator('text=视频不存在').count().then(count => count > 0).catch(() => false);
@@ -100,52 +159,88 @@ export async function scrapeDouyin(url: string) {
throw new ScrapeError('视频不存在或已被删除', 404, 'VIDEO_NOT_FOUND');
}
try {
// 优先尝试从内存读取图文数据
let { aweme, comments } = await readPostMem(context, page);
console.log(chalk.green('✓ 从内存读取图文数据成功'));
const uploads = await handleImagePost(context, aweme);
if (!comments) {
console.warn(chalk.yellow('⚠ 从内存读取评论数据失败,尝试从网络请求获取评论数据'));
const commentRes = await commentPromise;
comments = commentRes && await safeJson<DouyinCommentResponse>(commentRes);
if (!comments) {
console.warn(chalk.yellow('⚠ 无法从内存读取评论数据,且网络请求也未返回评论数据'));
comments = { comments: [], total: 0, status_code: 0 };
} else {
console.log(chalk.green('✓ 从网络请求获取评论数据成功'));
}
} else {
console.log(chalk.green('✓ 从内存读取评论数据成功'));
}
const saved = await saveImagePostToDB(context, aweme, comments, uploads); // 传递完整 JSON
console.log(chalk.green.bold('✓ 图文作品保存成功'));
return { type: "image", ...saved };
} catch {
}
const commentRes = await commentPromise;
// 等待作品类型判定
const firstType = await firstTypePromise;
if (!firstType) {
// 尝试从内存读取图文数据(如果是图文作品)
let memoryData: { aweme: any; comments: DouyinCommentResponse | null } | null = null;
try {
memoryData = await readPostMem(context, page);
console.log(chalk.green('✓ 从内存读取图文数据成功'));
} catch {
// 内存读取失败,稍后通过网络获取
}
if (!firstType && !memoryData) {
console.error(chalk.red('✗ 既无法从内存读取数据,也无法从网络获得数据'));
throw new ScrapeError('无法获取作品数据,可能是网络问题或作品已下架', 404, 'NO_DATA');
}
console.log(chalk.cyan(`📡 检测到作品类型: ${chalk.bold(firstType.key === 'post' ? '图文' : '视频')}`));
console.log(chalk.cyan(`📡 检测到作品类型: ${chalk.bold(firstType?.key === 'post' || memoryData ? '图文' : '视频')}`));
let comments = commentRes && await safeJson<DouyinCommentResponse>(commentRes);
if (!comments) {
console.warn(chalk.yellow('⚠ 无法从内存读取评论数据,且网络请求也未返回评论数据'));
comments = { comments: [], total: 0, status_code: 0 };
let allComments: DouyinComment[] = [];
try {
// 开始滚动并收集评论
const commentResponses = await scrollAndCollectComments(context, page);
// 解析所有收集到的评论响应
for (const commentRes of commentResponses) {
try {
const commentData = await safeJson<DouyinCommentResponse>(commentRes);
if (commentData?.comments?.length) {
allComments.push(...commentData.comments);
}
} catch (e) {
console.warn(chalk.yellow(`⚠ 解析评论响应失败: ${(e as Error)?.message}`));
}
}
} catch (error) {
console.warn(chalk.yellow(`⚠ 评论收集失败: ${(error as Error)?.message}`));
}
// 去重评论(根据 cid)
const uniqueComments = Array.from(
new Map(allComments.map(c => [c.cid, c])).values()
);
console.log(chalk.green(`✓ 共收集到 ${uniqueComments.length} 条独立评论(去重前: ${allComments.length})`));
// 如果从内存读取到了评论,合并进来作为兜底
let comments: DouyinCommentResponse;
if (memoryData?.comments?.comments?.length) {
console.log(chalk.blue(`📝 合并内存中的 ${memoryData.comments.comments.length} 条评论`));
const memComments = memoryData.comments.comments;
const mergedMap = new Map(uniqueComments.map(c => [c.cid, c]));
for (const c of memComments) {
if (!mergedMap.has(c.cid)) {
mergedMap.set(c.cid, c);
}
}
comments = {
comments: Array.from(mergedMap.values()),
total: mergedMap.size,
status_code: 0
};
console.log(chalk.green(`✓ 合并后共 ${comments.comments.length} 条评论`));
} else {
comments = {
comments: uniqueComments,
total: uniqueComments.length,
status_code: 0
};
}
// 分支:视频 or 图文(两者只会有一个命中,先到先得)
if (firstType.key === 'post') {
// 图文作品
// 优先处理内存数据(图文)
if (memoryData) {
const aweme = memoryData.aweme;
const uploads = await handleImagePost(context, aweme);
const saved = await saveImagePostToDB(context, aweme, comments, uploads); // 传递完整 JSON
console.log(chalk.green.bold('✓ 图文作品保存成功'));
return { type: "image", ...saved };
} else if (firstType?.key === 'post') {
// 图文作品(网络)
const postJson = await safeJson<DouyinPostListResponse>(firstType.response);
if (!postJson?.aweme_list?.length) throw new ScrapeError('图文作品响应为空', 404, 'EMPTY_POST_RESPONSE');
@@ -161,7 +256,7 @@ export async function scrapeDouyin(url: string) {
const saved = await saveImagePostToDB(context, aweme, comments, uploads, postJson); // 传递完整 JSON
console.log(chalk.green.bold('✓ 图文作品保存成功'));
return { type: "image", ...saved };
} else if (firstType.key === 'detail') {
} else if (firstType?.key === 'detail') {
// 视频作品
const detail = (await safeJson<DouyinVideoDetailResponse>(firstType.response))!;
@@ -216,11 +311,11 @@ export async function scrapeDouyin(url: string) {
if (error instanceof ScrapeError) {
throw error;
}
// 处理其他类型的错误
const errMsg = (error as Error)?.message || String(error);
console.error(chalk.red(`✗ 爬取失败: ${errMsg}`));
// 根据错误类型返回不同的状态码
if (errMsg.includes('timeout') || errMsg.includes('超时')) {
throw new ScrapeError('请求超时,请稍后重试', 408, 'TIMEOUT');
@@ -231,14 +326,14 @@ export async function scrapeDouyin(url: string) {
if (errMsg.includes('net::')) {
throw new ScrapeError('网络连接失败', 503, 'NETWORK_ERROR');
}
// 默认服务器错误
throw new ScrapeError(errMsg || '爬取过程中发生未知错误', 500, 'UNKNOWN_ERROR');
} finally {
console.log(chalk.gray('🧹 清理资源...'));
try { await page.close({ runBeforeUnload: true }); } catch {}
// 仅释放共享上下文的引用,不直接关闭窗口
await releaseBrowserContext();
try { await page.close({ runBeforeUnload: true }); } catch { }
// 关闭本次任务的隔离上下文与浏览器
await releaseIsolatedContext(context);
await prisma.$disconnect();
console.log(chalk.gray('✓ 资源清理完成'));
}
+2
View File
@@ -1,3 +1,5 @@
export const runtime = 'nodejs'
import { promises as fs } from 'fs';
import os from 'os';
import path from 'path';
+47 -3
View File
@@ -1,3 +1,5 @@
export const runtime = 'nodejs'
import type { BrowserContext, Response } from 'playwright';
export async function safeJson<T>(res: Response): Promise<T | null> {
@@ -20,12 +22,12 @@ export async function safeJson<T>(res: Response): Promise<T | null> {
*/
export async function downloadBinary(
context: BrowserContext,
url: string
url: string,
): Promise<{ buffer: Buffer; contentType: string; ext: string }> {
console.log('下载:', url);
const headers = {
referer: url,
referer: 'https://www.douyin.com/',
} as Record<string, string>;
const res = await context.request.get(url, {
@@ -93,7 +95,49 @@ export function waitForFirstResponse(
}
/**
* 等待符合条件的单个 Response,带短超时;用于评论等“可有可无”的数据。
* 在指定时间内持续收集所有符合条件的 Response
* 用于评论等需要滚动加载的数据
*/
export function collectResponsesWithinTime(
context: BrowserContext,
predicate: (r: Response) => boolean,
durationMs: number
): Promise<Response[]> {
return new Promise((resolve) => {
const collected: Response[] = [];
const seenUrls = new Set<string>();
let timer: NodeJS.Timeout | undefined;
const handler = (res: Response) => {
try {
if (predicate(res)) {
// 使用 URL 去重,避免重复收集同一个请求
const url = res.url();
if (!seenUrls.has(url)) {
seenUrls.add(url);
collected.push(res);
}
}
} catch {
// ignore predicate errors
}
};
const cleanup = () => {
context.off('response', handler);
if (timer) clearTimeout(timer);
};
context.on('response', handler);
timer = setTimeout(() => {
cleanup();
resolve(collected);
}, durationMs);
});
}
/**
* 等待符合条件的单个 Response,带短超时;用于评论等"可有可无"的数据。
*/
export function waitForResponseWithTimeout(
context: BrowserContext,
+91 -7
View File
@@ -1,6 +1,6 @@
import type { BrowserContext } from 'playwright';
import { prisma } from '@/lib/prisma';
import { uploadAvatarFromUrl } from './uploader';
import { uploadAvatarFromUrl, uploadImageFromUrl } from './uploader';
import { firstUrl } from './utils';
export async function saveToDB(
@@ -111,7 +111,7 @@ export async function saveToDB(
},
});
await prisma.comment.upsert({
const savedComment = await prisma.comment.upsert({
where: { cid: c.cid },
create: {
cid: c.cid,
@@ -129,6 +129,49 @@ export async function saveToDB(
userId: cu.id,
},
});
// 处理评论贴纸/配图上传与入库
try {
const sources: { url?: string | null; width?: number; height?: number }[] = [];
// 贴纸(当作第一张)
const stickerUrl = firstUrl(c.sticker?.animate_url?.url_list);
if (stickerUrl) {
sources.push({ url: stickerUrl, width: c.sticker?.animate_url?.width, height: c.sticker?.animate_url?.height });
}
// 配图列表
const imgs = c.image_list || [];
for (const it of imgs) {
const u = firstUrl(it?.origin_url?.url_list);
if (u) sources.push({ url: u, width: it?.origin_url?.width as any, height: it?.origin_url?.height as any });
}
for (let i = 0; i < sources.length; i++) {
const s = sources[i];
const uploaded = await uploadImageFromUrl(
context,
s.url ?? undefined,
`comments/${c.cid}/${i}`,
);
if (!uploaded) continue;
await prisma.commentImage.upsert({
where: { commentId_order: { commentId: savedComment.cid, order: i } },
create: {
commentId: savedComment.cid,
order: i,
url: uploaded,
width: typeof s.width === 'number' ? s.width : null,
height: typeof s.height === 'number' ? s.height : null,
},
update: {
url: uploaded,
width: typeof s.width === 'number' ? s.width : null,
height: typeof s.height === 'number' ? s.height : null,
},
});
}
} catch (e) {
console.warn('[comment-images] 保存失败:', (e as Error)?.message || e);
}
}
return { aweme_id: video.aweme_id, author_sec_uid: author.sec_uid, comment_count: comments.length };
@@ -138,7 +181,7 @@ export async function saveImagePostToDB(
context: BrowserContext,
aweme: DouyinImageAweme,
commentResp: DouyinCommentResponse,
uploads: { images: { url: string; width?: number; height?: number; video?: string; duration?: number }[]; musicUrl?: string },
uploads: { images: { url: string; width?: number; height?: number, video?: string }[]; musicUrl?: string },
rawJson?: any
) {
if (!aweme?.author?.sec_uid) throw new Error('作者 sec_uid 缺失');
@@ -205,7 +248,7 @@ export async function saveImagePostToDB(
// Upsert ImageFiles(按顺序)
for (let i = 0; i < uploads.images.length; i++) {
const { url, width, height, video, duration } = uploads.images[i];
const { url, width, height, video } = uploads.images[i];
await prisma.imageFile.upsert({
where: { postId_order: { postId: imagePost.aweme_id, order: i } },
create: {
@@ -215,14 +258,12 @@ export async function saveImagePostToDB(
width: typeof width === 'number' ? width : null,
height: typeof height === 'number' ? height : null,
animated: video || null,
duration: typeof duration === 'number' ? duration : null,
},
update: {
url,
width: typeof width === 'number' ? width : null,
height: typeof height === 'number' ? height : null,
animated: video || null,
duration: typeof duration === 'number' ? duration : null,
},
});
}
@@ -251,7 +292,7 @@ export async function saveImagePostToDB(
},
});
await prisma.comment.upsert({
const savedComment = await prisma.comment.upsert({
where: { cid: c.cid },
create: {
cid: c.cid,
@@ -269,6 +310,49 @@ export async function saveImagePostToDB(
userId: cu.id,
},
});
// 处理评论贴纸/配图上传与入库
try {
const sources: { url?: string | null; width?: number; height?: number }[] = [];
// 贴纸(当作第一张)
const stickerUrl = firstUrl(c.sticker?.animate_url?.url_list);
if (stickerUrl) {
sources.push({ url: stickerUrl, width: c.sticker?.animate_url?.width, height: c.sticker?.animate_url?.height });
}
// 配图列表
const imgs = c.image_list || [];
for (const it of imgs) {
const u = firstUrl(it?.origin_url?.url_list);
if (u) sources.push({ url: u, width: it?.origin_url?.width as any, height: it?.origin_url?.height as any });
}
for (let i = 0; i < sources.length; i++) {
const s = sources[i];
const uploaded = await uploadImageFromUrl(
context,
s.url ?? undefined,
`comments/${c.cid}/${i}`,
);
if (!uploaded) continue;
await prisma.commentImage.upsert({
where: { commentId_order: { commentId: savedComment.cid, order: i } },
create: {
commentId: savedComment.cid,
order: i,
url: uploaded,
width: typeof s.width === 'number' ? s.width : null,
height: typeof s.height === 'number' ? s.height : null,
},
update: {
url: uploaded,
width: typeof s.width === 'number' ? s.width : null,
height: typeof s.height === 'number' ? s.height : null,
},
});
}
} catch (e) {
console.warn('[comment-images] 保存失败:', (e as Error)?.message || e);
}
}
return { aweme_id: imagePost.aweme_id, author_sec_uid: author.sec_uid, image_count: uploads.images.length, comment_count: comments.length };
+2
View File
@@ -1,3 +1,5 @@
export const runtime = 'nodejs'
import { NextRequest, NextResponse } from 'next/server'
import { prisma } from '@/lib/prisma'
import { scrapeDouyin, ScrapeError } from '.';
+16
View File
@@ -12,6 +12,22 @@ interface DouyinComment {
digg_count: number; // 点赞数
create_time: number; // 创建时间(时间戳)
user: DouyinUser; // 评论用户
sticker?: {
id: number;
animate_url: {
width: number;
height: number;
url_list: string[]
}
},
image_list?: {
origin_url:{
width: number;
height: number;
url_list: string[]
}
}[]
}
/** 用户信息(精简版) */
+25 -1
View File
@@ -1,3 +1,5 @@
export const runtime = 'nodejs'
import type { BrowserContext } from 'playwright';
import { uploadFile, generateUniqueFileName } from '@/lib/minio';
import { downloadBinary } from './network';
@@ -10,7 +12,7 @@ import { getVideoDuration } from './media';
export async function uploadAvatarFromUrl(
context: BrowserContext,
srcUrl?: string | null,
nameHint?: string
nameHint?: string,
): Promise<string | undefined> {
if (!srcUrl) return undefined;
try {
@@ -26,6 +28,28 @@ export async function uploadAvatarFromUrl(
}
}
/**
* 下载任意图片并上传到 MinIO,返回外链;失败时回退为原始链接。
*/
export async function uploadImageFromUrl(
context: BrowserContext,
srcUrl?: string | null,
nameHint?: string,
): Promise<string | undefined> {
if (!srcUrl) return undefined;
try {
const { buffer, contentType, ext } = await downloadBinary(context, srcUrl);
const safeExt = ext || 'jpg';
const baseName = nameHint ? `${nameHint}.${safeExt}` : `image.${safeExt}`;
const fileName = generateUniqueFileName(baseName, 'douyin/comment-images');
const uploaded = await uploadFile(buffer, fileName, { 'Content-Type': contentType });
return uploaded;
} catch (e) {
console.warn('[image] 上传失败,使用原始链接:', (e as Error)?.message || e);
return srcUrl || undefined;
}
}
/** 下载图文作品的图片和音乐并上传到 MinIO */
export async function handleImagePost(
context: BrowserContext,
+2
View File
@@ -1,3 +1,5 @@
export const runtime = 'nodejs'
export function toCamelCaseKey(key: string): string {
return key.replace(/_([a-zA-Z])/g, (_, c: string) => c.toUpperCase());
}
+2 -2
View File
@@ -299,7 +299,7 @@ export default function AwemeDetailClient({ data, neighbors }: AwemeDetailClient
{/* 导航按钮 */}
<NavigationButtons
neighbors={neighbors}
commentsCount={data.comments.length}
commentsCount={data.commentsCount}
onNavigatePrev={() => neighbors.prev && router.push(`/aweme/${neighbors.prev.aweme_id}`)}
onNavigateNext={() => neighbors.next && router.push(`/aweme/${neighbors.next.aweme_id}`)}
onToggleComments={() => commentState.setOpen((v) => !v)}
@@ -312,7 +312,7 @@ export default function AwemeDetailClient({ data, neighbors }: AwemeDetailClient
onClose={() => commentState.setOpen(false)}
author={data.author}
createdAt={data.created_at}
comments={data.comments}
awemeId={data.aweme_id}
mounted={commentState.mounted}
/>
</div>
+53 -1
View File
@@ -1,4 +1,5 @@
import { ThumbsUp } from "lucide-react";
import { ThumbsUp, X } from "lucide-react";
import { useState } from "react";
import type { Comment, User } from "../types";
import { formatRelativeTime, formatAbsoluteUTC } from "../utils";
import { CommentText } from "./CommentText";
@@ -10,6 +11,8 @@ interface CommentListProps {
}
export function CommentList({ author, createdAt, comments }: CommentListProps) {
const [previewImage, setPreviewImage] = useState<string | null>(null);
return (
<>
<header className="flex items-center gap-4 mb-5">
@@ -42,6 +45,29 @@ export function CommentList({ author, createdAt, comments }: CommentListProps) {
<p className="mt-1 text-sm leading-relaxed text-white/90 break-words">
<CommentText text={c.text} />
</p>
{c.images && c.images.length > 0 ? (
<div className="mt-2 grid grid-cols-3 gap-1.5 sm:gap-2">
{c.images.map((img, idx) => (
<button
key={idx}
type="button"
onClick={() => setPreviewImage(img.url)}
className="block overflow-hidden rounded-md bg-zinc-800/60 cursor-pointer hover:opacity-80 transition-opacity"
title="点击预览大图"
>
{/* 以真实比例显示:利用 width/height 属性提供固有尺寸,CSS 设定宽度 100% 高度自适应 */}
<img
src={img.url}
alt="comment-image"
width={img.width}
height={img.height}
className="w-full h-auto"
loading="lazy"
/>
</button>
))}
</div>
) : null}
<div className="mt-2 inline-flex items-center gap-1 text-xs text-white/70">
<ThumbsUp size={14} />
<span>{c.digg_count}</span>
@@ -51,6 +77,32 @@ export function CommentList({ author, createdAt, comments }: CommentListProps) {
))}
{comments.length === 0 ? <li className="text-sm text-white/60">暂无评论</li> : null}
</ul>
{/* 图片预览灯箱 */}
{previewImage && (
<div
className="fixed inset-0 z-[100] flex items-center justify-center bg-black/90 backdrop-blur-sm"
onClick={() => setPreviewImage(null)}
>
{/* 关闭按钮 */}
<button
type="button"
className="absolute top-4 right-4 w-10 h-10 rounded-full bg-white/15 text-white border border-white/20 hover:bg-white/25 transition-colors flex items-center justify-center"
onClick={() => setPreviewImage(null)}
aria-label="关闭预览"
>
<X size={20} />
</button>
{/* 大图 */}
<img
src={previewImage}
alt="preview"
className="max-w-[90vw] max-h-[90vh] object-contain"
onClick={(e) => e.stopPropagation()}
/>
</div>
)}
</>
);
}
+177 -6
View File
@@ -1,4 +1,5 @@
import { X } from "lucide-react";
import { useState, useEffect, useRef, useCallback } from "react";
import type { Comment, User } from "../types";
import { CommentList } from "./CommentList";
@@ -7,13 +8,155 @@ interface CommentPanelProps {
onClose: () => void;
author: User;
createdAt: string | Date;
comments: Comment[];
awemeId: string;
mounted: boolean;
}
export function CommentPanel({ open, onClose, author, createdAt, comments, mounted }: CommentPanelProps) {
export function CommentPanel({ open, onClose, author, createdAt, awemeId, mounted }: CommentPanelProps) {
const [comments, setComments] = useState<Comment[]>([]);
const [total, setTotal] = useState(0);
const [loading, setLoading] = useState(false);
const [hasMore, setHasMore] = useState(true);
// ranked 排序的稳定参数(由后端返回,前端透传保证会话内稳定)
const [rankParams, setRankParams] = useState<null | { seed: string; snapshot: string }>(null);
// 两套滚动容器与哨兵,分别对应横屏与竖屏面板
const scrollRefLandscape = useRef<HTMLDivElement>(null);
const sentinelRefLandscape = useRef<HTMLDivElement>(null);
const scrollRefPortrait = useRef<HTMLDivElement>(null);
const sentinelRefPortrait = useRef<HTMLDivElement>(null);
const loadingRef = useRef(false);
// 加载评论
const loadComments = useCallback(async (reset = false) => {
if (loadingRef.current || (!reset && !hasMore)) return;
loadingRef.current = true;
setLoading(true);
try {
const skip = reset ? 0 : comments.length;
const query = new URLSearchParams({
skip: String(skip),
take: String(20),
mode: 'ranked',
});
if (rankParams) {
query.set('seed', rankParams.seed);
query.set('snapshot', rankParams.snapshot);
}
const response = await fetch(`/api/comments/${awemeId}?${query.toString()}`);
const data = await response.json();
// 统一做一次基于 cid 的去重,避免分页偶发重复
if (reset) {
setComments(() => {
const seen = new Set<string>();
return (data.comments as Comment[]).filter((c) => {
if (seen.has(c.cid)) return false;
seen.add(c.cid);
return true;
});
});
} else {
setComments((prev) => {
const merged = [...prev, ...(data.comments as Comment[])];
const seen = new Set<string>();
// 保留首次出现的项,既保证顺序也避免重复
return merged.filter((c) => {
if (seen.has(c.cid)) return false;
seen.add(c.cid);
return true;
});
});
}
setTotal(data.total);
setHasMore(data.hasMore);
if (data.mode === 'ranked' && data.seed && data.snapshot) {
// 初始化或重置时更新稳定参数
setRankParams((prev) => {
if (reset) return { seed: String(data.seed), snapshot: String(data.snapshot) };
return prev ?? { seed: String(data.seed), snapshot: String(data.snapshot) };
});
} else if (reset) {
setRankParams(null);
}
} catch (error) {
console.error("加载评论失败:", error);
} finally {
setLoading(false);
loadingRef.current = false;
}
}, [awemeId, comments.length, hasMore]);
// 面板打开时加载初始评论
useEffect(() => {
if (open && comments.length === 0) {
loadComments(true);
}
}, [open]);
// Intersection Observer 监听底部触发加载(针对横屏与竖屏两个容器分别监听)
useEffect(() => {
if (!open) return;
const observers: IntersectionObserver[] = [];
const setup = (rootEl: HTMLDivElement | null, targetEl: HTMLDivElement | null) => {
if (!rootEl || !targetEl) return;
const io = new IntersectionObserver(
(entries) => {
const entry = entries[0];
if (entry.isIntersecting && !loadingRef.current && hasMore) {
loadComments();
}
},
{
root: rootEl,
rootMargin: '0px 0px 200px 0px', // 距底部 200px 触发
threshold: 0,
}
);
io.observe(targetEl);
observers.push(io);
};
setup(scrollRefLandscape.current, sentinelRefLandscape.current);
setup(scrollRefPortrait.current, sentinelRefPortrait.current);
return () => {
for (const io of observers) io.disconnect();
};
}, [open, hasMore, loadComments]);
return (
<>
<style jsx global>{`
/* 自定义滚动条样式 */
.comment-scroll::-webkit-scrollbar {
width: 6px;
}
.comment-scroll::-webkit-scrollbar-track {
background: transparent;
}
.comment-scroll::-webkit-scrollbar-thumb {
background: rgba(255, 255, 255, 0.2);
border-radius: 3px;
transition: background 0.2s;
}
.comment-scroll::-webkit-scrollbar-thumb:hover {
background: rgba(255, 255, 255, 0.3);
}
/* Firefox */
.comment-scroll {
scrollbar-width: thin;
scrollbar-color: rgba(255, 255, 255, 0.2) transparent;
}
`}</style>
{/* 横屏评论面板:并排分栏 */}
<aside
className={`
@@ -32,11 +175,25 @@ export function CommentPanel({ open, onClose, author, createdAt, comments, mount
>
<X size={18} />
</button>
<div className="text-white font-semibold">评论 {comments.length > 0 ? `(${comments.length})` : ""}</div>
<div className="text-white font-semibold">
评论 {total > 0 ? `(${total})` : ""}
</div>
</div>
<div className="p-3 overflow-auto">
<div ref={scrollRefLandscape} className="p-3 overflow-auto comment-scroll">
<CommentList author={author} createdAt={createdAt} comments={comments} />
{/* 底部加载触发区 */}
{hasMore && (
<div ref={sentinelRefLandscape} className="h-1 w-full" />
)}
{loading && (
<div className="py-4 text-center text-white/60 text-sm">加载中...</div>
)}
{!hasMore && comments.length > 0 && (
<div className="py-4 text-center text-white/40 text-sm">没有更多评论了</div>
)}
</div>
</aside>
@@ -52,7 +209,9 @@ export function CommentPanel({ open, onClose, author, createdAt, comments, mount
`}
>
<div className="flex items-center justify-between px-3 py-3 border-b border-white/10">
<div className="text-white font-semibold">评论 {comments.length > 0 ? `(${comments.length})` : ""}</div>
<div className="text-white font-semibold">
评论 {total > 0 ? `(${total})` : ""}
</div>
<button
className="w-8 h-8 inline-flex items-center justify-center rounded-full bg-white/15 text-white/90 border border-white/20 hover:bg-white/25 transition-colors"
onClick={onClose}
@@ -62,8 +221,20 @@ export function CommentPanel({ open, onClose, author, createdAt, comments, mount
</button>
</div>
<div className="p-3 overflow-auto">
<div ref={scrollRefPortrait} className="p-3 overflow-auto comment-scroll">
<CommentList author={author} createdAt={createdAt} comments={comments} />
{/* 底部加载触发区 */}
{hasMore && (
<div ref={sentinelRefPortrait} className="h-1 w-full" />
)}
{loading && (
<div className="py-4 text-center text-white/60 text-sm">加载中...</div>
)}
{!hasMore && comments.length > 0 && (
<div className="py-4 text-center text-white/40 text-sm">没有更多评论了</div>
)}
</div>
</aside>
</>
+10 -16
View File
@@ -48,17 +48,23 @@ export default async function AwemeDetail({ params }: { params: Promise<{ awemeI
const [video, post] = await Promise.all([
prisma.video.findUnique({
where: { aweme_id: id },
include: { author: true, comments: { orderBy: { created_at: "desc" }, include: { user: true }, take: 50 } },
include: { author: true },
}),
prisma.imagePost.findUnique({
where: { aweme_id: id },
include: { author: true, images: { orderBy: { order: "asc" } }, comments: { orderBy: { created_at: "desc" }, include: { user: true }, take: 50 } },
include: { author: true, images: { orderBy: { order: "asc" } } },
})
]);
if (!video && !post) return <main className="p-8">找不到该作品</main>;
const isVideo = !!video;
// 获取评论总数
const commentsCount = await prisma.comment.count({
where: isVideo ? { videoId: id } : { imagePostId: id },
});
const data = isVideo
? {
type: "video" as const,
@@ -70,13 +76,7 @@ export default async function AwemeDetail({ params }: { params: Promise<{ awemeI
width: video!.width ?? null,
height: video!.height ?? null,
author: { nickname: video!.author.nickname, avatar_url: video!.author.avatar_url },
comments: video!.comments.map((c) => ({
cid: c.cid,
text: c.text,
created_at: c.created_at,
digg_count: c.digg_count,
user: { nickname: c.user.nickname, avatar_url: c.user.avatar_url },
})),
commentsCount,
}
: {
type: "image" as const,
@@ -86,13 +86,7 @@ export default async function AwemeDetail({ params }: { params: Promise<{ awemeI
images: post!.images,
music_url: post!.music_url,
author: { nickname: post!.author.nickname, avatar_url: post!.author.avatar_url },
comments: post!.comments.map((c) => ({
cid: c.cid,
text: c.text,
created_at: c.created_at,
digg_count: c.digg_count,
user: { nickname: c.user.nickname, avatar_url: c.user.avatar_url },
})),
commentsCount,
};
// Compute prev/next neighbors by created_at across videos and image posts
+3 -2
View File
@@ -6,6 +6,7 @@ export type Comment = {
digg_count: number;
created_at: string | Date;
user: User;
images?: { url: string; width?: number; height?: number }[];
};
export type VideoData = {
@@ -18,7 +19,7 @@ export type VideoData = {
width?: number | null;
height?: number | null;
author: User;
comments: Comment[];
commentsCount: number;
};
export type ImageData = {
@@ -29,7 +30,7 @@ export type ImageData = {
images: { id: string; url: string; width?: number; height?: number; animated?: string; duration?: number }[];
music_url?: string | null;
author: User;
comments: Comment[];
commentsCount: number;
};
export type AwemeData = VideoData | ImageData;
+5 -4
View File
@@ -147,9 +147,10 @@ export default function TasksPage() {
const extractedCount = useMemo(() => extractDouyinLinks(input).length, [input]);
const formatDuration = (t?: number) => {
if (!t) return "";
const secs = Math.max(0, Math.round((Date.now() - t) / 1000));
const formatDuration = (startTime?: number, endTime?: number) => {
if (!startTime) return "";
const end = endTime || Date.now();
const secs = Math.max(0, Math.round((end - startTime) / 1000));
if (secs < 60) return `${secs}s`;
const m = Math.floor(secs / 60);
const s = secs % 60;
@@ -228,7 +229,7 @@ export default function TasksPage() {
<span className="text-xs text-neutral-400">已耗时 {formatDuration(t.startedAt)}</span>
)}
{t.status === 'success' && (
<span className="text-xs text-neutral-400">用时 {t.startedAt ? formatDuration(t.startedAt) : '--'}</span>
<span className="text-xs text-neutral-400">用时 {formatDuration(t.startedAt, t.finishedAt)}</span>
)}
</div>
<div className="mt-1 flex items-center gap-2 text-sm text-neutral-200">