反爬虫优化，评论图片支持

app/api/fetcher/index.ts

@@ -1,19 +1,84 @@
+export const runtime = 'nodejs'
 // src/scrapeDouyin.ts
-import { BrowserContext, Page, chromium, type Response } from 'playwright';
+import { BrowserContext, Page, type Response } from 'playwright';
+import { chromium } from 'playwright-extra';
 import { prisma } from '@/lib/prisma';
 import { uploadFile, generateUniqueFileName } from '@/lib/minio';
 import { createCamelCompatibleProxy } from '@/app/api/fetcher/utils';
-import { waitForFirstResponse, waitForResponseWithTimeout, safeJson, downloadBinary } from '@/app/api/fetcher/network';
+import { waitForFirstResponse, waitForResponseWithTimeout, safeJson, downloadBinary, collectResponsesWithinTime } from '@/app/api/fetcher/network';
 import { pickBestPlayAddr, extractFirstFrame } from '@/app/api/fetcher/media';
 import { handleImagePost } from '@/app/api/fetcher/uploader';
 import { saveToDB, saveImagePostToDB } from '@/app/api/fetcher/persist';
 import chalk from 'chalk';
-import { acquireBrowserContext, releaseBrowserContext } from '@/app/api/fetcher/browser';
+import { acquireIsolatedContext, releaseIsolatedContext } from '@/app/api/fetcher/browser';
 
 const DETAIL_PATH = '/aweme/v1/web/aweme/detail/';
 const COMMENT_PATH = '/aweme/v1/web/comment/list/';
 const POST_PATH = '/aweme/v1/web/aweme/post/'
 
+/**
+ * 滚动页面并收集评论
+ * @param context 浏览器上下文
+ * @param page 页面对象
+ * @param durationMs 持续时间（毫秒）
+ * @returns 收集到的所有评论响应
+ */
+async function scrollAndCollectComments(
+    context: BrowserContext,
+    page: Page,
+    durationMs: number = 10_000
+): Promise<Response[]> {
+    console.log(chalk.blue(`📜 开始滚动页面收集评论（持续 ${durationMs / 1000} 秒）...`));
+
+    // 启动评论响应收集器
+    const commentResponsesPromise = collectResponsesWithinTime(
+        context,
+        (r: Response) => r.url().includes(COMMENT_PATH) && r.status() === 200 && r.request().frame()?.page() === page,
+        durationMs
+    );
+
+    // 在指定时间内持续滚动页面
+    const startTime = Date.now();
+    const scrollInterval = 500;
+    let scrollCount = 0;
+    const selector = "div[data-e2e='comment-list']";
+
+    // 1) 等元素出现并可见
+    await page.waitForSelector(selector, { state: 'visible', timeout: 5000 });
+
+    // 2) 确保滚动到可见区域
+    const list = page.locator(selector);
+    await list.scrollIntoViewIfNeeded();
+
+    // 3) 执行 hover（推荐用 locator 的 hover）
+    list.hover({ timeout: 5000 }).catch(() => { });
+    while (Date.now() - startTime < durationMs - 500) { // 留 500ms 缓冲
+        try {
+            list.hover({ timeout: 2000 }).catch(() => { });
+            // 使用 Playwright 的 mouse.wheel 方法滚动
+            // 每次滚动一大段距离
+            // await list.hover();
+            const scrollAmount = 1500;
+            await page.mouse.wheel(0, scrollAmount);
+
+            scrollCount++;
+            console.log(chalk.gray(`  ↓ 第 ${scrollCount} 次滚动`));
+
+            // 等待一段时间，让评论加载
+            await page.waitForTimeout(scrollInterval);
+
+        } catch (e) {
+            console.warn(chalk.yellow(`  ⚠ 滚动时出现警告: ${(e as Error)?.message}`));
+        }
+    }
+
+    // 等待收集器完成
+    const commentResponses = await commentResponsesPromise;
+    console.log(chalk.green(`✓ 评论收集完成，共收集到 ${commentResponses.length} 个评论响应`));
+
+    return commentResponses;
+}
+
 async function readPostMem(context: BrowserContext, page: Page) {
     const md = await page.evaluate(() => {
         // @ts-ignore
@@ -47,8 +112,7 @@ export class ScrapeError extends Error {
 
 export async function scrapeDouyin(url: string) {
     console.log(chalk.blue('🚀 启动共享 Chromium 浏览器...'));
-
-    const context = await acquireBrowserContext();
+    let context: BrowserContext | null = await acquireIsolatedContext();
     const page = await context.newPage();
     console.log(chalk.cyan(`📄 正在访问: ${chalk.underline(url)}`));
 
@@ -82,16 +146,11 @@ export async function scrapeDouyin(url: string) {
     try {
         // 先注册“先到先得”的监听，再导航，避免漏包
         const firstTypePromise = waitForFirstResponse(context, [
-            { key: 'detail', test: (r: Response) => r.url().includes(DETAIL_PATH) && r.status() === 200 },
-            { key: 'post', test: (r: Response) => r.url().includes(POST_PATH) && r.status() === 200 },
-        ], 40_000); 
+            { key: 'detail', test: (r: Response) => r.url().includes(DETAIL_PATH) && r.status() === 200 && r.request().frame()?.page() === page },
+            { key: 'post', test: (r: Response) => r.url().includes(POST_PATH) && r.status() === 200 && r.request().frame()?.page() === page },
+        ], 10_000);
 
-        // 评论只做短时“有就用、没有不等”的监听
-        const commentPromise = waitForResponseWithTimeout(
-            context, (r: Response) => r.url().includes(COMMENT_PATH) && r.status() === 200, 40_000
-        ).catch(() => null);
-
-        await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 60_000 });
+        await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 20_000 });
 
         // 查找页面中是否存在 "视频不存在" 的提示
         const isNotFound = await page.locator('text=视频不存在').count().then(count => count > 0).catch(() => false);
@@ -100,52 +159,88 @@ export async function scrapeDouyin(url: string) {
             throw new ScrapeError('视频不存在或已被删除', 404, 'VIDEO_NOT_FOUND');
         }
 
-        try {
-            // 优先尝试从内存读取图文数据
-            let { aweme, comments } = await readPostMem(context, page);
-            console.log(chalk.green('✓ 从内存读取图文数据成功'));
-
-            const uploads = await handleImagePost(context, aweme);
-            if (!comments) {
-                console.warn(chalk.yellow('⚠ 从内存读取评论数据失败，尝试从网络请求获取评论数据'));
-
-                const commentRes = await commentPromise;
-                comments = commentRes && await safeJson<DouyinCommentResponse>(commentRes);
-                if (!comments) {
-                    console.warn(chalk.yellow('⚠ 无法从内存读取评论数据，且网络请求也未返回评论数据'));
-                    comments = { comments: [], total: 0, status_code: 0 };
-                } else {
-                    console.log(chalk.green('✓ 从网络请求获取评论数据成功'));
-                }
-            } else {
-                console.log(chalk.green('✓ 从内存读取评论数据成功'));
-            }
-            const saved = await saveImagePostToDB(context, aweme, comments, uploads); // 传递完整 JSON
-            console.log(chalk.green.bold('✓ 图文作品保存成功'));
-            return { type: "image", ...saved };
-        } catch {
-
-        }
-
-        const commentRes = await commentPromise;
+        // 等待作品类型判定
         const firstType = await firstTypePromise;
 
-        if (!firstType) {
+        // 尝试从内存读取图文数据（如果是图文作品）
+        let memoryData: { aweme: any; comments: DouyinCommentResponse | null } | null = null;
+        try {
+            memoryData = await readPostMem(context, page);
+            console.log(chalk.green('✓ 从内存读取图文数据成功'));
+        } catch {
+            // 内存读取失败，稍后通过网络获取
+        }
+
+        if (!firstType && !memoryData) {
             console.error(chalk.red('✗ 既无法从内存读取数据,也无法从网络获得数据'));
             throw new ScrapeError('无法获取作品数据,可能是网络问题或作品已下架', 404, 'NO_DATA');
         }
 
-        console.log(chalk.cyan(`📡 检测到作品类型: ${chalk.bold(firstType.key === 'post' ? '图文' : '视频')}`));
+        console.log(chalk.cyan(`📡 检测到作品类型: ${chalk.bold(firstType?.key === 'post' || memoryData ? '图文' : '视频')}`));
 
-        let comments = commentRes && await safeJson<DouyinCommentResponse>(commentRes);
-        if (!comments) {
-            console.warn(chalk.yellow('⚠ 无法从内存读取评论数据，且网络请求也未返回评论数据'));
-            comments = { comments: [], total: 0, status_code: 0 };
+        let allComments: DouyinComment[] = [];
+        try {
+            // 开始滚动并收集评论
+            const commentResponses = await scrollAndCollectComments(context, page);
+
+            // 解析所有收集到的评论响应
+            for (const commentRes of commentResponses) {
+                try {
+                    const commentData = await safeJson<DouyinCommentResponse>(commentRes);
+                    if (commentData?.comments?.length) {
+                        allComments.push(...commentData.comments);
+                    }
+                } catch (e) {
+                    console.warn(chalk.yellow(`⚠ 解析评论响应失败: ${(e as Error)?.message}`));
+                }
+            }
+        } catch (error) {
+            console.warn(chalk.yellow(`⚠ 评论收集失败: ${(error as Error)?.message}`));
+        }
+
+
+        // 去重评论（根据 cid）
+        const uniqueComments = Array.from(
+            new Map(allComments.map(c => [c.cid, c])).values()
+        );
+
+        console.log(chalk.green(`✓ 共收集到 ${uniqueComments.length} 条独立评论（去重前: ${allComments.length}）`));
+
+        // 如果从内存读取到了评论，合并进来作为兜底
+        let comments: DouyinCommentResponse;
+        if (memoryData?.comments?.comments?.length) {
+            console.log(chalk.blue(`📝 合并内存中的 ${memoryData.comments.comments.length} 条评论`));
+            const memComments = memoryData.comments.comments;
+            const mergedMap = new Map(uniqueComments.map(c => [c.cid, c]));
+            for (const c of memComments) {
+                if (!mergedMap.has(c.cid)) {
+                    mergedMap.set(c.cid, c);
+                }
+            }
+            comments = {
+                comments: Array.from(mergedMap.values()),
+                total: mergedMap.size,
+                status_code: 0
+            };
+            console.log(chalk.green(`✓ 合并后共 ${comments.comments.length} 条评论`));
+        } else {
+            comments = {
+                comments: uniqueComments,
+                total: uniqueComments.length,
+                status_code: 0
+            };
         }
 
         // 分支：视频 or 图文（两者只会有一个命中，先到先得）
-        if (firstType.key === 'post') {
-            // 图文作品
+        // 优先处理内存数据（图文）
+        if (memoryData) {
+            const aweme = memoryData.aweme;
+            const uploads = await handleImagePost(context, aweme);
+            const saved = await saveImagePostToDB(context, aweme, comments, uploads); // 传递完整 JSON
+            console.log(chalk.green.bold('✓ 图文作品保存成功'));
+            return { type: "image", ...saved };
+        } else if (firstType?.key === 'post') {
+            // 图文作品（网络）
             const postJson = await safeJson<DouyinPostListResponse>(firstType.response);
             if (!postJson?.aweme_list?.length) throw new ScrapeError('图文作品响应为空', 404, 'EMPTY_POST_RESPONSE');
 
@@ -161,7 +256,7 @@ export async function scrapeDouyin(url: string) {
             const saved = await saveImagePostToDB(context, aweme, comments, uploads, postJson); // 传递完整 JSON
             console.log(chalk.green.bold('✓ 图文作品保存成功'));
             return { type: "image", ...saved };
-        } else if (firstType.key === 'detail') {
+        } else if (firstType?.key === 'detail') {
             // 视频作品
             const detail = (await safeJson<DouyinVideoDetailResponse>(firstType.response))!;
 
@@ -216,11 +311,11 @@ export async function scrapeDouyin(url: string) {
         if (error instanceof ScrapeError) {
             throw error;
         }
-        
+
         // 处理其他类型的错误
         const errMsg = (error as Error)?.message || String(error);
         console.error(chalk.red(`✗ 爬取失败: ${errMsg}`));
-        
+
         // 根据错误类型返回不同的状态码
         if (errMsg.includes('timeout') || errMsg.includes('超时')) {
             throw new ScrapeError('请求超时,请稍后重试', 408, 'TIMEOUT');
@@ -231,14 +326,14 @@ export async function scrapeDouyin(url: string) {
         if (errMsg.includes('net::')) {
             throw new ScrapeError('网络连接失败', 503, 'NETWORK_ERROR');
         }
-        
+
         // 默认服务器错误
         throw new ScrapeError(errMsg || '爬取过程中发生未知错误', 500, 'UNKNOWN_ERROR');
     } finally {
         console.log(chalk.gray('🧹 清理资源...'));
-        try { await page.close({ runBeforeUnload: true }); } catch {}
-        // 仅释放共享上下文的引用，不直接关闭窗口
-        await releaseBrowserContext();
+        try { await page.close({ runBeforeUnload: true }); } catch { }
+        // 关闭本次任务的隔离上下文与浏览器
+        await releaseIsolatedContext(context);
         await prisma.$disconnect();
         console.log(chalk.gray('✓ 资源清理完成'));
     }