反爬虫优化,评论图片支持

This commit is contained in:
2025-10-23 13:25:51 +08:00
parent 590b26c420
commit 5a5eba19e4
25 changed files with 1082 additions and 112 deletions
+47 -3
View File
@@ -1,3 +1,5 @@
export const runtime = 'nodejs'
import type { BrowserContext, Response } from 'playwright';
export async function safeJson<T>(res: Response): Promise<T | null> {
@@ -20,12 +22,12 @@ export async function safeJson<T>(res: Response): Promise<T | null> {
*/
export async function downloadBinary(
context: BrowserContext,
url: string
url: string,
): Promise<{ buffer: Buffer; contentType: string; ext: string }> {
console.log('下载:', url);
const headers = {
referer: url,
referer: 'https://www.douyin.com/',
} as Record<string, string>;
const res = await context.request.get(url, {
@@ -93,7 +95,49 @@ export function waitForFirstResponse(
}
/**
* 等待符合条件的单个 Response,带短超时;用于评论等“可有可无”的数据。
* 在指定时间内持续收集所有符合条件的 Response
* 用于评论等需要滚动加载的数据
*/
export function collectResponsesWithinTime(
context: BrowserContext,
predicate: (r: Response) => boolean,
durationMs: number
): Promise<Response[]> {
return new Promise((resolve) => {
const collected: Response[] = [];
const seenUrls = new Set<string>();
let timer: NodeJS.Timeout | undefined;
const handler = (res: Response) => {
try {
if (predicate(res)) {
// 使用 URL 去重,避免重复收集同一个请求
const url = res.url();
if (!seenUrls.has(url)) {
seenUrls.add(url);
collected.push(res);
}
}
} catch {
// ignore predicate errors
}
};
const cleanup = () => {
context.off('response', handler);
if (timer) clearTimeout(timer);
};
context.on('response', handler);
timer = setTimeout(() => {
cleanup();
resolve(collected);
}, durationMs);
});
}
/**
* 等待符合条件的单个 Response,带短超时;用于评论等"可有可无"的数据。
*/
export function waitForResponseWithTimeout(
context: BrowserContext,