反爬虫优化,评论图片支持
This commit is contained in:
@@ -1,3 +1,5 @@
|
||||
export const runtime = 'nodejs'
|
||||
|
||||
import type { BrowserContext, Response } from 'playwright';
|
||||
|
||||
export async function safeJson<T>(res: Response): Promise<T | null> {
|
||||
@@ -20,12 +22,12 @@ export async function safeJson<T>(res: Response): Promise<T | null> {
|
||||
*/
|
||||
export async function downloadBinary(
|
||||
context: BrowserContext,
|
||||
url: string
|
||||
url: string,
|
||||
): Promise<{ buffer: Buffer; contentType: string; ext: string }> {
|
||||
console.log('下载:', url);
|
||||
|
||||
const headers = {
|
||||
referer: url,
|
||||
referer: 'https://www.douyin.com/',
|
||||
} as Record<string, string>;
|
||||
|
||||
const res = await context.request.get(url, {
|
||||
@@ -93,7 +95,49 @@ export function waitForFirstResponse(
|
||||
}
|
||||
|
||||
/**
|
||||
* 等待符合条件的单个 Response,带短超时;用于评论等“可有可无”的数据。
|
||||
* 在指定时间内持续收集所有符合条件的 Response
|
||||
* 用于评论等需要滚动加载的数据
|
||||
*/
|
||||
export function collectResponsesWithinTime(
|
||||
context: BrowserContext,
|
||||
predicate: (r: Response) => boolean,
|
||||
durationMs: number
|
||||
): Promise<Response[]> {
|
||||
return new Promise((resolve) => {
|
||||
const collected: Response[] = [];
|
||||
const seenUrls = new Set<string>();
|
||||
let timer: NodeJS.Timeout | undefined;
|
||||
|
||||
const handler = (res: Response) => {
|
||||
try {
|
||||
if (predicate(res)) {
|
||||
// 使用 URL 去重,避免重复收集同一个请求
|
||||
const url = res.url();
|
||||
if (!seenUrls.has(url)) {
|
||||
seenUrls.add(url);
|
||||
collected.push(res);
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// ignore predicate errors
|
||||
}
|
||||
};
|
||||
|
||||
const cleanup = () => {
|
||||
context.off('response', handler);
|
||||
if (timer) clearTimeout(timer);
|
||||
};
|
||||
|
||||
context.on('response', handler);
|
||||
timer = setTimeout(() => {
|
||||
cleanup();
|
||||
resolve(collected);
|
||||
}, durationMs);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* 等待符合条件的单个 Response,带短超时;用于评论等"可有可无"的数据。
|
||||
*/
|
||||
export function waitForResponseWithTimeout(
|
||||
context: BrowserContext,
|
||||
|
||||
Reference in New Issue
Block a user