添加动图支持,优化图片和视频上传逻辑,调整超时设置,增强评论时间格式化
This commit is contained in:
@@ -9,10 +9,18 @@ let refCount = 0
|
||||
let idleCloseTimer: NodeJS.Timeout | null = null
|
||||
|
||||
const USER_DATA_DIR = 'chrome-profile/douyin'
|
||||
const DEFAULT_OPTIONS = { headless: true } as const
|
||||
|
||||
async function launchContext(): Promise<BrowserContext> {
|
||||
const ctx = await chromium.launchPersistentContext(USER_DATA_DIR, DEFAULT_OPTIONS)
|
||||
const ctx = await chromium.launchPersistentContext(
|
||||
USER_DATA_DIR,
|
||||
{
|
||||
headless: Boolean(process.env.CHROMIUM_HEADLESS ?? 'false'),
|
||||
viewport: {
|
||||
width: Number(process.env.CHROMIUM_VIEWPORT_WIDTH ?? 1280),
|
||||
height: Number(process.env.CHROMIUM_VIEWPORT_HEIGHT ?? 1080)
|
||||
}
|
||||
}
|
||||
)
|
||||
// When the context is closed externally, reset manager state
|
||||
ctx.on('close', () => {
|
||||
context = null
|
||||
|
||||
@@ -84,11 +84,11 @@ export async function scrapeDouyin(url: string) {
|
||||
const firstTypePromise = waitForFirstResponse(context, [
|
||||
{ key: 'detail', test: (r: Response) => r.url().includes(DETAIL_PATH) && r.status() === 200 },
|
||||
{ key: 'post', test: (r: Response) => r.url().includes(POST_PATH) && r.status() === 200 },
|
||||
], 9_000); // 整体 9s 兜底超时,不逐个等待
|
||||
], 40_000);
|
||||
|
||||
// 评论只做短时“有就用、没有不等”的监听
|
||||
const commentPromise = waitForResponseWithTimeout(
|
||||
context, (r: Response) => r.url().includes(COMMENT_PATH) && r.status() === 200, 8_000
|
||||
context, (r: Response) => r.url().includes(COMMENT_PATH) && r.status() === 200, 40_000
|
||||
).catch(() => null);
|
||||
|
||||
await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 60_000 });
|
||||
|
||||
@@ -138,7 +138,7 @@ export async function saveImagePostToDB(
|
||||
context: BrowserContext,
|
||||
aweme: DouyinImageAweme,
|
||||
commentResp: DouyinCommentResponse,
|
||||
uploads: { images: { url: string; width?: number; height?: number }[]; musicUrl?: string },
|
||||
uploads: { images: { url: string; width?: number; height?: number, video?: string }[]; musicUrl?: string },
|
||||
rawJson?: any
|
||||
) {
|
||||
if (!aweme?.author?.sec_uid) throw new Error('作者 sec_uid 缺失');
|
||||
@@ -205,7 +205,7 @@ export async function saveImagePostToDB(
|
||||
|
||||
// Upsert ImageFiles(按顺序)
|
||||
for (let i = 0; i < uploads.images.length; i++) {
|
||||
const { url, width, height } = uploads.images[i];
|
||||
const { url, width, height, video } = uploads.images[i];
|
||||
await prisma.imageFile.upsert({
|
||||
where: { postId_order: { postId: imagePost.aweme_id, order: i } },
|
||||
create: {
|
||||
@@ -214,11 +214,13 @@ export async function saveImagePostToDB(
|
||||
url,
|
||||
width: typeof width === 'number' ? width : null,
|
||||
height: typeof height === 'number' ? height : null,
|
||||
animated: video || null,
|
||||
},
|
||||
update: {
|
||||
url,
|
||||
width: typeof width === 'number' ? width : null,
|
||||
height: typeof height === 'number' ? height : null,
|
||||
animated: video || null,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
Vendored
+3
@@ -126,6 +126,9 @@ interface DouyinImageInfo {
|
||||
download_url_list?: string[]; // 可能带水印
|
||||
width: number;
|
||||
height: number;
|
||||
video: {
|
||||
play_addr: { src: string }[]
|
||||
} | null; // 如果是动图,会有 video 信息
|
||||
}
|
||||
|
||||
/** 音乐基本信息(精简) */
|
||||
|
||||
@@ -31,7 +31,7 @@ export async function handleImagePost(
|
||||
aweme: DouyinImageAweme
|
||||
): Promise<{ images: { url: string; width?: number; height?: number }[]; musicUrl?: string }> {
|
||||
const awemeId = aweme.aweme_id;
|
||||
const uploadedImages: { url: string; width?: number; height?: number }[] = [];
|
||||
const uploadedImages: { url: string; width?: number; height?: number, video?: string }[] = [];
|
||||
|
||||
// 下载图片(顺序保持)
|
||||
for (let i = 0; i < (aweme.images?.length || 0); i++) {
|
||||
@@ -42,7 +42,25 @@ export async function handleImagePost(
|
||||
const safeExt = ext || 'jpg';
|
||||
const fileName = generateUniqueFileName(`${awemeId}/${i}.${safeExt}`, 'douyin/images');
|
||||
const uploaded = await uploadFile(buffer, fileName, { 'Content-Type': contentType });
|
||||
uploadedImages.push({ url: uploaded, width: img?.width, height: img?.height });
|
||||
|
||||
if (img.video?.play_addr) {
|
||||
// 如果是动图,下载 video 并上传
|
||||
const videoUrl = img.video.play_addr[0]?.src;
|
||||
if (videoUrl) {
|
||||
try {
|
||||
const { buffer: videoBuffer, contentType: videoContentType, ext: videoExt } = await downloadBinary(context, videoUrl);
|
||||
const safeVideoExt = videoExt || 'mp4';
|
||||
const videoFileName = generateUniqueFileName(`${awemeId}/${i}_animated.${safeVideoExt}`, 'douyin/images');
|
||||
const uploadedVideo = await uploadFile(videoBuffer, videoFileName, { 'Content-Type': videoContentType });
|
||||
// 将动图的 video URL 也存储起来
|
||||
uploadedImages.push({ url: uploaded, width: img?.width, height: img?.height, video: uploadedVideo });
|
||||
} catch (e) {
|
||||
console.warn(`[image] 动图视频上传失败,跳过:`, (e as Error)?.message || e);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
uploadedImages.push({ url: uploaded, width: img?.width, height: img?.height });
|
||||
}
|
||||
}
|
||||
|
||||
// 下载音乐(可选)
|
||||
|
||||
Reference in New Issue
Block a user