添加动图支持,优化图片和视频上传逻辑,调整超时设置,增强评论时间格式化

This commit is contained in:
2025-10-23 00:58:56 +08:00
parent fe9bc8fd6c
commit 77bc2c5775
15 changed files with 82 additions and 31 deletions
+10 -2
View File
@@ -9,10 +9,18 @@ let refCount = 0
let idleCloseTimer: NodeJS.Timeout | null = null
const USER_DATA_DIR = 'chrome-profile/douyin'
const DEFAULT_OPTIONS = { headless: true } as const
async function launchContext(): Promise<BrowserContext> {
const ctx = await chromium.launchPersistentContext(USER_DATA_DIR, DEFAULT_OPTIONS)
const ctx = await chromium.launchPersistentContext(
USER_DATA_DIR,
{
headless: Boolean(process.env.CHROMIUM_HEADLESS ?? 'false'),
viewport: {
width: Number(process.env.CHROMIUM_VIEWPORT_WIDTH ?? 1280),
height: Number(process.env.CHROMIUM_VIEWPORT_HEIGHT ?? 1080)
}
}
)
// When the context is closed externally, reset manager state
ctx.on('close', () => {
context = null
+2 -2
View File
@@ -84,11 +84,11 @@ export async function scrapeDouyin(url: string) {
const firstTypePromise = waitForFirstResponse(context, [
{ key: 'detail', test: (r: Response) => r.url().includes(DETAIL_PATH) && r.status() === 200 },
{ key: 'post', test: (r: Response) => r.url().includes(POST_PATH) && r.status() === 200 },
], 9_000); // 整体 9s 兜底超时,不逐个等待
], 40_000);
// 评论只做短时“有就用、没有不等”的监听
const commentPromise = waitForResponseWithTimeout(
context, (r: Response) => r.url().includes(COMMENT_PATH) && r.status() === 200, 8_000
context, (r: Response) => r.url().includes(COMMENT_PATH) && r.status() === 200, 40_000
).catch(() => null);
await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 60_000 });
+4 -2
View File
@@ -138,7 +138,7 @@ export async function saveImagePostToDB(
context: BrowserContext,
aweme: DouyinImageAweme,
commentResp: DouyinCommentResponse,
uploads: { images: { url: string; width?: number; height?: number }[]; musicUrl?: string },
uploads: { images: { url: string; width?: number; height?: number, video?: string }[]; musicUrl?: string },
rawJson?: any
) {
if (!aweme?.author?.sec_uid) throw new Error('作者 sec_uid 缺失');
@@ -205,7 +205,7 @@ export async function saveImagePostToDB(
// Upsert ImageFiles(按顺序)
for (let i = 0; i < uploads.images.length; i++) {
const { url, width, height } = uploads.images[i];
const { url, width, height, video } = uploads.images[i];
await prisma.imageFile.upsert({
where: { postId_order: { postId: imagePost.aweme_id, order: i } },
create: {
@@ -214,11 +214,13 @@ export async function saveImagePostToDB(
url,
width: typeof width === 'number' ? width : null,
height: typeof height === 'number' ? height : null,
animated: video || null,
},
update: {
url,
width: typeof width === 'number' ? width : null,
height: typeof height === 'number' ? height : null,
animated: video || null,
},
});
}
+3
View File
@@ -126,6 +126,9 @@ interface DouyinImageInfo {
download_url_list?: string[]; // 可能带水印
width: number;
height: number;
video: {
play_addr: { src: string }[]
} | null; // 如果是动图,会有 video 信息
}
/** 音乐基本信息(精简) */
+20 -2
View File
@@ -31,7 +31,7 @@ export async function handleImagePost(
aweme: DouyinImageAweme
): Promise<{ images: { url: string; width?: number; height?: number }[]; musicUrl?: string }> {
const awemeId = aweme.aweme_id;
const uploadedImages: { url: string; width?: number; height?: number }[] = [];
const uploadedImages: { url: string; width?: number; height?: number, video?: string }[] = [];
// 下载图片(顺序保持)
for (let i = 0; i < (aweme.images?.length || 0); i++) {
@@ -42,7 +42,25 @@ export async function handleImagePost(
const safeExt = ext || 'jpg';
const fileName = generateUniqueFileName(`${awemeId}/${i}.${safeExt}`, 'douyin/images');
const uploaded = await uploadFile(buffer, fileName, { 'Content-Type': contentType });
uploadedImages.push({ url: uploaded, width: img?.width, height: img?.height });
if (img.video?.play_addr) {
// 如果是动图,下载 video 并上传
const videoUrl = img.video.play_addr[0]?.src;
if (videoUrl) {
try {
const { buffer: videoBuffer, contentType: videoContentType, ext: videoExt } = await downloadBinary(context, videoUrl);
const safeVideoExt = videoExt || 'mp4';
const videoFileName = generateUniqueFileName(`${awemeId}/${i}_animated.${safeVideoExt}`, 'douyin/images');
const uploadedVideo = await uploadFile(videoBuffer, videoFileName, { 'Content-Type': videoContentType });
// 将动图的 video URL 也存储起来
uploadedImages.push({ url: uploaded, width: img?.width, height: img?.height, video: uploadedVideo });
} catch (e) {
console.warn(`[image] 动图视频上传失败,跳过:`, (e as Error)?.message || e);
}
}
} else {
uploadedImages.push({ url: uploaded, width: img?.width, height: img?.height });
}
}
// 下载音乐(可选)