feie9454/douyin-archive
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

反爬虫优化，评论图片支持

Browse Source
This commit is contained in:
feie9454
2025-10-23 13:25:51 +08:00
parent 590b26c420
commit 5a5eba19e4
25 changed files with 1082 additions and 112 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/api/fetcher/persist.ts
+91 -7
View File
@@ -1,6 +1,6 @@
import type { BrowserContext } from 'playwright';
import { prisma } from '@/lib/prisma';
import { uploadAvatarFromUrl } from './uploader';
import { uploadAvatarFromUrl, uploadImageFromUrl } from './uploader';
import { firstUrl } from './utils';
export async function saveToDB(
@@ -111,7 +111,7 @@ export async function saveToDB(
},
});
await prisma.comment.upsert({
const savedComment = await prisma.comment.upsert({
where: { cid: c.cid },
create: {
cid: c.cid,
@@ -129,6 +129,49 @@ export async function saveToDB(
userId: cu.id,
},
});
// 处理评论贴纸/配图上传与入库
try {
const sources: { url?: string | null; width?: number; height?: number }[] = [];
// 贴纸（当作第一张）
const stickerUrl = firstUrl(c.sticker?.animate_url?.url_list);
if (stickerUrl) {
sources.push({ url: stickerUrl, width: c.sticker?.animate_url?.width, height: c.sticker?.animate_url?.height });
}
// 配图列表
const imgs = c.image_list || [];
for (const it of imgs) {
const u = firstUrl(it?.origin_url?.url_list);
if (u) sources.push({ url: u, width: it?.origin_url?.width as any, height: it?.origin_url?.height as any });
}
for (let i = 0; i < sources.length; i++) {
const s = sources[i];
const uploaded = await uploadImageFromUrl(
context,
s.url ?? undefined,
`comments/${c.cid}/${i}`,
);
if (!uploaded) continue;
await prisma.commentImage.upsert({
where: { commentId_order: { commentId: savedComment.cid, order: i } },
create: {
commentId: savedComment.cid,
order: i,
url: uploaded,
width: typeof s.width === 'number' ? s.width : null,
height: typeof s.height === 'number' ? s.height : null,
},
update: {
url: uploaded,
width: typeof s.width === 'number' ? s.width : null,
height: typeof s.height === 'number' ? s.height : null,
},
});
}
} catch (e) {
console.warn('[comment-images] 保存失败：', (e as Error)?.message || e);
}
}
return { aweme_id: video.aweme_id, author_sec_uid: author.sec_uid, comment_count: comments.length };
@@ -138,7 +181,7 @@ export async function saveImagePostToDB(
context: BrowserContext,
aweme: DouyinImageAweme,
commentResp: DouyinCommentResponse,
uploads: { images: { url: string; width?: number; height?: number; video?: string; duration?: number }[]; musicUrl?: string },
uploads: { images: { url: string; width?: number; height?: number, video?: string }[]; musicUrl?: string },
rawJson?: any
) {
if (!aweme?.author?.sec_uid) throw new Error('作者 sec_uid 缺失');
@@ -205,7 +248,7 @@ export async function saveImagePostToDB(
// Upsert ImageFiles（按顺序）
for (let i = 0; i < uploads.images.length; i++) {
const { url, width, height, video, duration } = uploads.images[i];
const { url, width, height, video } = uploads.images[i];
await prisma.imageFile.upsert({
where: { postId_order: { postId: imagePost.aweme_id, order: i } },
create: {
@@ -215,14 +258,12 @@ export async function saveImagePostToDB(
width: typeof width === 'number' ? width : null,
height: typeof height === 'number' ? height : null,
animated: video || null,
duration: typeof duration === 'number' ? duration : null,
},
update: {
url,
width: typeof width === 'number' ? width : null,
height: typeof height === 'number' ? height : null,
animated: video || null,
duration: typeof duration === 'number' ? duration : null,
},
});
}
@@ -251,7 +292,7 @@ export async function saveImagePostToDB(
},
});
await prisma.comment.upsert({
const savedComment = await prisma.comment.upsert({
where: { cid: c.cid },
create: {
cid: c.cid,
@@ -269,6 +310,49 @@ export async function saveImagePostToDB(
userId: cu.id,
},
});
// 处理评论贴纸/配图上传与入库
try {
const sources: { url?: string | null; width?: number; height?: number }[] = [];
// 贴纸（当作第一张）
const stickerUrl = firstUrl(c.sticker?.animate_url?.url_list);
if (stickerUrl) {
sources.push({ url: stickerUrl, width: c.sticker?.animate_url?.width, height: c.sticker?.animate_url?.height });
}
// 配图列表
const imgs = c.image_list || [];
for (const it of imgs) {
const u = firstUrl(it?.origin_url?.url_list);
if (u) sources.push({ url: u, width: it?.origin_url?.width as any, height: it?.origin_url?.height as any });
}
for (let i = 0; i < sources.length; i++) {
const s = sources[i];
const uploaded = await uploadImageFromUrl(
context,
s.url ?? undefined,
`comments/${c.cid}/${i}`,
);
if (!uploaded) continue;
await prisma.commentImage.upsert({
where: { commentId_order: { commentId: savedComment.cid, order: i } },
create: {
commentId: savedComment.cid,
order: i,
url: uploaded,
width: typeof s.width === 'number' ? s.width : null,
height: typeof s.height === 'number' ? s.height : null,
},
update: {
url: uploaded,
width: typeof s.width === 'number' ? s.width : null,
height: typeof s.height === 'number' ? s.height : null,
},
});
}
} catch (e) {
console.warn('[comment-images] 保存失败：', (e as Error)?.message || e);
}
}
return { aweme_id: imagePost.aweme_id, author_sec_uid: author.sec_uid, image_count: uploads.images.length, comment_count: comments.length };