init

app/fetcher/index.ts

@@ -0,0 +1,170 @@
+// src/scrapeDouyin.ts
+import { chromium, type Response } from 'playwright';
+import { prisma } from '@/lib/prisma';
+import { uploadFile, generateUniqueFileName } from '@/lib/minio';
+import { createCamelCompatibleProxy } from '@/app/fetcher/utils';
+import { waitForFirstResponse, waitForResponseWithTimeout, safeJson, downloadBinary } from '@/app/fetcher/network';
+import { pickBestPlayAddr, extractFirstFrame } from '@/app/fetcher/media';
+import { handleImagePost } from '@/app/fetcher/uploader';
+import { saveToDB, saveImagePostToDB } from '@/app/fetcher/persist';
+
+const DETAIL_PATH = '/aweme/v1/web/aweme/detail/';
+const COMMENT_PATH = '/aweme/v1/web/comment/list/';
+const POST_PATH = '/aweme/v1/web/aweme/post/'
+export async function scrapeDouyin(url: string) {
+    const browser = await chromium.launch({ headless: true });
+    console.log("Launch chromium");
+
+    const context = await chromium.launchPersistentContext('chrome-profile/douyin', { headless: true });
+    const page = await context.newPage();
+
+    await page.addInitScript(() => {
+        // 建一个全局容器存捕获的数据
+        (window as any).__pace_captured__ = [];
+
+        // 用 Proxy 包装一个数组，拦截 push
+        const captured = (window as any).__pace_captured__;
+        const proxyArr = new Proxy([] as any[], {
+            get(target, prop, receiver) {
+                if (prop === 'push') {
+                    return (...items: any[]) => {
+                        try { captured.push(...items); } catch { }
+                        return Array.prototype.push.apply(target, items);
+                    };
+                }
+                return Reflect.get(target, prop, receiver);
+            },
+            set(target, prop, value, receiver) {
+                // 兼容站点可能直接赋初始数组： self.__pace_f = [a,b]
+                if (prop === 'length') return Reflect.set(target, prop, value, receiver);
+                return Reflect.set(target, prop, value, receiver);
+            }
+        });
+
+        // 把 self/window 上的同名队列都指向我们的 proxy
+        // 有些站点用 self，有些用 window
+        (self as any).__pace_f = proxyArr;
+        (window as any).__pace_f = proxyArr;
+    });
+
+    try {
+        // 先注册“先到先得”的监听，再导航，避免漏包
+        const firstTypePromise = waitForFirstResponse(context, [
+            { key: 'detail', test: (r: Response) => r.url().includes(DETAIL_PATH) && r.status() === 200 },
+            { key: 'post', test: (r: Response) => r.url().includes(POST_PATH) && r.status() === 200 },
+        ], 20_000); // 整体 20s 兜底超时，不逐个等待
+
+        // 评论只做短时“有就用、没有不等”的监听
+        const commentPromise = waitForResponseWithTimeout(
+            context,
+            (r: Response) => r.url().includes(COMMENT_PATH) && r.status() === 200,
+            8_000
+        ).catch(() => null);
+
+        await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 60_000 });
+
+        const firstType = await firstTypePromise; // { key, response } | null
+        const commentRes = await commentPromise;  // Response | null
+
+        if (!firstType) {
+            console.warn('无法判定作品类型（未捕获详情或图文接口）');
+            const md = await page.evaluate(() => {
+                // @ts-ignore
+                let data = window.__pace_captured__.find(i => i[1] && i[1].includes(`"awemeId":`))[1]
+                return JSON.parse(data.slice(data.indexOf("{")).replaceAll("]\n", ''))
+                // return {aweme: { detail: {} } };
+            });
+            let aweme_mem = md.aweme.detail as DouyinImageAweme;
+            if (!aweme_mem) throw new Error('页面内存数据中未找到作品详情');
+
+            //@ts-ignore
+            aweme_mem.author = aweme_mem.authorInfo
+            const comments = commentRes ? (await safeJson<DouyinCommentResponse>(commentRes))! : { comments: [], total: 0, status_code: 0 };
+            
+            const aweme = createCamelCompatibleProxy(aweme_mem);
+
+            const uploads = await handleImagePost(context, aweme);
+            const saved = await saveImagePostToDB(context, aweme, comments, uploads);
+            return { type: "image", ...saved };
+        }
+
+        // 分支：视频 or 图文（两者只会有一个命中，先到先得）
+        if (firstType.key === 'post') {
+            // 图文作品
+            const postJson = await safeJson<DouyinPostListResponse>(firstType.response);
+            if (!postJson?.aweme_list?.length) throw new Error('图文作品响应为空');
+
+            const currentURL = page.url();
+            const target_aweme_id = currentURL.split('/').at(-1);
+            const awemeList = postJson.aweme_list as unknown as DouyinImageAweme[];
+            let aweme = awemeList.find((pt: DouyinImageAweme) => pt.aweme_id === target_aweme_id);
+            if (!aweme) {
+                console.warn(`图文作品响应中未找到对应作品，look for aweme_id=${target_aweme_id}, have ${postJson.aweme_list.map(pt => pt.aweme_id).join(', ')}`);
+                // Try read from memory
+                // await new Promise(resolve => setTimeout(resolve, 1000000));
+                const md = await page.evaluate(() => {
+                    // @ts-ignore
+                    let data = window.__pace_captured__.find(i => i[1] && i[1].includes(`"awemeId":`))[1]
+                    return JSON.parse(data.slice(data.indexOf("{")).replaceAll("]\n", ''))
+                    // return {aweme: { detail: {} } };
+                });
+                aweme = md.aweme.detail as DouyinImageAweme;
+            }
+            // console.log(aweme);
+            // await new Promise(resolve => setTimeout(resolve, 1000000)); 
+            console.log(aweme);
+
+
+            const comments = commentRes ? (await safeJson<DouyinCommentResponse>(commentRes))! : { comments: [], total: 0, status_code: 0 };
+
+            const uploads = await handleImagePost(context, aweme);
+            const saved = await saveImagePostToDB(context, aweme, comments, uploads);
+            return { type: "image", ...saved };
+        } else if (firstType.key === 'detail') {
+            // 视频作品
+            const detail = (await safeJson<DouyinVideoDetailResponse>(firstType.response))!;
+            const comments = commentRes ? (await safeJson<DouyinCommentResponse>(commentRes))! : { comments: [], total: 0, status_code: 0 };
+
+            // 找到比特率最高的 url
+            const bestPlayAddr = pickBestPlayAddr(
+                detail?.aweme_detail?.video.bit_rate
+            );
+            const bestVUrl = bestPlayAddr?.url_list?.[0];
+
+            console.log('Best video URL:', bestVUrl);
+
+            // 下载视频并上传至 MinIO，获取外链
+            let uploadedUrl: string | undefined;
+            let coverUrl: string | undefined;
+            if (bestVUrl && detail?.aweme_detail) {
+                const { buffer, contentType, ext } = await downloadBinary(context, bestVUrl);
+                const awemeId = detail.aweme_detail.aweme_id;
+                const fileName = generateUniqueFileName(`${awemeId}.${ext}`, 'douyin/videos');
+                uploadedUrl = await uploadFile(buffer, fileName, { 'Content-Type': contentType });
+                console.log('Uploaded to MinIO:', uploadedUrl);
+
+                // 提取首帧作为封面并上传
+                try {
+                    const cover = await extractFirstFrame(buffer);
+                    if (cover) {
+                        const coverName = generateUniqueFileName(`${awemeId}.jpg`, 'douyin/covers');
+                        coverUrl = await uploadFile(cover.buffer, coverName, { 'Content-Type': cover.contentType });
+                        console.log('Cover uploaded to MinIO:', coverUrl);
+                    }
+                } catch (e) {
+                    console.warn('Extract first frame failed, skip cover:', (e as Error)?.message || e);
+                }
+            }
+
+            const saved = await saveToDB(context, detail, comments, uploadedUrl, bestPlayAddr?.width, bestPlayAddr?.height, coverUrl);
+            return { type: "video", ...saved };
+        } else {
+            throw new Error('无法判定作品类型（未命中详情或图文接口）');
+        }
+    } finally {
+        await context.close();
+        await browser.close();
+        await prisma.$disconnect();
+    }
+}
+

app/fetcher/media.ts

@@ -0,0 +1,57 @@
+import { promises as fs } from 'fs';
+import os from 'os';
+import path from 'path';
+import { execFile } from 'child_process';
+import { promisify } from 'util';
+
+const execFileAsync = promisify(execFile);
+
+export function pickBestPlayAddr(variants: PlayVariant[] | undefined | null) {
+    if (!variants?.length) return null;
+
+    const best = variants.reduce((best, cur) => {
+        const b1 = best?.bit_rate ?? -1;
+        const b2 = cur?.bit_rate ?? -1;
+        return b2 > b1 ? cur : best;
+    });
+
+    return best?.play_addr ?? null;
+}
+
+/**
+ * 使用 ffmpeg 从视频二进制中提取第一帧，返回 JPEG buffer
+ */
+export async function extractFirstFrame(videoBuffer: Buffer): Promise<{ buffer: Buffer; contentType: string; ext: string } | null> {
+    const ffmpegCmd = process.env.FFMPEG_PATH || 'ffmpeg';
+    const tmpDir = os.tmpdir();
+    const base = `dy_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`;
+    const inPath = path.join(tmpDir, `${base}.mp4`);
+    const outPath = path.join(tmpDir, `${base}.jpg`);
+
+    try {
+        await fs.writeFile(inPath, videoBuffer);
+        const args = [
+            '-hide_banner',
+            '-loglevel', 'error',
+            '-ss', '0',
+            '-i', inPath,
+            '-frames:v', '1',
+            '-q:v', '2',
+            '-f', 'image2',
+            '-y',
+            outPath,
+        ];
+        await execFileAsync(ffmpegCmd, args, { windowsHide: true });
+        const img = await fs.readFile(outPath);
+        return { buffer: img, contentType: 'image/jpeg', ext: 'jpg' };
+    } catch (e: any) {
+        if (e && (e.code === 'ENOENT' || /not found|is not recognized/i.test(String(e.message)))) {
+            console.warn('系统未检测到 ffmpeg，可安装并配置 PATH 或设置 FFMPEG_PATH 后启用封面提取。');
+            return null;
+        }
+        throw e;
+    } finally {
+        try { await fs.unlink(inPath); } catch { }
+        try { await fs.unlink(outPath); } catch { }
+    }
+}

app/fetcher/network.ts

@@ -0,0 +1,130 @@
+import type { BrowserContext, Response } from 'playwright';
+
+export async function safeJson<T>(res: Response): Promise<T | null> {
+    const ctype = res.headers()['content-type'] || '';
+    if (ctype.includes('application/json')) {
+        return (await res.json()) as T;
+    }
+    const t = await res.text();
+    try {
+        return JSON.parse(t) as T;
+    } catch {
+        return null;
+    }
+}
+
+/**
+ * 使用 Playwright 的 APIRequestContext 下载二进制内容
+ * - 使用指定 headers 模拟浏览器请求
+ * - referrer 使用链接本身
+ */
+export async function downloadBinary(
+    context: BrowserContext,
+    url: string
+): Promise<{ buffer: Buffer; contentType: string; ext: string }> {
+    console.log('Download bin:', url);
+
+    const headers = {
+        referer: url,
+    } as Record<string, string>;
+
+    const res = await context.request.get(url, {
+        headers,
+        maxRedirects: 3,
+        timeout: 240_000,
+        failOnStatusCode: true,
+    });
+
+    if (!res.ok()) {
+        throw new Error(`下载内容失败: ${res.status()} ${res.statusText()}`);
+    }
+
+    const buffer = await res.body();
+    const contentType = res.headers()['content-type'] || 'application/octet-stream';
+    const ext = (contentType.split('/')[1] || 'bin').split(';')[0] || 'bin';
+    return { buffer, contentType, ext };
+}
+
+/**
+ * 在多个候选匹配中“先到先得”地返回首个命中的 Response。
+ * - 不为每个候选单独设长超时，改用整体兜底超时，避免无意义等待。
+ */
+export function waitForFirstResponse(
+    context: BrowserContext,
+    candidates: { key: string; test: (r: Response) => boolean }[],
+    timeoutMs = 20_000
+): Promise<{ key: string; response: Response } | null> {
+    return new Promise((resolve) => {
+        let resolved = false;
+        let timer: NodeJS.Timeout | undefined;
+
+        const handler = (res: Response) => {
+            if (resolved) return;
+            for (const c of candidates) {
+                try {
+                    if (c.test(res)) {
+                        resolved = true;
+                        cleanup();
+                        resolve({ key: c.key, response: res });
+                        return;
+                    }
+                } catch {
+                    // ignore predicate errors
+                }
+            }
+        };
+
+        const cleanup = () => {
+            context.off('response', handler);
+            if (timer) clearTimeout(timer);
+        };
+
+        context.on('response', handler);
+        if (timeoutMs > 0) {
+            timer = setTimeout(() => {
+                if (!resolved) {
+                    resolved = true;
+                    cleanup();
+                    resolve(null);
+                }
+            }, timeoutMs);
+        }
+    });
+}
+
+/**
+ * 等待符合条件的单个 Response，带短超时；用于评论等“可有可无”的数据。
+ */
+export function waitForResponseWithTimeout(
+    context: BrowserContext,
+    predicate: (r: Response) => boolean,
+    timeoutMs = 5_000
+): Promise<Response> {
+    return new Promise<Response>((resolve, reject) => {
+        let timer: NodeJS.Timeout | undefined;
+
+        const handler = (res: Response) => {
+            try {
+                if (predicate(res)) {
+                    cleanup();
+                    resolve(res);
+                }
+            } catch {
+                // ignore predicate errors
+            }
+        };
+
+        const cleanup = () => {
+            context.off('response', handler);
+            if (timer) clearTimeout(timer);
+        };
+
+        context.on('response', handler);
+        if (timeoutMs > 0) {
+            timer = setTimeout(() => {
+                cleanup();
+                reject(new Error('timeout'));
+            }, timeoutMs);
+        }
+    });
+}

app/fetcher/persist.ts

@@ -0,0 +1,263 @@
+import type { BrowserContext } from 'playwright';
+import { prisma } from '@/lib/prisma';
+import { uploadAvatarFromUrl } from './uploader';
+import { firstUrl } from './utils';
+
+export async function saveToDB(
+    context: BrowserContext,
+    detailResp: DouyinVideoDetailResponse,
+    commentResp: DouyinCommentResponse,
+    videoUrl?: string,
+    width?: number,
+    height?: number,
+    coverUrl?: string
+) {
+    if (!detailResp?.aweme_detail) throw new Error('视频详情为空');
+    const d = detailResp.aweme_detail;
+
+    // 1) Upsert Author
+    const authorAvatarSrc = firstUrl(d.author.avatar_thumb?.url_list);
+    const authorAvatarUploaded = await uploadAvatarFromUrl(context, authorAvatarSrc, `authors/${d.author.sec_uid}`);
+    const author = await prisma.author.upsert({
+        where: { sec_uid: d.author.sec_uid },
+        create: {
+            sec_uid: d.author.sec_uid,
+            uid: d.author.uid,
+            nickname: d.author.nickname,
+            signature: d.author.signature ?? null,
+            avatar_url: authorAvatarUploaded ?? null,
+            follower_count: BigInt(d.author.follower_count || 0),
+            total_favorited: BigInt(d.author.total_favorited || 0),
+            unique_id: d.author.unique_id ?? null,
+            short_id: d.author.short_id ?? null,
+        },
+        update: {
+            uid: d.author.uid,
+            nickname: d.author.nickname,
+            signature: d.author.signature ?? null,
+            avatar_url: authorAvatarUploaded ?? null,
+            follower_count: BigInt(d.author.follower_count || 0),
+            total_favorited: BigInt(d.author.total_favorited || 0),
+            unique_id: d.author.unique_id ?? null,
+            short_id: d.author.short_id ?? null,
+        },
+    });
+
+    // 2) Upsert Video
+    const video = await prisma.video.upsert({
+        where: { aweme_id: d.aweme_id },
+        create: {
+            aweme_id: d.aweme_id,
+            desc: d.desc,
+            preview_title: d.preview_title ?? null,
+            duration_ms: d.duration,
+            created_at: new Date((d.create_time || 0) * 1000),
+            share_url: d.share_url,
+            digg_count: BigInt(d.statistics?.digg_count || 0),
+            comment_count: BigInt(d.statistics?.comment_count || 0),
+            share_count: BigInt(d.statistics?.share_count || 0),
+            collect_count: BigInt(d.statistics?.collect_count || 0),
+            authorId: author.sec_uid,
+            tags: (d.tags?.map(t => t.tag_name) ?? []),
+            video_url: videoUrl ?? '',
+            width: width ?? null,
+            height: height ?? null,
+            cover_url: coverUrl ?? null,
+        },
+        update: {
+            desc: d.desc,
+            preview_title: d.preview_title ?? null,
+            duration_ms: d.duration,
+            created_at: new Date((d.create_time || 0) * 1000),
+            share_url: d.share_url,
+            digg_count: BigInt(d.statistics?.digg_count || 0),
+            comment_count: BigInt(d.statistics?.comment_count || 0),
+            share_count: BigInt(d.statistics?.share_count || 0),
+            collect_count: BigInt(d.statistics?.collect_count || 0),
+            authorId: author.sec_uid,
+            ...(videoUrl ? { video_url: videoUrl } : {}),
+            ...(width ? { width } : {}),
+            ...(height ? { height } : {}),
+            ...(coverUrl ? { cover_url: coverUrl } : {}),
+        },
+    });
+
+    // 3) Upsert Comments + CommentUser
+    const comments = commentResp?.comments ?? [];
+    for (const c of comments) {
+        const origAvatar: string | null = firstUrl(c.user?.avatar_thumb?.url_list) ?? null;
+        const nameHint = `comment-users/${(c.user?.nickname || 'unknown').replace(/\s+/g, '_')}-${c.cid}`;
+        const uploadedAvatar = await uploadAvatarFromUrl(context, origAvatar ?? undefined, nameHint);
+        const finalAvatar = uploadedAvatar ?? origAvatar; // string | null
+        const finalAvatarKey = finalAvatar ?? '';
+        const cu = await prisma.commentUser.upsert({
+            where: {
+                nickname_avatar_url: {
+                    nickname: c.user?.nickname || '未知用户',
+                    avatar_url: finalAvatarKey,
+                },
+            },
+            create: {
+                nickname: c.user?.nickname || '未知用户',
+                avatar_url: finalAvatar ?? null,
+            },
+            update: {
+                avatar_url: finalAvatar ?? null,
+            },
+        });
+
+        await prisma.comment.upsert({
+            where: { cid: c.cid },
+            create: {
+                cid: c.cid,
+                text: c.text,
+                digg_count: BigInt(c.digg_count || 0),
+                created_at: new Date((c.create_time || 0) * 1000),
+                videoId: video.aweme_id,
+                userId: cu.id,
+            },
+            update: {
+                text: c.text,
+                digg_count: BigInt(c.digg_count || 0),
+                created_at: new Date((c.create_time || 0) * 1000),
+                videoId: video.aweme_id,
+                userId: cu.id,
+            },
+        });
+    }
+
+    return { aweme_id: video.aweme_id, author_sec_uid: author.sec_uid, comment_count: comments.length };
+}
+
+export async function saveImagePostToDB(
+    context: BrowserContext,
+    aweme: DouyinImageAweme,
+    commentResp: DouyinCommentResponse,
+    uploads: { images: { url: string; width?: number; height?: number }[]; musicUrl?: string }
+) {
+    if (!aweme?.author?.sec_uid) throw new Error('作者 sec_uid 缺失');
+
+    // Upsert Author（与视频一致）
+    const authorAvatarSrc = firstUrl(aweme.author.avatar_thumb?.url_list);
+    const authorAvatarUploaded = await uploadAvatarFromUrl(context, authorAvatarSrc, `authors/${aweme.author.sec_uid}`);
+    const author = await prisma.author.upsert({
+        where: { sec_uid: aweme.author.sec_uid },
+        create: {
+            sec_uid: aweme.author.sec_uid,
+            uid: aweme.author.uid,
+            nickname: aweme.author.nickname,
+            signature: aweme.author.signature ?? null,
+            avatar_url: authorAvatarUploaded ?? null,
+            follower_count: BigInt((aweme.author as any).follower_count || 0),
+            total_favorited: BigInt((aweme.author as any).total_favorited || 0),
+            unique_id: (aweme.author as any).unique_id ?? null,
+            short_id: (aweme.author as any).short_id ?? null,
+        },
+        update: {
+            uid: aweme.author.uid,
+            nickname: aweme.author.nickname,
+            signature: aweme.author.signature ?? null,
+            avatar_url: authorAvatarUploaded ?? null,
+            follower_count: BigInt((aweme.author as any).follower_count || 0),
+            total_favorited: BigInt((aweme.author as any).total_favorited || 0),
+            unique_id: (aweme.author as any).unique_id ?? null,
+            short_id: (aweme.author as any).short_id ?? null,
+        },
+    });
+
+    // Upsert ImagePost
+    const imagePost = await prisma.imagePost.upsert({
+        where: { aweme_id: aweme.aweme_id },
+        create: {
+            aweme_id: aweme.aweme_id,
+            desc: aweme.desc,
+            created_at: new Date((aweme.create_time || 0) * 1000),
+            share_url: aweme.share_url || '',
+            digg_count: BigInt(aweme.statistics?.digg_count || 0),
+            comment_count: BigInt(aweme.statistics?.comment_count || 0),
+            share_count: BigInt(aweme.statistics?.share_count || 0),
+            collect_count: BigInt(aweme.statistics?.collect_count || 0),
+            authorId: author.sec_uid,
+            tags: (aweme.video_tag?.map(t => t.tag_name) ?? []),
+            music_url: uploads.musicUrl ?? null,
+        },
+        update: {
+            desc: aweme.desc,
+            created_at: new Date((aweme.create_time || 0) * 1000),
+            share_url: aweme.share_url,
+            digg_count: BigInt(aweme.statistics?.digg_count || 0),
+            comment_count: BigInt(aweme.statistics?.comment_count || 0),
+            share_count: BigInt(aweme.statistics?.share_count || 0),
+            collect_count: BigInt(aweme.statistics?.collect_count || 0),
+            authorId: author.sec_uid,
+            tags: (aweme.video_tag?.map(t => t.tag_name) ?? []),
+            music_url: uploads.musicUrl ?? undefined,
+        },
+    });
+
+    // Upsert ImageFiles（按顺序）
+    for (let i = 0; i < uploads.images.length; i++) {
+        const { url, width, height } = uploads.images[i];
+        await prisma.imageFile.upsert({
+            where: { postId_order: { postId: imagePost.aweme_id, order: i } },
+            create: {
+                postId: imagePost.aweme_id,
+                order: i,
+                url,
+                width: typeof width === 'number' ? width : null,
+                height: typeof height === 'number' ? height : null,
+            },
+            update: {
+                url,
+                width: typeof width === 'number' ? width : null,
+                height: typeof height === 'number' ? height : null,
+            },
+        });
+    }
+
+    // 评论入库：关联到 ImagePost
+    const comments = commentResp?.comments ?? [];
+    for (const c of comments) {
+        const origAvatar: string | null = firstUrl(c.user?.avatar_thumb?.url_list) ?? null;
+        const nameHint = `comment-users/${(c.user?.nickname || 'unknown').replace(/\s+/g, '_')}-${c.cid}`;
+        const uploadedAvatar = await uploadAvatarFromUrl(context, origAvatar ?? undefined, nameHint);
+        const finalAvatar = uploadedAvatar ?? origAvatar; // string | null
+        const finalAvatarKey = finalAvatar ?? '';
+        const cu = await prisma.commentUser.upsert({
+            where: {
+                nickname_avatar_url: {
+                    nickname: c.user?.nickname || '未知用户',
+                    avatar_url: finalAvatarKey,
+                },
+            },
+            create: {
+                nickname: c.user?.nickname || '未知用户',
+                avatar_url: finalAvatar ?? null,
+            },
+            update: {
+                avatar_url: finalAvatar ?? null,
+            },
+        });
+
+        await prisma.comment.upsert({
+            where: { cid: c.cid },
+            create: {
+                cid: c.cid,
+                text: c.text,
+                digg_count: BigInt(c.digg_count || 0),
+                created_at: new Date((c.create_time || 0) * 1000),
+                imagePostId: imagePost.aweme_id,
+                userId: cu.id,
+            },
+            update: {
+                text: c.text,
+                digg_count: BigInt(c.digg_count || 0),
+                created_at: new Date((c.create_time || 0) * 1000),
+                imagePostId: imagePost.aweme_id,
+                userId: cu.id,
+            },
+        });
+    }
+
+    return { aweme_id: imagePost.aweme_id, author_sec_uid: author.sec_uid, image_count: uploads.images.length, comment_count: comments.length };
+}

app/fetcher/route.ts

@@ -0,0 +1,17 @@
+import { NextRequest, NextResponse } from 'next/server'
+import { prisma } from '@/lib/prisma'
+import { scrapeDouyin } from '.';
+
+async function handleDouyinScrape(req: NextRequest) {
+    const { searchParams } = new URL(req.url);
+    const videoUrl = searchParams.get('url');
+    if (!videoUrl) {
+        return NextResponse.json({ error: '缺少视频URL' }, { status: 400 });
+    }
+
+    // 调用爬虫函数
+    const result = await scrapeDouyin(videoUrl);
+    return NextResponse.json(result);
+}
+
+export const GET = handleDouyinScrape

app/fetcher/types.d.ts

@@ -0,0 +1,138 @@
+/** 抖音评论响应（精简版） */
+interface DouyinCommentResponse {
+  status_code: number;
+  comments: DouyinComment[];
+  total: number;
+}
+
+/** 单条评论（精简版） */
+interface DouyinComment {
+  cid: string;
+  text: string; // 评论内容
+  digg_count: number; // 点赞数
+  create_time: number; // 创建时间（时间戳）
+  user: DouyinUser; // 评论用户
+}
+
+/** 用户信息（精简版） */
+interface DouyinUser {
+  nickname: string; // 用户昵称
+  avatar_thumb: {
+    url_list: string[]; // 头像链接（可取第一个）
+  };
+}
+
+/** 抖音视频详情响应（精简版） */
+interface DouyinVideoDetailResponse {
+  status_code: number;
+  aweme_detail: DouyinVideoDetail;
+}
+/** 作者信息（精简版） */
+interface DouyinAuthor {
+  uid: string;                   // 用户ID
+  sec_uid: string;               // 安全UID
+  nickname: string;              // 用户昵称
+  signature: string;             // 个性签名
+  avatar_thumb: {
+    url_list: string[];          // 头像URL（可取第一个）
+  };
+  follower_count: number;        // 粉丝数
+  total_favorited: number;       // 获赞总数
+  unique_id: string;             // 抖音号
+  short_id: string;              // 短ID
+}
+/** 视频详情 */
+interface DouyinVideoDetail {
+  aweme_id: string;              // 视频ID
+  desc: string;                  // 视频描述
+  preview_title?: string;        // 视频标题（有些字段中叫 preview_title）
+  duration: number;              // 视频时长（毫秒）
+  create_time: number;           // 创建时间（时间戳）
+  share_url: string;             // 视频分享链接
+
+  statistics: {
+    digg_count: number;          // 点赞数
+    comment_count: number;       // 评论数
+    share_count: number;         // 分享数
+    collect_count: number;       // 收藏数
+  };
+
+  author: DouyinAuthor;          // 作者信息
+  video: VideoPlayBasic;
+  tags: VideoTagBasic[];
+}
+
+/** 视频播放信息 */
+interface VideoPlayBasic {
+  /** 多种清晰度/编码的可播放变体（选其一即可） */
+  bit_rate: PlayVariant[];
+  /** 大缩略图（如果需要封面） */
+  big_thumb_url?: string;
+}
+
+/** 单个清晰度变体（来自 bit_rate[*] + play_addr） */
+interface PlayVariant {
+  format: string;           // mp4 等
+  FPS: number;
+  bit_rate: number;         // bit_rate.bit_rate
+
+  /** 直连播放地址（最关键） */
+  play_addr: {
+    uri: string;
+    url_list: string[];
+    width: number;
+    height: number;
+    data_size: number;
+  };
+}
+
+/** 视频标签（精简） */
+interface VideoTagBasic {
+  tag_id: number;
+  tag_name: string;
+  level?: number;
+}
+
+/** 图文作品列表响应（POST_PATH） */
+interface DouyinPostListResponse {
+  status_code: number;
+  aweme_list: DouyinImageAweme[];
+}
+
+/** 图文作品（精简必要字段） */
+interface DouyinImageAweme {
+  aweme_id: string;
+  desc: string;
+  create_time: number; // 秒
+  share_url: string;
+  statistics: {
+    digg_count: number;
+    comment_count: number;
+    share_count: number;
+    collect_count: number;
+  };
+  author: DouyinAuthor; // 复用视频作者类型（需包含 sec_uid）
+  images: DouyinImageInfo[]; // 图片列表
+  music?: DouyinMusicBasic;  // 背景音乐（可选）
+  video_tag?: VideoTagBasic[]; // 标签
+}
+
+/** 图文作品的单张图片信息（精简） */
+interface DouyinImageInfo {
+  url_list: string[]; // 多种格式（webp/jpeg）
+  download_url_list?: string[]; // 可能带水印
+  width: number;
+  height: number;
+}
+
+/** 音乐基本信息（精简） */
+interface DouyinMusicBasic {
+  id?: number | string;
+  title?: string;
+  author?: string;
+  album?: string;
+  play_url: {
+    uri?: string;
+    url_list: string[]; // 真实可下载地址
+  };
+}

app/fetcher/uploader.ts

@@ -0,0 +1,59 @@
+import type { BrowserContext } from 'playwright';
+import { uploadFile, generateUniqueFileName } from '@/lib/minio';
+import { downloadBinary } from './network';
+import { pickFirstUrl } from './utils';
+
+/**
+ * 下载头像并上传到 MinIO，返回外链；失败时回退为原始链接。
+ */
+export async function uploadAvatarFromUrl(
+    context: BrowserContext,
+    srcUrl?: string | null,
+    nameHint?: string
+): Promise<string | undefined> {
+    if (!srcUrl) return undefined;
+    try {
+        const { buffer, contentType, ext } = await downloadBinary(context, srcUrl);
+        const safeExt = ext || 'jpg';
+        const baseName = nameHint ? `${nameHint}.${safeExt}` : `avatar.${safeExt}`;
+        const fileName = generateUniqueFileName(baseName, 'douyin/avatars');
+        const uploaded = await uploadFile(buffer, fileName, { 'Content-Type': contentType });
+        return uploaded;
+    } catch (e) {
+        console.warn('[avatar] 上传失败，使用原始链接：', (e as Error)?.message || e);
+        return srcUrl || undefined;
+    }
+}
+
+/** 下载图文作品的图片和音乐并上传到 MinIO */
+export async function handleImagePost(
+    context: BrowserContext,
+    aweme: DouyinImageAweme
+): Promise<{ images: { url: string; width?: number; height?: number }[]; musicUrl?: string }> {
+    const awemeId = aweme.aweme_id;
+    const uploadedImages: { url: string; width?: number; height?: number }[] = [];
+
+    // 下载图片（顺序保持）
+    for (let i = 0; i < (aweme.images?.length || 0); i++) {
+        const img = aweme.images[i];
+        const url = pickFirstUrl(img?.url_list);
+        if (!url) continue;
+        const { buffer, contentType, ext } = await downloadBinary(context, url);
+        const safeExt = ext || 'jpg';
+        const fileName = generateUniqueFileName(`${awemeId}/${i}.${safeExt}`, 'douyin/images');
+        const uploaded = await uploadFile(buffer, fileName, { 'Content-Type': contentType });
+        uploadedImages.push({ url: uploaded, width: img?.width, height: img?.height });
+    }
+
+    // 下载音乐（可选）
+    let musicUrl: string | undefined;
+    const audioSrc = pickFirstUrl(aweme.music?.play_url?.url_list);
+    if (audioSrc) {
+        const { buffer, contentType, ext } = await downloadBinary(context, audioSrc);
+        const safeExt = ext || 'mp3';
+        const fileName = generateUniqueFileName(`${awemeId}.${safeExt}`, 'douyin/audios');
+        musicUrl = await uploadFile(buffer, fileName, { 'Content-Type': contentType });
+    }
+
+    return { images: uploadedImages, musicUrl };
+}

app/fetcher/utils.ts

@@ -0,0 +1,63 @@
+export function toCamelCaseKey(key: string): string {
+    return key.replace(/_([a-zA-Z])/g, (_, c: string) => c.toUpperCase());
+}
+
+export function toSnakeCaseKey(key: string): string {
+    return key.replace(/[A-Z]/g, (m) => `_${m.toLowerCase()}`);
+}
+
+/**
+ * 创建一个代理对象，使得读取属性时同时兼容 snake_case 与 camelCase 键名。
+ * 访问顺序：原名 -> camelCase -> snake_case
+ */
+export function createCamelCompatibleProxy<T extends object>(root: T): T {
+    const seen = new WeakMap<object, any>();
+
+    const wrap = (value: any): any => {
+        if (value === null || typeof value !== 'object') return value;
+        if (seen.has(value)) return seen.get(value);
+        const proxied = new Proxy(value, handler);
+        seen.set(value, proxied);
+        return proxied;
+    };
+
+    const handler: ProxyHandler<any> = {
+        get(target, prop, receiver) {
+            // 非字符串属性（如 Symbol、数字索引）直接透传
+            if (typeof prop !== 'string') {
+                return wrap(Reflect.get(target, prop, receiver));
+            }
+
+            const primary = prop;
+
+            if (primary in target) return wrap(Reflect.get(target, primary, receiver));
+
+            const camel = toCamelCaseKey(primary);
+            if (camel in target) return wrap(Reflect.get(target, camel, receiver));
+
+            const snake = toSnakeCaseKey(primary);
+            if (snake in target) return wrap(Reflect.get(target, snake, receiver));
+
+            return wrap(Reflect.get(target, prop, receiver));
+        },
+        has(target, prop) {
+            if (typeof prop !== 'string') return prop in target;
+            const primary = prop === 'auther' ? 'autherInfo' : prop;
+            return (
+                primary in target ||
+                toCamelCaseKey(primary) in target ||
+                toSnakeCaseKey(primary) in target
+            );
+        }
+    };
+
+    return wrap(root);
+}
+
+/** 选择首个可用 URL */
+export function pickFirstUrl(list?: string[]) {
+    return Array.isArray(list) && list.length ? list[0] : undefined;
+}
+
+// 别名，兼容旧命名
+export const firstUrl = pickFirstUrl;