修改api路径

app/api/fetcher/browser.ts

@@ -0,0 +1,71 @@
+import { chromium, type BrowserContext } from 'playwright'
+
+// A simple singleton manager for a persistent Chromium context with ref-counting.
+// Prevents concurrent tasks from closing the shared window prematurely.
+
+let contextPromise: Promise<BrowserContext> | null = null
+let context: BrowserContext | null = null
+let refCount = 0
+let idleCloseTimer: NodeJS.Timeout | null = null
+
+const USER_DATA_DIR = 'chrome-profile/douyin'
+const DEFAULT_OPTIONS = { headless: true } as const
+
+async function launchContext(): Promise<BrowserContext> {
+  const ctx = await chromium.launchPersistentContext(USER_DATA_DIR, DEFAULT_OPTIONS)
+  // When the context is closed externally, reset manager state
+  ctx.on('close', () => {
+    context = null
+    contextPromise = null
+    refCount = 0
+    if (idleCloseTimer) {
+      clearTimeout(idleCloseTimer)
+      idleCloseTimer = null
+    }
+  })
+  return ctx
+}
+
+export async function acquireBrowserContext(): Promise<BrowserContext> {
+  // Cancel any pending idle close if a new consumer arrives
+  if (idleCloseTimer) {
+    clearTimeout(idleCloseTimer)
+    idleCloseTimer = null
+  }
+
+  if (context) {
+    refCount += 1
+    return context
+  }
+
+  if (!contextPromise) {
+    contextPromise = launchContext()
+  }
+  context = await contextPromise
+  refCount += 1
+  return context
+}
+
+export async function releaseBrowserContext(options?: { idleMillis?: number }): Promise<void> {
+  const idleMillis = options?.idleMillis ?? 15_000
+  refCount = Math.max(0, refCount - 1)
+
+  if (refCount > 0 || !context) return
+
+  // Delay the close to allow bursty workloads to reuse the context
+  if (idleCloseTimer) {
+    clearTimeout(idleCloseTimer)
+    idleCloseTimer = null
+  }
+  idleCloseTimer = setTimeout(async () => {
+    try {
+      if (context && refCount === 0) {
+        await context.close()
+      }
+    } finally {
+      context = null
+      contextPromise = null
+      idleCloseTimer = null
+    }
+  }, idleMillis)
+}

app/api/fetcher/index.ts

@@ -0,0 +1,246 @@
+// src/scrapeDouyin.ts
+import { BrowserContext, Page, chromium, type Response } from 'playwright';
+import { prisma } from '@/lib/prisma';
+import { uploadFile, generateUniqueFileName } from '@/lib/minio';
+import { createCamelCompatibleProxy } from '@/app/api/fetcher/utils';
+import { waitForFirstResponse, waitForResponseWithTimeout, safeJson, downloadBinary } from '@/app/api/fetcher/network';
+import { pickBestPlayAddr, extractFirstFrame } from '@/app/api/fetcher/media';
+import { handleImagePost } from '@/app/api/fetcher/uploader';
+import { saveToDB, saveImagePostToDB } from '@/app/api/fetcher/persist';
+import chalk from 'chalk';
+import { acquireBrowserContext, releaseBrowserContext } from '@/app/api/fetcher/browser';
+
+const DETAIL_PATH = '/aweme/v1/web/aweme/detail/';
+const COMMENT_PATH = '/aweme/v1/web/comment/list/';
+const POST_PATH = '/aweme/v1/web/aweme/post/'
+
+async function readPostMem(context: BrowserContext, page: Page) {
+    const md = await page.evaluate(() => {
+        // @ts-ignore
+        let data = window.__pace_captured__.find(i => i[1] && i[1].includes(`"awemeId":`))[1]
+        return JSON.parse(data.slice(data.indexOf("{")).replaceAll("]\n", ''))
+        // return {aweme: { detail: {} } };
+    }).catch(() => null);
+
+    let aweme_mem = md?.aweme?.detail as DouyinImageAweme;
+    if (!aweme_mem) throw new Error('页面内存数据中未找到作品详情');
+
+    // @ts-ignore
+    aweme_mem.author = aweme_mem.authorInfo
+    const comments = md.comment ? createCamelCompatibleProxy<DouyinCommentResponse>(md.comment) : null;
+    const aweme = createCamelCompatibleProxy(aweme_mem);
+
+    return { aweme, comments }
+}
+
+
+export class ScrapeError extends Error {
+    constructor(
+        message: string,
+        public statusCode: number = 500,
+        public code?: string
+    ) {
+        super(message);
+        this.name = 'ScrapeError';
+    }
+}
+
+export async function scrapeDouyin(url: string) {
+    console.log(chalk.blue('🚀 启动共享 Chromium 浏览器...'));
+
+    const context = await acquireBrowserContext();
+    const page = await context.newPage();
+    console.log(chalk.cyan(`📄 正在访问: ${chalk.underline(url)}`));
+
+    await page.addInitScript(() => {
+        // 建一个全局容器存捕获的数据
+        (window as any).__pace_captured__ = [];
+
+        // 用 Proxy 包装一个数组，拦截 push
+        const captured = (window as any).__pace_captured__;
+        const proxyArr = new Proxy([] as any[], {
+            get(target, prop, receiver) {
+                if (prop === 'push') {
+                    return (...items: any[]) => {
+                        try { captured.push(...items); } catch { }
+                        return Array.prototype.push.apply(target, items);
+                    };
+                }
+                return Reflect.get(target, prop, receiver);
+            },
+            set(target, prop, value, receiver) {
+                // 兼容站点可能直接赋初始数组： self.__pace_f = [a,b]
+                if (prop === 'length') return Reflect.set(target, prop, value, receiver);
+                return Reflect.set(target, prop, value, receiver);
+            }
+        });
+
+        (self as any).__pace_f = proxyArr;
+        (window as any).__pace_f = proxyArr;
+    });
+
+    try {
+        // 先注册“先到先得”的监听，再导航，避免漏包
+        const firstTypePromise = waitForFirstResponse(context, [
+            { key: 'detail', test: (r: Response) => r.url().includes(DETAIL_PATH) && r.status() === 200 },
+            { key: 'post', test: (r: Response) => r.url().includes(POST_PATH) && r.status() === 200 },
+        ], 9_000); // 整体 9s 兜底超时，不逐个等待
+
+        // 评论只做短时“有就用、没有不等”的监听
+        const commentPromise = waitForResponseWithTimeout(
+            context, (r: Response) => r.url().includes(COMMENT_PATH) && r.status() === 200, 8_000
+        ).catch(() => null);
+
+        await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 60_000 });
+
+        // 查找页面中是否存在 "视频不存在" 的提示
+        const isNotFound = await page.locator('text=视频不存在').count().then(count => count > 0).catch(() => false);
+        if (isNotFound) {
+            console.error(chalk.red('✗ 视频不存在或已被删除'));
+            throw new ScrapeError('视频不存在或已被删除', 404, 'VIDEO_NOT_FOUND');
+        }
+
+        try {
+            // 优先尝试从内存读取图文数据
+            let { aweme, comments } = await readPostMem(context, page);
+            console.log(chalk.green('✓ 从内存读取图文数据成功'));
+
+            const uploads = await handleImagePost(context, aweme);
+            if (!comments) {
+                console.warn(chalk.yellow('⚠ 从内存读取评论数据失败，尝试从网络请求获取评论数据'));
+
+                const commentRes = await commentPromise;
+                comments = commentRes && await safeJson<DouyinCommentResponse>(commentRes);
+                if (!comments) {
+                    console.warn(chalk.yellow('⚠ 无法从内存读取评论数据，且网络请求也未返回评论数据'));
+                    comments = { comments: [], total: 0, status_code: 0 };
+                } else {
+                    console.log(chalk.green('✓ 从网络请求获取评论数据成功'));
+                }
+            } else {
+                console.log(chalk.green('✓ 从内存读取评论数据成功'));
+            }
+            const saved = await saveImagePostToDB(context, aweme, comments, uploads); // 传递完整 JSON
+            console.log(chalk.green.bold('✓ 图文作品保存成功'));
+            return { type: "image", ...saved };
+        } catch {
+
+        }
+
+        const commentRes = await commentPromise;
+        const firstType = await firstTypePromise;
+
+        if (!firstType) {
+            console.error(chalk.red('✗ 既无法从内存读取数据,也无法从网络获得数据'));
+            throw new ScrapeError('无法获取作品数据,可能是网络问题或作品已下架', 404, 'NO_DATA');
+        }
+
+        console.log(chalk.cyan(`📡 检测到作品类型: ${chalk.bold(firstType.key === 'post' ? '图文' : '视频')}`));
+
+        let comments = commentRes && await safeJson<DouyinCommentResponse>(commentRes);
+        if (!comments) {
+            console.warn(chalk.yellow('⚠ 无法从内存读取评论数据，且网络请求也未返回评论数据'));
+            comments = { comments: [], total: 0, status_code: 0 };
+        }
+
+        // 分支：视频 or 图文（两者只会有一个命中，先到先得）
+        if (firstType.key === 'post') {
+            // 图文作品
+            const postJson = await safeJson<DouyinPostListResponse>(firstType.response);
+            if (!postJson?.aweme_list?.length) throw new ScrapeError('图文作品响应为空', 404, 'EMPTY_POST_RESPONSE');
+
+            const currentURL = page.url();
+            const target_aweme_id = currentURL.split('/').at(-1);
+            const awemeList = postJson.aweme_list as unknown as DouyinImageAweme[];
+            let aweme = awemeList.find((pt: DouyinImageAweme) => pt.aweme_id === target_aweme_id);
+            if (!aweme) {
+                throw new ScrapeError('无法找到目标作品,可能已被删除', 404, 'POST_NOT_FOUND');
+            }
+
+            const uploads = await handleImagePost(context, aweme);
+            const saved = await saveImagePostToDB(context, aweme, comments, uploads, postJson); // 传递完整 JSON
+            console.log(chalk.green.bold('✓ 图文作品保存成功'));
+            return { type: "image", ...saved };
+        } else if (firstType.key === 'detail') {
+            // 视频作品
+            const detail = (await safeJson<DouyinVideoDetailResponse>(firstType.response))!;
+
+            // 找到比特率最高的 url
+            const bestPlayAddr = pickBestPlayAddr(
+                detail?.aweme_detail?.video.bit_rate
+            );
+            const bestVUrl = bestPlayAddr?.url_list?.[0];
+            const fps = bestPlayAddr?.FPS ?? null; // 提取 FPS
+
+            console.log(chalk.cyan(`📹 最佳视频 URL: ${chalk.dim(bestVUrl)}`));
+            console.log(chalk.cyan(`🎞️  视频帧率: ${chalk.bold(fps || 'N/A')} FPS`));
+            if (bestPlayAddr?.width && bestPlayAddr?.height) {
+                console.log(chalk.cyan(`📐 视频分辨率: ${chalk.bold(`${bestPlayAddr.width}x${bestPlayAddr.height}`)}`));
+            }
+
+            // 下载视频并上传至 MinIO，获取外链
+            let uploadedUrl: string | undefined;
+            let coverUrl: string | undefined;
+            if (bestVUrl && detail?.aweme_detail) {
+                console.log(chalk.blue('⬇️  正在下载视频...'));
+                const { buffer, contentType, ext } = await downloadBinary(context, bestVUrl);
+                const awemeId = detail.aweme_detail.aweme_id;
+                const fileName = generateUniqueFileName(`${awemeId}.${ext}`, 'douyin/videos');
+
+                console.log(chalk.blue('⬆️  正在上传视频到 MinIO...'));
+                uploadedUrl = await uploadFile(buffer, fileName, { 'Content-Type': contentType });
+                console.log(chalk.green(`✓ 视频上传成功: ${chalk.underline(uploadedUrl)}`));
+
+                // 提取首帧作为封面并上传
+                try {
+                    console.log(chalk.blue('🖼️  正在提取视频封面...'));
+                    const cover = await extractFirstFrame(buffer);
+                    if (cover) {
+                        const coverName = generateUniqueFileName(`${awemeId}.jpg`, 'douyin/covers');
+                        coverUrl = await uploadFile(cover.buffer, coverName, { 'Content-Type': cover.contentType });
+                        console.log(chalk.green(`✓ 封面上传成功: ${chalk.underline(coverUrl)}`));
+                    }
+                } catch (e) {
+                    console.warn(chalk.yellow(`⚠ 提取封面失败，跳过: ${(e as Error)?.message || e}`));
+                }
+            }
+
+            const saved = await saveToDB(context, detail, comments, uploadedUrl, bestPlayAddr?.width, bestPlayAddr?.height, coverUrl, fps ?? undefined);
+            console.log(chalk.green.bold('✓ 视频作品保存成功'));
+            return { type: "video", ...saved };
+        } else {
+            throw new ScrapeError('无法判定作品类型,接口响应异常', 500, 'UNKNOWN_TYPE');
+        }
+    } catch (error) {
+        // 如果是我们自定义的错误,直接抛出
+        if (error instanceof ScrapeError) {
+            throw error;
+        }
+        
+        // 处理其他类型的错误
+        const errMsg = (error as Error)?.message || String(error);
+        console.error(chalk.red(`✗ 爬取失败: ${errMsg}`));
+        
+        // 根据错误类型返回不同的状态码
+        if (errMsg.includes('timeout') || errMsg.includes('超时')) {
+            throw new ScrapeError('请求超时,请稍后重试', 408, 'TIMEOUT');
+        }
+        if (errMsg.includes('页面内存数据中未找到作品详情')) {
+            throw new ScrapeError('作品数据加载失败', 404, 'DATA_NOT_LOADED');
+        }
+        if (errMsg.includes('net::')) {
+            throw new ScrapeError('网络连接失败', 503, 'NETWORK_ERROR');
+        }
+        
+        // 默认服务器错误
+        throw new ScrapeError(errMsg || '爬取过程中发生未知错误', 500, 'UNKNOWN_ERROR');
+    } finally {
+        console.log(chalk.gray('🧹 清理资源...'));
+        try { await page.close({ runBeforeUnload: true }); } catch {}
+        // 仅释放共享上下文的引用，不直接关闭窗口
+        await releaseBrowserContext();
+        await prisma.$disconnect();
+        console.log(chalk.gray('✓ 资源清理完成'));
+    }
+}
+

app/api/fetcher/media.ts

@@ -0,0 +1,57 @@
+import { promises as fs } from 'fs';
+import os from 'os';
+import path from 'path';
+import { execFile } from 'child_process';
+import { promisify } from 'util';
+
+const execFileAsync = promisify(execFile);
+
+export function pickBestPlayAddr(variants: PlayVariant[] | undefined | null) {
+    if (!variants?.length) return null;
+
+    const best = variants.reduce((best, cur) => {
+        const b1 = best?.bit_rate ?? -1;
+        const b2 = cur?.bit_rate ?? -1;
+        return b2 > b1 ? cur : best;
+    });
+
+    return best?.play_addr ?? null;
+}
+
+/**
+ * 使用 ffmpeg 从视频二进制中提取第一帧，返回 JPEG buffer
+ */
+export async function extractFirstFrame(videoBuffer: Buffer): Promise<{ buffer: Buffer; contentType: string; ext: string } | null> {
+    const ffmpegCmd = process.env.FFMPEG_PATH || 'ffmpeg';
+    const tmpDir = os.tmpdir();
+    const base = `dy_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`;
+    const inPath = path.join(tmpDir, `${base}.mp4`);
+    const outPath = path.join(tmpDir, `${base}.jpg`);
+
+    try {
+        await fs.writeFile(inPath, videoBuffer);
+        const args = [
+            '-hide_banner',
+            '-loglevel', 'error',
+            '-ss', '0',
+            '-i', inPath,
+            '-frames:v', '1',
+            '-q:v', '2',
+            '-f', 'image2',
+            '-y',
+            outPath,
+        ];
+        await execFileAsync(ffmpegCmd, args, { windowsHide: true });
+        const img = await fs.readFile(outPath);
+        return { buffer: img, contentType: 'image/jpeg', ext: 'jpg' };
+    } catch (e: any) {
+        if (e && (e.code === 'ENOENT' || /not found|is not recognized/i.test(String(e.message)))) {
+            console.warn('系统未检测到 ffmpeg，可安装并配置 PATH 或设置 FFMPEG_PATH 后启用封面提取。');
+            return null;
+        }
+        throw e;
+    } finally {
+        try { await fs.unlink(inPath); } catch { }
+        try { await fs.unlink(outPath); } catch { }
+    }
+}

app/api/fetcher/network.ts

@@ -0,0 +1,130 @@
+import type { BrowserContext, Response } from 'playwright';
+
+export async function safeJson<T>(res: Response): Promise<T | null> {
+    const ctype = res.headers()['content-type'] || '';
+    if (ctype.includes('application/json')) {
+        return (await res.json()) as T;
+    }
+    const t = await res.text();
+    try {
+        return JSON.parse(t) as T;
+    } catch {
+        return null;
+    }
+}
+
+/**
+ * 使用 Playwright 的 APIRequestContext 下载二进制内容
+ * - 使用指定 headers 模拟浏览器请求
+ * - referrer 使用链接本身
+ */
+export async function downloadBinary(
+    context: BrowserContext,
+    url: string
+): Promise<{ buffer: Buffer; contentType: string; ext: string }> {
+    console.log('下载：', url);
+
+    const headers = {
+        referer: url,
+    } as Record<string, string>;
+
+    const res = await context.request.get(url, {
+        headers,
+        maxRedirects: 3,
+        timeout: 240_000,
+        failOnStatusCode: true,
+    });
+
+    if (!res.ok()) {
+        throw new Error(`下载内容失败: ${res.status()} ${res.statusText()}`);
+    }
+
+    const buffer = await res.body();
+    const contentType = res.headers()['content-type'] || 'application/octet-stream';
+    const ext = (contentType.split('/')[1] || 'bin').split(';')[0] || 'bin';
+    return { buffer, contentType, ext };
+}
+
+/**
+ * 在多个候选匹配中“先到先得”地返回首个命中的 Response。
+ * - 不为每个候选单独设长超时，改用整体兜底超时，避免无意义等待。
+ */
+export function waitForFirstResponse(
+    context: BrowserContext,
+    candidates: { key: string; test: (r: Response) => boolean }[],
+    timeoutMs = 20_000
+): Promise<{ key: string; response: Response } | null> {
+    return new Promise((resolve) => {
+        let resolved = false;
+        let timer: NodeJS.Timeout | undefined;
+
+        const handler = (res: Response) => {
+            if (resolved) return;
+            for (const c of candidates) {
+                try {
+                    if (c.test(res)) {
+                        resolved = true;
+                        cleanup();
+                        resolve({ key: c.key, response: res });
+                        return;
+                    }
+                } catch {
+                    // ignore predicate errors
+                }
+            }
+        };
+
+        const cleanup = () => {
+            context.off('response', handler);
+            if (timer) clearTimeout(timer);
+        };
+
+        context.on('response', handler);
+        if (timeoutMs > 0) {
+            timer = setTimeout(() => {
+                if (!resolved) {
+                    resolved = true;
+                    cleanup();
+                    resolve(null);
+                }
+            }, timeoutMs);
+        }
+    });
+}
+
+/**
+ * 等待符合条件的单个 Response，带短超时；用于评论等“可有可无”的数据。
+ */
+export function waitForResponseWithTimeout(
+    context: BrowserContext,
+    predicate: (r: Response) => boolean,
+    timeoutMs = 5_000
+): Promise<Response> {
+    return new Promise<Response>((resolve, reject) => {
+        let timer: NodeJS.Timeout | undefined;
+
+        const handler = (res: Response) => {
+            try {
+                if (predicate(res)) {
+                    cleanup();
+                    resolve(res);
+                }
+            } catch {
+                // ignore predicate errors
+            }
+        };
+
+        const cleanup = () => {
+            context.off('response', handler);
+            if (timer) clearTimeout(timer);
+        };
+
+        context.on('response', handler);
+        if (timeoutMs > 0) {
+            timer = setTimeout(() => {
+                cleanup();
+                reject(new Error('timeout'));
+            }, timeoutMs);
+        }
+    });
+}

app/api/fetcher/persist.ts

@@ -0,0 +1,271 @@
+import type { BrowserContext } from 'playwright';
+import { prisma } from '@/lib/prisma';
+import { uploadAvatarFromUrl } from './uploader';
+import { firstUrl } from './utils';
+
+export async function saveToDB(
+    context: BrowserContext,
+    detailResp: DouyinVideoDetailResponse,
+    commentResp: DouyinCommentResponse,
+    videoUrl?: string,
+    width?: number,
+    height?: number,
+    coverUrl?: string,
+    fps?: number
+) {
+    if (!detailResp?.aweme_detail) throw new Error('视频详情为空');
+    const d = detailResp.aweme_detail;
+
+    // 1) Upsert Author
+    const authorAvatarSrc = firstUrl(d.author.avatar_thumb?.url_list);
+    const authorAvatarUploaded = await uploadAvatarFromUrl(context, authorAvatarSrc, `authors/${d.author.sec_uid}`);
+    const author = await prisma.author.upsert({
+        where: { sec_uid: d.author.sec_uid },
+        create: {
+            sec_uid: d.author.sec_uid,
+            uid: d.author.uid,
+            nickname: d.author.nickname,
+            signature: d.author.signature ?? null,
+            avatar_url: authorAvatarUploaded ?? null,
+            follower_count: BigInt(d.author.follower_count || 0),
+            total_favorited: BigInt(d.author.total_favorited || 0),
+            unique_id: d.author.unique_id ?? null,
+            short_id: d.author.short_id ?? null,
+        },
+        update: {
+            uid: d.author.uid,
+            nickname: d.author.nickname,
+            signature: d.author.signature ?? null,
+            avatar_url: authorAvatarUploaded ?? null,
+            follower_count: BigInt(d.author.follower_count || 0),
+            total_favorited: BigInt(d.author.total_favorited || 0),
+            unique_id: d.author.unique_id ?? null,
+            short_id: d.author.short_id ?? null,
+        },
+    });
+
+    // 2) Upsert Video
+    const video = await prisma.video.upsert({
+        where: { aweme_id: d.aweme_id },
+        create: {
+            aweme_id: d.aweme_id,
+            desc: d.desc,
+            preview_title: d.preview_title ?? null,
+            duration_ms: d.duration,
+            created_at: new Date((d.create_time || 0) * 1000),
+            share_url: d.share_url,
+            digg_count: BigInt(d.statistics?.digg_count || 0),
+            comment_count: BigInt(d.statistics?.comment_count || 0),
+            share_count: BigInt(d.statistics?.share_count || 0),
+            collect_count: BigInt(d.statistics?.collect_count || 0),
+            authorId: author.sec_uid,
+            tags: (d.tags?.map(t => t.tag_name) ?? []),
+            video_url: videoUrl ?? '',
+            width: width ?? null,
+            height: height ?? null,
+            cover_url: coverUrl ?? null,
+            fps: fps ?? null,
+            raw_json: detailResp as any, // 保存完整接口 JSON
+        },
+        update: {
+            desc: d.desc,
+            preview_title: d.preview_title ?? null,
+            duration_ms: d.duration,
+            created_at: new Date((d.create_time || 0) * 1000),
+            share_url: d.share_url,
+            digg_count: BigInt(d.statistics?.digg_count || 0),
+            comment_count: BigInt(d.statistics?.comment_count || 0),
+            share_count: BigInt(d.statistics?.share_count || 0),
+            collect_count: BigInt(d.statistics?.collect_count || 0),
+            authorId: author.sec_uid,
+            ...(videoUrl ? { video_url: videoUrl } : {}),
+            ...(width ? { width } : {}),
+            ...(height ? { height } : {}),
+            ...(coverUrl ? { cover_url: coverUrl } : {}),
+            ...(fps ? { fps } : {}),
+            raw_json: detailResp as any, // 更新完整接口 JSON
+        },
+    });
+
+    // 3) Upsert Comments + CommentUser
+    const comments = commentResp?.comments ?? [];
+    for (const c of comments) {
+        const origAvatar: string | null = firstUrl(c.user?.avatar_thumb?.url_list) ?? null;
+        const nameHint = `comment-users/${(c.user?.nickname || 'unknown').replace(/\s+/g, '_')}-${c.cid}`;
+        const uploadedAvatar = await uploadAvatarFromUrl(context, origAvatar ?? undefined, nameHint);
+        const finalAvatar = uploadedAvatar ?? origAvatar; // string | null
+        const finalAvatarKey = finalAvatar ?? '';
+        const cu = await prisma.commentUser.upsert({
+            where: {
+                nickname_avatar_url: {
+                    nickname: c.user?.nickname || '未知用户',
+                    avatar_url: finalAvatarKey,
+                },
+            },
+            create: {
+                nickname: c.user?.nickname || '未知用户',
+                avatar_url: finalAvatar ?? null,
+            },
+            update: {
+                avatar_url: finalAvatar ?? null,
+            },
+        });
+
+        await prisma.comment.upsert({
+            where: { cid: c.cid },
+            create: {
+                cid: c.cid,
+                text: c.text,
+                digg_count: BigInt(c.digg_count || 0),
+                created_at: new Date((c.create_time || 0) * 1000),
+                videoId: video.aweme_id,
+                userId: cu.id,
+            },
+            update: {
+                text: c.text,
+                digg_count: BigInt(c.digg_count || 0),
+                created_at: new Date((c.create_time || 0) * 1000),
+                videoId: video.aweme_id,
+                userId: cu.id,
+            },
+        });
+    }
+
+    return { aweme_id: video.aweme_id, author_sec_uid: author.sec_uid, comment_count: comments.length };
+}
+
+export async function saveImagePostToDB(
+    context: BrowserContext,
+    aweme: DouyinImageAweme,
+    commentResp: DouyinCommentResponse,
+    uploads: { images: { url: string; width?: number; height?: number }[]; musicUrl?: string },
+    rawJson?: any
+) {
+    if (!aweme?.author?.sec_uid) throw new Error('作者 sec_uid 缺失');
+
+    // Upsert Author（与视频一致）
+    const authorAvatarSrc = firstUrl(aweme.author.avatar_thumb?.url_list);
+    const authorAvatarUploaded = await uploadAvatarFromUrl(context, authorAvatarSrc, `authors/${aweme.author.sec_uid}`);
+    const author = await prisma.author.upsert({
+        where: { sec_uid: aweme.author.sec_uid },
+        create: {
+            sec_uid: aweme.author.sec_uid,
+            uid: aweme.author.uid,
+            nickname: aweme.author.nickname,
+            signature: aweme.author.signature ?? null,
+            avatar_url: authorAvatarUploaded ?? null,
+            follower_count: BigInt((aweme.author as any).follower_count || 0),
+            total_favorited: BigInt((aweme.author as any).total_favorited || 0),
+            unique_id: (aweme.author as any).unique_id ?? null,
+            short_id: (aweme.author as any).short_id ?? null,
+        },
+        update: {
+            uid: aweme.author.uid,
+            nickname: aweme.author.nickname,
+            signature: aweme.author.signature ?? null,
+            avatar_url: authorAvatarUploaded ?? null,
+            follower_count: BigInt((aweme.author as any).follower_count || 0),
+            total_favorited: BigInt((aweme.author as any).total_favorited || 0),
+            unique_id: (aweme.author as any).unique_id ?? null,
+            short_id: (aweme.author as any).short_id ?? null,
+        },
+    });
+
+    // Upsert ImagePost
+    const imagePost = await prisma.imagePost.upsert({
+        where: { aweme_id: aweme.aweme_id },
+        create: {
+            aweme_id: aweme.aweme_id,
+            desc: aweme.desc,
+            created_at: new Date((aweme.create_time || 0) * 1000),
+            share_url: aweme.share_url || '',
+            digg_count: BigInt(aweme.statistics?.digg_count || 0),
+            comment_count: BigInt(aweme.statistics?.comment_count || 0),
+            share_count: BigInt(aweme.statistics?.share_count || 0),
+            collect_count: BigInt(aweme.statistics?.collect_count || 0),
+            authorId: author.sec_uid,
+            tags: (aweme.video_tag?.map(t => t.tag_name) ?? []),
+            music_url: uploads.musicUrl ?? null,
+            raw_json: rawJson ?? null, // 保存完整接口 JSON
+        },
+        update: {
+            desc: aweme.desc,
+            created_at: new Date((aweme.create_time || 0) * 1000),
+            share_url: aweme.share_url,
+            digg_count: BigInt(aweme.statistics?.digg_count || 0),
+            comment_count: BigInt(aweme.statistics?.comment_count || 0),
+            share_count: BigInt(aweme.statistics?.share_count || 0),
+            collect_count: BigInt(aweme.statistics?.collect_count || 0),
+            authorId: author.sec_uid,
+            tags: (aweme.video_tag?.map(t => t.tag_name) ?? []),
+            music_url: uploads.musicUrl ?? undefined,
+            raw_json: rawJson ?? undefined, // 更新完整接口 JSON
+        },
+    });
+
+    // Upsert ImageFiles（按顺序）
+    for (let i = 0; i < uploads.images.length; i++) {
+        const { url, width, height } = uploads.images[i];
+        await prisma.imageFile.upsert({
+            where: { postId_order: { postId: imagePost.aweme_id, order: i } },
+            create: {
+                postId: imagePost.aweme_id,
+                order: i,
+                url,
+                width: typeof width === 'number' ? width : null,
+                height: typeof height === 'number' ? height : null,
+            },
+            update: {
+                url,
+                width: typeof width === 'number' ? width : null,
+                height: typeof height === 'number' ? height : null,
+            },
+        });
+    }
+
+    // 评论入库：关联到 ImagePost
+    const comments = commentResp?.comments ?? [];
+    for (const c of comments) {
+        const origAvatar: string | null = firstUrl(c.user?.avatar_thumb?.url_list) ?? null;
+        const nameHint = `comment-users/${(c.user?.nickname || 'unknown').replace(/\s+/g, '_')}-${c.cid}`;
+        const uploadedAvatar = await uploadAvatarFromUrl(context, origAvatar ?? undefined, nameHint);
+        const finalAvatar = uploadedAvatar ?? origAvatar; // string | null
+        const finalAvatarKey = finalAvatar ?? '';
+        const cu = await prisma.commentUser.upsert({
+            where: {
+                nickname_avatar_url: {
+                    nickname: c.user?.nickname || '未知用户',
+                    avatar_url: finalAvatarKey,
+                },
+            },
+            create: {
+                nickname: c.user?.nickname || '未知用户',
+                avatar_url: finalAvatar ?? null,
+            },
+            update: {
+                avatar_url: finalAvatar ?? null,
+            },
+        });
+
+        await prisma.comment.upsert({
+            where: { cid: c.cid },
+            create: {
+                cid: c.cid,
+                text: c.text,
+                digg_count: BigInt(c.digg_count || 0),
+                created_at: new Date((c.create_time || 0) * 1000),
+                imagePostId: imagePost.aweme_id,
+                userId: cu.id,
+            },
+            update: {
+                text: c.text,
+                digg_count: BigInt(c.digg_count || 0),
+                created_at: new Date((c.create_time || 0) * 1000),
+                imagePostId: imagePost.aweme_id,
+                userId: cu.id,
+            },
+        });
+    }
+
+    return { aweme_id: imagePost.aweme_id, author_sec_uid: author.sec_uid, image_count: uploads.images.length, comment_count: comments.length };
+}

app/api/fetcher/route.ts

@@ -0,0 +1,49 @@
+import { NextRequest, NextResponse } from 'next/server'
+import { prisma } from '@/lib/prisma'
+import { scrapeDouyin, ScrapeError } from '.';
+
+async function handleDouyinScrape(req: NextRequest) {
+    const { searchParams } = new URL(req.url);
+    const videoUrl = searchParams.get('url');
+    
+    if (!videoUrl) {
+        return NextResponse.json(
+            { error: '缺少视频URL', code: 'MISSING_URL' }, 
+            { status: 400 }
+        );
+    }
+
+    try {
+        // 调用爬虫函数
+        const result = await scrapeDouyin(videoUrl);
+        return NextResponse.json({ 
+            success: true, 
+            data: result 
+        });
+    } catch (error) {
+        // 处理自定义的 ScrapeError
+        if (error instanceof ScrapeError) {
+            return NextResponse.json(
+                { 
+                    success: false,
+                    error: error.message,
+                    code: error.code
+                }, 
+                { status: error.statusCode }
+            );
+        }
+
+        // 处理未知错误
+        console.error('未捕获的错误:', error);
+        return NextResponse.json(
+            { 
+                success: false,
+                error: '服务器内部错误',
+                code: 'INTERNAL_ERROR'
+            }, 
+            { status: 500 }
+        );
+    }
+}
+
+export const GET = handleDouyinScrape

app/api/fetcher/types.d.ts

@@ -0,0 +1,141 @@
+/** 抖音评论响应（精简版） */
+interface DouyinCommentResponse {
+  status_code: number;
+  comments: DouyinComment[];
+  total: number;
+}
+
+/** 单条评论（精简版） */
+interface DouyinComment {
+  cid: string;
+  text: string; // 评论内容
+  digg_count: number; // 点赞数
+  create_time: number; // 创建时间（时间戳）
+  user: DouyinUser; // 评论用户
+}
+
+/** 用户信息（精简版） */
+interface DouyinUser {
+  nickname: string; // 用户昵称
+  avatar_thumb: {
+    url_list: string[]; // 头像链接（可取第一个）
+  };
+}
+
+/** 抖音视频详情响应（精简版） */
+interface DouyinVideoDetailResponse {
+  status_code: number;
+  aweme_detail: DouyinVideoDetail;
+}
+/** 作者信息（精简版） */
+interface DouyinAuthor {
+  uid: string;                   // 用户ID
+  sec_uid: string;               // 安全UID
+  nickname: string;              // 用户昵称
+  signature: string;             // 个性签名
+  avatar_thumb: {
+    url_list: string[];          // 头像URL（可取第一个）
+  };
+  follower_count: number;        // 粉丝数
+  total_favorited: number;       // 获赞总数
+  unique_id: string;             // 抖音号
+  short_id: string;              // 短ID
+}
+/** 视频详情 */
+interface DouyinVideoDetail {
+  aweme_id: string;              // 视频ID
+  desc: string;                  // 视频描述
+  preview_title?: string;        // 视频标题（有些字段中叫 preview_title）
+  duration: number;              // 视频时长（毫秒）
+  create_time: number;           // 创建时间（时间戳）
+  share_url: string;             // 视频分享链接
+
+  statistics: {
+    digg_count: number;          // 点赞数
+    comment_count: number;       // 评论数
+    share_count: number;         // 分享数
+    collect_count: number;       // 收藏数
+  };
+
+  author: DouyinAuthor;          // 作者信息
+  video: VideoPlayBasic;
+  tags: VideoTagBasic[];
+}
+
+/** 视频播放信息 */
+interface VideoPlayBasic {
+  /** 多种清晰度/编码的可播放变体（选其一即可） */
+  bit_rate: PlayVariant[];
+  /** 大缩略图（如果需要封面） */
+  big_thumb_url?: string;
+}
+
+/** 单个清晰度变体（来自 bit_rate[*] + play_addr） */
+interface PlayVariant {
+  format: string;           // mp4 等
+  FPS: number;
+  bit_rate: number;         // bit_rate.bit_rate
+
+  /** 直连播放地址（最关键） */
+  play_addr: {
+    uri: string;
+    url_list: string[];
+    width: number;
+    height: number;
+    data_size: number;
+    FPS: number;
+    is_bytevc1: number; // 0 or 1
+    is_h265: number;  // 0 or 1
+  };
+}
+
+/** 视频标签（精简） */
+interface VideoTagBasic {
+  tag_id: number;
+  tag_name: string;
+  level?: number;
+}
+
+/** 图文作品列表响应（POST_PATH） */
+interface DouyinPostListResponse {
+  status_code: number;
+  aweme_list: DouyinImageAweme[];
+}
+
+/** 图文作品（精简必要字段） */
+interface DouyinImageAweme {
+  aweme_id: string;
+  desc: string;
+  create_time: number; // 秒
+  share_url: string;
+  statistics: {
+    digg_count: number;
+    comment_count: number;
+    share_count: number;
+    collect_count: number;
+  };
+  author: DouyinAuthor; // 复用视频作者类型（需包含 sec_uid）
+  images: DouyinImageInfo[]; // 图片列表
+  music?: DouyinMusicBasic;  // 背景音乐（可选）
+  video_tag?: VideoTagBasic[]; // 标签
+}
+
+/** 图文作品的单张图片信息（精简） */
+interface DouyinImageInfo {
+  url_list: string[]; // 多种格式（webp/jpeg）
+  download_url_list?: string[]; // 可能带水印
+  width: number;
+  height: number;
+}
+
+/** 音乐基本信息（精简） */
+interface DouyinMusicBasic {
+  id?: number | string;
+  title?: string;
+  author?: string;
+  album?: string;
+  play_url: {
+    uri?: string;
+    url_list: string[]; // 真实可下载地址
+  };
+}

app/api/fetcher/uploader.ts

@@ -0,0 +1,59 @@
+import type { BrowserContext } from 'playwright';
+import { uploadFile, generateUniqueFileName } from '@/lib/minio';
+import { downloadBinary } from './network';
+import { pickFirstUrl } from './utils';
+
+/**
+ * 下载头像并上传到 MinIO，返回外链；失败时回退为原始链接。
+ */
+export async function uploadAvatarFromUrl(
+    context: BrowserContext,
+    srcUrl?: string | null,
+    nameHint?: string
+): Promise<string | undefined> {
+    if (!srcUrl) return undefined;
+    try {
+        const { buffer, contentType, ext } = await downloadBinary(context, srcUrl);
+        const safeExt = ext || 'jpg';
+        const baseName = nameHint ? `${nameHint}.${safeExt}` : `avatar.${safeExt}`;
+        const fileName = generateUniqueFileName(baseName, 'douyin/avatars');
+        const uploaded = await uploadFile(buffer, fileName, { 'Content-Type': contentType });
+        return uploaded;
+    } catch (e) {
+        console.warn('[avatar] 上传失败，使用原始链接：', (e as Error)?.message || e);
+        return srcUrl || undefined;
+    }
+}
+
+/** 下载图文作品的图片和音乐并上传到 MinIO */
+export async function handleImagePost(
+    context: BrowserContext,
+    aweme: DouyinImageAweme
+): Promise<{ images: { url: string; width?: number; height?: number }[]; musicUrl?: string }> {
+    const awemeId = aweme.aweme_id;
+    const uploadedImages: { url: string; width?: number; height?: number }[] = [];
+
+    // 下载图片（顺序保持）
+    for (let i = 0; i < (aweme.images?.length || 0); i++) {
+        const img = aweme.images[i];
+        const url = pickFirstUrl(img?.url_list);
+        if (!url) continue;
+        const { buffer, contentType, ext } = await downloadBinary(context, url);
+        const safeExt = ext || 'jpg';
+        const fileName = generateUniqueFileName(`${awemeId}/${i}.${safeExt}`, 'douyin/images');
+        const uploaded = await uploadFile(buffer, fileName, { 'Content-Type': contentType });
+        uploadedImages.push({ url: uploaded, width: img?.width, height: img?.height });
+    }
+
+    // 下载音乐（可选）
+    let musicUrl: string | undefined;
+    const audioSrc = pickFirstUrl(aweme.music?.play_url?.url_list);
+    if (audioSrc) {
+        const { buffer, contentType, ext } = await downloadBinary(context, audioSrc);
+        const safeExt = ext || 'mp3';
+        const fileName = generateUniqueFileName(`${awemeId}.${safeExt}`, 'douyin/audios');
+        musicUrl = await uploadFile(buffer, fileName, { 'Content-Type': contentType });
+    }
+
+    return { images: uploadedImages, musicUrl };
+}

app/api/fetcher/utils.ts

@@ -0,0 +1,63 @@
+export function toCamelCaseKey(key: string): string {
+    return key.replace(/_([a-zA-Z])/g, (_, c: string) => c.toUpperCase());
+}
+
+export function toSnakeCaseKey(key: string): string {
+    return key.replace(/[A-Z]/g, (m) => `_${m.toLowerCase()}`);
+}
+
+/**
+ * 创建一个代理对象，使得读取属性时同时兼容 snake_case 与 camelCase 键名。
+ * 访问顺序：原名 -> camelCase -> snake_case
+ */
+export function createCamelCompatibleProxy<T extends object>(root: T): T {
+    const seen = new WeakMap<object, any>();
+
+    const wrap = (value: any): any => {
+        if (value === null || typeof value !== 'object') return value;
+        if (seen.has(value)) return seen.get(value);
+        const proxied = new Proxy(value, handler);
+        seen.set(value, proxied);
+        return proxied;
+    };
+
+    const handler: ProxyHandler<any> = {
+        get(target, prop, receiver) {
+            // 非字符串属性（如 Symbol、数字索引）直接透传
+            if (typeof prop !== 'string') {
+                return wrap(Reflect.get(target, prop, receiver));
+            }
+
+            const primary = prop;
+
+            if (primary in target) return wrap(Reflect.get(target, primary, receiver));
+
+            const camel = toCamelCaseKey(primary);
+            if (camel in target) return wrap(Reflect.get(target, camel, receiver));
+
+            const snake = toSnakeCaseKey(primary);
+            if (snake in target) return wrap(Reflect.get(target, snake, receiver));
+
+            return wrap(Reflect.get(target, prop, receiver));
+        },
+        has(target, prop) {
+            if (typeof prop !== 'string') return prop in target;
+            const primary = prop === 'auther' ? 'autherInfo' : prop;
+            return (
+                primary in target ||
+                toCamelCaseKey(primary) in target ||
+                toSnakeCaseKey(primary) in target
+            );
+        }
+    };
+
+    return wrap(root);
+}
+
+/** 选择首个可用 URL */
+export function pickFirstUrl(list?: string[]) {
+    return Array.isArray(list) && list.length ? list[0] : undefined;
+}
+
+// 别名，兼容旧命名
+export const firstUrl = pickFirstUrl;