优化图文页
This commit is contained in:
@@ -1,6 +1,6 @@
|
||||
// index.js
|
||||
import { chromium } from 'playwright-extra';
|
||||
import stealth from 'puppeteer-extra-plugin-stealth'
|
||||
import { chromium } from "playwright-extra";
|
||||
import stealth from "puppeteer-extra-plugin-stealth";
|
||||
|
||||
chromium.use(stealth());
|
||||
|
||||
@@ -8,9 +8,9 @@ chromium.use(stealth());
|
||||
const browser = await chromium.launch({ headless: false }); // 需要可视化就 false
|
||||
const context = await browser.newContext();
|
||||
const page = await context.newPage();
|
||||
await page.goto('https://bot.sannysoft.com/'); // 常用自测页
|
||||
console.log('Title:', await page.title());
|
||||
await page.screenshot({ path: 'stealth.png', fullPage: true });
|
||||
await page.goto("https://bot.sannysoft.com/"); // 常用自测页
|
||||
console.log("Title:", await page.title());
|
||||
await page.screenshot({ path: "stealth.png", fullPage: true });
|
||||
setTimeout(async () => {
|
||||
await browser.close();
|
||||
}, 1000_000);
|
||||
|
||||
+72
-71
@@ -1,99 +1,98 @@
|
||||
export const runtime = 'nodejs'
|
||||
export const runtime = "nodejs";
|
||||
|
||||
import { type Browser, type BrowserContext } from 'playwright'
|
||||
import { chromium } from 'playwright-extra' // A simple singleton manager for a persistent Chromium context with ref-counting. // Prevents concurrent tasks from closing the shared window prematurely.
|
||||
import stealth from 'puppeteer-extra-plugin-stealth'
|
||||
import { type Browser, type BrowserContext } from "playwright";
|
||||
import { chromium } from "playwright-extra"; // A simple singleton manager for a persistent Chromium context with ref-counting. // Prevents concurrent tasks from closing the shared window prematurely.
|
||||
import stealth from "puppeteer-extra-plugin-stealth";
|
||||
chromium.use(stealth());
|
||||
|
||||
let contextPromise: Promise<BrowserContext> | null = null
|
||||
let context: BrowserContext | null = null
|
||||
let refCount = 0
|
||||
let idleCloseTimer: NodeJS.Timeout | null = null
|
||||
let contextPromise: Promise<BrowserContext> | null = null;
|
||||
let context: BrowserContext | null = null;
|
||||
let refCount = 0;
|
||||
let idleCloseTimer: NodeJS.Timeout | null = null;
|
||||
|
||||
const USER_DATA_DIR = process.env.USER_DATA_DIR ?? 'chrome-profile/douyin'
|
||||
const USER_DATA_DIR = process.env.USER_DATA_DIR ?? "chrome-profile/douyin";
|
||||
|
||||
async function launchContext(): Promise<BrowserContext> {
|
||||
const ctx = await chromium.launchPersistentContext(
|
||||
USER_DATA_DIR,
|
||||
{
|
||||
headless: process.env.CHROMIUM_HEADLESS === 'true',
|
||||
viewport: {
|
||||
width: Number(process.env.CHROMIUM_VIEWPORT_WIDTH ?? 1280),
|
||||
height: Number(process.env.CHROMIUM_VIEWPORT_HEIGHT ?? 1080)
|
||||
}
|
||||
}
|
||||
)
|
||||
const ctx = await chromium.launchPersistentContext(USER_DATA_DIR, {
|
||||
headless: process.env.CHROMIUM_HEADLESS === "true",
|
||||
viewport: {
|
||||
width: Number(process.env.CHROMIUM_VIEWPORT_WIDTH ?? 1280),
|
||||
height: Number(process.env.CHROMIUM_VIEWPORT_HEIGHT ?? 1080),
|
||||
},
|
||||
});
|
||||
// When the context is closed externally, reset manager state
|
||||
ctx.on('close', () => {
|
||||
context = null
|
||||
contextPromise = null
|
||||
refCount = 0
|
||||
ctx.on("close", () => {
|
||||
context = null;
|
||||
contextPromise = null;
|
||||
refCount = 0;
|
||||
if (idleCloseTimer) {
|
||||
clearTimeout(idleCloseTimer)
|
||||
idleCloseTimer = null
|
||||
clearTimeout(idleCloseTimer);
|
||||
idleCloseTimer = null;
|
||||
}
|
||||
})
|
||||
return ctx
|
||||
});
|
||||
return ctx;
|
||||
}
|
||||
|
||||
export async function acquireBrowserContext(): Promise<BrowserContext> {
|
||||
// Cancel any pending idle close if a new consumer arrives
|
||||
if (idleCloseTimer) {
|
||||
clearTimeout(idleCloseTimer)
|
||||
idleCloseTimer = null
|
||||
clearTimeout(idleCloseTimer);
|
||||
idleCloseTimer = null;
|
||||
}
|
||||
|
||||
if (context) {
|
||||
refCount += 1
|
||||
return context
|
||||
refCount += 1;
|
||||
return context;
|
||||
}
|
||||
|
||||
if (!contextPromise) {
|
||||
contextPromise = launchContext()
|
||||
contextPromise = launchContext();
|
||||
}
|
||||
context = await contextPromise
|
||||
refCount += 1
|
||||
return context
|
||||
context = await contextPromise;
|
||||
refCount += 1;
|
||||
return context;
|
||||
}
|
||||
|
||||
export async function releaseBrowserContext(options?: { idleMillis?: number }): Promise<void> {
|
||||
const idleMillis = options?.idleMillis ?? 15_000
|
||||
refCount = Math.max(0, refCount - 1)
|
||||
export async function releaseBrowserContext(options?: {
|
||||
idleMillis?: number;
|
||||
}): Promise<void> {
|
||||
const idleMillis = options?.idleMillis ?? 15_000;
|
||||
refCount = Math.max(0, refCount - 1);
|
||||
|
||||
if (refCount > 0 || !context) return
|
||||
if (refCount > 0 || !context) return;
|
||||
|
||||
// Delay the close to allow bursty workloads to reuse the context
|
||||
if (idleCloseTimer) {
|
||||
clearTimeout(idleCloseTimer)
|
||||
idleCloseTimer = null
|
||||
clearTimeout(idleCloseTimer);
|
||||
idleCloseTimer = null;
|
||||
}
|
||||
idleCloseTimer = setTimeout(async () => {
|
||||
try {
|
||||
if (context && refCount === 0) {
|
||||
await context.close()
|
||||
await context.close();
|
||||
}
|
||||
} finally {
|
||||
context = null
|
||||
contextPromise = null
|
||||
idleCloseTimer = null
|
||||
context = null;
|
||||
contextPromise = null;
|
||||
idleCloseTimer = null;
|
||||
}
|
||||
}, idleMillis)
|
||||
}, idleMillis);
|
||||
}
|
||||
|
||||
// --- Isolated context support for per-scrape independence ---
|
||||
|
||||
const isolatedMap = new WeakMap<BrowserContext, Browser>()
|
||||
const isolatedMap = new WeakMap<BrowserContext, Browser>();
|
||||
|
||||
async function getSharedStorageState(): Promise<any | undefined> {
|
||||
try {
|
||||
const shared = await acquireBrowserContext()
|
||||
const state = await shared.storageState()
|
||||
const shared = await acquireBrowserContext();
|
||||
const state = await shared.storageState();
|
||||
// Do not force-close immediately; keep ref-counting behavior
|
||||
await releaseBrowserContext()
|
||||
return state
|
||||
await releaseBrowserContext();
|
||||
return state;
|
||||
} catch {
|
||||
// If shared context not available, proceed without storageState
|
||||
return undefined
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -103,37 +102,39 @@ async function getSharedStorageState(): Promise<any | undefined> {
|
||||
* so you remain logged-in, but isolates network events, cache and listeners.
|
||||
*/
|
||||
export async function acquireIsolatedContext(): Promise<BrowserContext> {
|
||||
const storageState = await getSharedStorageState()
|
||||
const storageState = await getSharedStorageState();
|
||||
const browser = await chromium.launch({
|
||||
headless: process.env.CHROMIUM_HEADLESS === 'true'
|
||||
})
|
||||
headless: process.env.CHROMIUM_HEADLESS === "true",
|
||||
});
|
||||
const ctx = await browser.newContext({
|
||||
storageState,
|
||||
viewport: {
|
||||
width: Number(process.env.CHROMIUM_VIEWPORT_WIDTH ?? 1280),
|
||||
height: Number(process.env.CHROMIUM_VIEWPORT_HEIGHT ?? 1080)
|
||||
}
|
||||
})
|
||||
isolatedMap.set(ctx, browser)
|
||||
ctx.on('close', () => {
|
||||
const b = isolatedMap.get(ctx)
|
||||
height: Number(process.env.CHROMIUM_VIEWPORT_HEIGHT ?? 1080),
|
||||
},
|
||||
});
|
||||
isolatedMap.set(ctx, browser);
|
||||
ctx.on("close", () => {
|
||||
const b = isolatedMap.get(ctx);
|
||||
if (b) {
|
||||
b.close().catch(() => {})
|
||||
isolatedMap.delete(ctx)
|
||||
b.close().catch(() => {});
|
||||
isolatedMap.delete(ctx);
|
||||
}
|
||||
})
|
||||
return ctx
|
||||
});
|
||||
return ctx;
|
||||
}
|
||||
|
||||
export async function releaseIsolatedContext(ctx: BrowserContext | null | undefined): Promise<void> {
|
||||
if (!ctx) return
|
||||
export async function releaseIsolatedContext(
|
||||
ctx: BrowserContext | null | undefined,
|
||||
): Promise<void> {
|
||||
if (!ctx) return;
|
||||
try {
|
||||
await ctx.close()
|
||||
await ctx.close();
|
||||
} finally {
|
||||
const b = isolatedMap.get(ctx)
|
||||
const b = isolatedMap.get(ctx);
|
||||
if (b) {
|
||||
await b.close().catch(() => {})
|
||||
isolatedMap.delete(ctx)
|
||||
await b.close().catch(() => {});
|
||||
isolatedMap.delete(ctx);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+416
-287
@@ -1,22 +1,31 @@
|
||||
export const runtime = 'nodejs'
|
||||
export const runtime = "nodejs";
|
||||
// src/scrapeDouyin.ts
|
||||
import { BrowserContext, Page, type Response } from 'playwright';
|
||||
import { chromium } from 'playwright-extra';
|
||||
import { prisma } from '@/lib/prisma';
|
||||
import { uploadFile, generateUniqueFileName } from '@/lib/minio';
|
||||
import { createCamelCompatibleProxy } from '@/app/api/fetcher/utils';
|
||||
import { waitForFirstResponse, waitForResponseWithTimeout, safeJson, downloadBinary, collectResponsesWithinTime } from '@/app/api/fetcher/network';
|
||||
import { pickBestPlayAddr } from '@/app/api/fetcher/media';
|
||||
import { handleImagePost } from '@/app/api/fetcher/uploader';
|
||||
import { saveToDB, saveImagePostToDB } from '@/app/api/fetcher/persist';
|
||||
import chalk from 'chalk';
|
||||
import { acquireIsolatedContext, releaseIsolatedContext } from '@/app/api/fetcher/browser';
|
||||
import { extractFirstFrame } from '@/app/api/media';
|
||||
import { transcriptAweme } from '../stt';
|
||||
import { BrowserContext, Page, type Response } from "playwright";
|
||||
import { chromium } from "playwright-extra";
|
||||
import { prisma } from "@/lib/prisma";
|
||||
import { uploadFile, generateUniqueFileName } from "@/lib/minio";
|
||||
import { createCamelCompatibleProxy } from "@/app/api/fetcher/utils";
|
||||
import {
|
||||
waitForFirstResponse,
|
||||
waitForResponseWithTimeout,
|
||||
safeJson,
|
||||
downloadBinary,
|
||||
collectResponsesWithinTime,
|
||||
} from "@/app/api/fetcher/network";
|
||||
import { pickBestPlayAddr } from "@/app/api/fetcher/media";
|
||||
import { handleImagePost } from "@/app/api/fetcher/uploader";
|
||||
import { saveToDB, saveImagePostToDB } from "@/app/api/fetcher/persist";
|
||||
import chalk from "chalk";
|
||||
import {
|
||||
acquireIsolatedContext,
|
||||
releaseIsolatedContext,
|
||||
} from "@/app/api/fetcher/browser";
|
||||
import { extractFirstFrame } from "@/app/api/media";
|
||||
import { transcriptAweme } from "../stt";
|
||||
|
||||
const DETAIL_PATH = '/aweme/v1/web/aweme/detail/';
|
||||
const COMMENT_PATH = '/aweme/v1/web/comment/list/';
|
||||
const POST_PATH = '/aweme/v1/web/aweme/post/'
|
||||
const DETAIL_PATH = "/aweme/v1/web/aweme/detail/";
|
||||
const COMMENT_PATH = "/aweme/v1/web/comment/list/";
|
||||
const POST_PATH = "/aweme/v1/web/aweme/post/";
|
||||
|
||||
/**
|
||||
* 滚动页面并收集评论
|
||||
@@ -26,324 +35,444 @@ const POST_PATH = '/aweme/v1/web/aweme/post/'
|
||||
* @returns 收集到的所有评论响应
|
||||
*/
|
||||
async function scrollAndCollectComments(
|
||||
context: BrowserContext,
|
||||
page: Page,
|
||||
durationMs: number = 10_000
|
||||
context: BrowserContext,
|
||||
page: Page,
|
||||
durationMs: number = 10_000,
|
||||
): Promise<Response[]> {
|
||||
console.log(chalk.blue(`📜 开始滚动页面收集评论（持续 ${durationMs / 1000} 秒）...`));
|
||||
console.log(
|
||||
chalk.blue(`📜 开始滚动页面收集评论（持续 ${durationMs / 1000} 秒）...`),
|
||||
);
|
||||
|
||||
// 启动评论响应收集器
|
||||
const commentResponsesPromise = collectResponsesWithinTime(
|
||||
context,
|
||||
(r: Response) => r.url().includes(COMMENT_PATH) && r.status() === 200 && r.request().frame()?.page() === page,
|
||||
durationMs
|
||||
);
|
||||
// 启动评论响应收集器
|
||||
const commentResponsesPromise = collectResponsesWithinTime(
|
||||
context,
|
||||
(r: Response) =>
|
||||
r.url().includes(COMMENT_PATH) &&
|
||||
r.status() === 200 &&
|
||||
r.request().frame()?.page() === page,
|
||||
durationMs,
|
||||
);
|
||||
|
||||
// 在指定时间内持续滚动页面
|
||||
const startTime = Date.now();
|
||||
const scrollInterval = 500;
|
||||
let scrollCount = 0;
|
||||
const selector = "div[data-e2e='comment-list']";
|
||||
// 在指定时间内持续滚动页面
|
||||
const startTime = Date.now();
|
||||
const scrollInterval = 500;
|
||||
let scrollCount = 0;
|
||||
const selector = "div[data-e2e='comment-list']";
|
||||
|
||||
// 1) 等元素出现并可见
|
||||
await page.waitForSelector(selector, { state: 'visible', timeout: 5000 });
|
||||
// 1) 等元素出现并可见
|
||||
await page.waitForSelector(selector, { state: "visible", timeout: 5000 });
|
||||
|
||||
// 2) 确保滚动到可见区域
|
||||
const list = page.locator(selector);
|
||||
await list.scrollIntoViewIfNeeded();
|
||||
// 2) 确保滚动到可见区域
|
||||
const list = page.locator(selector);
|
||||
await list.scrollIntoViewIfNeeded();
|
||||
|
||||
// 3) 执行 hover（推荐用 locator 的 hover）
|
||||
list.hover({ timeout: 5000 }).catch(() => { });
|
||||
while (Date.now() - startTime < durationMs - 500) { // 留 500ms 缓冲
|
||||
try {
|
||||
list.hover({ timeout: 2000 }).catch(() => { });
|
||||
// 使用 Playwright 的 mouse.wheel 方法滚动
|
||||
// 每次滚动一大段距离
|
||||
// await list.hover();
|
||||
const scrollAmount = 1500;
|
||||
await page.mouse.wheel(0, scrollAmount);
|
||||
// 3) 执行 hover（推荐用 locator 的 hover）
|
||||
list.hover({ timeout: 5000 }).catch(() => {});
|
||||
while (Date.now() - startTime < durationMs - 500) {
|
||||
// 留 500ms 缓冲
|
||||
try {
|
||||
list.hover({ timeout: 2000 }).catch(() => {});
|
||||
// 使用 Playwright 的 mouse.wheel 方法滚动
|
||||
// 每次滚动一大段距离
|
||||
// await list.hover();
|
||||
const scrollAmount = 1500;
|
||||
await page.mouse.wheel(0, scrollAmount);
|
||||
|
||||
scrollCount++;
|
||||
console.log(chalk.gray(` ↓ 第 ${scrollCount} 次滚动`));
|
||||
scrollCount++;
|
||||
console.log(chalk.gray(` ↓ 第 ${scrollCount} 次滚动`));
|
||||
|
||||
// 等待一段时间，让评论加载
|
||||
await page.waitForTimeout(scrollInterval);
|
||||
|
||||
} catch (e) {
|
||||
console.warn(chalk.yellow(` ⚠ 滚动时出现警告: ${(e as Error)?.message}`));
|
||||
}
|
||||
// 等待一段时间，让评论加载
|
||||
await page.waitForTimeout(scrollInterval);
|
||||
} catch (e) {
|
||||
console.warn(
|
||||
chalk.yellow(` ⚠ 滚动时出现警告: ${(e as Error)?.message}`),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// 等待收集器完成
|
||||
const commentResponses = await commentResponsesPromise;
|
||||
console.log(chalk.green(`✓ 评论收集完成，共收集到 ${commentResponses.length} 个评论响应`));
|
||||
// 等待收集器完成
|
||||
const commentResponses = await commentResponsesPromise;
|
||||
console.log(
|
||||
chalk.green(
|
||||
`✓ 评论收集完成，共收集到 ${commentResponses.length} 个评论响应`,
|
||||
),
|
||||
);
|
||||
|
||||
return commentResponses;
|
||||
return commentResponses;
|
||||
}
|
||||
|
||||
async function readPostMem(context: BrowserContext, page: Page) {
|
||||
const md = await page.evaluate(() => {
|
||||
// @ts-ignore
|
||||
let data = window.__pace_captured__.find(i => i[1] && i[1].includes(`"awemeId":`))[1]
|
||||
return JSON.parse(data.slice(data.indexOf("{")).replaceAll("]\n", ''))
|
||||
// return {aweme: { detail: {} } };
|
||||
}).catch(() => null);
|
||||
const md = await page
|
||||
.evaluate(() => {
|
||||
const captured = (window as any).__pace_captured__ as Array<unknown[]>;
|
||||
let data = captured.find(
|
||||
(item) => typeof item[1] === "string" && item[1].includes(`"awemeId":`),
|
||||
)?.[1] as string | undefined;
|
||||
if (!data) return null;
|
||||
return JSON.parse(data.slice(data.indexOf("{")).replaceAll("]\n", ""));
|
||||
// return {aweme: { detail: {} } };
|
||||
})
|
||||
.catch(() => null);
|
||||
|
||||
// await new Promise((res) => setTimeout(res, 1000000));
|
||||
// await new Promise((res) => setTimeout(res, 1000000));
|
||||
|
||||
let aweme_mem = md?.aweme?.detail as DouyinImageAweme;
|
||||
if (!aweme_mem) throw new Error('页面内存数据中未找到作品详情');
|
||||
let aweme_mem = md?.aweme?.detail as DouyinImageAweme;
|
||||
if (!aweme_mem) throw new Error("页面内存数据中未找到作品详情");
|
||||
|
||||
// @ts-ignore
|
||||
aweme_mem.author = aweme_mem.authorInfo
|
||||
// @ts-ignore
|
||||
aweme_mem.statistics = aweme_mem.stats
|
||||
// @ts-ignore
|
||||
aweme_mem.author = aweme_mem.authorInfo;
|
||||
// @ts-ignore
|
||||
aweme_mem.statistics = aweme_mem.stats;
|
||||
|
||||
const comments = md.comment ? createCamelCompatibleProxy<DouyinCommentResponse>(md.comment) : null;
|
||||
const aweme = createCamelCompatibleProxy(aweme_mem);
|
||||
const comments = md.comment
|
||||
? createCamelCompatibleProxy<DouyinCommentResponse>(md.comment)
|
||||
: null;
|
||||
const aweme = createCamelCompatibleProxy(aweme_mem);
|
||||
|
||||
return { aweme, comments }
|
||||
return { aweme, comments };
|
||||
}
|
||||
|
||||
|
||||
export class ScrapeError extends Error {
|
||||
constructor(
|
||||
message: string,
|
||||
public statusCode: number = 500,
|
||||
public code?: string
|
||||
) {
|
||||
super(message);
|
||||
this.name = 'ScrapeError';
|
||||
}
|
||||
constructor(
|
||||
message: string,
|
||||
public statusCode: number = 500,
|
||||
public code?: string,
|
||||
) {
|
||||
super(message);
|
||||
this.name = "ScrapeError";
|
||||
}
|
||||
}
|
||||
|
||||
export async function scrapeDouyin(url: string) {
|
||||
console.log(chalk.blue('🚀 启动共享 Chromium 浏览器...'));
|
||||
let context: BrowserContext | null = await acquireIsolatedContext();
|
||||
const page = await context.newPage();
|
||||
console.log(chalk.cyan(`📄 正在访问: ${chalk.underline(url)}`));
|
||||
console.log(chalk.blue("🚀 启动共享 Chromium 浏览器..."));
|
||||
let context: BrowserContext | null = await acquireIsolatedContext();
|
||||
const page = await context.newPage();
|
||||
console.log(chalk.cyan(`📄 正在访问: ${chalk.underline(url)}`));
|
||||
|
||||
await page.addInitScript(() => {
|
||||
// 建一个全局容器存捕获的数据
|
||||
(window as any).__pace_captured__ = [];
|
||||
await page.addInitScript(() => {
|
||||
// 建一个全局容器存捕获的数据
|
||||
(window as any).__pace_captured__ = [];
|
||||
|
||||
// 用 Proxy 包装一个数组，拦截 push
|
||||
const captured = (window as any).__pace_captured__;
|
||||
const proxyArr = new Proxy([] as any[], {
|
||||
get(target, prop, receiver) {
|
||||
if (prop === 'push') {
|
||||
return (...items: any[]) => {
|
||||
try { captured.push(...items); } catch { }
|
||||
return Array.prototype.push.apply(target, items);
|
||||
};
|
||||
}
|
||||
return Reflect.get(target, prop, receiver);
|
||||
},
|
||||
set(target, prop, value, receiver) {
|
||||
// 兼容站点可能直接赋初始数组： self.__pace_f = [a,b]
|
||||
if (prop === 'length') return Reflect.set(target, prop, value, receiver);
|
||||
return Reflect.set(target, prop, value, receiver);
|
||||
}
|
||||
});
|
||||
|
||||
(self as any).__pace_f = proxyArr;
|
||||
(window as any).__pace_f = proxyArr;
|
||||
// 用 Proxy 包装一个数组，拦截 push
|
||||
const captured = (window as any).__pace_captured__;
|
||||
const proxyArr = new Proxy([] as any[], {
|
||||
get(target, prop, receiver) {
|
||||
if (prop === "push") {
|
||||
return (...items: any[]) => {
|
||||
try {
|
||||
captured.push(...items);
|
||||
} catch {}
|
||||
return Array.prototype.push.apply(target, items);
|
||||
};
|
||||
}
|
||||
return Reflect.get(target, prop, receiver);
|
||||
},
|
||||
set(target, prop, value, receiver) {
|
||||
// 兼容站点可能直接赋初始数组： self.__pace_f = [a,b]
|
||||
if (prop === "length")
|
||||
return Reflect.set(target, prop, value, receiver);
|
||||
return Reflect.set(target, prop, value, receiver);
|
||||
},
|
||||
});
|
||||
|
||||
(self as any).__pace_f = proxyArr;
|
||||
(window as any).__pace_f = proxyArr;
|
||||
});
|
||||
|
||||
try {
|
||||
// 先注册“先到先得”的监听，再导航，避免漏包
|
||||
const firstTypePromise = waitForFirstResponse(
|
||||
context,
|
||||
[
|
||||
{
|
||||
key: "detail",
|
||||
test: (r: Response) =>
|
||||
r.url().includes(DETAIL_PATH) &&
|
||||
r.status() === 200 &&
|
||||
r.request().frame()?.page() === page,
|
||||
},
|
||||
{
|
||||
key: "post",
|
||||
test: (r: Response) =>
|
||||
r.url().includes(POST_PATH) &&
|
||||
r.status() === 200 &&
|
||||
r.request().frame()?.page() === page,
|
||||
},
|
||||
],
|
||||
10_000,
|
||||
);
|
||||
|
||||
await page.goto(url, { waitUntil: "domcontentloaded", timeout: 20_000 });
|
||||
|
||||
// 查找页面中是否存在 "视频不存在" 的提示
|
||||
const isNotFound = await page
|
||||
.locator("text=视频不存在")
|
||||
.count()
|
||||
.then((count) => count > 0)
|
||||
.catch(() => false);
|
||||
if (isNotFound) {
|
||||
console.error(chalk.red("✗ 视频不存在或已被删除"));
|
||||
throw new ScrapeError("视频不存在或已被删除", 404, "VIDEO_NOT_FOUND");
|
||||
}
|
||||
|
||||
// 等待作品类型判定
|
||||
const firstType = await firstTypePromise;
|
||||
|
||||
// 尝试从内存读取图文数据（如果是图文作品）
|
||||
let memoryData: {
|
||||
aweme: any;
|
||||
comments: DouyinCommentResponse | null;
|
||||
} | null = null;
|
||||
try {
|
||||
// 先注册“先到先得”的监听，再导航，避免漏包
|
||||
const firstTypePromise = waitForFirstResponse(context, [
|
||||
{ key: 'detail', test: (r: Response) => r.url().includes(DETAIL_PATH) && r.status() === 200 && r.request().frame()?.page() === page },
|
||||
{ key: 'post', test: (r: Response) => r.url().includes(POST_PATH) && r.status() === 200 && r.request().frame()?.page() === page },
|
||||
], 10_000);
|
||||
memoryData = await readPostMem(context, page);
|
||||
console.log(chalk.green("✓ 从内存读取图文数据成功"));
|
||||
} catch {
|
||||
// 内存读取失败，稍后通过网络获取
|
||||
}
|
||||
|
||||
await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 20_000 });
|
||||
if (!firstType && !memoryData) {
|
||||
console.error(chalk.red("✗ 既无法从内存读取数据,也无法从网络获得数据"));
|
||||
throw new ScrapeError(
|
||||
"无法获取作品数据,可能是网络问题或作品已下架",
|
||||
404,
|
||||
"NO_DATA",
|
||||
);
|
||||
}
|
||||
|
||||
// 查找页面中是否存在 "视频不存在" 的提示
|
||||
const isNotFound = await page.locator('text=视频不存在').count().then(count => count > 0).catch(() => false);
|
||||
if (isNotFound) {
|
||||
console.error(chalk.red('✗ 视频不存在或已被删除'));
|
||||
throw new ScrapeError('视频不存在或已被删除', 404, 'VIDEO_NOT_FOUND');
|
||||
}
|
||||
console.log(
|
||||
chalk.cyan(
|
||||
`📡 检测到作品类型: ${chalk.bold(firstType?.key === "post" || memoryData ? "图文" : "视频")}`,
|
||||
),
|
||||
);
|
||||
|
||||
// 等待作品类型判定
|
||||
const firstType = await firstTypePromise;
|
||||
let allComments: DouyinComment[] = [];
|
||||
try {
|
||||
// 开始滚动并收集评论
|
||||
const commentResponses = await scrollAndCollectComments(context, page);
|
||||
|
||||
// 尝试从内存读取图文数据（如果是图文作品）
|
||||
let memoryData: { aweme: any; comments: DouyinCommentResponse | null } | null = null;
|
||||
// 解析所有收集到的评论响应
|
||||
for (const commentRes of commentResponses) {
|
||||
try {
|
||||
memoryData = await readPostMem(context, page);
|
||||
console.log(chalk.green('✓ 从内存读取图文数据成功'));
|
||||
} catch {
|
||||
// 内存读取失败，稍后通过网络获取
|
||||
const commentData = await safeJson<DouyinCommentResponse>(commentRes);
|
||||
if (commentData?.comments?.length) {
|
||||
allComments.push(...commentData.comments);
|
||||
}
|
||||
} catch (e) {
|
||||
console.warn(
|
||||
chalk.yellow(`⚠ 解析评论响应失败: ${(e as Error)?.message}`),
|
||||
);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.warn(
|
||||
chalk.yellow(`⚠ 评论收集失败: ${(error as Error)?.message}`),
|
||||
);
|
||||
}
|
||||
|
||||
if (!firstType && !memoryData) {
|
||||
console.error(chalk.red('✗ 既无法从内存读取数据,也无法从网络获得数据'));
|
||||
throw new ScrapeError('无法获取作品数据,可能是网络问题或作品已下架', 404, 'NO_DATA');
|
||||
// 去重评论（根据 cid）
|
||||
const uniqueComments = Array.from(
|
||||
new Map(allComments.map((c) => [c.cid, c])).values(),
|
||||
);
|
||||
|
||||
console.log(
|
||||
chalk.green(
|
||||
`✓ 共收集到 ${uniqueComments.length} 条独立评论（去重前: ${allComments.length}）`,
|
||||
),
|
||||
);
|
||||
|
||||
// 如果从内存读取到了评论，合并进来作为兜底
|
||||
let comments: DouyinCommentResponse;
|
||||
if (memoryData?.comments?.comments?.length) {
|
||||
console.log(
|
||||
chalk.blue(
|
||||
`📝 合并内存中的 ${memoryData.comments.comments.length} 条评论`,
|
||||
),
|
||||
);
|
||||
const memComments = memoryData.comments.comments;
|
||||
const mergedMap = new Map(uniqueComments.map((c) => [c.cid, c]));
|
||||
for (const c of memComments) {
|
||||
if (!mergedMap.has(c.cid)) {
|
||||
mergedMap.set(c.cid, c);
|
||||
}
|
||||
}
|
||||
comments = {
|
||||
comments: Array.from(mergedMap.values()),
|
||||
total: mergedMap.size,
|
||||
status_code: 0,
|
||||
};
|
||||
console.log(chalk.green(`✓ 合并后共 ${comments.comments.length} 条评论`));
|
||||
} else {
|
||||
comments = {
|
||||
comments: uniqueComments,
|
||||
total: uniqueComments.length,
|
||||
status_code: 0,
|
||||
};
|
||||
}
|
||||
|
||||
console.log(chalk.cyan(`📡 检测到作品类型: ${chalk.bold(firstType?.key === 'post' || memoryData ? '图文' : '视频')}`));
|
||||
// 分支：视频 or 图文（两者只会有一个命中，先到先得）
|
||||
// 优先处理内存数据（图文）
|
||||
if (memoryData) {
|
||||
const aweme = memoryData.aweme;
|
||||
const uploads = await handleImagePost(context, aweme);
|
||||
const saved = await saveImagePostToDB(context, aweme, comments, uploads); // 传递完整 JSON
|
||||
console.log(chalk.green.bold("✓ 图文作品保存成功"));
|
||||
return { type: "image", ...saved };
|
||||
} else if (firstType?.key === "post") {
|
||||
// 图文作品（网络）
|
||||
const postJson = await safeJson<DouyinPostListResponse>(
|
||||
firstType.response,
|
||||
);
|
||||
if (!postJson?.aweme_list?.length)
|
||||
throw new ScrapeError("图文作品响应为空", 404, "EMPTY_POST_RESPONSE");
|
||||
|
||||
let allComments: DouyinComment[] = [];
|
||||
try {
|
||||
// 开始滚动并收集评论
|
||||
const commentResponses = await scrollAndCollectComments(context, page);
|
||||
const currentURL = page.url();
|
||||
const target_aweme_id = currentURL.split("/").at(-1);
|
||||
const awemeList = postJson.aweme_list as unknown as DouyinImageAweme[];
|
||||
let aweme = awemeList.find(
|
||||
(pt: DouyinImageAweme) => pt.aweme_id === target_aweme_id,
|
||||
);
|
||||
if (!aweme) {
|
||||
throw new ScrapeError(
|
||||
"无法找到目标作品,可能已被删除",
|
||||
404,
|
||||
"POST_NOT_FOUND",
|
||||
);
|
||||
}
|
||||
|
||||
// 解析所有收集到的评论响应
|
||||
for (const commentRes of commentResponses) {
|
||||
try {
|
||||
const commentData = await safeJson<DouyinCommentResponse>(commentRes);
|
||||
if (commentData?.comments?.length) {
|
||||
allComments.push(...commentData.comments);
|
||||
}
|
||||
} catch (e) {
|
||||
console.warn(chalk.yellow(`⚠ 解析评论响应失败: ${(e as Error)?.message}`));
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.warn(chalk.yellow(`⚠ 评论收集失败: ${(error as Error)?.message}`));
|
||||
}
|
||||
const uploads = await handleImagePost(context, aweme);
|
||||
const saved = await saveImagePostToDB(
|
||||
context,
|
||||
aweme,
|
||||
comments,
|
||||
uploads,
|
||||
postJson,
|
||||
); // 传递完整 JSON
|
||||
console.log(chalk.green.bold("✓ 图文作品保存成功"));
|
||||
return { type: "image", ...saved };
|
||||
} else if (firstType?.key === "detail") {
|
||||
// 视频作品
|
||||
const detail = (await safeJson<DouyinVideoDetailResponse>(
|
||||
firstType.response,
|
||||
))!;
|
||||
|
||||
// 找到比特率最高的 url
|
||||
const bestPlayAddr = pickBestPlayAddr(
|
||||
detail?.aweme_detail?.video.bit_rate,
|
||||
);
|
||||
const bestVUrl = bestPlayAddr?.url_list?.[0];
|
||||
const fps = bestPlayAddr?.FPS ?? null; // 提取 FPS
|
||||
|
||||
// 去重评论（根据 cid）
|
||||
const uniqueComments = Array.from(
|
||||
new Map(allComments.map(c => [c.cid, c])).values()
|
||||
console.log(chalk.cyan(`📹 最佳视频 URL: ${chalk.dim(bestVUrl)}`));
|
||||
console.log(chalk.cyan(`🎞️ 视频帧率: ${chalk.bold(fps || "N/A")} FPS`));
|
||||
if (bestPlayAddr?.width && bestPlayAddr?.height) {
|
||||
console.log(
|
||||
chalk.cyan(
|
||||
`📐 视频分辨率: ${chalk.bold(`${bestPlayAddr.width}x${bestPlayAddr.height}`)}`,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
// 下载视频并上传至 MinIO，获取外链
|
||||
let uploadedUrl: string | undefined;
|
||||
let coverUrl: string | undefined;
|
||||
if (bestVUrl && detail?.aweme_detail) {
|
||||
console.log(chalk.blue("⬇️ 正在下载视频..."));
|
||||
const { buffer, contentType, ext } = await downloadBinary(
|
||||
context,
|
||||
bestVUrl,
|
||||
);
|
||||
const awemeId = detail.aweme_detail.aweme_id;
|
||||
const fileName = generateUniqueFileName(
|
||||
`${awemeId}.${ext}`,
|
||||
"douyin/videos",
|
||||
);
|
||||
|
||||
console.log(chalk.green(`✓ 共收集到 ${uniqueComments.length} 条独立评论（去重前: ${allComments.length}）`));
|
||||
console.log(chalk.blue("⬆️ 正在上传视频到 MinIO..."));
|
||||
uploadedUrl = await uploadFile(buffer, fileName, {
|
||||
"Content-Type": contentType,
|
||||
});
|
||||
console.log(
|
||||
chalk.green(`✓ 视频上传成功: ${chalk.underline(uploadedUrl)}`),
|
||||
);
|
||||
|
||||
// 如果从内存读取到了评论，合并进来作为兜底
|
||||
let comments: DouyinCommentResponse;
|
||||
if (memoryData?.comments?.comments?.length) {
|
||||
console.log(chalk.blue(`📝 合并内存中的 ${memoryData.comments.comments.length} 条评论`));
|
||||
const memComments = memoryData.comments.comments;
|
||||
const mergedMap = new Map(uniqueComments.map(c => [c.cid, c]));
|
||||
for (const c of memComments) {
|
||||
if (!mergedMap.has(c.cid)) {
|
||||
mergedMap.set(c.cid, c);
|
||||
}
|
||||
}
|
||||
comments = {
|
||||
comments: Array.from(mergedMap.values()),
|
||||
total: mergedMap.size,
|
||||
status_code: 0
|
||||
};
|
||||
console.log(chalk.green(`✓ 合并后共 ${comments.comments.length} 条评论`));
|
||||
} else {
|
||||
comments = {
|
||||
comments: uniqueComments,
|
||||
total: uniqueComments.length,
|
||||
status_code: 0
|
||||
};
|
||||
}
|
||||
|
||||
// 分支：视频 or 图文（两者只会有一个命中，先到先得）
|
||||
// 优先处理内存数据（图文）
|
||||
if (memoryData) {
|
||||
const aweme = memoryData.aweme;
|
||||
const uploads = await handleImagePost(context, aweme);
|
||||
const saved = await saveImagePostToDB(context, aweme, comments, uploads); // 传递完整 JSON
|
||||
console.log(chalk.green.bold('✓ 图文作品保存成功'));
|
||||
return { type: "image", ...saved };
|
||||
} else if (firstType?.key === 'post') {
|
||||
// 图文作品（网络）
|
||||
const postJson = await safeJson<DouyinPostListResponse>(firstType.response);
|
||||
if (!postJson?.aweme_list?.length) throw new ScrapeError('图文作品响应为空', 404, 'EMPTY_POST_RESPONSE');
|
||||
|
||||
const currentURL = page.url();
|
||||
const target_aweme_id = currentURL.split('/').at(-1);
|
||||
const awemeList = postJson.aweme_list as unknown as DouyinImageAweme[];
|
||||
let aweme = awemeList.find((pt: DouyinImageAweme) => pt.aweme_id === target_aweme_id);
|
||||
if (!aweme) {
|
||||
throw new ScrapeError('无法找到目标作品,可能已被删除', 404, 'POST_NOT_FOUND');
|
||||
}
|
||||
|
||||
const uploads = await handleImagePost(context, aweme);
|
||||
const saved = await saveImagePostToDB(context, aweme, comments, uploads, postJson); // 传递完整 JSON
|
||||
console.log(chalk.green.bold('✓ 图文作品保存成功'));
|
||||
return { type: "image", ...saved };
|
||||
} else if (firstType?.key === 'detail') {
|
||||
// 视频作品
|
||||
const detail = (await safeJson<DouyinVideoDetailResponse>(firstType.response))!;
|
||||
|
||||
// 找到比特率最高的 url
|
||||
const bestPlayAddr = pickBestPlayAddr(
|
||||
detail?.aweme_detail?.video.bit_rate
|
||||
// 提取首帧作为封面并上传
|
||||
try {
|
||||
console.log(chalk.blue("🖼️ 正在提取视频封面..."));
|
||||
const cover = await extractFirstFrame(buffer);
|
||||
if (cover) {
|
||||
const coverName = generateUniqueFileName(
|
||||
`${awemeId}.jpg`,
|
||||
"douyin/covers",
|
||||
);
|
||||
const bestVUrl = bestPlayAddr?.url_list?.[0];
|
||||
const fps = bestPlayAddr?.FPS ?? null; // 提取 FPS
|
||||
|
||||
console.log(chalk.cyan(`📹 最佳视频 URL: ${chalk.dim(bestVUrl)}`));
|
||||
console.log(chalk.cyan(`🎞️ 视频帧率: ${chalk.bold(fps || 'N/A')} FPS`));
|
||||
if (bestPlayAddr?.width && bestPlayAddr?.height) {
|
||||
console.log(chalk.cyan(`📐 视频分辨率: ${chalk.bold(`${bestPlayAddr.width}x${bestPlayAddr.height}`)}`));
|
||||
}
|
||||
|
||||
// 下载视频并上传至 MinIO，获取外链
|
||||
let uploadedUrl: string | undefined;
|
||||
let coverUrl: string | undefined;
|
||||
if (bestVUrl && detail?.aweme_detail) {
|
||||
console.log(chalk.blue('⬇️ 正在下载视频...'));
|
||||
const { buffer, contentType, ext } = await downloadBinary(context, bestVUrl);
|
||||
const awemeId = detail.aweme_detail.aweme_id;
|
||||
const fileName = generateUniqueFileName(`${awemeId}.${ext}`, 'douyin/videos');
|
||||
|
||||
console.log(chalk.blue('⬆️ 正在上传视频到 MinIO...'));
|
||||
uploadedUrl = await uploadFile(buffer, fileName, { 'Content-Type': contentType });
|
||||
console.log(chalk.green(`✓ 视频上传成功: ${chalk.underline(uploadedUrl)}`));
|
||||
|
||||
// 提取首帧作为封面并上传
|
||||
try {
|
||||
console.log(chalk.blue('🖼️ 正在提取视频封面...'));
|
||||
const cover = await extractFirstFrame(buffer);
|
||||
if (cover) {
|
||||
const coverName = generateUniqueFileName(`${awemeId}.jpg`, 'douyin/covers');
|
||||
coverUrl = await uploadFile(cover.buffer, coverName, { 'Content-Type': cover.contentType });
|
||||
console.log(chalk.green(`✓ 封面上传成功: ${chalk.underline(coverUrl)}`));
|
||||
}
|
||||
} catch (e) {
|
||||
console.warn(chalk.yellow(`⚠ 提取封面失败，跳过: ${(e as Error)?.message || e}`));
|
||||
}
|
||||
}
|
||||
|
||||
const saved = await saveToDB(context, detail, comments, uploadedUrl, bestPlayAddr?.width, bestPlayAddr?.height, coverUrl, fps ?? undefined);
|
||||
console.log(chalk.green.bold('✓ 视频作品保存成功'));
|
||||
transcriptAweme(detail.aweme_detail.aweme_id).catch((e) => {}); // 异步转写，不阻塞主流程
|
||||
return { type: "video", ...saved };
|
||||
} else {
|
||||
throw new ScrapeError('无法判定作品类型,接口响应异常', 500, 'UNKNOWN_TYPE');
|
||||
}
|
||||
} catch (error) {
|
||||
// 如果是我们自定义的错误,直接抛出
|
||||
if (error instanceof ScrapeError) {
|
||||
throw error;
|
||||
coverUrl = await uploadFile(cover.buffer, coverName, {
|
||||
"Content-Type": cover.contentType,
|
||||
});
|
||||
console.log(
|
||||
chalk.green(`✓ 封面上传成功: ${chalk.underline(coverUrl)}`),
|
||||
);
|
||||
}
|
||||
} catch (e) {
|
||||
console.warn(
|
||||
chalk.yellow(`⚠ 提取封面失败，跳过: ${(e as Error)?.message || e}`),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// 处理其他类型的错误
|
||||
const errMsg = (error as Error)?.message || String(error);
|
||||
console.error(chalk.red(`✗ 爬取失败: ${errMsg}`));
|
||||
|
||||
// 根据错误类型返回不同的状态码
|
||||
if (errMsg.includes('timeout') || errMsg.includes('超时')) {
|
||||
throw new ScrapeError('请求超时,请稍后重试', 408, 'TIMEOUT');
|
||||
}
|
||||
if (errMsg.includes('页面内存数据中未找到作品详情')) {
|
||||
throw new ScrapeError('作品数据加载失败', 404, 'DATA_NOT_LOADED');
|
||||
}
|
||||
if (errMsg.includes('net::')) {
|
||||
throw new ScrapeError('网络连接失败', 503, 'NETWORK_ERROR');
|
||||
}
|
||||
|
||||
// 默认服务器错误
|
||||
throw new ScrapeError(errMsg || '爬取过程中发生未知错误', 500, 'UNKNOWN_ERROR');
|
||||
} finally {
|
||||
console.log(chalk.gray('🧹 清理资源...'));
|
||||
try { await page.close({ runBeforeUnload: true }); } catch { }
|
||||
// 关闭本次任务的隔离上下文与浏览器
|
||||
await releaseIsolatedContext(context);
|
||||
await prisma.$disconnect();
|
||||
console.log(chalk.gray('✓ 资源清理完成'));
|
||||
const saved = await saveToDB(
|
||||
context,
|
||||
detail,
|
||||
comments,
|
||||
uploadedUrl,
|
||||
bestPlayAddr?.width,
|
||||
bestPlayAddr?.height,
|
||||
coverUrl,
|
||||
fps ?? undefined,
|
||||
);
|
||||
console.log(chalk.green.bold("✓ 视频作品保存成功"));
|
||||
transcriptAweme(detail.aweme_detail.aweme_id).catch((e) => {}); // 异步转写，不阻塞主流程
|
||||
return { type: "video", ...saved };
|
||||
} else {
|
||||
throw new ScrapeError(
|
||||
"无法判定作品类型,接口响应异常",
|
||||
500,
|
||||
"UNKNOWN_TYPE",
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
// 如果是我们自定义的错误,直接抛出
|
||||
if (error instanceof ScrapeError) {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
// 处理其他类型的错误
|
||||
const errMsg = (error as Error)?.message || String(error);
|
||||
console.error(chalk.red(`✗ 爬取失败: ${errMsg}`));
|
||||
|
||||
// 根据错误类型返回不同的状态码
|
||||
if (errMsg.includes("timeout") || errMsg.includes("超时")) {
|
||||
throw new ScrapeError("请求超时,请稍后重试", 408, "TIMEOUT");
|
||||
}
|
||||
if (errMsg.includes("页面内存数据中未找到作品详情")) {
|
||||
throw new ScrapeError("作品数据加载失败", 404, "DATA_NOT_LOADED");
|
||||
}
|
||||
if (errMsg.includes("net::")) {
|
||||
throw new ScrapeError("网络连接失败", 503, "NETWORK_ERROR");
|
||||
}
|
||||
|
||||
// 默认服务器错误
|
||||
throw new ScrapeError(
|
||||
errMsg || "爬取过程中发生未知错误",
|
||||
500,
|
||||
"UNKNOWN_ERROR",
|
||||
);
|
||||
} finally {
|
||||
console.log(chalk.gray("🧹 清理资源..."));
|
||||
try {
|
||||
await page.close({ runBeforeUnload: true });
|
||||
} catch {}
|
||||
// 关闭本次任务的隔离上下文与浏览器
|
||||
await releaseIsolatedContext(context);
|
||||
await prisma.$disconnect();
|
||||
console.log(chalk.gray("✓ 资源清理完成"));
|
||||
}
|
||||
}
|
||||
|
||||
+10
-11
@@ -1,17 +1,16 @@
|
||||
export const runtime = 'nodejs'
|
||||
|
||||
import { execFile } from 'child_process';
|
||||
import { promisify } from 'util';
|
||||
export const runtime = "nodejs";
|
||||
|
||||
import { execFile } from "child_process";
|
||||
import { promisify } from "util";
|
||||
|
||||
export function pickBestPlayAddr(variants: PlayVariant[] | undefined | null) {
|
||||
if (!variants?.length) return null;
|
||||
if (!variants?.length) return null;
|
||||
|
||||
const best = variants.reduce((best, cur) => {
|
||||
const b1 = best?.bit_rate ?? -1;
|
||||
const b2 = cur?.bit_rate ?? -1;
|
||||
return b2 > b1 ? cur : best;
|
||||
});
|
||||
const best = variants.reduce((best, cur) => {
|
||||
const b1 = best?.bit_rate ?? -1;
|
||||
const b2 = cur?.bit_rate ?? -1;
|
||||
return b2 > b1 ? cur : best;
|
||||
});
|
||||
|
||||
return best?.play_addr ?? null;
|
||||
return best?.play_addr ?? null;
|
||||
}
|
||||
|
||||
+128
-127
@@ -1,18 +1,18 @@
|
||||
export const runtime = 'nodejs'
|
||||
export const runtime = "nodejs";
|
||||
|
||||
import type { BrowserContext, Response } from 'playwright';
|
||||
import type { BrowserContext, Response } from "playwright";
|
||||
|
||||
export async function safeJson<T>(res: Response): Promise<T | null> {
|
||||
const ctype = res.headers()['content-type'] || '';
|
||||
if (ctype.includes('application/json')) {
|
||||
return (await res.json()) as T;
|
||||
}
|
||||
const t = await res.text();
|
||||
try {
|
||||
return JSON.parse(t) as T;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
const ctype = res.headers()["content-type"] || "";
|
||||
if (ctype.includes("application/json")) {
|
||||
return (await res.json()) as T;
|
||||
}
|
||||
const t = await res.text();
|
||||
try {
|
||||
return JSON.parse(t) as T;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -21,30 +21,31 @@ export async function safeJson<T>(res: Response): Promise<T | null> {
|
||||
* - referrer 使用链接本身
|
||||
*/
|
||||
export async function downloadBinary(
|
||||
context: BrowserContext,
|
||||
url: string,
|
||||
context: BrowserContext,
|
||||
url: string,
|
||||
): Promise<{ buffer: Buffer; contentType: string; ext: string }> {
|
||||
console.log('下载：', url);
|
||||
console.log("下载：", url);
|
||||
|
||||
const headers = {
|
||||
referer: 'https://www.douyin.com/',
|
||||
} as Record<string, string>;
|
||||
const headers = {
|
||||
referer: "https://www.douyin.com/",
|
||||
} as Record<string, string>;
|
||||
|
||||
const res = await context.request.get(url, {
|
||||
headers,
|
||||
maxRedirects: 3,
|
||||
timeout: 240_000,
|
||||
failOnStatusCode: true,
|
||||
});
|
||||
const res = await context.request.get(url, {
|
||||
headers,
|
||||
maxRedirects: 3,
|
||||
timeout: 240_000,
|
||||
failOnStatusCode: true,
|
||||
});
|
||||
|
||||
if (!res.ok()) {
|
||||
throw new Error(`下载内容失败: ${res.status()} ${res.statusText()}`);
|
||||
}
|
||||
if (!res.ok()) {
|
||||
throw new Error(`下载内容失败: ${res.status()} ${res.statusText()}`);
|
||||
}
|
||||
|
||||
const buffer = await res.body();
|
||||
const contentType = res.headers()['content-type'] || 'application/octet-stream';
|
||||
const ext = (contentType.split('/')[1] || 'bin').split(';')[0] || 'bin';
|
||||
return { buffer, contentType, ext };
|
||||
const buffer = await res.body();
|
||||
const contentType =
|
||||
res.headers()["content-type"] || "application/octet-stream";
|
||||
const ext = (contentType.split("/")[1] || "bin").split(";")[0] || "bin";
|
||||
return { buffer, contentType, ext };
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -52,46 +53,46 @@ export async function downloadBinary(
|
||||
* - 不为每个候选单独设长超时，改用整体兜底超时，避免无意义等待。
|
||||
*/
|
||||
export function waitForFirstResponse(
|
||||
context: BrowserContext,
|
||||
candidates: { key: string; test: (r: Response) => boolean }[],
|
||||
timeoutMs = 20_000
|
||||
context: BrowserContext,
|
||||
candidates: { key: string; test: (r: Response) => boolean }[],
|
||||
timeoutMs = 20_000,
|
||||
): Promise<{ key: string; response: Response } | null> {
|
||||
return new Promise((resolve) => {
|
||||
let resolved = false;
|
||||
let timer: NodeJS.Timeout | undefined;
|
||||
return new Promise((resolve) => {
|
||||
let resolved = false;
|
||||
let timer: NodeJS.Timeout | undefined;
|
||||
|
||||
const handler = (res: Response) => {
|
||||
if (resolved) return;
|
||||
for (const c of candidates) {
|
||||
try {
|
||||
if (c.test(res)) {
|
||||
resolved = true;
|
||||
cleanup();
|
||||
resolve({ key: c.key, response: res });
|
||||
return;
|
||||
}
|
||||
} catch {
|
||||
// ignore predicate errors
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const cleanup = () => {
|
||||
context.off('response', handler);
|
||||
if (timer) clearTimeout(timer);
|
||||
};
|
||||
|
||||
context.on('response', handler);
|
||||
if (timeoutMs > 0) {
|
||||
timer = setTimeout(() => {
|
||||
if (!resolved) {
|
||||
resolved = true;
|
||||
cleanup();
|
||||
resolve(null);
|
||||
}
|
||||
}, timeoutMs);
|
||||
const handler = (res: Response) => {
|
||||
if (resolved) return;
|
||||
for (const c of candidates) {
|
||||
try {
|
||||
if (c.test(res)) {
|
||||
resolved = true;
|
||||
cleanup();
|
||||
resolve({ key: c.key, response: res });
|
||||
return;
|
||||
}
|
||||
} catch {
|
||||
// ignore predicate errors
|
||||
}
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
const cleanup = () => {
|
||||
context.off("response", handler);
|
||||
if (timer) clearTimeout(timer);
|
||||
};
|
||||
|
||||
context.on("response", handler);
|
||||
if (timeoutMs > 0) {
|
||||
timer = setTimeout(() => {
|
||||
if (!resolved) {
|
||||
resolved = true;
|
||||
cleanup();
|
||||
resolve(null);
|
||||
}
|
||||
}, timeoutMs);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -99,76 +100,76 @@ export function waitForFirstResponse(
|
||||
* 用于评论等需要滚动加载的数据
|
||||
*/
|
||||
export function collectResponsesWithinTime(
|
||||
context: BrowserContext,
|
||||
predicate: (r: Response) => boolean,
|
||||
durationMs: number
|
||||
context: BrowserContext,
|
||||
predicate: (r: Response) => boolean,
|
||||
durationMs: number,
|
||||
): Promise<Response[]> {
|
||||
return new Promise((resolve) => {
|
||||
const collected: Response[] = [];
|
||||
const seenUrls = new Set<string>();
|
||||
let timer: NodeJS.Timeout | undefined;
|
||||
return new Promise((resolve) => {
|
||||
const collected: Response[] = [];
|
||||
const seenUrls = new Set<string>();
|
||||
let timer: NodeJS.Timeout | undefined;
|
||||
|
||||
const handler = (res: Response) => {
|
||||
try {
|
||||
if (predicate(res)) {
|
||||
// 使用 URL 去重，避免重复收集同一个请求
|
||||
const url = res.url();
|
||||
if (!seenUrls.has(url)) {
|
||||
seenUrls.add(url);
|
||||
collected.push(res);
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// ignore predicate errors
|
||||
}
|
||||
};
|
||||
const handler = (res: Response) => {
|
||||
try {
|
||||
if (predicate(res)) {
|
||||
// 使用 URL 去重，避免重复收集同一个请求
|
||||
const url = res.url();
|
||||
if (!seenUrls.has(url)) {
|
||||
seenUrls.add(url);
|
||||
collected.push(res);
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// ignore predicate errors
|
||||
}
|
||||
};
|
||||
|
||||
const cleanup = () => {
|
||||
context.off('response', handler);
|
||||
if (timer) clearTimeout(timer);
|
||||
};
|
||||
const cleanup = () => {
|
||||
context.off("response", handler);
|
||||
if (timer) clearTimeout(timer);
|
||||
};
|
||||
|
||||
context.on('response', handler);
|
||||
timer = setTimeout(() => {
|
||||
cleanup();
|
||||
resolve(collected);
|
||||
}, durationMs);
|
||||
});
|
||||
context.on("response", handler);
|
||||
timer = setTimeout(() => {
|
||||
cleanup();
|
||||
resolve(collected);
|
||||
}, durationMs);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* 等待符合条件的单个 Response，带短超时；用于评论等"可有可无"的数据。
|
||||
*/
|
||||
export function waitForResponseWithTimeout(
|
||||
context: BrowserContext,
|
||||
predicate: (r: Response) => boolean,
|
||||
timeoutMs = 5_000
|
||||
context: BrowserContext,
|
||||
predicate: (r: Response) => boolean,
|
||||
timeoutMs = 5_000,
|
||||
): Promise<Response> {
|
||||
return new Promise<Response>((resolve, reject) => {
|
||||
let timer: NodeJS.Timeout | undefined;
|
||||
return new Promise<Response>((resolve, reject) => {
|
||||
let timer: NodeJS.Timeout | undefined;
|
||||
|
||||
const handler = (res: Response) => {
|
||||
try {
|
||||
if (predicate(res)) {
|
||||
cleanup();
|
||||
resolve(res);
|
||||
}
|
||||
} catch {
|
||||
// ignore predicate errors
|
||||
}
|
||||
};
|
||||
|
||||
const cleanup = () => {
|
||||
context.off('response', handler);
|
||||
if (timer) clearTimeout(timer);
|
||||
};
|
||||
|
||||
context.on('response', handler);
|
||||
if (timeoutMs > 0) {
|
||||
timer = setTimeout(() => {
|
||||
cleanup();
|
||||
reject(new Error('timeout'));
|
||||
}, timeoutMs);
|
||||
const handler = (res: Response) => {
|
||||
try {
|
||||
if (predicate(res)) {
|
||||
cleanup();
|
||||
resolve(res);
|
||||
}
|
||||
});
|
||||
} catch {
|
||||
// ignore predicate errors
|
||||
}
|
||||
};
|
||||
|
||||
const cleanup = () => {
|
||||
context.off("response", handler);
|
||||
if (timer) clearTimeout(timer);
|
||||
};
|
||||
|
||||
context.on("response", handler);
|
||||
if (timeoutMs > 0) {
|
||||
timer = setTimeout(() => {
|
||||
cleanup();
|
||||
reject(new Error("timeout"));
|
||||
}, timeoutMs);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
+399
-331
@@ -1,359 +1,427 @@
|
||||
import type { BrowserContext } from 'playwright';
|
||||
import { prisma } from '@/lib/prisma';
|
||||
import { uploadAvatarFromUrl, uploadImageFromUrl } from './uploader';
|
||||
import { firstUrl } from './utils';
|
||||
import type { BrowserContext } from "playwright";
|
||||
import { prisma } from "@/lib/prisma";
|
||||
import { uploadAvatarFromUrl, uploadImageFromUrl } from "./uploader";
|
||||
import { firstUrl } from "./utils";
|
||||
|
||||
export async function saveToDB(
|
||||
context: BrowserContext,
|
||||
detailResp: DouyinVideoDetailResponse,
|
||||
commentResp: DouyinCommentResponse,
|
||||
videoUrl?: string,
|
||||
width?: number,
|
||||
height?: number,
|
||||
coverUrl?: string,
|
||||
fps?: number
|
||||
context: BrowserContext,
|
||||
detailResp: DouyinVideoDetailResponse,
|
||||
commentResp: DouyinCommentResponse,
|
||||
videoUrl?: string,
|
||||
width?: number,
|
||||
height?: number,
|
||||
coverUrl?: string,
|
||||
fps?: number,
|
||||
) {
|
||||
if (!detailResp?.aweme_detail) throw new Error('视频详情为空');
|
||||
const d = detailResp.aweme_detail;
|
||||
if (!detailResp?.aweme_detail) throw new Error("视频详情为空");
|
||||
const d = detailResp.aweme_detail;
|
||||
|
||||
// 1) Upsert Author
|
||||
const authorAvatarSrc = firstUrl(d.author.avatar_thumb?.url_list);
|
||||
const authorAvatarUploaded = await uploadAvatarFromUrl(context, authorAvatarSrc, `authors/${d.author.sec_uid}`);
|
||||
const author = await prisma.author.upsert({
|
||||
where: { sec_uid: d.author.sec_uid },
|
||||
create: {
|
||||
sec_uid: d.author.sec_uid,
|
||||
uid: d.author.uid,
|
||||
nickname: d.author.nickname,
|
||||
signature: d.author.signature ?? null,
|
||||
avatar_url: authorAvatarUploaded ?? null,
|
||||
follower_count: BigInt(d.author.follower_count || 0),
|
||||
total_favorited: BigInt(d.author.total_favorited || 0),
|
||||
unique_id: d.author.unique_id ?? null,
|
||||
short_id: d.author.short_id ?? null,
|
||||
},
|
||||
update: {
|
||||
uid: d.author.uid,
|
||||
nickname: d.author.nickname,
|
||||
signature: d.author.signature ?? null,
|
||||
avatar_url: authorAvatarUploaded ?? null,
|
||||
follower_count: BigInt(d.author.follower_count || 0),
|
||||
total_favorited: BigInt(d.author.total_favorited || 0),
|
||||
unique_id: d.author.unique_id ?? null,
|
||||
short_id: d.author.short_id ?? null,
|
||||
// 1) Upsert Author
|
||||
const authorAvatarSrc = firstUrl(d.author.avatar_thumb?.url_list);
|
||||
const authorAvatarUploaded = await uploadAvatarFromUrl(
|
||||
context,
|
||||
authorAvatarSrc,
|
||||
`authors/${d.author.sec_uid}`,
|
||||
);
|
||||
const author = await prisma.author.upsert({
|
||||
where: { sec_uid: d.author.sec_uid },
|
||||
create: {
|
||||
sec_uid: d.author.sec_uid,
|
||||
uid: d.author.uid,
|
||||
nickname: d.author.nickname,
|
||||
signature: d.author.signature ?? null,
|
||||
avatar_url: authorAvatarUploaded ?? null,
|
||||
follower_count: BigInt(d.author.follower_count || 0),
|
||||
total_favorited: BigInt(d.author.total_favorited || 0),
|
||||
unique_id: d.author.unique_id ?? null,
|
||||
short_id: d.author.short_id ?? null,
|
||||
},
|
||||
update: {
|
||||
uid: d.author.uid,
|
||||
nickname: d.author.nickname,
|
||||
signature: d.author.signature ?? null,
|
||||
avatar_url: authorAvatarUploaded ?? null,
|
||||
follower_count: BigInt(d.author.follower_count || 0),
|
||||
total_favorited: BigInt(d.author.total_favorited || 0),
|
||||
unique_id: d.author.unique_id ?? null,
|
||||
short_id: d.author.short_id ?? null,
|
||||
},
|
||||
});
|
||||
|
||||
// 2) Upsert Video
|
||||
const video = await prisma.video.upsert({
|
||||
where: { aweme_id: d.aweme_id },
|
||||
create: {
|
||||
aweme_id: d.aweme_id,
|
||||
desc: d.desc,
|
||||
preview_title: d.preview_title ?? null,
|
||||
duration_ms: d.duration,
|
||||
created_at: new Date((d.create_time || 0) * 1000),
|
||||
share_url: d.share_url,
|
||||
digg_count: BigInt(d.statistics?.digg_count || 0),
|
||||
comment_count: BigInt(d.statistics?.comment_count || 0),
|
||||
share_count: BigInt(d.statistics?.share_count || 0),
|
||||
collect_count: BigInt(d.statistics?.collect_count || 0),
|
||||
authorId: author.sec_uid,
|
||||
tags: d.tags?.map((t) => t.tag_name) ?? [],
|
||||
video_url: videoUrl ?? "",
|
||||
width: width ?? null,
|
||||
height: height ?? null,
|
||||
cover_url: coverUrl ?? null,
|
||||
fps: fps ?? null,
|
||||
raw_json: detailResp as any, // 保存完整接口 JSON
|
||||
},
|
||||
update: {
|
||||
desc: d.desc,
|
||||
preview_title: d.preview_title ?? null,
|
||||
duration_ms: d.duration,
|
||||
created_at: new Date((d.create_time || 0) * 1000),
|
||||
share_url: d.share_url,
|
||||
digg_count: BigInt(d.statistics?.digg_count || 0),
|
||||
comment_count: BigInt(d.statistics?.comment_count || 0),
|
||||
share_count: BigInt(d.statistics?.share_count || 0),
|
||||
collect_count: BigInt(d.statistics?.collect_count || 0),
|
||||
authorId: author.sec_uid,
|
||||
...(videoUrl ? { video_url: videoUrl } : {}),
|
||||
...(width ? { width } : {}),
|
||||
...(height ? { height } : {}),
|
||||
...(coverUrl ? { cover_url: coverUrl } : {}),
|
||||
...(fps ? { fps } : {}),
|
||||
raw_json: detailResp as any, // 更新完整接口 JSON
|
||||
},
|
||||
});
|
||||
|
||||
// 3) Upsert Comments + CommentUser
|
||||
const comments = commentResp?.comments ?? [];
|
||||
for (const c of comments) {
|
||||
const origAvatar: string | null =
|
||||
firstUrl(c.user?.avatar_thumb?.url_list) ?? null;
|
||||
const nameHint = `comment-users/${(c.user?.nickname || "unknown").replace(/\s+/g, "_")}-${c.cid}`;
|
||||
const uploadedAvatar = await uploadAvatarFromUrl(
|
||||
context,
|
||||
origAvatar ?? undefined,
|
||||
nameHint,
|
||||
);
|
||||
const finalAvatar = uploadedAvatar ?? origAvatar; // string | null
|
||||
const finalAvatarKey = finalAvatar ?? "";
|
||||
const cu = await prisma.commentUser.upsert({
|
||||
where: {
|
||||
nickname_avatar_url: {
|
||||
nickname: c.user?.nickname || "未知用户",
|
||||
avatar_url: finalAvatarKey,
|
||||
},
|
||||
},
|
||||
create: {
|
||||
nickname: c.user?.nickname || "未知用户",
|
||||
avatar_url: finalAvatar ?? null,
|
||||
},
|
||||
update: {
|
||||
avatar_url: finalAvatar ?? null,
|
||||
},
|
||||
});
|
||||
|
||||
// 2) Upsert Video
|
||||
const video = await prisma.video.upsert({
|
||||
where: { aweme_id: d.aweme_id },
|
||||
create: {
|
||||
aweme_id: d.aweme_id,
|
||||
desc: d.desc,
|
||||
preview_title: d.preview_title ?? null,
|
||||
duration_ms: d.duration,
|
||||
created_at: new Date((d.create_time || 0) * 1000),
|
||||
share_url: d.share_url,
|
||||
digg_count: BigInt(d.statistics?.digg_count || 0),
|
||||
comment_count: BigInt(d.statistics?.comment_count || 0),
|
||||
share_count: BigInt(d.statistics?.share_count || 0),
|
||||
collect_count: BigInt(d.statistics?.collect_count || 0),
|
||||
authorId: author.sec_uid,
|
||||
tags: (d.tags?.map(t => t.tag_name) ?? []),
|
||||
video_url: videoUrl ?? '',
|
||||
width: width ?? null,
|
||||
height: height ?? null,
|
||||
cover_url: coverUrl ?? null,
|
||||
fps: fps ?? null,
|
||||
raw_json: detailResp as any, // 保存完整接口 JSON
|
||||
},
|
||||
update: {
|
||||
desc: d.desc,
|
||||
preview_title: d.preview_title ?? null,
|
||||
duration_ms: d.duration,
|
||||
created_at: new Date((d.create_time || 0) * 1000),
|
||||
share_url: d.share_url,
|
||||
digg_count: BigInt(d.statistics?.digg_count || 0),
|
||||
comment_count: BigInt(d.statistics?.comment_count || 0),
|
||||
share_count: BigInt(d.statistics?.share_count || 0),
|
||||
collect_count: BigInt(d.statistics?.collect_count || 0),
|
||||
authorId: author.sec_uid,
|
||||
...(videoUrl ? { video_url: videoUrl } : {}),
|
||||
...(width ? { width } : {}),
|
||||
...(height ? { height } : {}),
|
||||
...(coverUrl ? { cover_url: coverUrl } : {}),
|
||||
...(fps ? { fps } : {}),
|
||||
raw_json: detailResp as any, // 更新完整接口 JSON
|
||||
},
|
||||
const savedComment = await prisma.comment.upsert({
|
||||
where: { cid: c.cid },
|
||||
create: {
|
||||
cid: c.cid,
|
||||
text: c.text,
|
||||
digg_count: BigInt(c.digg_count || 0),
|
||||
created_at: new Date((c.create_time || 0) * 1000),
|
||||
videoId: video.aweme_id,
|
||||
userId: cu.id,
|
||||
},
|
||||
update: {
|
||||
text: c.text,
|
||||
digg_count: BigInt(c.digg_count || 0),
|
||||
created_at: new Date((c.create_time || 0) * 1000),
|
||||
videoId: video.aweme_id,
|
||||
userId: cu.id,
|
||||
},
|
||||
});
|
||||
|
||||
// 3) Upsert Comments + CommentUser
|
||||
const comments = commentResp?.comments ?? [];
|
||||
for (const c of comments) {
|
||||
const origAvatar: string | null = firstUrl(c.user?.avatar_thumb?.url_list) ?? null;
|
||||
const nameHint = `comment-users/${(c.user?.nickname || 'unknown').replace(/\s+/g, '_')}-${c.cid}`;
|
||||
const uploadedAvatar = await uploadAvatarFromUrl(context, origAvatar ?? undefined, nameHint);
|
||||
const finalAvatar = uploadedAvatar ?? origAvatar; // string | null
|
||||
const finalAvatarKey = finalAvatar ?? '';
|
||||
const cu = await prisma.commentUser.upsert({
|
||||
where: {
|
||||
nickname_avatar_url: {
|
||||
nickname: c.user?.nickname || '未知用户',
|
||||
avatar_url: finalAvatarKey,
|
||||
},
|
||||
},
|
||||
create: {
|
||||
nickname: c.user?.nickname || '未知用户',
|
||||
avatar_url: finalAvatar ?? null,
|
||||
},
|
||||
update: {
|
||||
avatar_url: finalAvatar ?? null,
|
||||
},
|
||||
// 处理评论贴纸/配图上传与入库
|
||||
try {
|
||||
const sources: {
|
||||
url?: string | null;
|
||||
width?: number;
|
||||
height?: number;
|
||||
}[] = [];
|
||||
// 贴纸（当作第一张）
|
||||
const stickerUrl = firstUrl(c.sticker?.animate_url?.url_list);
|
||||
if (stickerUrl) {
|
||||
sources.push({
|
||||
url: stickerUrl,
|
||||
width: c.sticker?.animate_url?.width,
|
||||
height: c.sticker?.animate_url?.height,
|
||||
});
|
||||
}
|
||||
// 配图列表
|
||||
const imgs = c.image_list || [];
|
||||
for (const it of imgs) {
|
||||
const u = firstUrl(it?.origin_url?.url_list);
|
||||
if (u)
|
||||
sources.push({
|
||||
url: u,
|
||||
width: it?.origin_url?.width as any,
|
||||
height: it?.origin_url?.height as any,
|
||||
});
|
||||
}
|
||||
|
||||
const savedComment = await prisma.comment.upsert({
|
||||
where: { cid: c.cid },
|
||||
create: {
|
||||
cid: c.cid,
|
||||
text: c.text,
|
||||
digg_count: BigInt(c.digg_count || 0),
|
||||
created_at: new Date((c.create_time || 0) * 1000),
|
||||
videoId: video.aweme_id,
|
||||
userId: cu.id,
|
||||
},
|
||||
update: {
|
||||
text: c.text,
|
||||
digg_count: BigInt(c.digg_count || 0),
|
||||
created_at: new Date((c.create_time || 0) * 1000),
|
||||
videoId: video.aweme_id,
|
||||
userId: cu.id,
|
||||
},
|
||||
for (let i = 0; i < sources.length; i++) {
|
||||
const s = sources[i];
|
||||
const uploaded = await uploadImageFromUrl(
|
||||
context,
|
||||
s.url ?? undefined,
|
||||
`comments/${c.cid}/${i}`,
|
||||
);
|
||||
if (!uploaded) continue;
|
||||
await prisma.commentImage.upsert({
|
||||
where: { commentId_order: { commentId: savedComment.cid, order: i } },
|
||||
create: {
|
||||
commentId: savedComment.cid,
|
||||
order: i,
|
||||
url: uploaded,
|
||||
width: typeof s.width === "number" ? s.width : null,
|
||||
height: typeof s.height === "number" ? s.height : null,
|
||||
},
|
||||
update: {
|
||||
url: uploaded,
|
||||
width: typeof s.width === "number" ? s.width : null,
|
||||
height: typeof s.height === "number" ? s.height : null,
|
||||
},
|
||||
});
|
||||
|
||||
// 处理评论贴纸/配图上传与入库
|
||||
try {
|
||||
const sources: { url?: string | null; width?: number; height?: number }[] = [];
|
||||
// 贴纸（当作第一张）
|
||||
const stickerUrl = firstUrl(c.sticker?.animate_url?.url_list);
|
||||
if (stickerUrl) {
|
||||
sources.push({ url: stickerUrl, width: c.sticker?.animate_url?.width, height: c.sticker?.animate_url?.height });
|
||||
}
|
||||
// 配图列表
|
||||
const imgs = c.image_list || [];
|
||||
for (const it of imgs) {
|
||||
const u = firstUrl(it?.origin_url?.url_list);
|
||||
if (u) sources.push({ url: u, width: it?.origin_url?.width as any, height: it?.origin_url?.height as any });
|
||||
}
|
||||
|
||||
for (let i = 0; i < sources.length; i++) {
|
||||
const s = sources[i];
|
||||
const uploaded = await uploadImageFromUrl(
|
||||
context,
|
||||
s.url ?? undefined,
|
||||
`comments/${c.cid}/${i}`,
|
||||
);
|
||||
if (!uploaded) continue;
|
||||
await prisma.commentImage.upsert({
|
||||
where: { commentId_order: { commentId: savedComment.cid, order: i } },
|
||||
create: {
|
||||
commentId: savedComment.cid,
|
||||
order: i,
|
||||
url: uploaded,
|
||||
width: typeof s.width === 'number' ? s.width : null,
|
||||
height: typeof s.height === 'number' ? s.height : null,
|
||||
},
|
||||
update: {
|
||||
url: uploaded,
|
||||
width: typeof s.width === 'number' ? s.width : null,
|
||||
height: typeof s.height === 'number' ? s.height : null,
|
||||
},
|
||||
});
|
||||
}
|
||||
} catch (e) {
|
||||
console.warn('[comment-images] 保存失败：', (e as Error)?.message || e);
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
console.warn("[comment-images] 保存失败：", (e as Error)?.message || e);
|
||||
}
|
||||
}
|
||||
|
||||
return { aweme_id: video.aweme_id, author_sec_uid: author.sec_uid, comment_count: comments.length };
|
||||
return {
|
||||
aweme_id: video.aweme_id,
|
||||
author_sec_uid: author.sec_uid,
|
||||
comment_count: comments.length,
|
||||
};
|
||||
}
|
||||
|
||||
export async function saveImagePostToDB(
|
||||
context: BrowserContext,
|
||||
aweme: DouyinImageAweme,
|
||||
commentResp: DouyinCommentResponse,
|
||||
uploads: { images: { url: string; width?: number; height?: number, video?: string }[]; musicUrl?: string },
|
||||
rawJson?: any
|
||||
context: BrowserContext,
|
||||
aweme: DouyinImageAweme,
|
||||
commentResp: DouyinCommentResponse,
|
||||
uploads: {
|
||||
images: {
|
||||
url: string;
|
||||
width?: number;
|
||||
height?: number;
|
||||
video?: string;
|
||||
duration?: number;
|
||||
}[];
|
||||
musicUrl?: string;
|
||||
},
|
||||
rawJson?: any,
|
||||
) {
|
||||
if (!aweme?.author?.sec_uid) throw new Error('作者 sec_uid 缺失');
|
||||
if (!aweme?.author?.sec_uid) throw new Error("作者 sec_uid 缺失");
|
||||
|
||||
// Upsert Author（与视频一致）
|
||||
const authorAvatarSrc = firstUrl(aweme.author.avatar_thumb?.url_list);
|
||||
const authorAvatarUploaded = await uploadAvatarFromUrl(context, authorAvatarSrc, `authors/${aweme.author.sec_uid}`);
|
||||
const author = await prisma.author.upsert({
|
||||
where: { sec_uid: aweme.author.sec_uid },
|
||||
create: {
|
||||
sec_uid: aweme.author.sec_uid,
|
||||
uid: aweme.author.uid,
|
||||
nickname: aweme.author.nickname,
|
||||
signature: aweme.author.signature ?? null,
|
||||
avatar_url: authorAvatarUploaded ?? null,
|
||||
follower_count: BigInt((aweme.author as any).follower_count || 0),
|
||||
total_favorited: BigInt((aweme.author as any).total_favorited || 0),
|
||||
unique_id: (aweme.author as any).unique_id ?? null,
|
||||
short_id: (aweme.author as any).short_id ?? null,
|
||||
},
|
||||
update: {
|
||||
uid: aweme.author.uid,
|
||||
nickname: aweme.author.nickname,
|
||||
signature: aweme.author.signature ?? null,
|
||||
avatar_url: authorAvatarUploaded ?? null,
|
||||
follower_count: BigInt((aweme.author as any).follower_count || 0),
|
||||
total_favorited: BigInt((aweme.author as any).total_favorited || 0),
|
||||
unique_id: (aweme.author as any).unique_id ?? null,
|
||||
short_id: (aweme.author as any).short_id ?? null,
|
||||
// Upsert Author（与视频一致）
|
||||
const authorAvatarSrc = firstUrl(aweme.author.avatar_thumb?.url_list);
|
||||
const authorAvatarUploaded = await uploadAvatarFromUrl(
|
||||
context,
|
||||
authorAvatarSrc,
|
||||
`authors/${aweme.author.sec_uid}`,
|
||||
);
|
||||
const author = await prisma.author.upsert({
|
||||
where: { sec_uid: aweme.author.sec_uid },
|
||||
create: {
|
||||
sec_uid: aweme.author.sec_uid,
|
||||
uid: aweme.author.uid,
|
||||
nickname: aweme.author.nickname,
|
||||
signature: aweme.author.signature ?? null,
|
||||
avatar_url: authorAvatarUploaded ?? null,
|
||||
follower_count: BigInt((aweme.author as any).follower_count || 0),
|
||||
total_favorited: BigInt((aweme.author as any).total_favorited || 0),
|
||||
unique_id: (aweme.author as any).unique_id ?? null,
|
||||
short_id: (aweme.author as any).short_id ?? null,
|
||||
},
|
||||
update: {
|
||||
uid: aweme.author.uid,
|
||||
nickname: aweme.author.nickname,
|
||||
signature: aweme.author.signature ?? null,
|
||||
avatar_url: authorAvatarUploaded ?? null,
|
||||
follower_count: BigInt((aweme.author as any).follower_count || 0),
|
||||
total_favorited: BigInt((aweme.author as any).total_favorited || 0),
|
||||
unique_id: (aweme.author as any).unique_id ?? null,
|
||||
short_id: (aweme.author as any).short_id ?? null,
|
||||
},
|
||||
});
|
||||
|
||||
// Upsert ImagePost
|
||||
const imagePost = await prisma.imagePost.upsert({
|
||||
where: { aweme_id: aweme.aweme_id },
|
||||
create: {
|
||||
aweme_id: aweme.aweme_id,
|
||||
desc: aweme.desc,
|
||||
created_at: new Date((aweme.create_time || 0) * 1000),
|
||||
share_url: aweme.share_url || "",
|
||||
digg_count: BigInt(aweme.statistics?.digg_count || 0),
|
||||
comment_count: BigInt(aweme.statistics?.comment_count || 0),
|
||||
share_count: BigInt(aweme.statistics?.share_count || 0),
|
||||
collect_count: BigInt(aweme.statistics?.collect_count || 0),
|
||||
authorId: author.sec_uid,
|
||||
tags: aweme.video_tag?.map((t) => t.tag_name) ?? [],
|
||||
music_url: uploads.musicUrl ?? null,
|
||||
raw_json: rawJson ?? null, // 保存完整接口 JSON
|
||||
},
|
||||
update: {
|
||||
desc: aweme.desc,
|
||||
created_at: new Date((aweme.create_time || 0) * 1000),
|
||||
share_url: aweme.share_url,
|
||||
digg_count: BigInt(aweme.statistics?.digg_count || 0),
|
||||
comment_count: BigInt(aweme.statistics?.comment_count || 0),
|
||||
share_count: BigInt(aweme.statistics?.share_count || 0),
|
||||
collect_count: BigInt(aweme.statistics?.collect_count || 0),
|
||||
authorId: author.sec_uid,
|
||||
tags: aweme.video_tag?.map((t) => t.tag_name) ?? [],
|
||||
music_url: uploads.musicUrl ?? undefined,
|
||||
raw_json: rawJson ?? undefined, // 更新完整接口 JSON
|
||||
},
|
||||
});
|
||||
|
||||
// Upsert ImageFiles（按顺序）
|
||||
for (let i = 0; i < uploads.images.length; i++) {
|
||||
const { url, width, height, video, duration } = uploads.images[i];
|
||||
const durationMs =
|
||||
typeof duration === "number" && Number.isFinite(duration)
|
||||
? Math.max(1, Math.round(duration))
|
||||
: null;
|
||||
await prisma.imageFile.upsert({
|
||||
where: { postId_order: { postId: imagePost.aweme_id, order: i } },
|
||||
create: {
|
||||
postId: imagePost.aweme_id,
|
||||
order: i,
|
||||
url,
|
||||
width: typeof width === "number" ? width : null,
|
||||
height: typeof height === "number" ? height : null,
|
||||
animated: video || null,
|
||||
duration: durationMs,
|
||||
},
|
||||
update: {
|
||||
url,
|
||||
width: typeof width === "number" ? width : null,
|
||||
height: typeof height === "number" ? height : null,
|
||||
animated: video || null,
|
||||
duration: durationMs,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
// 评论入库：关联到 ImagePost
|
||||
const comments = commentResp?.comments ?? [];
|
||||
for (const c of comments) {
|
||||
const origAvatar: string | null =
|
||||
firstUrl(c.user?.avatar_thumb?.url_list) ?? null;
|
||||
const nameHint = `comment-users/${(c.user?.nickname || "unknown").replace(/\s+/g, "_")}-${c.cid}`;
|
||||
const uploadedAvatar = await uploadAvatarFromUrl(
|
||||
context,
|
||||
origAvatar ?? undefined,
|
||||
nameHint,
|
||||
);
|
||||
const finalAvatar = uploadedAvatar ?? origAvatar; // string | null
|
||||
const finalAvatarKey = finalAvatar ?? "";
|
||||
const cu = await prisma.commentUser.upsert({
|
||||
where: {
|
||||
nickname_avatar_url: {
|
||||
nickname: c.user?.nickname || "未知用户",
|
||||
avatar_url: finalAvatarKey,
|
||||
},
|
||||
},
|
||||
create: {
|
||||
nickname: c.user?.nickname || "未知用户",
|
||||
avatar_url: finalAvatar ?? null,
|
||||
},
|
||||
update: {
|
||||
avatar_url: finalAvatar ?? null,
|
||||
},
|
||||
});
|
||||
|
||||
// Upsert ImagePost
|
||||
const imagePost = await prisma.imagePost.upsert({
|
||||
where: { aweme_id: aweme.aweme_id },
|
||||
create: {
|
||||
aweme_id: aweme.aweme_id,
|
||||
desc: aweme.desc,
|
||||
created_at: new Date((aweme.create_time || 0) * 1000),
|
||||
share_url: aweme.share_url || '',
|
||||
digg_count: BigInt(aweme.statistics?.digg_count || 0),
|
||||
comment_count: BigInt(aweme.statistics?.comment_count || 0),
|
||||
share_count: BigInt(aweme.statistics?.share_count || 0),
|
||||
collect_count: BigInt(aweme.statistics?.collect_count || 0),
|
||||
authorId: author.sec_uid,
|
||||
tags: (aweme.video_tag?.map(t => t.tag_name) ?? []),
|
||||
music_url: uploads.musicUrl ?? null,
|
||||
raw_json: rawJson ?? null, // 保存完整接口 JSON
|
||||
},
|
||||
update: {
|
||||
desc: aweme.desc,
|
||||
created_at: new Date((aweme.create_time || 0) * 1000),
|
||||
share_url: aweme.share_url,
|
||||
digg_count: BigInt(aweme.statistics?.digg_count || 0),
|
||||
comment_count: BigInt(aweme.statistics?.comment_count || 0),
|
||||
share_count: BigInt(aweme.statistics?.share_count || 0),
|
||||
collect_count: BigInt(aweme.statistics?.collect_count || 0),
|
||||
authorId: author.sec_uid,
|
||||
tags: (aweme.video_tag?.map(t => t.tag_name) ?? []),
|
||||
music_url: uploads.musicUrl ?? undefined,
|
||||
raw_json: rawJson ?? undefined, // 更新完整接口 JSON
|
||||
},
|
||||
const savedComment = await prisma.comment.upsert({
|
||||
where: { cid: c.cid },
|
||||
create: {
|
||||
cid: c.cid,
|
||||
text: c.text,
|
||||
digg_count: BigInt(c.digg_count || 0),
|
||||
created_at: new Date((c.create_time || 0) * 1000),
|
||||
imagePostId: imagePost.aweme_id,
|
||||
userId: cu.id,
|
||||
},
|
||||
update: {
|
||||
text: c.text,
|
||||
digg_count: BigInt(c.digg_count || 0),
|
||||
created_at: new Date((c.create_time || 0) * 1000),
|
||||
imagePostId: imagePost.aweme_id,
|
||||
userId: cu.id,
|
||||
},
|
||||
});
|
||||
|
||||
// Upsert ImageFiles（按顺序）
|
||||
for (let i = 0; i < uploads.images.length; i++) {
|
||||
const { url, width, height, video } = uploads.images[i];
|
||||
await prisma.imageFile.upsert({
|
||||
where: { postId_order: { postId: imagePost.aweme_id, order: i } },
|
||||
create: {
|
||||
postId: imagePost.aweme_id,
|
||||
order: i,
|
||||
url,
|
||||
width: typeof width === 'number' ? width : null,
|
||||
height: typeof height === 'number' ? height : null,
|
||||
animated: video || null,
|
||||
},
|
||||
update: {
|
||||
url,
|
||||
width: typeof width === 'number' ? width : null,
|
||||
height: typeof height === 'number' ? height : null,
|
||||
animated: video || null,
|
||||
},
|
||||
// 处理评论贴纸/配图上传与入库
|
||||
try {
|
||||
const sources: {
|
||||
url?: string | null;
|
||||
width?: number;
|
||||
height?: number;
|
||||
}[] = [];
|
||||
// 贴纸（当作第一张）
|
||||
const stickerUrl = firstUrl(c.sticker?.animate_url?.url_list);
|
||||
if (stickerUrl) {
|
||||
sources.push({
|
||||
url: stickerUrl,
|
||||
width: c.sticker?.animate_url?.width,
|
||||
height: c.sticker?.animate_url?.height,
|
||||
});
|
||||
}
|
||||
// 配图列表
|
||||
const imgs = c.image_list || [];
|
||||
for (const it of imgs) {
|
||||
const u = firstUrl(it?.origin_url?.url_list);
|
||||
if (u)
|
||||
sources.push({
|
||||
url: u,
|
||||
width: it?.origin_url?.width as any,
|
||||
height: it?.origin_url?.height as any,
|
||||
});
|
||||
}
|
||||
|
||||
for (let i = 0; i < sources.length; i++) {
|
||||
const s = sources[i];
|
||||
const uploaded = await uploadImageFromUrl(
|
||||
context,
|
||||
s.url ?? undefined,
|
||||
`comments/${c.cid}/${i}`,
|
||||
);
|
||||
if (!uploaded) continue;
|
||||
await prisma.commentImage.upsert({
|
||||
where: { commentId_order: { commentId: savedComment.cid, order: i } },
|
||||
create: {
|
||||
commentId: savedComment.cid,
|
||||
order: i,
|
||||
url: uploaded,
|
||||
width: typeof s.width === "number" ? s.width : null,
|
||||
height: typeof s.height === "number" ? s.height : null,
|
||||
},
|
||||
update: {
|
||||
url: uploaded,
|
||||
width: typeof s.width === "number" ? s.width : null,
|
||||
height: typeof s.height === "number" ? s.height : null,
|
||||
},
|
||||
});
|
||||
}
|
||||
} catch (e) {
|
||||
console.warn("[comment-images] 保存失败：", (e as Error)?.message || e);
|
||||
}
|
||||
}
|
||||
|
||||
// 评论入库：关联到 ImagePost
|
||||
const comments = commentResp?.comments ?? [];
|
||||
for (const c of comments) {
|
||||
const origAvatar: string | null = firstUrl(c.user?.avatar_thumb?.url_list) ?? null;
|
||||
const nameHint = `comment-users/${(c.user?.nickname || 'unknown').replace(/\s+/g, '_')}-${c.cid}`;
|
||||
const uploadedAvatar = await uploadAvatarFromUrl(context, origAvatar ?? undefined, nameHint);
|
||||
const finalAvatar = uploadedAvatar ?? origAvatar; // string | null
|
||||
const finalAvatarKey = finalAvatar ?? '';
|
||||
const cu = await prisma.commentUser.upsert({
|
||||
where: {
|
||||
nickname_avatar_url: {
|
||||
nickname: c.user?.nickname || '未知用户',
|
||||
avatar_url: finalAvatarKey,
|
||||
},
|
||||
},
|
||||
create: {
|
||||
nickname: c.user?.nickname || '未知用户',
|
||||
avatar_url: finalAvatar ?? null,
|
||||
},
|
||||
update: {
|
||||
avatar_url: finalAvatar ?? null,
|
||||
},
|
||||
});
|
||||
|
||||
const savedComment = await prisma.comment.upsert({
|
||||
where: { cid: c.cid },
|
||||
create: {
|
||||
cid: c.cid,
|
||||
text: c.text,
|
||||
digg_count: BigInt(c.digg_count || 0),
|
||||
created_at: new Date((c.create_time || 0) * 1000),
|
||||
imagePostId: imagePost.aweme_id,
|
||||
userId: cu.id,
|
||||
},
|
||||
update: {
|
||||
text: c.text,
|
||||
digg_count: BigInt(c.digg_count || 0),
|
||||
created_at: new Date((c.create_time || 0) * 1000),
|
||||
imagePostId: imagePost.aweme_id,
|
||||
userId: cu.id,
|
||||
},
|
||||
});
|
||||
|
||||
// 处理评论贴纸/配图上传与入库
|
||||
try {
|
||||
const sources: { url?: string | null; width?: number; height?: number }[] = [];
|
||||
// 贴纸（当作第一张）
|
||||
const stickerUrl = firstUrl(c.sticker?.animate_url?.url_list);
|
||||
if (stickerUrl) {
|
||||
sources.push({ url: stickerUrl, width: c.sticker?.animate_url?.width, height: c.sticker?.animate_url?.height });
|
||||
}
|
||||
// 配图列表
|
||||
const imgs = c.image_list || [];
|
||||
for (const it of imgs) {
|
||||
const u = firstUrl(it?.origin_url?.url_list);
|
||||
if (u) sources.push({ url: u, width: it?.origin_url?.width as any, height: it?.origin_url?.height as any });
|
||||
}
|
||||
|
||||
for (let i = 0; i < sources.length; i++) {
|
||||
const s = sources[i];
|
||||
const uploaded = await uploadImageFromUrl(
|
||||
context,
|
||||
s.url ?? undefined,
|
||||
`comments/${c.cid}/${i}`,
|
||||
);
|
||||
if (!uploaded) continue;
|
||||
await prisma.commentImage.upsert({
|
||||
where: { commentId_order: { commentId: savedComment.cid, order: i } },
|
||||
create: {
|
||||
commentId: savedComment.cid,
|
||||
order: i,
|
||||
url: uploaded,
|
||||
width: typeof s.width === 'number' ? s.width : null,
|
||||
height: typeof s.height === 'number' ? s.height : null,
|
||||
},
|
||||
update: {
|
||||
url: uploaded,
|
||||
width: typeof s.width === 'number' ? s.width : null,
|
||||
height: typeof s.height === 'number' ? s.height : null,
|
||||
},
|
||||
});
|
||||
}
|
||||
} catch (e) {
|
||||
console.warn('[comment-images] 保存失败：', (e as Error)?.message || e);
|
||||
}
|
||||
}
|
||||
|
||||
return { aweme_id: imagePost.aweme_id, author_sec_uid: author.sec_uid, image_count: uploads.images.length, comment_count: comments.length };
|
||||
return {
|
||||
aweme_id: imagePost.aweme_id,
|
||||
author_sec_uid: author.sec_uid,
|
||||
image_count: uploads.images.length,
|
||||
comment_count: comments.length,
|
||||
};
|
||||
}
|
||||
|
||||
+44
-44
@@ -1,51 +1,51 @@
|
||||
export const runtime = 'nodejs'
|
||||
export const runtime = "nodejs";
|
||||
|
||||
import { NextRequest, NextResponse } from 'next/server'
|
||||
import { prisma } from '@/lib/prisma'
|
||||
import { scrapeDouyin, ScrapeError } from '.';
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { prisma } from "@/lib/prisma";
|
||||
import { scrapeDouyin, ScrapeError } from ".";
|
||||
|
||||
async function handleDouyinScrape(req: NextRequest) {
|
||||
const { searchParams } = new URL(req.url);
|
||||
const videoUrl = searchParams.get('url');
|
||||
|
||||
if (!videoUrl) {
|
||||
return NextResponse.json(
|
||||
{ error: '缺少视频URL', code: 'MISSING_URL' },
|
||||
{ status: 400 }
|
||||
);
|
||||
const { searchParams } = new URL(req.url);
|
||||
const videoUrl = searchParams.get("url");
|
||||
|
||||
if (!videoUrl) {
|
||||
return NextResponse.json(
|
||||
{ error: "缺少视频URL", code: "MISSING_URL" },
|
||||
{ status: 400 },
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
// 调用爬虫函数
|
||||
const result = await scrapeDouyin(videoUrl);
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
data: result,
|
||||
});
|
||||
} catch (error) {
|
||||
// 处理自定义的 ScrapeError
|
||||
if (error instanceof ScrapeError) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: error.message,
|
||||
code: error.code,
|
||||
},
|
||||
{ status: error.statusCode },
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
// 调用爬虫函数
|
||||
const result = await scrapeDouyin(videoUrl);
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
data: result
|
||||
});
|
||||
} catch (error) {
|
||||
// 处理自定义的 ScrapeError
|
||||
if (error instanceof ScrapeError) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: error.message,
|
||||
code: error.code
|
||||
},
|
||||
{ status: error.statusCode }
|
||||
);
|
||||
}
|
||||
|
||||
// 处理未知错误
|
||||
console.error('未捕获的错误:', error);
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: '服务器内部错误',
|
||||
code: 'INTERNAL_ERROR'
|
||||
},
|
||||
{ status: 500 }
|
||||
);
|
||||
}
|
||||
// 处理未知错误
|
||||
console.error("未捕获的错误:", error);
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: "服务器内部错误",
|
||||
code: "INTERNAL_ERROR",
|
||||
},
|
||||
{ status: 500 },
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export const GET = handleDouyinScrape
|
||||
export const GET = handleDouyinScrape;
|
||||
|
||||
Vendored
+33
-33
@@ -17,17 +17,17 @@ interface DouyinComment {
|
||||
animate_url: {
|
||||
width: number;
|
||||
height: number;
|
||||
url_list: string[]
|
||||
}
|
||||
},
|
||||
url_list: string[];
|
||||
};
|
||||
};
|
||||
|
||||
image_list?: {
|
||||
origin_url:{
|
||||
origin_url: {
|
||||
width: number;
|
||||
height: number;
|
||||
url_list: string[]
|
||||
}
|
||||
}[]
|
||||
url_list: string[];
|
||||
};
|
||||
}[];
|
||||
}
|
||||
|
||||
/** 用户信息（精简版） */
|
||||
@@ -45,35 +45,35 @@ interface DouyinVideoDetailResponse {
|
||||
}
|
||||
/** 作者信息（精简版） */
|
||||
interface DouyinAuthor {
|
||||
uid: string; // 用户ID
|
||||
sec_uid: string; // 安全UID
|
||||
nickname: string; // 用户昵称
|
||||
signature: string; // 个性签名
|
||||
uid: string; // 用户ID
|
||||
sec_uid: string; // 安全UID
|
||||
nickname: string; // 用户昵称
|
||||
signature: string; // 个性签名
|
||||
avatar_thumb: {
|
||||
url_list: string[]; // 头像URL（可取第一个）
|
||||
url_list: string[]; // 头像URL（可取第一个）
|
||||
};
|
||||
follower_count: number; // 粉丝数
|
||||
total_favorited: number; // 获赞总数
|
||||
unique_id: string; // 抖音号
|
||||
short_id: string; // 短ID
|
||||
follower_count: number; // 粉丝数
|
||||
total_favorited: number; // 获赞总数
|
||||
unique_id: string; // 抖音号
|
||||
short_id: string; // 短ID
|
||||
}
|
||||
/** 视频详情 */
|
||||
interface DouyinVideoDetail {
|
||||
aweme_id: string; // 视频ID
|
||||
desc: string; // 视频描述
|
||||
preview_title?: string; // 视频标题（有些字段中叫 preview_title）
|
||||
duration: number; // 视频时长（毫秒）
|
||||
create_time: number; // 创建时间（时间戳）
|
||||
share_url: string; // 视频分享链接
|
||||
aweme_id: string; // 视频ID
|
||||
desc: string; // 视频描述
|
||||
preview_title?: string; // 视频标题（有些字段中叫 preview_title）
|
||||
duration: number; // 视频时长（毫秒）
|
||||
create_time: number; // 创建时间（时间戳）
|
||||
share_url: string; // 视频分享链接
|
||||
|
||||
statistics: {
|
||||
digg_count: number; // 点赞数
|
||||
comment_count: number; // 评论数
|
||||
share_count: number; // 分享数
|
||||
collect_count: number; // 收藏数
|
||||
digg_count: number; // 点赞数
|
||||
comment_count: number; // 评论数
|
||||
share_count: number; // 分享数
|
||||
collect_count: number; // 收藏数
|
||||
};
|
||||
|
||||
author: DouyinAuthor; // 作者信息
|
||||
author: DouyinAuthor; // 作者信息
|
||||
video: VideoPlayBasic;
|
||||
tags: VideoTagBasic[];
|
||||
}
|
||||
@@ -88,9 +88,9 @@ interface VideoPlayBasic {
|
||||
|
||||
/** 单个清晰度变体（来自 bit_rate[*] + play_addr） */
|
||||
interface PlayVariant {
|
||||
format: string; // mp4 等
|
||||
format: string; // mp4 等
|
||||
FPS: number;
|
||||
bit_rate: number; // bit_rate.bit_rate
|
||||
bit_rate: number; // bit_rate.bit_rate
|
||||
|
||||
/** 直连播放地址（最关键） */
|
||||
play_addr: {
|
||||
@@ -101,7 +101,7 @@ interface PlayVariant {
|
||||
data_size: number;
|
||||
FPS: number;
|
||||
is_bytevc1: number; // 0 or 1
|
||||
is_h265: number; // 0 or 1
|
||||
is_h265: number; // 0 or 1
|
||||
};
|
||||
}
|
||||
|
||||
@@ -132,7 +132,7 @@ interface DouyinImageAweme {
|
||||
};
|
||||
author: DouyinAuthor; // 复用视频作者类型（需包含 sec_uid）
|
||||
images: DouyinImageInfo[]; // 图片列表
|
||||
music?: DouyinMusicBasic; // 背景音乐（可选）
|
||||
music?: DouyinMusicBasic; // 背景音乐（可选）
|
||||
video_tag?: VideoTagBasic[]; // 标签
|
||||
}
|
||||
|
||||
@@ -143,7 +143,7 @@ interface DouyinImageInfo {
|
||||
width: number;
|
||||
height: number;
|
||||
video: {
|
||||
play_addr: { src: string }[]
|
||||
play_addr: { src: string }[];
|
||||
} | null; // 如果是动图，会有 video 信息
|
||||
}
|
||||
|
||||
@@ -157,4 +157,4 @@ interface DouyinMusicBasic {
|
||||
uri?: string;
|
||||
url_list: string[]; // 真实可下载地址
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
+150
-92
@@ -1,117 +1,175 @@
|
||||
export const runtime = 'nodejs'
|
||||
export const runtime = "nodejs";
|
||||
|
||||
import type { BrowserContext } from 'playwright';
|
||||
import { uploadFile, generateUniqueFileName } from '@/lib/minio';
|
||||
import { downloadBinary } from './network';
|
||||
import { pickFirstUrl } from './utils';
|
||||
import { getVideoDuration } from '@/app/api/media';
|
||||
import type { BrowserContext } from "playwright";
|
||||
import { uploadFile, generateUniqueFileName } from "@/lib/minio";
|
||||
import { downloadBinary } from "./network";
|
||||
import { pickFirstUrl } from "./utils";
|
||||
import { getVideoDuration } from "@/app/api/media";
|
||||
|
||||
/**
|
||||
* 下载头像并上传到 MinIO，返回外链；失败时回退为原始链接。
|
||||
*/
|
||||
export async function uploadAvatarFromUrl(
|
||||
context: BrowserContext,
|
||||
srcUrl?: string | null,
|
||||
nameHint?: string,
|
||||
context: BrowserContext,
|
||||
srcUrl?: string | null,
|
||||
nameHint?: string,
|
||||
): Promise<string | undefined> {
|
||||
if (!srcUrl) return undefined;
|
||||
try {
|
||||
const { buffer, contentType, ext } = await downloadBinary(context, srcUrl);
|
||||
const safeExt = ext || 'jpg';
|
||||
const baseName = nameHint ? `${nameHint}.${safeExt}` : `avatar.${safeExt}`;
|
||||
const fileName = generateUniqueFileName(baseName, 'douyin/avatars');
|
||||
const uploaded = await uploadFile(buffer, fileName, { 'Content-Type': contentType });
|
||||
return uploaded;
|
||||
} catch (e) {
|
||||
console.warn('[avatar] 上传失败，使用原始链接：', (e as Error)?.message || e);
|
||||
return srcUrl || undefined;
|
||||
}
|
||||
if (!srcUrl) return undefined;
|
||||
try {
|
||||
const { buffer, contentType, ext } = await downloadBinary(context, srcUrl);
|
||||
const safeExt = ext || "jpg";
|
||||
const baseName = nameHint ? `${nameHint}.${safeExt}` : `avatar.${safeExt}`;
|
||||
const fileName = generateUniqueFileName(baseName, "douyin/avatars");
|
||||
const uploaded = await uploadFile(buffer, fileName, {
|
||||
"Content-Type": contentType,
|
||||
});
|
||||
return uploaded;
|
||||
} catch (e) {
|
||||
console.warn(
|
||||
"[avatar] 上传失败，使用原始链接：",
|
||||
(e as Error)?.message || e,
|
||||
);
|
||||
return srcUrl || undefined;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 下载任意图片并上传到 MinIO，返回外链；失败时回退为原始链接。
|
||||
*/
|
||||
export async function uploadImageFromUrl(
|
||||
context: BrowserContext,
|
||||
srcUrl?: string | null,
|
||||
nameHint?: string,
|
||||
context: BrowserContext,
|
||||
srcUrl?: string | null,
|
||||
nameHint?: string,
|
||||
): Promise<string | undefined> {
|
||||
if (!srcUrl) return undefined;
|
||||
try {
|
||||
const { buffer, contentType, ext } = await downloadBinary(context, srcUrl);
|
||||
const safeExt = ext || 'jpg';
|
||||
const baseName = nameHint ? `${nameHint}.${safeExt}` : `image.${safeExt}`;
|
||||
const fileName = generateUniqueFileName(baseName, 'douyin/comment-images');
|
||||
const uploaded = await uploadFile(buffer, fileName, { 'Content-Type': contentType });
|
||||
return uploaded;
|
||||
} catch (e) {
|
||||
console.warn('[image] 上传失败，使用原始链接：', (e as Error)?.message || e);
|
||||
return srcUrl || undefined;
|
||||
}
|
||||
if (!srcUrl) return undefined;
|
||||
try {
|
||||
const { buffer, contentType, ext } = await downloadBinary(context, srcUrl);
|
||||
const safeExt = ext || "jpg";
|
||||
const baseName = nameHint ? `${nameHint}.${safeExt}` : `image.${safeExt}`;
|
||||
const fileName = generateUniqueFileName(baseName, "douyin/comment-images");
|
||||
const uploaded = await uploadFile(buffer, fileName, {
|
||||
"Content-Type": contentType,
|
||||
});
|
||||
return uploaded;
|
||||
} catch (e) {
|
||||
console.warn(
|
||||
"[image] 上传失败，使用原始链接：",
|
||||
(e as Error)?.message || e,
|
||||
);
|
||||
return srcUrl || undefined;
|
||||
}
|
||||
}
|
||||
|
||||
/** 下载图文作品的图片和音乐并上传到 MinIO */
|
||||
export async function handleImagePost(
|
||||
context: BrowserContext,
|
||||
aweme: DouyinImageAweme
|
||||
): Promise<{ images: { url: string; width?: number; height?: number; video?: string; duration?: number }[]; musicUrl?: string }> {
|
||||
const awemeId = aweme.aweme_id;
|
||||
const uploadedImages: { url: string; width?: number; height?: number; video?: string; duration?: number }[] = [];
|
||||
context: BrowserContext,
|
||||
aweme: DouyinImageAweme,
|
||||
): Promise<{
|
||||
images: {
|
||||
url: string;
|
||||
width?: number;
|
||||
height?: number;
|
||||
video?: string;
|
||||
duration?: number;
|
||||
}[];
|
||||
musicUrl?: string;
|
||||
}> {
|
||||
const awemeId = aweme.aweme_id;
|
||||
const uploadedImages: {
|
||||
url: string;
|
||||
width?: number;
|
||||
height?: number;
|
||||
video?: string;
|
||||
duration?: number;
|
||||
}[] = [];
|
||||
|
||||
// 下载图片（顺序保持）
|
||||
for (let i = 0; i < (aweme.images?.length || 0); i++) {
|
||||
const img = aweme.images[i];
|
||||
const url = pickFirstUrl(img?.url_list);
|
||||
if (!url) continue;
|
||||
const { buffer, contentType, ext } = await downloadBinary(context, url);
|
||||
const safeExt = ext || 'jpg';
|
||||
const fileName = generateUniqueFileName(`${awemeId}/${i}.${safeExt}`, 'douyin/images');
|
||||
const uploaded = await uploadFile(buffer, fileName, { 'Content-Type': contentType });
|
||||
// 下载图片（顺序保持）
|
||||
for (let i = 0; i < (aweme.images?.length || 0); i++) {
|
||||
const img = aweme.images[i];
|
||||
const url = pickFirstUrl(img?.url_list);
|
||||
if (!url) continue;
|
||||
const { buffer, contentType, ext } = await downloadBinary(context, url);
|
||||
const safeExt = ext || "jpg";
|
||||
const fileName = generateUniqueFileName(
|
||||
`${awemeId}/${i}.${safeExt}`,
|
||||
"douyin/images",
|
||||
);
|
||||
const uploaded = await uploadFile(buffer, fileName, {
|
||||
"Content-Type": contentType,
|
||||
});
|
||||
|
||||
if (img.video?.play_addr) {
|
||||
// 如果是动图，下载 video 并上传
|
||||
const videoUrl = img.video.play_addr[0]?.src;
|
||||
if (videoUrl) {
|
||||
try {
|
||||
const { buffer: videoBuffer, contentType: videoContentType, ext: videoExt } = await downloadBinary(context, videoUrl);
|
||||
const safeVideoExt = videoExt || 'mp4';
|
||||
const videoFileName = generateUniqueFileName(`${awemeId}/${i}_animated.${safeVideoExt}`, 'douyin/images');
|
||||
const uploadedVideo = await uploadFile(videoBuffer, videoFileName, { 'Content-Type': videoContentType });
|
||||
|
||||
// 获取动图时长
|
||||
const duration = await getVideoDuration(videoBuffer);
|
||||
|
||||
// 将动图的 video URL 和 duration 也存储起来
|
||||
uploadedImages.push({
|
||||
url: uploaded,
|
||||
width: img?.width,
|
||||
height: img?.height,
|
||||
video: uploadedVideo,
|
||||
duration: duration ?? undefined
|
||||
});
|
||||
|
||||
if (duration) {
|
||||
console.log(`[image] 动图 ${i} 时长: ${duration}ms`);
|
||||
}
|
||||
} catch (e) {
|
||||
console.warn(`[image] 动图视频上传失败，跳过：`, (e as Error)?.message || e);
|
||||
uploadedImages.push({ url: uploaded, width: img?.width, height: img?.height });
|
||||
}
|
||||
}
|
||||
} else {
|
||||
uploadedImages.push({ url: uploaded, width: img?.width, height: img?.height });
|
||||
if (img.video?.play_addr) {
|
||||
// 如果是动图，下载 video 并上传
|
||||
const videoUrl = img.video.play_addr[0]?.src;
|
||||
if (videoUrl) {
|
||||
try {
|
||||
const {
|
||||
buffer: videoBuffer,
|
||||
contentType: videoContentType,
|
||||
ext: videoExt,
|
||||
} = await downloadBinary(context, videoUrl);
|
||||
const safeVideoExt = videoExt || "mp4";
|
||||
const videoFileName = generateUniqueFileName(
|
||||
`${awemeId}/${i}_animated.${safeVideoExt}`,
|
||||
"douyin/images",
|
||||
);
|
||||
const uploadedVideo = await uploadFile(videoBuffer, videoFileName, {
|
||||
"Content-Type": videoContentType,
|
||||
});
|
||||
|
||||
// 获取动图时长
|
||||
const duration = await getVideoDuration(videoBuffer);
|
||||
|
||||
// 将动图的 video URL 和 duration 也存储起来
|
||||
uploadedImages.push({
|
||||
url: uploaded,
|
||||
width: img?.width,
|
||||
height: img?.height,
|
||||
video: uploadedVideo,
|
||||
duration: duration ?? undefined,
|
||||
});
|
||||
|
||||
if (duration) {
|
||||
console.log(`[image] 动图 ${i} 时长: ${duration}ms`);
|
||||
}
|
||||
} catch (e) {
|
||||
console.warn(
|
||||
`[image] 动图视频上传失败，跳过：`,
|
||||
(e as Error)?.message || e,
|
||||
);
|
||||
uploadedImages.push({
|
||||
url: uploaded,
|
||||
width: img?.width,
|
||||
height: img?.height,
|
||||
});
|
||||
}
|
||||
}
|
||||
} else {
|
||||
uploadedImages.push({
|
||||
url: uploaded,
|
||||
width: img?.width,
|
||||
height: img?.height,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// 下载音乐（可选）
|
||||
let musicUrl: string | undefined;
|
||||
const audioSrc = pickFirstUrl(aweme.music?.play_url?.url_list);
|
||||
if (audioSrc) {
|
||||
const { buffer, contentType, ext } = await downloadBinary(context, audioSrc);
|
||||
const safeExt = ext || 'mp3';
|
||||
const fileName = generateUniqueFileName(`${awemeId}.${safeExt}`, 'douyin/audios');
|
||||
musicUrl = await uploadFile(buffer, fileName, { 'Content-Type': contentType });
|
||||
}
|
||||
// 下载音乐（可选）
|
||||
let musicUrl: string | undefined;
|
||||
const audioSrc = pickFirstUrl(aweme.music?.play_url?.url_list);
|
||||
if (audioSrc) {
|
||||
const { buffer, contentType, ext } = await downloadBinary(
|
||||
context,
|
||||
audioSrc,
|
||||
);
|
||||
const safeExt = ext || "mp3";
|
||||
const fileName = generateUniqueFileName(
|
||||
`${awemeId}.${safeExt}`,
|
||||
"douyin/audios",
|
||||
);
|
||||
musicUrl = await uploadFile(buffer, fileName, {
|
||||
"Content-Type": contentType,
|
||||
});
|
||||
}
|
||||
|
||||
return { images: uploadedImages, musicUrl };
|
||||
return { images: uploadedImages, musicUrl };
|
||||
}
|
||||
|
||||
+38
-37
@@ -1,11 +1,11 @@
|
||||
export const runtime = 'nodejs'
|
||||
export const runtime = "nodejs";
|
||||
|
||||
export function toCamelCaseKey(key: string): string {
|
||||
return key.replace(/_([a-zA-Z])/g, (_, c: string) => c.toUpperCase());
|
||||
return key.replace(/_([a-zA-Z])/g, (_, c: string) => c.toUpperCase());
|
||||
}
|
||||
|
||||
export function toSnakeCaseKey(key: string): string {
|
||||
return key.replace(/[A-Z]/g, (m) => `_${m.toLowerCase()}`);
|
||||
return key.replace(/[A-Z]/g, (m) => `_${m.toLowerCase()}`);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -13,52 +13,53 @@ export function toSnakeCaseKey(key: string): string {
|
||||
* 访问顺序：原名 -> camelCase -> snake_case
|
||||
*/
|
||||
export function createCamelCompatibleProxy<T extends object>(root: T): T {
|
||||
const seen = new WeakMap<object, any>();
|
||||
const seen = new WeakMap<object, any>();
|
||||
|
||||
const wrap = (value: any): any => {
|
||||
if (value === null || typeof value !== 'object') return value;
|
||||
if (seen.has(value)) return seen.get(value);
|
||||
const proxied = new Proxy(value, handler);
|
||||
seen.set(value, proxied);
|
||||
return proxied;
|
||||
};
|
||||
const wrap = (value: any): any => {
|
||||
if (value === null || typeof value !== "object") return value;
|
||||
if (seen.has(value)) return seen.get(value);
|
||||
const proxied = new Proxy(value, handler);
|
||||
seen.set(value, proxied);
|
||||
return proxied;
|
||||
};
|
||||
|
||||
const handler: ProxyHandler<any> = {
|
||||
get(target, prop, receiver) {
|
||||
// 非字符串属性（如 Symbol、数字索引）直接透传
|
||||
if (typeof prop !== 'string') {
|
||||
return wrap(Reflect.get(target, prop, receiver));
|
||||
}
|
||||
const handler: ProxyHandler<any> = {
|
||||
get(target, prop, receiver) {
|
||||
// 非字符串属性（如 Symbol、数字索引）直接透传
|
||||
if (typeof prop !== "string") {
|
||||
return wrap(Reflect.get(target, prop, receiver));
|
||||
}
|
||||
|
||||
const primary = prop;
|
||||
const primary = prop;
|
||||
|
||||
if (primary in target) return wrap(Reflect.get(target, primary, receiver));
|
||||
if (primary in target)
|
||||
return wrap(Reflect.get(target, primary, receiver));
|
||||
|
||||
const camel = toCamelCaseKey(primary);
|
||||
if (camel in target) return wrap(Reflect.get(target, camel, receiver));
|
||||
const camel = toCamelCaseKey(primary);
|
||||
if (camel in target) return wrap(Reflect.get(target, camel, receiver));
|
||||
|
||||
const snake = toSnakeCaseKey(primary);
|
||||
if (snake in target) return wrap(Reflect.get(target, snake, receiver));
|
||||
const snake = toSnakeCaseKey(primary);
|
||||
if (snake in target) return wrap(Reflect.get(target, snake, receiver));
|
||||
|
||||
return wrap(Reflect.get(target, prop, receiver));
|
||||
},
|
||||
has(target, prop) {
|
||||
if (typeof prop !== 'string') return prop in target;
|
||||
const primary = prop === 'auther' ? 'autherInfo' : prop;
|
||||
return (
|
||||
primary in target ||
|
||||
toCamelCaseKey(primary) in target ||
|
||||
toSnakeCaseKey(primary) in target
|
||||
);
|
||||
}
|
||||
};
|
||||
return wrap(Reflect.get(target, prop, receiver));
|
||||
},
|
||||
has(target, prop) {
|
||||
if (typeof prop !== "string") return prop in target;
|
||||
const primary = prop === "auther" ? "autherInfo" : prop;
|
||||
return (
|
||||
primary in target ||
|
||||
toCamelCaseKey(primary) in target ||
|
||||
toSnakeCaseKey(primary) in target
|
||||
);
|
||||
},
|
||||
};
|
||||
|
||||
return wrap(root);
|
||||
return wrap(root);
|
||||
}
|
||||
|
||||
/** 选择首个可用 URL */
|
||||
export function pickFirstUrl(list?: string[]) {
|
||||
return Array.isArray(list) && list.length ? list[0] : undefined;
|
||||
return Array.isArray(list) && list.length ? list[0] : undefined;
|
||||
}
|
||||
|
||||
// 别名，兼容旧命名
|
||||
|
||||
Reference in New Issue
Block a user