feie9454/douyin-archive
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

优化前端，修改返回逻辑，添加视频实时背景，音量倍速持久化

Browse Source
This commit is contained in:
feie9454
2025-10-21 16:33:16 +08:00
parent 9d1598a5ab
commit 200283b21b
231 changed files with 744 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/fetcher/browser.ts
+71
View File
@@ -0,0 +1,71 @@
import { chromium, type BrowserContext } from 'playwright'
// A simple singleton manager for a persistent Chromium context with ref-counting.
// Prevents concurrent tasks from closing the shared window prematurely.
let contextPromise: Promise<BrowserContext> | null = null
let context: BrowserContext | null = null
let refCount = 0
let idleCloseTimer: NodeJS.Timeout | null = null
const USER_DATA_DIR = 'chrome-profile/douyin'
const DEFAULT_OPTIONS = { headless: true } as const
async function launchContext(): Promise<BrowserContext> {
const ctx = await chromium.launchPersistentContext(USER_DATA_DIR, DEFAULT_OPTIONS)
// When the context is closed externally, reset manager state
ctx.on('close', () => {
context = null
contextPromise = null
refCount = 0
if (idleCloseTimer) {
clearTimeout(idleCloseTimer)
idleCloseTimer = null
}
})
return ctx
}
export async function acquireBrowserContext(): Promise<BrowserContext> {
// Cancel any pending idle close if a new consumer arrives
if (idleCloseTimer) {
clearTimeout(idleCloseTimer)
idleCloseTimer = null
}
if (context) {
refCount += 1
return context
}
if (!contextPromise) {
contextPromise = launchContext()
}
context = await contextPromise
refCount += 1
return context
}
export async function releaseBrowserContext(options?: { idleMillis?: number }): Promise<void> {
const idleMillis = options?.idleMillis ?? 15_000
refCount = Math.max(0, refCount - 1)
if (refCount > 0 || !context) return
// Delay the close to allow bursty workloads to reuse the context
if (idleCloseTimer) {
clearTimeout(idleCloseTimer)
idleCloseTimer = null
}
idleCloseTimer = setTimeout(async () => {
try {
if (context && refCount === 0) {
await context.close()
}
} finally {
context = null
contextPromise = null
idleCloseTimer = null
}
}, idleMillis)
}
app/fetcher/index.ts
+54 -12
View File
@@ -1,5 +1,5 @@
// src/scrapeDouyin.ts
import { BrowserContext, chromium, Page, type Response } from 'playwright';
import { BrowserContext, Page, chromium, type Response } from 'playwright';
import { prisma } from '@/lib/prisma';
import { uploadFile, generateUniqueFileName } from '@/lib/minio';
import { createCamelCompatibleProxy } from '@/app/fetcher/utils';
@@ -8,6 +8,7 @@ import { pickBestPlayAddr, extractFirstFrame } from '@/app/fetcher/media';
import { handleImagePost } from '@/app/fetcher/uploader';
import { saveToDB, saveImagePostToDB } from '@/app/fetcher/persist';
import chalk from 'chalk';
import { acquireBrowserContext, releaseBrowserContext } from '@/app/fetcher/browser';
const DETAIL_PATH = '/aweme/v1/web/aweme/detail/';
const COMMENT_PATH = '/aweme/v1/web/comment/list/';
@@ -33,11 +34,21 @@ async function readPostMem(context: BrowserContext, page: Page) {
}
export async function scrapeDouyin(url: string) {
const browser = await chromium.launch({ headless: true });
console.log(chalk.blue('🚀 启动 Chromium 浏览器...'));
export class ScrapeError extends Error {
constructor(
message: string,
public statusCode: number = 500,
public code?: string
) {
super(message);
this.name = 'ScrapeError';
}
}
const context = await chromium.launchPersistentContext('chrome-profile/douyin', { headless: false });
export async function scrapeDouyin(url: string) {
console.log(chalk.blue('🚀 启动共享 Chromium 浏览器...'));
const context = await acquireBrowserContext();
const page = await context.newPage();
console.log(chalk.cyan(`📄 正在访问: ${chalk.underline(url)}`));
@@ -82,6 +93,13 @@ export async function scrapeDouyin(url: string) {
await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 60_000 });
// 查找页面中是否存在 "视频不存在" 的提示
const isNotFound = await page.locator('text=视频不存在').count().then(count => count > 0).catch(() => false);
if (isNotFound) {
console.error(chalk.red('✗ 视频不存在或已被删除'));
throw new ScrapeError('视频不存在或已被删除', 404, 'VIDEO_NOT_FOUND');
}
try {
// 优先尝试从内存读取图文数据
let { aweme, comments } = await readPostMem(context, page);
@@ -113,8 +131,8 @@ export async function scrapeDouyin(url: string) {
const firstType = await firstTypePromise;
if (!firstType) {
console.error(chalk.red('✗ 既无法从内存读取数据，也无法从网络获得数据'));
throw new Error('既无法从内存读取数据，也无法从网络获得数据。');
console.error(chalk.red('✗ 既无法从内存读取数据,也无法从网络获得数据'));
throw new ScrapeError('无法获取作品数据,可能是网络问题或作品已下架', 404, 'NO_DATA');
}
console.log(chalk.cyan(`📡 检测到作品类型: ${chalk.bold(firstType.key === 'post' ? '图文' : '视频')}`));
@@ -129,14 +147,14 @@ export async function scrapeDouyin(url: string) {
if (firstType.key === 'post') {
// 图文作品
const postJson = await safeJson<DouyinPostListResponse>(firstType.response);
if (!postJson?.aweme_list?.length) throw new Error('图文作品响应为空');
if (!postJson?.aweme_list?.length) throw new ScrapeError('图文作品响应为空', 404, 'EMPTY_POST_RESPONSE');
const currentURL = page.url();
const target_aweme_id = currentURL.split('/').at(-1);
const awemeList = postJson.aweme_list as unknown as DouyinImageAweme[];
let aweme = awemeList.find((pt: DouyinImageAweme) => pt.aweme_id === target_aweme_id);
if (!aweme) {
throw new Error('既无法从内存读取数据，Post 列表中也不包含需要爬取的作品。');
throw new ScrapeError('无法找到目标作品,可能已被删除', 404, 'POST_NOT_FOUND');
}
const uploads = await handleImagePost(context, aweme);
@@ -191,12 +209,36 @@ export async function scrapeDouyin(url: string) {
console.log(chalk.green.bold('✓ 视频作品保存成功'));
return { type: "video", ...saved };
} else {
throw new Error('无法判定作品类型（未命中详情或图文接口）');
throw new ScrapeError('无法判定作品类型,接口响应异常', 500, 'UNKNOWN_TYPE');
}
} catch (error) {
// 如果是我们自定义的错误,直接抛出
if (error instanceof ScrapeError) {
throw error;
}
// 处理其他类型的错误
const errMsg = (error as Error)?.message || String(error);
console.error(chalk.red(`✗ 爬取失败: ${errMsg}`));
// 根据错误类型返回不同的状态码
if (errMsg.includes('timeout') || errMsg.includes('超时')) {
throw new ScrapeError('请求超时,请稍后重试', 408, 'TIMEOUT');
}
if (errMsg.includes('页面内存数据中未找到作品详情')) {
throw new ScrapeError('作品数据加载失败', 404, 'DATA_NOT_LOADED');
}
if (errMsg.includes('net::')) {
throw new ScrapeError('网络连接失败', 503, 'NETWORK_ERROR');
}
// 默认服务器错误
throw new ScrapeError(errMsg || '爬取过程中发生未知错误', 500, 'UNKNOWN_ERROR');
} finally {
console.log(chalk.gray('🧹 清理资源...'));
await context.close();
await browser.close();
try { await page.close({ runBeforeUnload: true }); } catch {}
// 仅释放共享上下文的引用，不直接关闭窗口
await releaseBrowserContext();
await prisma.$disconnect();
console.log(chalk.gray('✓ 资源清理完成'));
}
app/fetcher/network.ts
+1 -1
View File
@@ -22,7 +22,7 @@ export async function downloadBinary(
context: BrowserContext,
url: string
): Promise<{ buffer: Buffer; contentType: string; ext: string }> {
console.log('Download bin:', url);
console.log('下载：', url);
const headers = {
referer: url,
app/fetcher/route.ts
+37 -5
View File
@@ -1,17 +1,49 @@
import { NextRequest, NextResponse } from 'next/server'
import { prisma } from '@/lib/prisma'
import { scrapeDouyin } from '.';
import { scrapeDouyin, ScrapeError } from '.';
async function handleDouyinScrape(req: NextRequest) {
const { searchParams } = new URL(req.url);
const videoUrl = searchParams.get('url');
if (!videoUrl) {
return NextResponse.json({ error: '缺少视频URL' }, { status: 400 });
return NextResponse.json(
{ error: '缺少视频URL', code: 'MISSING_URL' },
{ status: 400 }
);
}
// 调用爬虫函数
const result = await scrapeDouyin(videoUrl);
return NextResponse.json(result);
try {
// 调用爬虫函数
const result = await scrapeDouyin(videoUrl);
return NextResponse.json({
success: true,
data: result
});
} catch (error) {
// 处理自定义的 ScrapeError
if (error instanceof ScrapeError) {
return NextResponse.json(
{
success: false,
error: error.message,
code: error.code
},
{ status: error.statusCode }
);
}
// 处理未知错误
console.error('未捕获的错误:', error);
return NextResponse.json(
{
success: false,
error: '服务器内部错误',
code: 'INTERNAL_ERROR'
},
{ status: 500 }
);
}
}
export const GET = handleDouyinScrape