修改资源URL模式,不再存储https://domain 前缀

This commit is contained in:
feie9454 2025-10-25 12:19:53 +08:00
parent f94ef73518
commit e4339a5b91
27 changed files with 347 additions and 137 deletions

49
app/api/stt/index.ts Normal file
View File

@ -0,0 +1,49 @@
import fs from "fs";
import OpenAI from "openai";
import prompt from "./prompt.md";
import { prisma } from "@/lib/prisma";
const client = new OpenAI({
apiKey: process.env.OPENAI_API_KEY,
baseURL: process.env.OPENAI_API_BASE_URL,
});
async function transcriptAudio(audio: Buffer | string) {
if (typeof audio === "string") {
audio = fs.readFileSync(audio);
}
const base64Audio = Buffer.from(audio).toString("base64");
const response = await client.chat.completions.create({
model: "gemini-2.5-flash-lite",
messages: [
{
role: "user",
content: [
{
type: "text",
text: prompt,
},
{
type: "input_audio",
input_audio: {
data: base64Audio,
format: "wav",
},
},
],
},
],
});
console.log(response.choices[0].message.content);
}
async function transcriptAweme(awemeId: string) {
const aweme = await prisma.video.findUnique({
where: { aweme_id: awemeId },
});
if (!aweme) {
throw new Error("Aweme not found or aweme is not a video post");
}
}

51
app/api/stt/prompt.md Normal file
View File

@ -0,0 +1,51 @@
你将接收一段音频。请完成:
A.语音活动检测VAD与声源分类
B.条件式处理:
- 若包含可辨识的人类发言:** 进行转录 **(保留原语言,不翻译),并尽可能给出说话人分离与时间戳;
- 若不包含人类发言:** 不转录 **,仅返回音频类型与简要描述。
C.严格输出为下方 JSON字段不得缺失或额外编造。听不清处用“[听不清]”。
** 输出 JSON Schema示例**
```json
{
"speech_detected": true,
"language": "zh-CN",
"audio_type": null,
"background": "music | ambience | none | unknown",
"transcript": [
{
"start": 0.00,
"end": 3.42,
"text": "大家好,我是……"
},
{
"start": 3.50,
"end": 6.10,
"text": "欢迎来到今天的节目。"
}
],
"non_speech_summary": null,
}
```
>
** 当无发言时返回:**
```json
{
"speech_detected": false,
"language": null,
"audio_type": "music | ambience | animal | mechanical | other",
"background": "none",
"transcript": [],
"non_speech_summary": "示例:纯音乐-钢琴独奏,节奏舒缓;或 环境声-雨声伴随雷鸣。",
}
```
** 规则补充 **
* 只要存在可理解的人类发言(即便有音乐 / 噪声),就执行转录,并在 `background` 标注“music / ambience”。
* 不要将唱词 / 哼唱视为“发言”;若仅有人声演唱且无口语发言,视为 ** 音乐 **
* 不要臆测未听清内容;不要添加与音频无关的信息。
* 时间单位统一为秒,保留两位小数。
* 允许`language` 为多标签(如 "zh-CN,en")或为 `null`(无发言时)。

View File

@ -3,7 +3,7 @@
import { useRouter } from "next/navigation"; import { useRouter } from "next/navigation";
import { useMemo, useRef } from "react"; import { useMemo, useRef } from "react";
import { Pause, Play } from "lucide-react"; import { Pause, Play } from "lucide-react";
import type { AwemeData, ImageData, Neighbors, VideoData } from "./types"; import type { AwemeData, ImageData, Neighbors, VideoData } from "./types.ts";
import { BackgroundCanvas } from "./components/BackgroundCanvas"; import { BackgroundCanvas } from "./components/BackgroundCanvas";
import { CommentPanel } from "./components/CommentPanel"; import { CommentPanel } from "./components/CommentPanel";
import { ImageCarousel } from "./components/ImageCarousel"; import { ImageCarousel } from "./components/ImageCarousel";

View File

@ -1,6 +1,6 @@
import { ThumbsUp, X } from "lucide-react"; import { ThumbsUp, X } from "lucide-react";
import { useState } from "react"; import { useState } from "react";
import type { Comment, User } from "../types"; import type { Comment, User } from "../types.ts";
import { formatRelativeTime, formatAbsoluteUTC } from "../utils"; import { formatRelativeTime, formatAbsoluteUTC } from "../utils";
import { CommentText } from "./CommentText"; import { CommentText } from "./CommentText";

View File

@ -1,6 +1,6 @@
import { X } from "lucide-react"; import { X } from "lucide-react";
import { useState, useEffect, useRef, useCallback } from "react"; import { useState, useEffect, useRef, useCallback } from "react";
import type { Comment, User } from "../types"; import type { Comment, User } from "../types.ts";
import { CommentList } from "./CommentList"; import { CommentList } from "./CommentList";
interface CommentPanelProps { interface CommentPanelProps {

View File

@ -1,5 +1,5 @@
import { forwardRef, useEffect, useRef, useState } from "react"; import { forwardRef, useEffect, useRef, useState } from "react";
import type { ImageData } from "../types"; import type { ImageData } from "../types.ts";
interface ImageCarouselProps { interface ImageCarouselProps {
images: ImageData["images"]; images: ImageData["images"];

View File

@ -14,7 +14,7 @@ import {
VolumeX, VolumeX,
} from "lucide-react"; } from "lucide-react";
import type { RefObject } from "react"; import type { RefObject } from "react";
import type { LoopMode, ObjectFit, User } from "../types"; import type { LoopMode, ObjectFit, User } from "../types.ts";
import { formatRelativeTime, formatTime, formatAbsoluteUTC } from "../utils"; import { formatRelativeTime, formatTime, formatAbsoluteUTC } from "../utils";
import { ProgressBar } from "./ProgressBar"; import { ProgressBar } from "./ProgressBar";
import { SegmentedProgressBar } from "./SegmentedProgressBar"; import { SegmentedProgressBar } from "./SegmentedProgressBar";

View File

@ -1,5 +1,5 @@
import { ChevronDown, ChevronUp, MessageSquareText, ThumbsUp } from "lucide-react"; import { ChevronDown, ChevronUp, MessageSquareText, ThumbsUp } from "lucide-react";
import type { Neighbors } from "../types"; import type { Neighbors } from "../types.ts";
interface NavigationButtonsProps { interface NavigationButtonsProps {
neighbors: Neighbors; neighbors: Neighbors;

View File

@ -1,5 +1,5 @@
import { forwardRef } from "react"; import { forwardRef } from "react";
import type { ObjectFit } from "../types"; import type { ObjectFit } from "../types.ts";
interface VideoPlayerProps { interface VideoPlayerProps {
videoUrl: string; videoUrl: string;

View File

@ -89,7 +89,7 @@ export function useBackgroundCanvas({
ctx.drawImage(sourceElement, offsetX, offsetY, drawWidth, drawHeight); ctx.drawImage(sourceElement, offsetX, offsetY, drawWidth, drawHeight);
}; };
const intervalId = setInterval(drawMediaToCanvas, 20); const intervalId = setInterval(drawMediaToCanvas, 34);
return () => { return () => {
clearInterval(intervalId); clearInterval(intervalId);

View File

@ -1,7 +1,7 @@
import { useEffect, useRef, useState } from "react"; import { useEffect, useRef, useState } from "react";
import type { RefObject } from "react"; import type { RefObject } from "react";
import { useRouter } from "next/navigation"; import { useRouter } from "next/navigation";
import type { ImageData, LoopMode, Neighbors } from "../types"; import type { ImageData, LoopMode, Neighbors } from "../types.ts";
interface UseImageCarouselProps { interface UseImageCarouselProps {
images: ImageData["images"]; images: ImageData["images"];

View File

@ -1,7 +1,7 @@
import { useEffect, useRef } from "react"; import { useEffect, useRef } from "react";
import type { RefObject } from "react"; import type { RefObject } from "react";
import { useRouter } from "next/navigation"; import { useRouter } from "next/navigation";
import type { Neighbors } from "../types"; import type { Neighbors } from "../types.ts";
interface UseNavigationProps { interface UseNavigationProps {
neighbors: Neighbors; neighbors: Neighbors;

View File

@ -1,5 +1,5 @@
import { useEffect, useState } from "react"; import { useEffect, useState } from "react";
import type { LoopMode, ObjectFit } from "../types"; import type { LoopMode, ObjectFit } from "../types.ts";
import { getNumberFromStorage, getStringFromStorage, saveToStorage } from "../utils"; import { getNumberFromStorage, getStringFromStorage, saveToStorage } from "../utils";
export function usePlayerState() { export function usePlayerState() {

View File

@ -1,7 +1,7 @@
import { useEffect, useRef } from "react"; import { useEffect, useRef } from "react";
import type { RefObject } from "react"; import type { RefObject } from "react";
import { useRouter } from "next/navigation"; import { useRouter } from "next/navigation";
import type { LoopMode, Neighbors } from "../types"; import type { LoopMode, Neighbors } from "../types.ts";
interface UseVideoPlayerProps { interface UseVideoPlayerProps {
awemeId: string; awemeId: string;

View File

@ -2,14 +2,8 @@ import { prisma } from "@/lib/prisma";
import BackButton from "@/app/components/BackButton"; import BackButton from "@/app/components/BackButton";
import AwemeDetailClient from "./Client"; import AwemeDetailClient from "./Client";
import type { Metadata } from "next"; import type { Metadata } from "next";
import { getFileUrl } from "@/lib/minio";
function ms(v?: number | null) { import { AwemeData } from "./types";
if (!v) return "";
const s = Math.round(v / 1000);
const m = Math.floor(s / 60);
const r = s % 60;
return `${m}:${r.toString().padStart(2, "0")}`;
}
export async function generateMetadata({ params }: { params: Promise<{ awemeId: string }> }): Promise<Metadata> { export async function generateMetadata({ params }: { params: Promise<{ awemeId: string }> }): Promise<Metadata> {
const id = (await params).awemeId; const id = (await params).awemeId;
@ -65,34 +59,39 @@ export default async function AwemeDetail({ params }: { params: Promise<{ awemeI
where: isVideo ? { videoId: id } : { imagePostId: id }, where: isVideo ? { videoId: id } : { imagePostId: id },
}); });
const data = isVideo const aweme = isVideo ? video : post;
? {
const data: AwemeData = {
aweme_id: aweme!.aweme_id,
desc: aweme!.desc,
created_at: aweme!.created_at,
likesCount: Number(aweme!.digg_count),
commentsCount,
author: { nickname: aweme!.author.nickname, avatar_url: getFileUrl(aweme!.author.avatar_url || 'default-avatar.png') },
...(() => {
if (isVideo) {
const aweme = video!
return {
type: "video" as const, type: "video" as const,
aweme_id: video!.aweme_id, duration_ms: aweme!.duration_ms,
desc: video!.desc, video_url: aweme!.video_url,
created_at: video!.created_at, width: aweme!.width ?? null,
duration_ms: video!.duration_ms, height: aweme!.height ?? null,
video_url: video!.video_url,
width: video!.width ?? null,
height: video!.height ?? null,
author: { nickname: video!.author.nickname, avatar_url: video!.author.avatar_url },
commentsCount,
likesCount: Number(video!.digg_count),
} }
: { } else {
const aweme = post!
return {
type: "image" as const, type: "image" as const,
aweme_id: post!.aweme_id, images: aweme!.images.map(img => ({ ...img, url: getFileUrl(img.url) })),
desc: post!.desc, music_url: aweme!.music_url,
created_at: post!.created_at,
images: post!.images,
music_url: post!.music_url,
author: { nickname: post!.author.nickname, avatar_url: post!.author.avatar_url },
commentsCount,
likesCount: Number(post!.digg_count),
}; };
}
})()
}
// Compute prev/next neighbors by created_at across videos and image posts // Compute prev/next neighbors by created_at across videos and image posts
const currentCreatedAt = (isVideo ? video!.created_at : post!.created_at) as unknown as Date; const currentCreatedAt = (isVideo ? video!.created_at : post!.created_at);
const [newerVideo, newerPost, olderVideo, olderPost] = await Promise.all([ const [newerVideo, newerPost, olderVideo, olderPost] = await Promise.all([
prisma.video.findFirst({ where: { created_at: { gt: currentCreatedAt } }, orderBy: { created_at: "asc" }, select: { aweme_id: true, created_at: true } }), prisma.video.findFirst({ where: { created_at: { gt: currentCreatedAt } }, orderBy: { created_at: "asc" }, select: { aweme_id: true, created_at: true } }),
prisma.imagePost.findFirst({ where: { created_at: { gt: currentCreatedAt } }, orderBy: { created_at: "asc" }, select: { aweme_id: true, created_at: true } }), prisma.imagePost.findFirst({ where: { created_at: { gt: currentCreatedAt } }, orderBy: { created_at: "asc" }, select: { aweme_id: true, created_at: true } }),
@ -101,16 +100,16 @@ export default async function AwemeDetail({ params }: { params: Promise<{ awemeI
]); ]);
const pickPrev = (() => { const pickPrev = (() => {
const cands: { aweme_id: string; created_at: Date }[] = []; const cands: { aweme_id: string; created_at: Date }[] = [];
if (newerVideo) cands.push({ aweme_id: newerVideo.aweme_id, created_at: newerVideo.created_at as unknown as Date }); if (newerVideo) cands.push({ aweme_id: newerVideo.aweme_id, created_at: newerVideo.created_at });
if (newerPost) cands.push({ aweme_id: newerPost.aweme_id, created_at: newerPost.created_at as unknown as Date }); if (newerPost) cands.push({ aweme_id: newerPost.aweme_id, created_at: newerPost.created_at });
if (cands.length === 0) return null; if (cands.length === 0) return null;
cands.sort((a, b) => +a.created_at - +b.created_at); cands.sort((a, b) => +a.created_at - +b.created_at);
return { aweme_id: cands[0].aweme_id }; return { aweme_id: cands[0].aweme_id };
})(); })();
const pickNext = (() => { const pickNext = (() => {
const cands: { aweme_id: string; created_at: Date }[] = []; const cands: { aweme_id: string; created_at: Date }[] = [];
if (olderVideo) cands.push({ aweme_id: olderVideo.aweme_id, created_at: olderVideo.created_at as unknown as Date }); if (olderVideo) cands.push({ aweme_id: olderVideo.aweme_id, created_at: olderVideo.created_at });
if (olderPost) cands.push({ aweme_id: olderPost.aweme_id, created_at: olderPost.created_at as unknown as Date }); if (olderPost) cands.push({ aweme_id: olderPost.aweme_id, created_at: olderPost.created_at });
if (cands.length === 0) return null; if (cands.length === 0) return null;
cands.sort((a, b) => +b.created_at - +a.created_at); cands.sort((a, b) => +b.created_at - +a.created_at);
return { aweme_id: cands[0].aweme_id }; return { aweme_id: cands[0].aweme_id };
@ -126,7 +125,7 @@ export default async function AwemeDetail({ params }: { params: Promise<{ awemeI
className="inline-flex items-center justify-center w-9 h-9 rounded-full bg-white/15 text-white border border-white/20 backdrop-blur hover:bg-white/25" className="inline-flex items-center justify-center w-9 h-9 rounded-full bg-white/15 text-white border border-white/20 backdrop-blur hover:bg-white/25"
/> />
</div> </div>
<AwemeDetailClient data={data as any} neighbors={neighbors as any} /> <AwemeDetailClient data={data} neighbors={neighbors} />
</main> </main>
); );
} }

View File

@ -14,10 +14,10 @@ export type VideoData = {
aweme_id: string; aweme_id: string;
desc: string; desc: string;
created_at: string | Date; created_at: string | Date;
duration_ms?: number | null; duration_ms: number | null;
video_url: string; video_url: string;
width?: number | null; width: number | null;
height?: number | null; height: number | null;
author: User; author: User;
commentsCount: number; commentsCount: number;
likesCount: number; likesCount: number;
@ -28,8 +28,8 @@ export type ImageData = {
aweme_id: string; aweme_id: string;
desc: string; desc: string;
created_at: string | Date; created_at: string | Date;
images: { id: string; url: string; width?: number; height?: number; animated?: string; duration?: number }[]; images: { id: string; url: string; width: number | null; height: number | null; animated: string | null; duration: number | null }[];
music_url?: string | null; music_url: string | null;
author: User; author: User;
commentsCount: number; commentsCount: number;
likesCount: number; likesCount: number;

View File

@ -28,9 +28,13 @@ export default function FeedMasonry({ initialItems, initialCursor }: Props) {
if (w >= 640) return 2; // sm if (w >= 640) return 2; // sm
return 1; return 1;
}, []); }, []);
const [columnCount, setColumnCount] = useState<number>(getColumnCount()); // 为避免 SSR 与客户端初次渲染不一致window 未定义导致服务端为 1 列,客户端首次渲染为多列),
// 这里将初始列数固定为 1待挂载后再根据窗口宽度更新。
const [columnCount, setColumnCount] = useState<number>(1);
useEffect(() => { useEffect(() => {
// 挂载后立即根据当前窗口宽度更新一次列数
setColumnCount(getColumnCount());
const onResize = () => setColumnCount(getColumnCount()); const onResize = () => setColumnCount(getColumnCount());
window.addEventListener('resize', onResize); window.addEventListener('resize', onResize);
return () => window.removeEventListener('resize', onResize); return () => window.removeEventListener('resize', onResize);

View File

@ -2,6 +2,7 @@ import { prisma } from "@/lib/prisma";
import FeedMasonry from "./components/FeedMasonry"; import FeedMasonry from "./components/FeedMasonry";
import type { FeedItem } from "./types/feed"; import type { FeedItem } from "./types/feed";
import type { Metadata } from "next"; import type { Metadata } from "next";
import { getFileUrl } from "@/lib/minio";
export const metadata: Metadata = { export const metadata: Metadata = {
title: "作品集 - 抖歪", title: "作品集 - 抖歪",
@ -28,11 +29,11 @@ export default async function Home() {
aweme_id: v.aweme_id, aweme_id: v.aweme_id,
created_at: v.created_at, created_at: v.created_at,
desc: v.desc, desc: v.desc,
video_url: v.video_url, video_url: getFileUrl(v.video_url),
cover_url: v.cover_url ?? null, cover_url: getFileUrl(v.cover_url ?? ''),
width: v.width ?? null, width: v.width ?? null,
height: v.height ?? null, height: v.height ?? null,
author: { nickname: v.author.nickname, avatar_url: v.author.avatar_url ?? null }, author: { nickname: v.author.nickname, avatar_url: getFileUrl(v.author.avatar_url ?? '') },
likes: Number(v.digg_count) likes: Number(v.digg_count)
})), })),
...posts.map((p) => ({ ...posts.map((p) => ({
@ -40,10 +41,10 @@ export default async function Home() {
aweme_id: p.aweme_id, aweme_id: p.aweme_id,
created_at: p.created_at, created_at: p.created_at,
desc: p.desc, desc: p.desc,
cover_url: p.images?.[0]?.url ?? null, cover_url: getFileUrl(p.images?.[0]?.url ?? null),
width: p.images?.[0]?.width ?? null, width: p.images?.[0]?.width ?? null,
height: p.images?.[0]?.height ?? null, height: p.images?.[0]?.height ?? null,
author: { nickname: p.author.nickname, avatar_url: p.author.avatar_url ?? null }, author: { nickname: p.author.nickname, avatar_url: getFileUrl(p.author.avatar_url ?? '') },
likes: Number(p.digg_count) likes: Number(p.digg_count)
})), })),
] ]

View File

@ -9,6 +9,7 @@
"lucide-react": "^0.546.0", "lucide-react": "^0.546.0",
"minio": "^8.0.6", "minio": "^8.0.6",
"next": "15.5.6", "next": "15.5.6",
"openai": "^6.7.0",
"playwright": "1.56.1", "playwright": "1.56.1",
"playwright-extra": "^4.3.6", "playwright-extra": "^4.3.6",
"puppeteer-extra-plugin-stealth": "^2.11.2", "puppeteer-extra-plugin-stealth": "^2.11.2",
@ -402,6 +403,8 @@
"once": ["once@1.4.0", "", { "dependencies": { "wrappy": "1" } }, "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w=="], "once": ["once@1.4.0", "", { "dependencies": { "wrappy": "1" } }, "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w=="],
"openai": ["openai@6.7.0", "https://registry.npmmirror.com/openai/-/openai-6.7.0.tgz", { "peerDependencies": { "ws": "^8.18.0", "zod": "^3.25 || ^4.0" }, "optionalPeers": ["ws", "zod"], "bin": { "openai": "bin/cli" } }, "sha512-mgSQXa3O/UXTbA8qFzoa7aydbXBJR5dbLQXCRapAOtoNT+v69sLdKMZzgiakpqhclRnhPggPAXoniVGn2kMY2A=="],
"path-is-absolute": ["path-is-absolute@1.0.1", "", {}, "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg=="], "path-is-absolute": ["path-is-absolute@1.0.1", "", {}, "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg=="],
"pathe": ["pathe@2.0.3", "", {}, "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w=="], "pathe": ["pathe@2.0.3", "", {}, "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w=="],

132
fix-asset-urls.ts Normal file
View File

@ -0,0 +1,132 @@
// scripts/fix-asset-urls.ts
import { PrismaClient } from '@prisma/client';
const prisma = new PrismaClient();
const FROM = 'douyin-archive/';
const TO = '';
function escapeForPgRegex(s: string) {
return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
}
const FROM_RE = `^${escapeForPgRegex(FROM)}`; // 只替换“以旧前缀开头”的字符串
const dryRun = false; // true: 只统计,不修改
async function main() {
if (dryRun) {
const rows = await prisma.$queryRawUnsafe<any[]>(`
WITH c AS (
SELECT 'Author.avatar_url' AS col, COUNT(*) AS n FROM "Author" WHERE avatar_url LIKE '${FROM}%'
UNION ALL
SELECT 'CommentUser.avatar_url' , COUNT(*) FROM "CommentUser" WHERE avatar_url LIKE '${FROM}%'
UNION ALL
SELECT 'CommentImage.url' , COUNT(*) FROM "CommentImage" WHERE url LIKE '${FROM}%'
UNION ALL
SELECT 'Video.cover_url' , COUNT(*) FROM "Video" WHERE cover_url LIKE '${FROM}%'
UNION ALL
SELECT 'Video.video_url' , COUNT(*) FROM "Video" WHERE video_url LIKE '${FROM}%'
UNION ALL
SELECT 'ImageFile.url' , COUNT(*) FROM "ImageFile" WHERE url LIKE '${FROM}%'
UNION ALL
SELECT 'ImageFile.animated' , COUNT(*) FROM "ImageFile" WHERE animated LIKE '${FROM}%'
UNION ALL
SELECT 'ImagePost.music_url' , COUNT(*) FROM "ImagePost" WHERE music_url LIKE '${FROM}%'
UNION ALL
SELECT 'Video.raw_json' , COUNT(*) FROM "Video" WHERE raw_json::text LIKE '%${FROM}%'
UNION ALL
SELECT 'ImagePost.raw_json' , COUNT(*) FROM "ImagePost" WHERE raw_json::text LIKE '%${FROM}%'
)
SELECT * FROM c ORDER BY col;
`);
console.table(rows);
return;
}
await prisma.$transaction(async (tx) => {
// CommentUser.avatar_url
await tx.$executeRawUnsafe(`
UPDATE "CommentUser"
SET avatar_url = regexp_replace(avatar_url, '${FROM_RE}', '${TO}')
WHERE avatar_url LIKE '${FROM}%'
`);
// CommentImage.url
await tx.$executeRawUnsafe(`
UPDATE "CommentImage"
SET url = regexp_replace(url, '${FROM_RE}', '${TO}')
WHERE url LIKE '${FROM}%'
`);
// Author.avatar_url
await tx.$executeRawUnsafe(`
UPDATE "Author"
SET avatar_url = regexp_replace(avatar_url, '${FROM_RE}', '${TO}')
WHERE avatar_url LIKE '${FROM}%'
`);
// Video.cover_url
await tx.$executeRawUnsafe(`
UPDATE "Video"
SET cover_url = regexp_replace(cover_url, '${FROM_RE}', '${TO}')
WHERE cover_url LIKE '${FROM}%'
`);
// Video.video_url
await tx.$executeRawUnsafe(`
UPDATE "Video"
SET video_url = regexp_replace(video_url, '${FROM_RE}', '${TO}')
WHERE video_url LIKE '${FROM}%'
`);
// ImageFile.url
await tx.$executeRawUnsafe(`
UPDATE "ImageFile"
SET url = regexp_replace(url, '${FROM_RE}', '${TO}')
WHERE url LIKE '${FROM}%'
`);
// ImageFile.animated
await tx.$executeRawUnsafe(`
UPDATE "ImageFile"
SET animated = regexp_replace(animated, '${FROM_RE}', '${TO}')
WHERE animated LIKE '${FROM}%'
`);
// ImagePost.music_url
await tx.$executeRawUnsafe(`
UPDATE "ImagePost"
SET music_url = regexp_replace(music_url, '${FROM_RE}', '${TO}')
WHERE music_url LIKE '${FROM}%'
`);
// 可选raw_json简单文本整体替换若想更精细可用递归 JSONB 方法
await tx.$executeRawUnsafe(`
UPDATE "Video"
SET raw_json = to_jsonb(replace(raw_json::text, '${FROM}', '${TO}'))
WHERE raw_json::text LIKE '%${FROM}%'
`);
await tx.$executeRawUnsafe(`
UPDATE "ImagePost"
SET raw_json = to_jsonb(replace(raw_json::text, '${FROM}', '${TO}'))
WHERE raw_json::text LIKE '%${FROM}%'
`);
});
// 快速抽样
const sample = await prisma.$queryRawUnsafe<any[]>(`
(
SELECT 'CommentImage.url' AS col, url AS v FROM "CommentImage" WHERE url LIKE '${TO}%' LIMIT 2
) UNION ALL (
SELECT 'CommentUser.avatar_url', avatar_url FROM "CommentUser" WHERE avatar_url LIKE '${TO}%' LIMIT 2
) UNION ALL (
SELECT 'Video.video_url', video_url FROM "Video" WHERE video_url LIKE '${TO}%' LIMIT 2
)
`);
console.log('Sample after update:', sample);
}
main().catch((e) => {
console.error(e);
process.exit(1);
}).finally(async () => {
await prisma.$disconnect();
});

9
global.d.ts vendored Normal file
View File

@ -0,0 +1,9 @@
declare module '*.md' {
const content: string;
export default content;
}
declare module '*.txt' {
const content: string;
export default content;
}

View File

@ -54,7 +54,7 @@ export async function initBucket(): Promise<void> {
* @param file - File Buffer * @param file - File Buffer
* @param path - : 'avatars/user123.jpg' 'posts/2024/image.png' * @param path - : 'avatars/user123.jpg' 'posts/2024/image.png'
* @param metadata - * @param metadata -
* @returns 访URL * @returns path = args.path
*/ */
export async function uploadFile( export async function uploadFile(
file: File | Buffer, file: File | Buffer,
@ -82,7 +82,7 @@ export async function uploadFile(
await minioClient.putObject(BUCKET_NAME, path, buffer, buffer.length, metaData); await minioClient.putObject(BUCKET_NAME, path, buffer, buffer.length, metaData);
return getFileUrl(path); return path;
} catch (error) { } catch (error) {
console.error('Error uploading file:', error); console.error('Error uploading file:', error);
throw error; throw error;
@ -96,29 +96,11 @@ export async function uploadFile(
* @returns 访URL * @returns 访URL
*/ */
export function getFileUrl(path: string): string { export function getFileUrl(path: string): string {
const endpoint = process.env.MINIO_REAL_DOMAIN; const endpoint = process.env.MINIO_PUBLIC_DOMAIN;
return `${endpoint}/${BUCKET_NAME}/${path}`; return `${endpoint}/${BUCKET_NAME}/${path}`;
} }
/**
* 访URL
* @param path -
* @param expirySeconds - 7
* @returns 访URL
*/
export async function getPresignedUrl(
path: string,
expirySeconds: number = 7 * 24 * 60 * 60
): Promise<string> {
try {
return await minioClient.presignedGetObject(BUCKET_NAME, path, expirySeconds);
} catch (error) {
console.error('Error generating presigned URL:', error);
throw error;
}
}
/** /**
* *
* @param path - * @param path -
@ -225,7 +207,7 @@ export async function listFiles(
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
stream.on('data', (obj) => { stream.on('data', (obj) => {
if (obj.name) { if (obj.name) {
files.push({ endpoint: `${process.env.MINIO_REAL_DOMAIN}/${BUCKET_NAME}`, ...obj, } as Minio.BucketItem & { endpoint: string }); files.push({ endpoint: `${process.env.MINIO_PUBLIC_DOMAIN}/${BUCKET_NAME}`, ...obj, } as Minio.BucketItem & { endpoint: string });
} }
}); });
stream.on('end', () => resolve(files)); stream.on('end', () => resolve(files));
@ -323,7 +305,6 @@ export default {
initBucket, initBucket,
uploadFile, uploadFile,
getFileUrl, getFileUrl,
getPresignedUrl,
downloadFile, downloadFile,
getFileStream, getFileStream,
deleteFile, deleteFile,

View File

@ -5,7 +5,13 @@ const nextConfig: NextConfig = {
'playwright-extra', 'playwright-extra',
'puppeteer-extra-plugin-stealth', 'puppeteer-extra-plugin-stealth',
'puppeteer-extra-plugin', 'puppeteer-extra-plugin',
], ], webpack: (config) => {
config.module.rules.push({
test: /\.(md|txt)$/i,
type: 'asset/source', // 让这些文件作为纯文本注入
});
return config;
},
/* config options here */ /* config options here */
}; };

View File

@ -15,6 +15,7 @@
"lucide-react": "^0.546.0", "lucide-react": "^0.546.0",
"minio": "^8.0.6", "minio": "^8.0.6",
"next": "15.5.6", "next": "15.5.6",
"openai": "^6.7.0",
"playwright": "1.56.1", "playwright": "1.56.1",
"playwright-extra": "^4.3.6", "playwright-extra": "^4.3.6",
"puppeteer-extra-plugin-stealth": "^2.11.2", "puppeteer-extra-plugin-stealth": "^2.11.2",

View File

@ -8,22 +8,28 @@ module.exports = {
apps: [ apps: [
{ {
name: "DouyinArchive", name: "DouyinArchive",
script: 'index.ts', script: 'npm',
args: 'run start',
cwd: __dirname, cwd: __dirname,
autorestart: true, autorestart: true,
restart_delay: 4000, restart_delay: 4000,
kill_timeout: 5000, kill_timeout: 5000,
instances, instances,
exec_mode: instances > 1 ? 'cluster' : 'fork', exec_mode: instances > 1 ? 'cluster' : 'fork',
watch: process.env.NODE_ENV !== 'production', // 注意:不要在生产环境 watch否则 Next.js 写入 .next 会触发重启风暴,导致 Playwright 进程被提前关闭
ignore_watch: ['generated', 'node_modules', '.git'], watch: false,
ignore_watch: ['.next', '.turbo', 'generated', 'node_modules', '.git'],
env: { env: {
// 明确开发环境可选项(如需)
NODE_ENV: process.env.NODE_ENV || 'development',
...envFromFile, ...envFromFile,
NODE_ENV: 'development'
}, },
env_production: { env_production: {
// 关键:确保应用进程中的 NODE_ENV=production从而禁用 Next.js 的开发特性
NODE_ENV: 'production',
// 为避免多个实例同时拉起共享浏览器,默认单实例;如需并发,请改为独立浏览器服务
WEB_CONCURRENCY: '1',
...envFromFile, ...envFromFile,
NODE_ENV: 'production'
}, },
time: true time: true
} }

34
test.ts
View File

@ -1,36 +1,4 @@
import { createWriteStream, writeFileSync } from "node:fs"; import { createWriteStream, writeFileSync } from "node:fs";
import { initBucket } from "./lib/minio";
initBucket() initBucket()
const response = await fetch("https://v3-web.douyinvod.com/95ee4b6a1f064d04653f506024e3d493/68f4daca/video/tos/cn/tos-cn-ve-15c000-ce/oojxjQ2XsXPPgPniGM4kz8M3BSEahiGNcA0AI/?a=6383\\u0026ch=26\\u0026cr=3\\u0026dr=0\\u0026lr=all\\u0026cd=0%7C0%7C0%7C3\\u0026cv=1\\u0026br=6662\\u0026bt=6662\\u0026cs=2\\u0026ds=10\\u0026ft=khyHAB1UiiuGzJrZ~~OC~49Zyo3nOz7HQNaLpMyC6LZjrKQ2B22E1J6kcKb2oPd.o~\\u0026mime_type=video_mp4\\u0026qs=15\\u0026rc=ODNoPGk2MzRnNDg7NDdnNUBpM3ZzbXA5cjNyNjMzbGkzNEAzMy41MzU1XzMxNTY2Ll9jYSNmYy1tMmQ0b2xhLS1kLWJzcw%3D%3D\\u0026btag=c0000e00028000\\u0026cquery=100w_100B_100x_100z_100o\\u0026dy_q=1760866258\\u0026feature_id=10cf95ef75b4f3e7eac623e4ea0ea691\\u0026l=20251019173058DF8022814E036CA4C097", {
"headers": {
"accept": "*/*",
"accept-language": "zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6",
"sec-ch-ua": "\"Microsoft Edge\";v=\"141\", \"Not?A_Brand\";v=\"8\", \"Chromium\";v=\"141\"",
"sec-ch-ua-mobile": "?0",
"sec-ch-ua-platform": "\"Windows\"",
"sec-fetch-dest": "video",
"sec-fetch-mode": "no-cors",
"sec-fetch-site": "same-origin"
},
"referrer": "https://v3-web.douyinvod.com/95ee4b6a1f064d04653f506024e3d493/68f4daca/video/tos/cn/tos-cn-ve-15c000-ce/oojxjQ2XsXPPgPniGM4kz8M3BSEahiGNcA0AI/?a=6383\\u0026ch=26\\u0026cr=3\\u0026dr=0\\u0026lr=all\\u0026cd=0%7C0%7C0%7C3\\u0026cv=1\\u0026br=6662\\u0026bt=6662\\u0026cs=2\\u0026ds=10\\u0026ft=khyHAB1UiiuGzJrZ~~OC~49Zyo3nOz7HQNaLpMyC6LZjrKQ2B22E1J6kcKb2oPd.o~\\u0026mime_type=video_mp4\\u0026qs=15\\u0026rc=ODNoPGk2MzRnNDg7NDdnNUBpM3ZzbXA5cjNyNjMzbGkzNEAzMy41MzU1XzMxNTY2Ll9jYSNmYy1tMmQ0b2xhLS1kLWJzcw%3D%3D\\u0026btag=c0000e00028000\\u0026cquery=100w_100B_100x_100z_100o\\u0026dy_q=1760866258\\u0026feature_id=10cf95ef75b4f3e7eac623e4ea0ea691\\u0026l=20251019173058DF8022814E036CA4C097",
"body": null,
"method": "GET",
"mode": "cors",
"credentials": "omit"
});
import { pipeline, Readable } from 'stream';
import { promisify } from 'util';
import { initBucket } from "./lib/minio";
const streamPipeline = promisify(pipeline);
if(!response.body) {
throw new Error("No response body");
}
// 将 Web ReadableStream 转换为 Node.js Readable
const nodeStream = Readable.fromWeb(response.body as any);
const contentLength = response.headers.get('content-length');
await uploadFileStream(nodeStream, 'test-video.mp4', Number(contentLength));
export { };

View File

@ -22,6 +22,6 @@
"@/*": ["./*"] "@/*": ["./*"]
} }
}, },
"include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"], "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts", "app/aweme/[awemeId]/types.ts"],
"exclude": ["node_modules"] "exclude": ["node_modules"]
} }