语音转文本接口，bgm路径修复

2025-10-25 13:06:17 +08:00
parent e4339a5b91
commit d1b4fb6be0
12 changed files with 423 additions and 172 deletions
@@ -1,3 +1,4 @@
+import { getFileUrl } from "@/lib/minio";
 import { prisma } from "@/lib/prisma";
 import { Prisma } from "@prisma/client";
 import { NextRequest, NextResponse } from "next/server";
@@ -10,7 +11,6 @@ export async function GET(
  const searchParams = request.nextUrl.searchParams;
  const skip = parseInt(searchParams.get("skip") || "0", 10);
  const take = parseInt(searchParams.get("take") || "20", 10);
-  const mode = (searchParams.get("mode") || "").toLowerCase();

  // ranked 模式参数（均为可选，提供合理默认值）
  const seed = searchParams.get("seed") || new Date().toISOString().slice(0, 10); // 默认按日期稳定
@@ -21,6 +21,8 @@ export async function GET(
  const wTime = parseFloat(searchParams.get("wTime") || "2"); // 时间衰减权重
  const wJit = parseFloat(searchParams.get("wJit") || "10"); // 随机扰动权重（稳定随机）

+
+
  try {
    // 查找是视频还是图文
    const [video, post] = await Promise.all([
@@ -43,12 +45,11 @@ export async function GET(
      ? { videoId: awemeId }
      : { imagePostId: awemeId };

-    // ranked 模式：按「热度 + 时间衰减 + 稳定随机扰动」打分排序；否则使用稳定的时间排序
-    if (mode === "ranked") {
-      const total = await prisma.comment.count({ where });
+    // 按「热度 + 时间衰减 + 稳定随机扰动」打分排序；否则使用稳定的时间排序
+    const total = await prisma.comment.count({ where });

-      // 计算稳定随机扰动：基于 (cid, seed) 的 md5 -> 取前4字节 -> 映射到 [0,1)
-      const jitterExpr = Prisma.sql`(
+    // 计算稳定随机扰动：基于 (cid, seed) 的 md5 -> 取前4字节 -> 映射到 [0,1)
+    const jitterExpr = Prisma.sql`(
        (
          get_byte(decode(md5(c."cid" || ':' || ${seed}), 'hex'), 0)::double precision * 16777216.0 +
          get_byte(decode(md5(c."cid" || ':' || ${seed}), 'hex'), 1)::double precision * 65536.0 +
@@ -57,27 +58,27 @@ export async function GET(
        ) / 4294967295.0
      )`;

-      const snapshotTs = snapshot.toISOString();
+    const snapshotTs = snapshot.toISOString();

-      // popularity: ln(1 + digg_count)
-      // time decay: exp(- age_hours / halfLifeHours)，使用固定 snapshot 基准保证会话内稳定
-      const scoreExpr = Prisma.sql`(
+    // popularity: ln(1 + digg_count)
+    // time decay: exp(- age_hours / halfLifeHours)，使用固定 snapshot 基准保证会话内稳定
+    const scoreExpr = Prisma.sql`(
        ${wPop} * ln(1 + c."digg_count"::numeric) +
        ${wTime} * exp(- (extract(epoch from (${snapshotTs}::timestamptz - c."created_at")) / 3600.0) / ${halfLifeHours}) +
        ${wJit} * ${jitterExpr}
      )`;

-      const whereField = (await video) ? Prisma.sql`c."videoId"` : Prisma.sql`c."imagePostId"`;
+    const whereField = (await video) ? Prisma.sql`c."videoId"` : Prisma.sql`c."imagePostId"`;

-      const rows: Array<{
-        cid: string;
-        text: string;
-        created_at: Date;
-        digg_count: bigint;
-        nickname: string;
-        avatar_url: string | null;
-      }> = await prisma.$queryRaw(
-        Prisma.sql`
+    const rows: Array<{
+      cid: string;
+      text: string;
+      created_at: Date;
+      digg_count: bigint;
+      nickname: string;
+      avatar_url: string | null;
+    }> = await prisma.$queryRaw(
+      Prisma.sql`
          SELECT
            c."cid",
            c."text",
@@ -92,78 +93,44 @@ export async function GET(
          OFFSET ${skip}
          LIMIT ${take}
        `
-      );
+    );

-      // 批量查询每条评论的配图/贴纸
-      const cids = rows.map(r => r.cid);
-      const images = cids.length
-        ? await prisma.commentImage.findMany({
-            where: { commentId: { in: cids } },
-            orderBy: { order: 'asc' },
-            select: { commentId: true, url: true, width: true, height: true, order: true },
-          })
-        : [];
-      const group = new Map<string, { url: string; width?: number | null; height?: number | null; order: number }[]>();
-      for (const img of images) {
-        const arr = group.get(img.commentId) || [];
-        arr.push({ url: img.url, width: img.width, height: img.height, order: img.order });
-        group.set(img.commentId, arr);
-      }
-      const formattedComments = rows.map((c) => ({
-        cid: c.cid,
-        text: c.text,
-        created_at: c.created_at,
-        digg_count: Number(c.digg_count),
-        user: {
-          nickname: c.nickname,
-          avatar_url: c.avatar_url || undefined,
-        },
-        images: (group.get(c.cid) || []).sort((a,b)=>a.order-b.order).map(i=>({ url: i.url, width: i.width ?? undefined, height: i.height ?? undefined })),
-      }));
-
-      return NextResponse.json({
-        comments: formattedComments,
-        total,
-        hasMore: skip + take < total,
-        // 回传用于稳定排序的参数，前端可在后续请求中透传
-        seed,
-        snapshot: snapshotTs,
-        weights: { wPop, wTime, wJit, halfLifeHours },
-        mode: "ranked",
-      });
+    // 批量查询每条评论的配图/贴纸
+    const cids = rows.map(r => r.cid);
+    const images = cids.length
+      ? await prisma.commentImage.findMany({
+        where: { commentId: { in: cids } },
+        orderBy: { order: 'asc' },
+        select: { commentId: true, url: true, width: true, height: true, order: true },
+      })
+      : [];
+    const group = new Map<string, { url: string; width?: number | null; height?: number | null; order: number }[]>();
+    for (const img of images) {
+      const arr = group.get(img.commentId) || [];
+      arr.push({ url: img.url, width: img.width, height: img.height, order: img.order });
+      group.set(img.commentId, arr);
    }
-
-    // 默认模式：获取评论总数和分页数据（使用稳定且不可变的排序，避免因 digg_count 变化导致翻页重复/遗漏）
-    const [total, comments] = await Promise.all([
-      prisma.comment.count({ where }),
-      prisma.comment.findMany({
-        where,
-        orderBy: [
-          { created_at: "desc" },
-          { cid: "asc" },
-        ],
-        include: { user: true, images: { orderBy: { order: 'asc' }, select: { url: true, width: true, height: true } } },
-        skip,
-        take,
-      }),
-    ]);
-
-    const formattedComments = comments.map((c) => ({
+    const formattedComments = rows.map((c) => ({
      cid: c.cid,
      text: c.text,
      created_at: c.created_at,
      digg_count: Number(c.digg_count),
      user: {
-        nickname: c.user.nickname,
-        avatar_url: c.user.avatar_url,
+        nickname: c.nickname,
+        avatar_url: getFileUrl(c.avatar_url || 'default_avatar.png'),
      },
-      images: (c.images || []).map(i=>({ url: i.url, width: i.width ?? undefined, height: i.height ?? undefined })),
+      images: (group.get(c.cid) || []).sort((a, b) => a.order - b.order).map(i => ({ url: getFileUrl(i.url), width: i.width ?? undefined, height: i.height ?? undefined })),
    }));

    return NextResponse.json({
      comments: formattedComments,
      total,
      hasMore: skip + take < total,
+      // 回传用于稳定排序的参数，前端可在后续请求中透传
+      seed,
+      snapshot: snapshotTs,
+      weights: { wPop, wTime, wJit, halfLifeHours },
+      mode: "ranked",
    });
  } catch (error) {
    console.error("获取评论失败:", error);
@@ -1,6 +1,7 @@
 import { NextRequest, NextResponse } from 'next/server';
 import { prisma } from '@/lib/prisma';
 import type { FeedItem, FeedResponse } from '@/app/types/feed';
+import { getFileUrl } from '@/lib/minio';

 // Contract
 // Inputs: search params { before?: ISOString, limit?: number }
@@ -32,30 +33,29 @@ export async function GET(req: NextRequest) {

  const merged: FeedItem[] = [
    ...videos.map((v) => ({
-      type: 'video' as const,
+      type: "video" as const,
      aweme_id: v.aweme_id,
      created_at: v.created_at,
      desc: v.desc,
-      video_url: v.video_url,
-      cover_url: v.cover_url ?? null,
+      video_url: getFileUrl(v.video_url),
+      cover_url: getFileUrl(v.cover_url ?? 'default_cover.png'),
      width: v.width ?? null,
      height: v.height ?? null,
-      author: { nickname: v.author.nickname, avatar_url: v.author.avatar_url ?? null },
-      likes: Number(v.digg_count),
+      author: { nickname: v.author.nickname, avatar_url: getFileUrl(v.author.avatar_url ?? '') },
+      likes: Number(v.digg_count)
    })),
    ...posts.map((p) => ({
-      type: 'image' as const,
+      type: "image" as const,
      aweme_id: p.aweme_id,
      created_at: p.created_at,
      desc: p.desc,
-      cover_url: p.images?.[0]?.url ?? null,
+      cover_url: getFileUrl(p.images?.[0]?.url ?? null),
      width: p.images?.[0]?.width ?? null,
      height: p.images?.[0]?.height ?? null,
-      author: { nickname: p.author.nickname, avatar_url: p.author.avatar_url ?? null },
-      likes: Number(p.digg_count),
+      author: { nickname: p.author.nickname, avatar_url: getFileUrl(p.author.avatar_url ?? '') },
+      likes: Number(p.digg_count)
    })),
-  ]
-    .sort((a, b) => +new Date(b.created_at as any) - +new Date(a.created_at as any))
+  ].sort((a, b) => +new Date(b.created_at) - +new Date(a.created_at))
    .slice(0, limit);

  const nextCursor = merged.length > 0 ? new Date(merged[merged.length - 1].created_at as any).toISOString() : null;
@@ -6,11 +6,12 @@ import { prisma } from '@/lib/prisma';
 import { uploadFile, generateUniqueFileName } from '@/lib/minio';
 import { createCamelCompatibleProxy } from '@/app/api/fetcher/utils';
 import { waitForFirstResponse, waitForResponseWithTimeout, safeJson, downloadBinary, collectResponsesWithinTime } from '@/app/api/fetcher/network';
-import { pickBestPlayAddr, extractFirstFrame } from '@/app/api/fetcher/media';
+import { pickBestPlayAddr } from '@/app/api/fetcher/media';
 import { handleImagePost } from '@/app/api/fetcher/uploader';
 import { saveToDB, saveImagePostToDB } from '@/app/api/fetcher/persist';
 import chalk from 'chalk';
 import { acquireIsolatedContext, releaseIsolatedContext } from '@/app/api/fetcher/browser';
+import { extractFirstFrame } from '@/app/api/media';

 const DETAIL_PATH = '/aweme/v1/web/aweme/detail/';
 const COMMENT_PATH = '/aweme/v1/web/comment/list/';
@@ -1,12 +1,8 @@
 export const runtime = 'nodejs'

-import { promises as fs } from 'fs';
-import os from 'os';
-import path from 'path';
 import { execFile } from 'child_process';
 import { promisify } from 'util';

-const execFileAsync = promisify(execFile);

 export function pickBestPlayAddr(variants: PlayVariant[] | undefined | null) {
    if (!variants?.length) return null;
@@ -19,74 +15,3 @@ export function pickBestPlayAddr(variants: PlayVariant[] | undefined | null) {

    return best?.play_addr ?? null;
 }
-
-/**
- * 使用 ffmpeg 从视频二进制中提取第一帧，返回 JPEG buffer
- */
-export async function extractFirstFrame(videoBuffer: Buffer): Promise<{ buffer: Buffer; contentType: string; ext: string } | null> {
-    const ffmpegCmd = process.env.FFMPEG_PATH || 'ffmpeg';
-    const tmpDir = os.tmpdir();
-    const base = `dy_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`;
-    const inPath = path.join(tmpDir, `${base}.mp4`);
-    const outPath = path.join(tmpDir, `${base}.jpg`);
-
-    try {
-        await fs.writeFile(inPath, videoBuffer);
-        const args = [
-            '-hide_banner',
-            '-loglevel', 'error',
-            '-ss', '0',
-            '-i', inPath,
-            '-frames:v', '1',
-            '-q:v', '2',
-            '-f', 'image2',
-            '-y',
-            outPath,
-        ];
-        await execFileAsync(ffmpegCmd, args, { windowsHide: true });
-        const img = await fs.readFile(outPath);
-        return { buffer: img, contentType: 'image/jpeg', ext: 'jpg' };
-    } catch (e: any) {
-        if (e && (e.code === 'ENOENT' || /not found|is not recognized/i.test(String(e.message)))) {
-            console.warn('系统未检测到 ffmpeg，可安装并配置 PATH 或设置 FFMPEG_PATH 后启用封面提取。');
-            return null;
-        }
-        throw e;
-    } finally {
-        try { await fs.unlink(inPath); } catch { }
-        try { await fs.unlink(outPath); } catch { }
-    }
-}
-
-/**
- * 使用 ffprobe 获取视频时长（毫秒）
- */
-export async function getVideoDuration(videoBuffer: Buffer): Promise<number | null> {
-    const ffprobeCmd = process.env.FFPROBE_PATH || 'ffprobe';
-    const tmpDir = os.tmpdir();
-    const base = `dy_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`;
-    const inPath = path.join(tmpDir, `${base}.mp4`);
-
-    try {
-        await fs.writeFile(inPath, videoBuffer);
-        const args = [
-            '-v', 'error',
-            '-show_entries', 'format=duration',
-            '-of', 'default=noprint_wrappers=1:nokey=1',
-            inPath,
-        ];
-        const { stdout } = await execFileAsync(ffprobeCmd, args, { windowsHide: true });
-        const durationSeconds = parseFloat(stdout.trim());
-        if (isNaN(durationSeconds)) return null;
-        return Math.round(durationSeconds * 1000); // 转换为毫秒
-    } catch (e: any) {
-        if (e && (e.code === 'ENOENT' || /not found|is not recognized/i.test(String(e.message)))) {
-            console.warn('系统未检测到 ffprobe，可安装并配置 PATH 或设置 FFPROBE_PATH 后启用时长提取。');
-            return null;
-        }
-        console.warn(`获取视频时长失败: ${e?.message || e}`);
-        return null;
-    } finally {
-        try { await fs.unlink(inPath); } catch { }
-    }
-}
@@ -4,7 +4,7 @@ import type { BrowserContext } from 'playwright';
 import { uploadFile, generateUniqueFileName } from '@/lib/minio';
 import { downloadBinary } from './network';
 import { pickFirstUrl } from './utils';
-import { getVideoDuration } from './media';
+import { getVideoDuration } from '@/app/api/media';

 /**
 * 下载头像并上传到 MinIO，返回外链；失败时回退为原始链接。
@@ -0,0 +1,155 @@
+import { execFile } from 'child_process';
+import { promises as fs } from 'fs';
+import os from 'os';
+import path from 'path';
+import { promisify } from 'util';
+
+const execFileAsync = promisify(execFile);
+
+/**
+ * 使用 ffmpeg 从视频二进制中提取第一帧，返回 JPEG buffer
+ */
+export async function extractFirstFrame(videoBuffer: Buffer): Promise<{ buffer: Buffer; contentType: string; ext: string } | null> {
+    const ffmpegCmd = process.env.FFMPEG_PATH || 'ffmpeg';
+    const tmpDir = os.tmpdir();
+    const base = `dy_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`;
+    const inPath = path.join(tmpDir, `${base}.mp4`);
+    const outPath = path.join(tmpDir, `${base}.jpg`);
+
+    try {
+        await fs.writeFile(inPath, videoBuffer);
+        const args = [
+            '-hide_banner',
+            '-loglevel', 'error',
+            '-ss', '0',
+            '-i', inPath,
+            '-frames:v', '1',
+            '-q:v', '2',
+            '-f', 'image2',
+            '-y',
+            outPath,
+        ];
+        await execFileAsync(ffmpegCmd, args, { windowsHide: true });
+        const img = await fs.readFile(outPath);
+        return { buffer: img, contentType: 'image/jpeg', ext: 'jpg' };
+    } catch (e: any) {
+        if (e && (e.code === 'ENOENT' || /not found|is not recognized/i.test(String(e.message)))) {
+            console.warn('系统未检测到 ffmpeg，可安装并配置 PATH 或设置 FFMPEG_PATH 后启用封面提取。');
+            return null;
+        }
+        throw e;
+    } finally {
+        try { await fs.unlink(inPath); } catch { }
+        try { await fs.unlink(outPath); } catch { }
+    }
+}
+
+/**
+ * 使用 ffprobe 获取视频时长（毫秒）
+ */
+export async function getVideoDuration(videoBuffer: Buffer): Promise<number | null> {
+    const ffprobeCmd = process.env.FFPROBE_PATH || 'ffprobe';
+    const tmpDir = os.tmpdir();
+    const base = `dy_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`;
+    const inPath = path.join(tmpDir, `${base}.mp4`);
+
+    try {
+        await fs.writeFile(inPath, videoBuffer);
+        const args = [
+            '-v', 'error',
+            '-show_entries', 'format=duration',
+            '-of', 'default=noprint_wrappers=1:nokey=1',
+            inPath,
+        ];
+        const { stdout } = await execFileAsync(ffprobeCmd, args, { windowsHide: true });
+        const durationSeconds = parseFloat(stdout.trim());
+        if (isNaN(durationSeconds)) return null;
+        return Math.round(durationSeconds * 1000); // 转换为毫秒
+    } catch (e: any) {
+        if (e && (e.code === 'ENOENT' || /not found|is not recognized/i.test(String(e.message)))) {
+            console.warn('系统未检测到 ffprobe，可安装并配置 PATH 或设置 FFPROBE_PATH 后启用时长提取。');
+            return null;
+        }
+        console.warn(`获取视频时长失败: ${e?.message || e}`);
+        return null;
+    } finally {
+        try { await fs.unlink(inPath); } catch { }
+    }
+}
+
+/**
+ * 使用 ffmpeg 将视频二进制转换为音频，返回音频 Buffer
+ *
+ * 默认输出 mp3，可通过 opts.format 指定 'mp3' | 'aac' | 'wav'
+ * - mp3: 使用 libmp3lame，contentType 为 'audio/mpeg'
+ * - aac: 输出 m4a 容器（AAC 编码），contentType 为 'audio/mp4'
+ * - wav: PCM 16-bit，44100Hz，双声道，contentType 为 'audio/wav'
+ *
+ * 若系统未安装 ffmpeg 或未在 PATH 中，返回 null 并给出提示。
+ */
+export async function extractAudio(
+    videoBuffer: Buffer,
+    opts?: { format?: 'mp3' | 'aac' | 'wav'; bitrateKbps?: number }
+): Promise<{ buffer: Buffer; contentType: string; ext: string } | null> {
+    const ffmpegCmd = process.env.FFMPEG_PATH || 'ffmpeg';
+    const tmpDir = os.tmpdir();
+    const base = `dy_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`;
+    const inPath = path.join(tmpDir, `${base}.mp4`);
+
+    const format = opts?.format ?? 'mp3';
+    const bitrate = Math.max(32, Math.min(512, opts?.bitrateKbps ?? 192)); // 安全范围 32~512 kbps
+
+    // 根据目标格式设置输出路径、MIME 与编码参数
+    let outPath = '';
+    let contentType = '';
+    let ext = '';
+    let codecArgs: string[] = [];
+
+    if (format === 'mp3') {
+        ext = 'mp3';
+        contentType = 'audio/mpeg';
+        outPath = path.join(tmpDir, `${base}.${ext}`);
+        codecArgs = ['-c:a', 'libmp3lame', '-b:a', `${bitrate}k`];
+    } else if (format === 'aac') {
+        // 使用 m4a 容器更通用
+        ext = 'm4a';
+        contentType = 'audio/mp4';
+        outPath = path.join(tmpDir, `${base}.${ext}`);
+        codecArgs = ['-c:a', 'aac', '-b:a', `${bitrate}k`, '-movflags', '+faststart'];
+    } else {
+        // wav
+        ext = 'wav';
+        contentType = 'audio/wav';
+        outPath = path.join(tmpDir, `${base}.${ext}`);
+        codecArgs = ['-f', 'wav', '-acodec', 'pcm_s16le', '-ar', '44100', '-ac', '2'];
+    }
+
+    try {
+        await fs.writeFile(inPath, videoBuffer);
+        const args = [
+            '-hide_banner',
+            '-loglevel', 'error',
+            '-i', inPath,
+            '-vn', // 丢弃视频流
+            ...codecArgs,
+            '-y',
+            outPath,
+        ];
+        await execFileAsync(ffmpegCmd, args, { windowsHide: true });
+        const audio = await fs.readFile(outPath);
+        return { buffer: audio, contentType, ext };
+    } catch (e: any) {
+        if (e && (e.code === 'ENOENT' || /not found|is not recognized/i.test(String(e.message)))) {
+            console.warn('系统未检测到 ffmpeg，可安装并配置 PATH 或设置 FFMPEG_PATH 后启用音频提取。');
+            return null;
+        }
+        // 一些环境可能缺少特定编码器（如 libmp3lame），提示并抛出原始错误
+        console.warn(`提取音频失败: ${e?.message || e}`);
+        throw e;
+    } finally {
+        try { await fs.unlink(inPath); } catch { }
+        try {
+            if (outPath) await fs.unlink(outPath);
+        } catch { }
+    }
+}
@@ -2,6 +2,8 @@ import fs from "fs";
 import OpenAI from "openai";
 import prompt from "./prompt.md";
 import { prisma } from "@/lib/prisma";
+import { downloadFile } from "@/lib/minio";
+import { extractAudio } from "../media";

 const client = new OpenAI({
  apiKey: process.env.OPENAI_API_KEY,
@@ -13,6 +15,7 @@ async function transcriptAudio(audio: Buffer | string) {
    audio = fs.readFileSync(audio);
  }
  const base64Audio = Buffer.from(audio).toString("base64");
+
  const response = await client.chat.completions.create({
    model: "gemini-2.5-flash-lite",
    messages: [
@@ -27,7 +30,7 @@ async function transcriptAudio(audio: Buffer | string) {
            type: "input_audio",
            input_audio: {
              data: base64Audio,
-              format: "wav",
+              format: "mp3",
            },
          },
        ],
@@ -36,14 +39,24 @@ async function transcriptAudio(audio: Buffer | string) {
  });

  console.log(response.choices[0].message.content);
+  return response.choices[0].message.content;
 }

-async function transcriptAweme(awemeId: string) {
+export async function transcriptAweme(awemeId: string) {
  const aweme = await prisma.video.findUnique({
    where: { aweme_id: awemeId },
  });
  if (!aweme) {
    throw new Error("Aweme not found or aweme is not a video post");
  }
-  
+  const vPath = aweme.video_url
+  const buffer = await downloadFile(vPath);
+
+  const audioDat = await extractAudio(buffer, { format: "mp3", bitrateKbps: 96 });
+
+  if (!audioDat || !audioDat.buffer) {
+    throw new Error("Failed to extract audio from video");
+  }
+
+  return await transcriptAudio(audioDat.buffer);
 }
@@ -0,0 +1,19 @@
+import { NextRequest, NextResponse } from 'next/server';
+import { prisma } from '@/lib/prisma';
+import type { FeedItem, FeedResponse } from '@/app/types/feed';
+import { getFileUrl } from '@/lib/minio';
+import { transcriptAweme } from '.';
+
+// Contract
+// Inputs: search params { before?: ISOString, limit?: number }
+// Output: { items: FeedItem[], nextCursor: ISOString | null }
+
+export async function GET(req: NextRequest) {
+  const { searchParams } = new URL(req.url)
+  const awemeId = searchParams.get('awemeId');
+  if (!awemeId) {
+    return NextResponse.json({ error: "Missing awemeId parameter" }, { status: 400 });
+  }
+  const script = await transcriptAweme(awemeId);
+  return NextResponse.json(script);
+}