diff --git a/app/admin/stt/page.tsx b/app/admin/asr/page.tsx similarity index 94% rename from app/admin/stt/page.tsx rename to app/admin/asr/page.tsx index 5214bff..49b7813 100644 --- a/app/admin/stt/page.tsx +++ b/app/admin/asr/page.tsx @@ -3,26 +3,27 @@ import Link from "next/link"; import BackButton from "@/app/components/BackButton"; -export default function SttAdminPage() { +export default function AsrAdminPage() { return (
-

STT 管理中心

-

管理和配置视频语音转写功能

+

ASR 管理中心

+

管理和配置视频语音识别功能

{/* 视频转写管理 */}

错误: {error}

+ +
{batchTranscribing && (
- 正在转写: {batchProgress.current} / {batchProgress.total} + 正在生成: {batchProgress.current} / {batchProgress.total}
)}
@@ -319,7 +364,7 @@ export default function SttVideosPage() { 时长 - 转写状态 + 识别/标签状态 操作 @@ -331,9 +376,12 @@ export default function SttVideosPage() {
-
@@ -367,7 +415,7 @@ export default function SttVideosPage() {
{video.transcript.speech_detected ? (
- {video.transcript.transcript} + {video.transcript.transcript.join(" ")}
) : (
@@ -381,6 +429,23 @@ export default function SttVideosPage() { )}
)} + {video.transcript.content_tags.length > 0 && ( +
+ {video.transcript.content_tags.map((tag) => ( + + {tag} + + ))} +
+ )} + {video.transcript.content_tags.length === 0 && ( + + 未生成标签 + + )}
) : ( @@ -391,6 +456,7 @@ export default function SttVideosPage() { {video.transcript ? ( ) : ( )} diff --git a/app/api/admin/stt/videos/route.ts b/app/api/admin/asr/videos/route.ts similarity index 94% rename from app/api/admin/stt/videos/route.ts rename to app/api/admin/asr/videos/route.ts index 26b104c..a62c0a3 100644 --- a/app/api/admin/stt/videos/route.ts +++ b/app/api/admin/asr/videos/route.ts @@ -1,6 +1,6 @@ import { NextResponse } from "next/server"; -import { prisma } from "@/lib/prisma"; import { getFileUrl } from "@/lib/minio"; +import { prisma } from "@/lib/prisma"; export async function GET() { try { @@ -21,6 +21,7 @@ export async function GET() { audio_type: true, transcript: true, non_speech_summary: true, + content_tags: true, }, }, }, @@ -42,6 +43,7 @@ export async function GET() { audio_type: v.transcript.audio_type, transcript: v.transcript.transcript, non_speech_summary: v.transcript.non_speech_summary, + content_tags: v.transcript.content_tags, } : null, })); diff --git a/app/api/asr/index.ts b/app/api/asr/index.ts new file mode 100644 index 0000000..b042e09 --- /dev/null +++ b/app/api/asr/index.ts @@ -0,0 +1,227 @@ +import OpenAI from "openai"; +import { z } from "zod"; +import { ASR_CONTENT_TAGS, normalizeAsrContentTags } from "@/lib/asr-tags"; +import { downloadFile } from "@/lib/minio"; +import { prisma } from "@/lib/prisma"; +import { + type ExtractedVideoFrame, + extractAudio, + extractVideoFrames, +} from "../media"; +import prompt from "./prompt.md"; + +export type AsrLogger = (message: string) => void; + +type TranscriptAwemeOptions = { + log?: AsrLogger; +}; + +let qwenClient: OpenAI | null = null; + +function getQwenClient() { + if (qwenClient) return qwenClient; + + const apiKey = process.env.QWEN_API_KEY || process.env.DASHSCOPE_API_KEY; + if (!apiKey) { + throw new Error("Missing QWEN_API_KEY environment variable"); + } + + qwenClient = new OpenAI({ + apiKey, + baseURL: + process.env.QWEN_API_BASE_URL || + "https://dashscope.aliyuncs.com/compatible-mode/v1", + }); + + return qwenClient; +} + +// Zod schema aligned with prompt.md +export const AsrSchema = z + .object({ + speech_detected: z.boolean(), + language: z.string().min(1).nullable(), + audio_type: z.string().nullable(), + transcript: z.array(z.string()), + non_speech_summary: z.string().nullable(), + content_tags: z.array(z.enum(ASR_CONTENT_TAGS)).min(1).max(3), + }) + .strict(); + +const RawAsrSchema = z.object({ + speech_detected: z.boolean(), + language: z.string().min(1).nullable(), + audio_type: z.string().nullable(), + transcript: z.array(z.string()), + non_speech_summary: z.string().nullable(), + content_tags: z.unknown().optional(), +}); + +export type AsrResult = z.infer; + +function extractJsonObject(text: string) { + const trimmed = text.trim(); + const fenced = trimmed.match(/```(?:json)?\s*([\s\S]*?)\s*```/i); + if (fenced?.[1]) return fenced[1].trim(); + + const start = trimmed.indexOf("{"); + const end = trimmed.lastIndexOf("}"); + if (start >= 0 && end > start) { + return trimmed.slice(start, end + 1); + } + + return trimmed; +} + +function parseAsrResult(content: string): AsrResult { + const parsed = RawAsrSchema.parse(JSON.parse(extractJsonObject(content))); + return AsrSchema.parse({ + ...parsed, + content_tags: normalizeAsrContentTags(parsed.content_tags), + }); +} + +async function transcriptAudio( + audio: Buffer, + frames: ExtractedVideoFrame[], + videoDesc: string, + log?: AsrLogger, +) { + const base64Audio = Buffer.from(audio).toString("base64"); + const client = getQwenClient(); + const model = process.env.QWEN_OMNI_MODEL || "qwen3.5-omni-plus"; + const frameContentParts = frames.map((frame, index) => ({ + type: "image_url" as const, + image_url: { + url: `data:${frame.contentType};base64,${frame.buffer.toString("base64")}`, + }, + index, + })); + + log?.( + `调用 Qwen Omni: model=${model}, audio=${Math.round(audio.length / 1024)}KB, frames=${frames.length}`, + ); + + const completion = await client.chat.completions.create({ + model, + messages: [ + { + role: "user", + content: [ + ...frameContentParts.map(({ type, image_url }) => ({ + type, + image_url, + })), + { + type: "input_audio", + input_audio: { data: `data:;base64,${base64Audio}`, format: "mp3" }, + }, + { + type: "text", + text: `${prompt}\n\n视频简介：\n${videoDesc || "无"}\n\n已附带视频三帧，时间点约为：${ + frames + .map((frame) => `${(frame.timestampMs / 1000).toFixed(1)}s`) + .join("、") || "无" + }。请结合画面和音频判断内容标签。`, + }, + ], + }, + ], + modalities: ["text"], + stream: true, + stream_options: { include_usage: true }, + }); + + let content = ""; + for await (const chunk of completion) { + const deltaContent = chunk.choices?.[0]?.delta?.content; + if (typeof deltaContent === "string") content += deltaContent; + } + + log?.(`Qwen Omni 原始输出: ${content}`); + + if (!content.trim()) { + throw new Error("No ASR result returned from Qwen Omni"); + } + + try { + return parseAsrResult(content); + } catch { + throw new Error("Failed to parse ASR result JSON"); + } +} + +export async function transcriptAweme( + awemeId: string, + options: TranscriptAwemeOptions = {}, +): Promise { + const log = options.log; + log?.(`查询视频: ${awemeId}`); + const aweme = await prisma.video.findUnique({ + where: { aweme_id: awemeId }, + }); + if (!aweme) { + throw new Error("Aweme not found or aweme is not a video post"); + } + const vPath = aweme.video_url; + log?.(`下载视频文件: ${vPath}`); + const buffer = await downloadFile(vPath); + log?.(`视频下载完成: ${Math.round(buffer.length / 1024)}KB`); + + log?.("提取视频三帧"); + const frames = await extractVideoFrames(buffer, { + count: 3, + durationMs: aweme.duration_ms, + }); + log?.( + `视频帧提取完成: ${frames + .map((frame) => `${(frame.timestampMs / 1000).toFixed(1)}s`) + .join("、")}`, + ); + + log?.("提取音频"); + const audioDat = await extractAudio(buffer, { + format: "mp3", + bitrateKbps: 128, + }); + + if (!audioDat || !audioDat.buffer) { + throw new Error("Failed to extract audio from video"); + } + log?.(`音频提取完成: ${Math.round(audioDat.buffer.length / 1024)}KB`); + + // 调用 Qwen Omni 生成转写与固定分类标签 + const result = await transcriptAudio( + audioDat.buffer, + frames, + aweme.desc, + log, + ); + log?.(`解析完成: tags=${result.content_tags.join("、")}`); + + // 将转写结果持久化到数据库 + log?.("写入数据库 VideoTranscript"); + await prisma.videoTranscript.upsert({ + where: { videoId: awemeId }, + update: { + speech_detected: result.speech_detected, + language: result.language, + audio_type: result.audio_type, + transcript: result.transcript, + non_speech_summary: result.non_speech_summary, + content_tags: result.content_tags, + }, + create: { + videoId: awemeId, + speech_detected: result.speech_detected, + language: result.language, + audio_type: result.audio_type, + transcript: result.transcript, + non_speech_summary: result.non_speech_summary, + content_tags: result.content_tags, + }, + }); + log?.("数据库写入完成"); + + return result; +} diff --git a/app/api/asr/omni-docs/index.md b/app/api/asr/omni-docs/index.md new file mode 100644 index 0000000..2bdd581 --- /dev/null +++ b/app/api/asr/omni-docs/index.md @@ -0,0 +1,2505 @@ +Qwen-Omni 模型能够接收多模态输入并生成文本或语音形式的回复，提供多种拟人音色，支持多语言和方言的语音输出，可应用于内容审核、文本创作、视觉识别、音视频交互助手等场景。 + +**支持的地域：**北京、新加坡，需使用各地域的 [API Key](https://help.aliyun.com/zh/model-studio/get-api-key)。 + +## **快速开始** + +**前提条件** + +- 已[配置 API Key](https://help.aliyun.com/zh/model-studio/get-api-key)并[配置API Key到环境变量](https://help.aliyun.com/zh/model-studio/configure-api-key-through-environment-variables)。 + +- Qwen-Omni 模型仅支持 OpenAI 兼容方式调用，需要[安装最新版SDK](https://help.aliyun.com/zh/model-studio/install-sdk)。OpenAI Python SDK 最低版本为 1.52.0， Node.js SDK 最低版本为 4.68.0。 + + +以下示例将一段文本发送至 Qwen-Omni的API接口，并流式返回文本和音频的回复。 + +Python + +``` +# 运行前的准备工作: +# 运行下列命令安装第三方依赖 +# pip install numpy soundfile openai + +import os +import base64 +import soundfile as sf +import numpy as np +from openai import OpenAI + +# 1. 初始化客户端 +client = OpenAI( + api_key=os.getenv("DASHSCOPE_API_KEY"), # 确认已配置环境变量 + base_url="https://dashscope.aliyuncs.com/compatible-mode/v1", +) + +# 2. 发起请求 +try: + completion = client.chat.completions.create( + model="qwen3.5-omni-plus", + messages=[{"role": "user", "content": "你是谁"}], + modalities=["text", "audio"], # 指定输出文本和音频 + audio={"voice": "Tina", "format": "wav"}, + stream=True, # 必须设置为 True + stream_options={"include_usage": True}, + ) + + # 3. 处理流式响应并解码音频 + print("模型回复：") + audio_base64_string = "" + for chunk in completion: + # 处理文本部分 + if chunk.choices and chunk.choices[0].delta.content: + print(chunk.choices[0].delta.content, end="") + + # 收集音频部分 + if chunk.choices and hasattr(chunk.choices[0].delta, "audio") and chunk.choices[0].delta.audio: + audio_base64_string += chunk.choices[0].delta.audio.get("data", "") + + # 4. 保存音频文件 + if audio_base64_string: + wav_bytes = base64.b64decode(audio_base64_string) + audio_np = np.frombuffer(wav_bytes, dtype=np.int16) + sf.write("audio_assistant.wav", audio_np, samplerate=24000) + print("\n音频文件已保存至：audio_assistant.wav") + +except Exception as e: + print(f"请求失败: {e}") +``` + +Node.js + +``` +// 运行前的准备工作: +// Windows/Mac/Linux 通用: +// 1. 确保已安装 Node.js (建议版本 >= 14) +// 2. 运行以下命令安装必要的依赖: +// npm install openai wav + +import OpenAI from "openai"; +import { createWriteStream } from 'node:fs'; +import { Writer } from 'wav'; + +// 定义音频转换函数：将Base64字符串转换并保存为标准的 WAV 音频文件 +async function convertAudio(audioString, audioPath) { + try { + // 解码Base64字符串为Buffer + const wavBuffer = Buffer.from(audioString, 'base64'); + // 创建WAV文件写入流 + const writer = new Writer({ + sampleRate: 24000, // 采样率 + channels: 1, // 单声道 + bitDepth: 16 // 16位深度 + }); + // 创建输出文件流并建立管道连接 + const outputStream = createWriteStream(audioPath); + writer.pipe(outputStream); + + // 写入PCM数据并结束写入 + writer.write(wavBuffer); + writer.end(); + + // 使用Promise等待文件写入完成 + await new Promise((resolve, reject) => { + outputStream.on('finish', resolve); + outputStream.on('error', reject); + }); + + // 添加额外等待时间确保音频完整 + await new Promise(resolve => setTimeout(resolve, 800)); + + console.log(`\n音频文件已成功保存为 ${audioPath}`); + } catch (error) { + console.error('处理过程中发生错误:', error); + } +} + +// 1. 初始化客户端 +const openai = new OpenAI( + { + // 若没有配置环境变量，请用阿里云百炼API Key将下行替换为：apiKey: "sk-xxx", + apiKey: process.env.DASHSCOPE_API_KEY, + baseURL: "https://dashscope.aliyuncs.com/compatible-mode/v1" + } +); +// 2. 发起请求 +const completion = await openai.chat.completions.create({ + model: "qwen3.5-omni-plus", + messages: [ + { + "role": "user", + "content": "你是谁？" + }], + stream: true, + stream_options: { + include_usage: true + }, + modalities: ["text", "audio"], + audio: { voice: "Tina", format: "wav" } +}); + +let audioString = ""; +console.log("大模型的回复：") + +// 3. 处理流式响应并解码音频 +for await (const chunk of completion) { + if (Array.isArray(chunk.choices) && chunk.choices.length > 0) { + // 处理文本内容 + if (chunk.choices[0].delta.content) { + process.stdout.write(chunk.choices[0].delta.content); + } + // 处理音频内容 + if (chunk.choices[0].delta.audio) { + if (chunk.choices[0].delta.audio["data"]) { + audioString += chunk.choices[0].delta.audio["data"]; + } + } + } +} +// 4. 保存音频文件 +convertAudio(audioString, "audio_assistant.wav"); +``` + +curl + +``` +curl -X POST https://dashscope.aliyuncs.com/compatible-mode/v1/chat/completions \ +-H "Authorization: Bearer $DASHSCOPE_API_KEY" \ +-H "Content-Type: application/json" \ +-d '{ + "model": "qwen3.5-omni-plus", + "messages": [ + { + "role": "user", + "content": "你是谁？" + } + ], + "stream":true, + "stream_options":{ + "include_usage":true + }, + "modalities":["text","audio"], + "audio":{"voice":"Tina","format":"wav"} +}' +``` + +**返回结果** + +运行`Python`和`Node.js`代码后，将在控制台看到模型的文本回复，并在代码文件目录下找到一个名为`audio_assistant.wav` 的音频文件。 + +``` +大模型的回复： +我是阿里云研发的大规模语言模型，我叫千问。有什么我可以帮助你的吗？ +``` + +运行`HTTP`代码将直接返回文本和`Base64`编码（`audio`字段）的音频数据。 + +``` +data: {"choices":[{"delta":{"content":"我"},"finish_reason":null,"index":0,"logprobs":null}],"object":"chat.completion.chunk","usage":null,"created":1757647879,"system_fingerprint":null,"model":"qwen3.5-omni-plus","id":"chatcmpl-a68eca3b-c67e-4666-a72f-73c0b4919860"} +data: {"choices":[{"delta":{"content":"是"},"finish_reason":null,"index":0,"logprobs":null}],"object":"chat.completion.chunk","usage":null,"created":1757647879,"system_fingerprint":null,"model":"qwen3.5-omni-plus","id":"chatcmpl-a68eca3b-c67e-4666-a72f-73c0b4919860"} +...... +data: {"choices":[{"delta":{"audio":{"data":"/v8AAAAAAAAAAAAAAA...","expires_at":1757647879,"id":"audio_a68eca3b-c67e-4666-a72f-73c0b4919860"}},"finish_reason":null,"index":0,"logprobs":null}],"object":"chat.completion.chunk","usage":null,"created":1757647879,"system_fingerprint":null,"model":"qwen3.5-omni-plus","id":"chatcmpl-a68eca3b-c67e-4666-a72f-73c0b4919860"} +data: {"choices":[{"finish_reason":"stop","delta":{"content":""},"index":0,"logprobs":null}],"object":"chat.completion.chunk","usage":null,"created":1764763585,"system_fingerprint":null,"model":"qwen3.5-omni-plus","id":"chatcmpl-e8c82e9e-073e-4289-a786-a20eb444ac9c"} +data: {"choices":[],"object":"chat.completion.chunk","usage":{"prompt_tokens":207,"completion_tokens":103,"total_tokens":310,"completion_tokens_details":{"audio_tokens":83,"text_tokens":20},"prompt_tokens_details":{"text_tokens":207}},"created":1757940330,"system_fingerprint":null,"model":"qwen3.5-omni-plus","id":"chatcmpl-9cdd5a26-f9e9-4eff-9dcc-93a878165afc"} +``` + +## **模型选型** + +- **Qwen3.5-Omni 系列：**适用于长视频分析、会议纪要、字幕生成、内容审核、音视频交互等场景。 + + - 输入限制：3 小时音频或 1 小时视频 + + - 音频控制：支持通过指令调节音量、语速、情绪 + + - 视觉能力：与 Qwen3.5 同等水平，可理解画面、语音、音效等多模态信息 + + - 多模态组合输入：支持文本与图片、音频、视频的任意组合同时输入，不限于单一模态 + +- **Qwen3-Omni-Flash系列：**适用于短视频分析、成本敏感场景。 + + - 输入限制：150 秒以内音视频 + + - 思考模式：Qwen-Omni 系列中唯一支持思考模式的系列 + + - 输入模态：仅支持文本与单一其他模态（图片、音频或视频）的组合输入 + +- **Qwen-Omni-Turbo系列** + + 已停止更新，功能受限，建议迁移至 Qwen3.5-Omni 系列或 Qwen3-Omni-Flash 系列。 + + +| **模型系列** | **音视频描述能力** | **深度思考** | **联网搜索** | **输入音频语种** | **输出音频语种** | **音色数量** | +| --- | --- | --- | --- | --- | --- | --- | +| Qwen3.5-Omni 最新一代全模态模型 | 强 | 不支持 | 支持 | 113 种 含74 种语言、39 种方言 **语言：**中文、英语、德语、法语、意大利语、捷克语、印尼语、泰语、韩语、波兰语、日语、越南语、芬兰语、葡萄牙语、西班牙语、荷兰语、俄语、马来语、加泰罗尼亚语、瑞典语、土耳其语、乌克兰语、罗马尼亚语、斯洛伐克语、丹麦语、冰岛语、挪威语（博克马尔）、马其顿语、希腊语、匈牙利语、加利西亚语、菲律宾语、克罗地亚语、波斯尼亚语、斯洛文尼亚语、保加利亚语、哈萨克语、白俄罗斯语、拉脱维亚语、爱沙尼亚语、阿塞拜疆语、维吾尔语、斯瓦希里语、印地语、世界语、柯尔克孜语、塔吉克语、宿务语、南非语、阿拉伯语、立陶宛语、爪哇语、孟加拉语、波斯语、希伯来语、旁遮普语、古吉拉特语、蒙古语、阿斯图里亚斯语、卡纳达语、马拉地语、国际语、马拉雅拉姆语、马耳他语、新挪威语、泰卢固语、乌尔都语、格鲁吉亚语、巴斯克语、泰米尔语、奥里亚语、塞尔维亚语、毛利语 **方言：** 东北话、贵州话、粤语、河南话、香港粤语、上海话、陕西话、天津话、台湾话、云南话、安徽话、福建话、甘肃话、广东话、湖北话、湖南话、江西话、山东话、山西话、四川话、广西话、海南话、重庆话、长沙话、杭州话、合肥话、银川话、郑州话、沈阳话、温州话、武汉话、昆明话、太原话、南昌话、济南话、兰州话、南京话、客家话、闽南语 | 36 种 含 29 种语言、7 种方言 **语言：** 中文、英语、德语、意大利语、葡萄牙语、西班牙语、日语、韩语、法语、俄语、泰语、印度尼西亚语、阿拉伯语、越南语、土耳其语、芬兰语、波兰语、印地语、荷兰语、捷克语、乌尔都语、他加禄语、瑞典语、丹麦语、希伯来语、冰岛语、马来语、挪威语、波斯语 **方言：** 四川话、北京话、天津话、南京话、陕西话、粤语、闽南语 | 55 种 | +| Qwen3-Omni-Flash 混合思考模型 | 较弱 | 支持 | 不支持 | 19 种 含 11 种语言、8 种方言 **语言：** 中文、英语、德语、法语、意大利语、泰语、韩语、日语、俄语、西班牙语、葡萄牙语 **方言：** 四川话、上海话、粤语、闽南语、陕西话、南京话、天津话、北京话 | 19 种 含 11 种语言、8 种方言 **语言：** 中文、英语、德语、法语、意大利语、泰语、韩语、日语、俄语、西班牙语、葡萄牙语 **方言：** 四川话、上海话、粤语、闽南语、陕西话、南京话、天津话、北京话 | 17~49 种 > 因版本而异 | +| Qwen-Omni-Turbo **已停止更新** | 无 | 不支持 | 不支持 | 中文、英语 | 中文、英语 | 4 种 | + +> 模型的名称、上下文、价格、快照版本等信息请参见百炼控制台；并发限流条件请参考[限流](https://help.aliyun.com/zh/model-studio/rate-limit)。 + +## **模型效果** + +### **音视频内容分析** + +| > 请你对这个视频按时间戳生成全面的描述。 | **00:00.000 – 00:02.500** 雨夜中的城市街道填满宽银幕画面。长时间曝光下，红蓝车灯拖出光轨滑过，在湿漉漉的路面上 闪烁。一位身穿深色及膝风衣、内搭浅色衬衫领带的孤独男子，沿着右侧人行道向镜头走来。雨滴凝结在他的肩膀和头发上；每一步都在潮湿的混凝土上发出沉闷的啪嗒声。低沉不祥的电子嗡鸣支撑着环境音，同时稳定的雨声四处噼啪作响。他身旁的砖墙上贴满彩色涂鸦，街区远处几块霓虹灯招牌发光——一块清晰显示橙红色草书体 "Nice"，另一块竖置招牌的末端露出字母 "AT"。 ...... **00:12.300 – 00:14.533** 镜头向下倾斜。在手电筒光圈内，一张 泛黄老化的单页纸躺在浅水洼中，边缘烧焦且边角卷曲。墨迹段落覆盖大部分表面，但潮湿和焦痕使文字无法辨认。男子跪下，戴手套的手指伸向文件；手靠近时水波向外荡漾。警笛达到峰值，随后开始略微消退，而 环境嗡鸣保持稳定。画面在颤抖的纸页即将被触碰前 定格，片段结束。 > 仅展示部分时间戳内容。 | +| --- | --- | + +### **多方言理解** + +| > 给出不同方言歌手的声音特征、演唱时的情绪状态、以及对应的完整歌词 | 这段音频由多位来自中国不同地区的歌手用各自的方言进行说唱，展现了中国方言的丰富多样性。以下是每位歌手的声音特征、情绪状态及对应歌词的详细描述： **杭州话（吴语）** **声音特征：** 这位男歌手的声音较为柔和，带有吴语特有的软糯和温婉。他的语调起伏平缓，节奏轻快，给人一种娓娓道来的感觉。 **情绪状态：** 悠闲、惬意且充满生活气息。他描绘了杭州的慢生活和城市变迁，情绪中带着对家乡的眷恋和对现代发展的感慨，整体感觉舒适而亲切。 **对应歌词：** \\> 杭州话，听听看，我们格文化，包括家门口的儿化音 \\> 像花开在西湖边，不搞文化调查，只有我们搞得懂 \\> 你们都不懂，三不搭门，格儿，老儿，粉儿，伢儿，这外人分得清 \\> 出发，变化快，地铁交通发达，据说要去看看 \\> 包容新马泰，看十三太，为城市骄傲，性格直给 \\> 感情你只为过桥，而且味道不一样，你不来嘞 > 仅展示部分效果。 | +| --- | --- | + +### 歌词字幕生成 + +| > 请你识别这首歌的歌词，并分句给出歌词的时间戳，输出格式如下：\\[00:00:15,020 --> 00:00:28,085\\] ：When you walk through a storm, hold your head up high.\\[00:00:28,085 --> 00:00:40,200\\] And don't be afraid of the dark. ...... | \\[00:00:12,680 --> 00:00:16,960\\] 猫线绕过树摇晃的月光 \\[00:00:18,400 --> 00:00:22,800\\] 暖气片哼着九八年排行 \\[00:00:24,160 --> 00:00:28,080\\] 时间拨开云雾般的热浪 \\[00:00:28,920 --> 00:00:33,000\\] 屏幕里的霓虹晒在鼻梁 ...... \\[00:03:16,720 --> 00:03:21,680\\] 我们窝在年轮最柔软一墙 \\[00:03:22,400 --> 00:03:27,000\\] 呼吸被余温酿成蜂蜜糖 \\[00:03:28,160 --> 00:03:33,200\\] 沙发陷落成云絮的形状 \\[00:03:34,000 --> 00:03:38,800\\] 每个毛孔都晒着晴朗 \\[00:04:09,000 --> 00:04:10,020\\] (End) > 仅展示部分效果。 | +| --- | --- | + +### **音视频编程** + +| | | +| --- | --- | + +## **使用方式** + +#### **流式输出** + +所有对 Qwen-Omni 模型的请求都必须设置 `stream=True`。 + +#### **模型配置** + +根据使用场景，为模型配置合适的参数、提示词、音视频长度，可在成本、速度和效果之间找到平衡。 + +## 音视频理解 + +| **使用场景** | **推荐视频长度** | **Prompt 建议** | **max\\_pixels 推荐参数值** | +| --- | --- | --- | --- | +| 快速审核，成本低 | ≤60分钟 | 50 个词以内的简单 Prompt | 230,400 | +| 内容提取（长视频分段） | ≤60分钟 | 921,600~2,073,600 | +| 标准分析（短视频打标） | ≤4分钟 | 使用下方的结构化 Prompt **建议Prompt** ```` Provide a detailed description of the video. It should explicitly include three sections: 1. A structured chronological storyline of **every noticeable audio and visual details** 2. A structured list of all visible text. For each text element, include start timestamp, end timestamp, the exact text content, the appearance characteristics. If no text appears, explicitly state so. 3. A structured speech-to-text transcription, include speaker（Corresponding to the character or voice‑over in Section 1, including their accent and tone）, exact spoken content, start timestamp, end timestamp, and speaking state (prosody, emotion, and style). If no speech appears, explicitly state so. Aside from these three required sections, you are free to organize any additional content in any way you find helpful. This additional content can include global information about the entire video or localized information about specific moments. You may choose the topic of this extra content freely. Output Format: ``` ## Storyline - - - ... ## Visible Text - “”: “”: - “”: “”: “”: - “”: ... ## Speakers and Transcript Speaker profiles: - - - ... - Speaker: State: Content: “” - Speaker: State: Content: “” - Speaker: State: Content: “” ... ## ## ... ``` ```` | 921,600~2,073,600 | +| 精细分析（多说话人/复杂场景） | ≤2分钟 | 2,073,600 | + +**说明** + +长视频如需获得细粒度的描述，建议分段处理。 + +## 音频理解 + +通过控制音频长度和 Prompt 复杂度来平衡成本与效果。 + +| **使用场景** | **推荐音频长度** | **Prompt 建议** | +| --- | --- | --- | +| 快速审核、低成本 | ≤60分钟 | 50 个词以内的简单 Prompt | +| 内容提取（长音频分段） | ≤60分钟 | +| 标准分析（音频打标） | ≤2分钟 | 使用结构化 Prompt **结构化Prompt** ```` Provide a detailed description of the audio. It should explicitly include two sections: 1. A structured chronological storyline of **every noticeable audio details** 2. A structured speech-to-text transcription, include speaker（Corresponding to the character or voice‑over in Section 1, including their accent and tone）, exact spoken content, start timestamp, end timestamp, and speaking state (prosody, emotion, and style). If no speech appears, explicitly state so. Aside from these two required components, you are free to organize any additional content in any way you find helpful. This additional content can include global information about the entire audio or localized information about specific moments. You may choose the topic of this extra content freely. Output Format: ``` ## Storyline - - - ... ... ## Speakers and Transcript Speaker profiles: - - - ... - Speaker: State: Content: “” - Speaker: State: Content: “” - Speaker: State: Content: “” ... ## ## ... ``` ```` | +| 精细分析（多说话人/复杂场景） | ≤1分钟 | + +**说明** + +长音频如需获得细粒度的描述，建议分段处理。 + +## **多模态组合输入** + +**说明** + +多模态组合输入仅 Qwen3.5-Omni 系列支持。您可以在同一请求中同时传入多种模态的数据（例如图片+音频+文本、视频+图片+文本等任意组合）。 + +以下示例展示如何在一个请求中同时传入图片和音频，并让模型综合分析多种模态的内容。 + +## OpenAI 兼容 + +Python + +``` +import os +from openai import OpenAI + +client = OpenAI( + api_key=os.getenv("DASHSCOPE_API_KEY"), + base_url="https://dashscope.aliyuncs.com/compatible-mode/v1", +) + +completion = client.chat.completions.create( + model="qwen3.5-omni-plus", + messages=[ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": { + "url": "https://help-static-aliyun-doc.aliyuncs.com/file-manage-files/zh-CN/20241022/emyrja/dog_and_girl.jpeg" + }, + }, + { + "type": "input_audio", + "input_audio": { + "data": "https://help-static-aliyun-doc.aliyuncs.com/file-manage-files/zh-CN/20250211/tixcef/cherry.wav", + "format": "wav" + }, + }, + {"type": "text", "text": "请描述图片内容，并告诉我音频在说什么。"}, + ], + }, + ], + modalities=["text", "audio"], + audio={"voice": "Tina", "format": "wav"}, + stream=True, + stream_options={"include_usage": True}, +) + +for chunk in completion: + if chunk.choices: + print(chunk.choices[0].delta) + else: + print(chunk.usage) +``` + +Node.js + +``` +import OpenAI from "openai"; + +const openai = new OpenAI( + { + apiKey: process.env.DASHSCOPE_API_KEY, + baseURL: "https://dashscope.aliyuncs.com/compatible-mode/v1" + } +); + +const completion = await openai.chat.completions.create({ + model: "qwen3.5-omni-plus", + messages: [ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": { "url": "https://help-static-aliyun-doc.aliyuncs.com/file-manage-files/zh-CN/20241022/emyrja/dog_and_girl.jpeg" }, + }, + { + "type": "input_audio", + "input_audio": { + "data": "https://help-static-aliyun-doc.aliyuncs.com/file-manage-files/zh-CN/20250211/tixcef/cherry.wav", + "format": "wav" + }, + }, + { "type": "text", "text": "请描述图片内容，并告诉我音频在说什么。" } + ] + } + ], + stream: true, + stream_options: { + include_usage: true + }, + modalities: ["text", "audio"], + audio: { voice: "Tina", format: "wav" } +}); + +for await (const chunk of completion) { + if (Array.isArray(chunk.choices) && chunk.choices.length > 0) { + console.log(chunk.choices[0].delta); + } else { + console.log(chunk.usage); + } +} +``` + +curl + +``` +curl -X POST https://dashscope.aliyuncs.com/compatible-mode/v1/chat/completions \ +-H "Authorization: Bearer $DASHSCOPE_API_KEY" \ +-H "Content-Type: application/json" \ +-d '{ + "model": "qwen3.5-omni-plus", + "messages": [ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": { + "url": "https://help-static-aliyun-doc.aliyuncs.com/file-manage-files/zh-CN/20241022/emyrja/dog_and_girl.jpeg" + } + }, + { + "type": "input_audio", + "input_audio": { + "data": "https://help-static-aliyun-doc.aliyuncs.com/file-manage-files/zh-CN/20250211/tixcef/cherry.wav", + "format": "wav" + } + }, + { + "type": "text", + "text": "请描述图片内容，并告诉我音频在说什么。" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "modalities": ["text", "audio"], + "audio": {"voice": "Tina", "format": "wav"} +}' +``` + +## **单一模态输入** + +以下场景中，每次请求仅传入文本与一种其他模态（视频、音频或图片）的组合，所有 Qwen-Omni 模型均支持。 + +## **视频+文本输入** + +视频的传入方式可以为[图片列表形式](#0f4360d63a8nk)或[视频文件形式（可理解视频中的音频）](#5ed48035d09so)。 + +#### **视频文件形式（可理解视频中的音频）** + +- 文件数量： + + - Qwen3.5-Omni系列：使用公网URL方式，最多可传入 512 个；使用Base64编码方式，最多可传入 250 个。 + + - Qwen3-Omni-Flash系列、Qwen-Omni-Turbo系列：仅支持输入一个； + +- 文件大小： + + - 使用公网 URL 方式： + + - Qwen3.5-Omni系列：限制为 2GB + + - Qwen3-Omni-Flash：限制为 256 MB + + - Qwen-Omni-Turbo：限制为 150 MB + + - 使用 Base64 编码方式：编码后的 Base64 字符串大小必须小于 10MB + +- 时长限制： + + - Qwen3.5-Omni系列：1 小时 + + - Qwen3-Omni-Flash：150 秒 + + - Qwen-Omni-Turbo：40 秒 + +- 文件格式：MP4、AVI、MKV、MOV、FLV、WMV 等。 + +- 视频文件中的视觉信息与音频信息会分开计费。 + + +## OpenAI 兼容 + +Python + +``` +import os +from openai import OpenAI + +client = OpenAI( + # 若没有配置环境变量，请用阿里云百炼API Key将下行替换为：api_key="sk-xxx", + # 新加坡和北京地域的API Key不同。获取API Key：https://help.aliyun.com/zh/model-studio/get-api-key + api_key=os.getenv("DASHSCOPE_API_KEY"), + # 以下是北京地域base_url，如果使用新加坡地域的模型，需要将base_url替换为：https://dashscope-intl.aliyuncs.com/compatible-mode/v1 + base_url="https://dashscope.aliyuncs.com/compatible-mode/v1", +) + +completion = client.chat.completions.create( + model="qwen3.5-omni-plus", # 模型为Qwen3-Omni-Flash时，请在非思考模式下运行 + messages=[ + { + "role": "user", + "content": [ + { + "type": "video_url", + "video_url": { + "url": "https://help-static-aliyun-doc.aliyuncs.com/file-manage-files/zh-CN/20241115/cqqkru/1.mp4" + }, + }, + {"type": "text", "text": "视频的内容是什么?"}, + ], + }, + ], + # 设置输出数据的模态，当前支持两种：["text","audio"]、["text"] + modalities=["text", "audio"], + audio={"voice": "Tina", "format": "wav"}, + # stream 必须设置为 True，否则会报错 + stream=True, + stream_options={"include_usage": True}, +) + +for chunk in completion: + if chunk.choices: + print(chunk.choices[0].delta) + else: + print(chunk.usage) +``` + +Node.js + +curl + +``` +# ======= 重要提示 ======= +# 新加坡和北京地域的API Key不同。获取API Key：https://help.aliyun.com/zh/model-studio/get-api-key +# 以下是北京地域base_url，如果使用新加坡地域的模型，需要将base_url替换为：https://dashscope-intl.aliyuncs.com/compatible-mode/v1/chat/completions +# === 执行时请删除该注释 === + +curl -X POST https://dashscope.aliyuncs.com/compatible-mode/v1/chat/completions \ +-H "Authorization: Bearer $DASHSCOPE_API_KEY" \ +-H "Content-Type: application/json" \ +-d '{ + "model": "qwen3.5-omni-plus", + "messages": [ + { + "role": "user", + "content": [ + { + "type": "video_url", + "video_url": { + "url": "https://help-static-aliyun-doc.aliyuncs.com/file-manage-files/zh-CN/20241115/cqqkru/1.mp4" + } + }, + { + "type": "text", + "text": "视频的内容是什么" + } + ] + } + ], + "stream":true, + "stream_options": { + "include_usage": true + }, + "modalities":["text","audio"], + "audio":{"voice":"Tina","format":"wav"} +}' +``` + +#### **图片列表形式** + +**图片数量** + +- Qwen3.5-Omni系列：最少传入 2 张图片，最多可传入 2048 张图片 + +- Qwen3-Omni-Flash：最少传入 2 张图片，最多可传入 128 张图片 + +- Qwen-Omni-Turbo：最少传入 4 张图片，最多可传入 80 张图片 + + +## OpenAI 兼容 + +Python + +``` +import os +from openai import OpenAI + +# 初始化OpenAI客户端 +client = OpenAI( + # 若没有配置环境变量，请用阿里云百炼API Key将下行替换为：api_key="sk-xxx", + # 新加坡和北京地域的API Key不同。获取API Key：https://help.aliyun.com/zh/model-studio/get-api-key + api_key=os.getenv("DASHSCOPE_API_KEY"), + # 以下是北京地域base_url，如果使用新加坡地域的模型，需要将base_url替换为：https://dashscope-intl.aliyuncs.com/compatible-mode/v1 + base_url="https://dashscope.aliyuncs.com/compatible-mode/v1", +) + +completion = client.chat.completions.create( + model="qwen3.5-omni-plus", # 模型为Qwen3-Omni-Flash时，请在非思考模式下运行 + messages=[ + { + "role": "user", + "content": [ + { + "type": "video", + "video": [ + "https://help-static-aliyun-doc.aliyuncs.com/file-manage-files/zh-CN/20241108/xzsgiz/football1.jpg", + "https://help-static-aliyun-doc.aliyuncs.com/file-manage-files/zh-CN/20241108/tdescd/football2.jpg", + "https://help-static-aliyun-doc.aliyuncs.com/file-manage-files/zh-CN/20241108/zefdja/football3.jpg", + "https://help-static-aliyun-doc.aliyuncs.com/file-manage-files/zh-CN/20241108/aedbqh/football4.jpg", + ], + }, + {"type": "text", "text": "描述这个视频的具体过程"}, + ], + } + ], + # 设置输出数据的模态，当前支持两种：["text","audio"]、["text"] + modalities=["text", "audio"], + audio={"voice": "Tina", "format": "wav"}, + # stream 必须设置为 True，否则会报错 + stream=True, + stream_options={"include_usage": True}, +) + +for chunk in completion: + if chunk.choices: + print(chunk.choices[0].delta) + else: + print(chunk.usage) +``` + +Node.js + +``` +import OpenAI from "openai"; + +const openai = new OpenAI({ + // 若没有配置环境变量，请用阿里云百炼API Key将下行替换为：apiKey: "sk-xxx", + // 新加坡和北京地域的API Key不同。获取API Key：https://help.aliyun.com/zh/model-studio/get-api-key + apiKey: process.env.DASHSCOPE_API_KEY, + // 以下是北京地域base_url，如果使用新加坡地域的模型，需要将base_url替换为：https://dashscope-intl.aliyuncs.com/compatible-mode/v1 + baseURL: 'https://dashscope.aliyuncs.com/compatible-mode/v1' +}); + + +const completion = await openai.chat.completions.create({ + model: "qwen3.5-omni-plus", // 模型为Qwen3-Omni-Flash时，请在非思考模式下运行 + messages: [{ + role: "user", + content: [ + { + type: "video", + video: [ + "https://help-static-aliyun-doc.aliyuncs.com/file-manage-files/zh-CN/20241108/xzsgiz/football1.jpg", + "https://help-static-aliyun-doc.aliyuncs.com/file-manage-files/zh-CN/20241108/tdescd/football2.jpg", + "https://help-static-aliyun-doc.aliyuncs.com/file-manage-files/zh-CN/20241108/zefdja/football3.jpg", + "https://help-static-aliyun-doc.aliyuncs.com/file-manage-files/zh-CN/20241108/aedbqh/football4.jpg" + ] + }, + { + type: "text", + text: "描述这个视频的具体过程" + } + ] + }], + stream: true, + stream_options: { + include_usage: true + }, + modalities: ["text", "audio"], + audio: { voice: "Tina", format: "wav" } +}); + +for await (const chunk of completion) { + if (Array.isArray(chunk.choices) && chunk.choices.length > 0) { + console.log(chunk.choices[0].delta); + } else { + console.log(chunk.usage); + } +} +``` + +curl + +``` +# ======= 重要提示 ======= +# 新加坡和北京地域的API Key不同。获取API Key：https://help.aliyun.com/zh/model-studio/get-api-key +# 以下是北京地域base_url，如果使用新加坡地域的模型，需要将base_url替换为：https://dashscope-intl.aliyuncs.com/compatible-mode/v1/chat/completions +# === 执行时请删除该注释 === + +curl -X POST https://dashscope.aliyuncs.com/compatible-mode/v1/chat/completions \ +-H "Authorization: Bearer $DASHSCOPE_API_KEY" \ +-H "Content-Type: application/json" \ +-d '{ + "model": "qwen3.5-omni-plus", + "messages": [ + { + "role": "user", + "content": [ + { + "type": "video", + "video": [ + "https://help-static-aliyun-doc.aliyuncs.com/file-manage-files/zh-CN/20241108/xzsgiz/football1.jpg", + "https://help-static-aliyun-doc.aliyuncs.com/file-manage-files/zh-CN/20241108/tdescd/football2.jpg", + "https://help-static-aliyun-doc.aliyuncs.com/file-manage-files/zh-CN/20241108/zefdja/football3.jpg", + "https://help-static-aliyun-doc.aliyuncs.com/file-manage-files/zh-CN/20241108/aedbqh/football4.jpg" + ] + }, + { + "type": "text", + "text": "描述这个视频的具体过程" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "modalities": ["text", "audio"], + "audio": { + "voice": "Tina", + "format": "wav" + } +}' +``` + +## **音频+文本输入** + +- 文件数量： + + - Qwen3.5-Omni系列：使用公网URL方式，最多可传入 2048 个；使用Base64编码方式，最多可传入 250 个； + + - Qwen3-Omni-Flash系列、Qwen-Omni-Turbo系列：仅支持输入一个； + +- 文件大小： + + - 使用公网 URL 方式： + + - Qwen3.5-Omni系列：不超过 2GB + + - Qwen3-Omni-Flash：不超过 100MB + + - Qwen-Omni-Turbo：不超过 10MB + + - 使用 Base64 编码方式：编码后的 Base64 字符串大小必须小于 10MB + +- 时长限制： + + - Qwen3.5-Omni系列：最长 3 小时 + + - Qwen3-Omni-Flash：最长 20 分钟 + + - Qwen-Omni-Turbo：最长 3 分钟 + +- 文件格式：支持AMR、 WAV、 3GP、 3GPP、 AAC、 MP3等主流格式 + + +以下示例代码以传入音频公网 URL 为例，传入本地音频请参见：[输入 Base64 编码的本地文件](#c516d1e824x03)。当前只支持以流式输出的方式进行调用。 + +## OpenAI 兼容 + +Python + +``` +import os +from openai import OpenAI + +# 初始化OpenAI客户端 +client = OpenAI( + # 若没有配置环境变量，请用阿里云百炼API Key将下行替换为：api_key="sk-xxx", + # 新加坡和北京地域的API Key不同。获取API Key：https://help.aliyun.com/zh/model-studio/get-api-key + api_key=os.getenv("DASHSCOPE_API_KEY"), + # 以下是北京地域base_url，如果使用新加坡地域的模型，需要将base_url替换为：https://dashscope-intl.aliyuncs.com/compatible-mode/v1 + base_url="https://dashscope.aliyuncs.com/compatible-mode/v1", +) + +completion = client.chat.completions.create( + model="qwen3.5-omni-plus", # 模型为Qwen3-Omni-Flash时，请在非思考模式下运行 + messages=[ + { + "role": "user", + "content": [ + { + "type": "input_audio", + "input_audio": { + "data": "https://help-static-aliyun-doc.aliyuncs.com/file-manage-files/zh-CN/20250211/tixcef/cherry.wav", + "format": "wav", + }, + }, + {"type": "text", "text": "这段音频在说什么"}, + ], + }, + ], + # 设置输出数据的模态，当前支持两种：["text","audio"]、["text"] + modalities=["text", "audio"], + audio={"voice": "Tina", "format": "wav"}, + # stream 必须设置为 True，否则会报错 + stream=True, + stream_options={"include_usage": True}, +) + +for chunk in completion: + print(chunk) + # if chunk.choices: + # print(chunk.choices[0].delta) + # else: + # print(chunk.usage) +``` + +Node.js + +``` +import OpenAI from "openai"; + +// 初始化 openai 客户端 +const openai = new OpenAI({ + // 若没有配置环境变量，请用阿里云百炼API Key将下行替换为：apiKey:"sk-xxx", + // 新加坡和北京地域的API Key不同。获取API Key：https://help.aliyun.com/zh/model-studio/get-api-key + apiKey: process.env.DASHSCOPE_API_KEY, + // 以下是北京地域base_url，如果使用新加坡地域的模型，需要将base_url替换为：https://dashscope-intl.aliyuncs.com/compatible-mode/v1 + baseURL: 'https://dashscope.aliyuncs.com/compatible-mode/v1' +}); + +const completion = await openai.chat.completions.create({ + model: "qwen3.5-omni-plus", // 模型为Qwen3-Omni-Flash时，请在非思考模式下运行 + messages: [ + { + "role": "user", + "content": [{ + "type": "input_audio", + "input_audio": { "data": "https://help-static-aliyun-doc.aliyuncs.com/file-manage-files/zh-CN/20250211/tixcef/cherry.wav", "format": "wav" }, + }, + { "type": "text", "text": "这段音频在说什么" }] + }], + stream: true, + stream_options: { + include_usage: true + }, + modalities: ["text", "audio"], + audio: { voice: "Tina", format: "wav" } +}); + +for await (const chunk of completion) { + if (Array.isArray(chunk.choices) && chunk.choices.length > 0) { + console.log(chunk.choices[0].delta); + } else { + console.log(chunk.usage); + } +} +``` + +curl + +``` +# ======= 重要提示 ======= +# 新加坡和北京地域的API Key不同。获取API Key：https://help.aliyun.com/zh/model-studio/get-api-key +# 以下是北京地域base_url，如果使用新加坡地域的模型，需要将base_url替换为：https://dashscope-intl.aliyuncs.com/compatible-mode/v1/chat/completions +# === 执行时请删除该注释 === + +curl -X POST https://dashscope.aliyuncs.com/compatible-mode/v1/chat/completions \ +-H "Authorization: Bearer $DASHSCOPE_API_KEY" \ +-H "Content-Type: application/json" \ +-d '{ + "model": "qwen3.5-omni-plus", + "messages": [ + { + "role": "user", + "content": [ + { + "type": "input_audio", + "input_audio": { + "data": "https://help-static-aliyun-doc.aliyuncs.com/file-manage-files/zh-CN/20250211/tixcef/cherry.wav", + "format": "wav" + } + }, + { + "type": "text", + "text": "这段音频在说什么" + } + ] + } + ], + "stream":true, + "stream_options":{ + "include_usage":true + }, + "modalities":["text","audio"], + "audio":{"voice":"Tina","format":"wav"} +}' +``` + +## **图片+文本输入** + +Qwen-Omni 模型支持传入多张图片。对输入图片的要求如下： + +- 图片数量： + + - 公网URL传入：最多可传入 2048 张 + + - Base64 编码：最多可传入 250 张 + +- 图像大小： + + - 使用公网 URL 方式： + + - Qwen3.5-Omni系列：单个图片文件的大小不超过 20MB + + - Qwen3-Omni-Flash系列、Qwen-Omni-Turbo系列：单个图片文件的大小不超过 10MB + + - 使用 Base64 编码方式：编码后的 Base64 字符串大小必须小于 10MB； + +- 图片的宽度和高度均应大于 10 像素，宽高比不应超过 200:1 或 1:200 + +- 支持的图片类型请参见[图像与视频理解](https://help.aliyun.com/zh/model-studio/vision#afa499b5b1rl5) + + +以下示例代码以传入图片公网 URL 为例，传入本地图片请参见：[输入 Base64 编码的本地文件](#c516d1e824x03)。当前只支持以流式输出的方式进行调用。 + +## OpenAI 兼容 + +Python + +``` +import os +from openai import OpenAI + +client = OpenAI( + # 若没有配置环境变量，请用阿里云百炼API Key将下行替换为：api_key="sk-xxx", + # 新加坡和北京地域的API Key不同。获取API Key：https://help.aliyun.com/zh/model-studio/get-api-key + api_key=os.getenv("DASHSCOPE_API_KEY"), + # 以下是北京地域base_url，如果使用新加坡地域的模型，需要将base_url替换为：https://dashscope-intl.aliyuncs.com/compatible-mode/v1 + base_url="https://dashscope.aliyuncs.com/compatible-mode/v1", +) + +completion = client.chat.completions.create( + model="qwen3.5-omni-plus", # 模型为Qwen3-Omni-Flash时，请在非思考模式下运行 + messages=[ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": { + "url": "https://help-static-aliyun-doc.aliyuncs.com/file-manage-files/zh-CN/20241022/emyrja/dog_and_girl.jpeg" + }, + }, + {"type": "text", "text": "图中描绘的是什么景象？"}, + ], + }, + ], + # 设置输出数据的模态，当前支持两种：["text","audio"]、["text"] + modalities=["text", "audio"], + audio={"voice": "Tina", "format": "wav"}, + # stream 必须设置为 True，否则会报错 + stream=True, + stream_options={ + "include_usage": True + } +) + +for chunk in completion: + if chunk.choices: + print(chunk.choices[0].delta) + else: + print(chunk.usage) +``` + +Node.js + +``` +import OpenAI from "openai"; + +const openai = new OpenAI( + { + // 若没有配置环境变量，请用阿里云百炼API Key将下行替换为：apiKey: "sk-xxx", + // 新加坡和北京地域的API Key不同。获取API Key：https://help.aliyun.com/zh/model-studio/get-api-key + apiKey: process.env.DASHSCOPE_API_KEY, + // 以下是北京地域base_url，如果使用新加坡地域的模型，需要将base_url替换为：https://dashscope-intl.aliyuncs.com/compatible-mode/v1 + baseURL: "https://dashscope.aliyuncs.com/compatible-mode/v1" + } +); +const completion = await openai.chat.completions.create({ + // 模型为Qwen3-Omni-Flash时，请在非思考模式下运行 + model: "qwen3.5-omni-plus", + messages: [ + { + "role": "user", + "content": [{ + "type": "image_url", + "image_url": { "url": "https://help-static-aliyun-doc.aliyuncs.com/file-manage-files/zh-CN/20241022/emyrja/dog_and_girl.jpeg" }, + }, + { "type": "text", "text": "图中描绘的是什么景象？" }] + }], + stream: true, + stream_options: { + include_usage: true + }, + modalities: ["text", "audio"], + audio: { voice: "Tina", format: "wav" } +}); + +for await (const chunk of completion) { + if (Array.isArray(chunk.choices) && chunk.choices.length > 0) { + console.log(chunk.choices[0].delta); + } else { + console.log(chunk.usage); + } +} +``` + +curl + +``` +# ======= 重要提示 ======= +# 新加坡和北京地域的API Key不同。获取API Key：https://help.aliyun.com/zh/model-studio/get-api-key +# 以下是北京地域base_url，如果使用新加坡地域的模型，需要将base_url替换为：https://dashscope-intl.aliyuncs.com/compatible-mode/v1/chat/completions +# === 执行时请删除该注释 === + +curl -X POST https://dashscope.aliyuncs.com/compatible-mode/v1/chat/completions \ +-H "Authorization: Bearer $DASHSCOPE_API_KEY" \ +-H "Content-Type: application/json" \ +-d '{ + "model": "qwen3.5-omni-plus", + "messages": [ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": { + "url": "https://help-static-aliyun-doc.aliyuncs.com/file-manage-files/zh-CN/20241022/emyrja/dog_and_girl.jpeg" + } + }, + { + "type": "text", + "text": "图中描绘的是什么景象？" + } + ] + } + ], + "stream":true, + "stream_options":{ + "include_usage":true + }, + "modalities":["text","audio"], + "audio":{"voice":"Tina","format":"wav"} +}' +``` + +## **联网搜索** + +Qwen3.5-Omni系列 系列模型支持联网搜索功能，可以获取实时信息并进行推理分析。 + +- 联网搜索功能仅在 **Qwen3.5-Omni系列** 系列模型中支持，仅支持 `agent`搜索策略。 + +- 计费请参考[计费说明](https://help.aliyun.com/zh/model-studio/web-search#92ce83df3a599)中的`agent`策略。 + + +联网搜索功能需要通过 `enable_search` 参数开启，以下示例展示如何开启联网搜索功能查询实时信息： + +## OpenAI 兼容 + +Python + +``` +# 运行前的准备工作: +# pip install openai + +import os +from openai import OpenAI + +# 初始化客户端 +client = OpenAI( + api_key=os.getenv("DASHSCOPE_API_KEY"), + base_url="https://dashscope.aliyuncs.com/compatible-mode/v1", +) + +# 发起请求（开启联网搜索） +try: + completion = client.chat.completions.create( + model="qwen3.5-omni-plus", + messages=[{ + "role": "user", + "content": "请查询今天的日期和星期，并告诉我今天有哪些重要节日" + }], + stream=True, + stream_options={"include_usage": True}, + # 开启联网搜索 + extra_body={ + "enable_search": True + } + ) + + print("模型回复（包含实时信息）：") + for chunk in completion: + if chunk.choices and chunk.choices[0].delta.content: + print(chunk.choices[0].delta.content, end="") + print() + +except Exception as e: + print(f"请求失败: {e}") +``` + +Node.js + +``` +// 运行前的准备工作: +// npm install openai + +import OpenAI from "openai"; + +// 初始化客户端 +const openai = new OpenAI({ + apiKey: process.env.DASHSCOPE_API_KEY, + baseURL: "https://dashscope.aliyuncs.com/compatible-mode/v1" +}); + +// 发起请求（开启联网搜索） +const completion = await openai.chat.completions.create({ + model: "qwen3.5-omni-plus", + messages: [{ + "role": "user", + "content": "请查询今天的日期和星期，并告诉我今天有哪些重要节日" + }], + stream: true, + stream_options: { + include_usage: true + }, + // 开启联网搜索 + extra_body: { + enable_search: true + } +}); + +console.log("模型回复（包含实时信息）："); + +for await (const chunk of completion) { + if (Array.isArray(chunk.choices) && chunk.choices.length > 0) { + if (chunk.choices[0].delta.content) { + process.stdout.write(chunk.choices[0].delta.content); + } + } +} +console.log(); +``` + +curl + +``` +curl -X POST https://dashscope.aliyuncs.com/compatible-mode/v1/chat/completions \ +-H "Authorization: Bearer $DASHSCOPE_API_KEY" \ +-H "Content-Type: application/json" \ +-d '{ + "model": "qwen3.5-omni-plus", + "messages": [ + { + "role": "user", + "content": "请查询今天的日期和星期，并告诉我今天有哪些重要节日" + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "enable_search": true +}' +``` + +## **开启/关闭思考模式** + +Qwen-Omni系列模型中，仅Qwen3-Omni-Flash 模型属于混合思考模型，通过`enable_thinking`参数控制是否开启思考模式： + +- `true`：开启思考模式 + +- `false`（默认）：关闭思考模式 + + +> 在思考模式下，**不支持输出音频。** + +## OpenAI 兼容 + +Python + +``` +import os +from openai import OpenAI + +client = OpenAI( + # 若没有配置环境变量，请用阿里云百炼API Key将下行替换为：api_key="sk-xxx", + # 新加坡和北京地域的API Key不同。获取API Key：https://help.aliyun.com/zh/model-studio/get-api-key + api_key=os.getenv("DASHSCOPE_API_KEY"), + # 以下是北京地域base_url，如果使用新加坡地域的模型，需要将base_url替换为：https://dashscope-intl.aliyuncs.com/compatible-mode/v1 + base_url="https://dashscope.aliyuncs.com/compatible-mode/v1", +) + +completion = client.chat.completions.create( + model="qwen3-omni-flash", + messages=[{"role": "user", "content": "你是谁"}], + + # 开启/关闭思考模式，在思考模式下不支持输出音频；qwen-omni-turbo不支持设置enable_thinking。 + extra_body={'enable_thinking': True}, + + # 设置输出数据的模态，非思考模式下当前支持两种：["text","audio"]、["text"]，思考模式仅支持：["text"] + modalities=["text"], + + # 设置音色，思考模式下不支持设置audio参数 + # audio={"voice": "Tina", "format": "wav"}, + # stream 必须设置为 True，否则会报错 + stream=True, + stream_options={"include_usage": True}, +) + +for chunk in completion: + if chunk.choices: + print(chunk.choices[0].delta) + else: + print(chunk.usage) +``` + +Node.js + +``` +import OpenAI from "openai"; + +const openai = new OpenAI({ + // 若没有配置环境变量，请用阿里云百炼API Key将下行替换为：apiKey:"sk-xxx", + // 新加坡和北京地域的API Key不同。获取API Key：https://help.aliyun.com/zh/model-studio/get-api-key + apiKey: process.env.DASHSCOPE_API_KEY, + // 以下是北京地域base_url，如果使用新加坡地域的模型，需要将base_url替换为：https://dashscope-intl.aliyuncs.com/compatible-mode/v1 + baseURL: 'https://dashscope.aliyuncs.com/compatible-mode/v1' +}); + +const completion = await openai.chat.completions.create({ + model: "qwen3-omni-flash", + messages: [ + { role: "user", content: "你是谁？" } + ], + // stream 必须设置为 True，否则会报错 + stream: true, + stream_options: { + include_usage: true + }, + // 开启/关闭思考模式，在思考模式下不支持输出音频；qwen-omni-turbo不支持设置enable_thinking。 + extra_body:{'enable_thinking': true}, + // 设置输出数据的模态，非思考模式下当前支持两种：["text","audio"]、["text"]，思考模式仅支持：["text"] + modalities: ["text"], + // 设置音色，思考模式下不支持设置audio参数 + //audio: { voice: "Tina", format: "wav" } +}); + +for await (const chunk of completion) { + if (Array.isArray(chunk.choices) && chunk.choices.length > 0) { + console.log(chunk.choices[0].delta); + } else { + console.log(chunk.usage); + } +} +``` + +curl + +``` +# ======= 重要提示 ======= +# 新加坡和北京地域的API Key不同。获取API Key：https://help.aliyun.com/zh/model-studio/get-api-key +# 以下是北京地域base_url，如果使用新加坡地域的模型，需要将base_url替换为：https://dashscope-intl.aliyuncs.com/compatible-mode/v1/chat/completions +# === 执行时请删除该注释 === + +curl -X POST https://dashscope.aliyuncs.com/compatible-mode/v1/chat/completions \ +-H "Authorization: Bearer $DASHSCOPE_API_KEY" \ +-H "Content-Type: application/json" \ +-d '{ + "model": "qwen3-omni-flash", + "messages": [ + { + "role": "user", + "content": "你是谁？" + } + ], + "stream":true, + "stream_options":{ + "include_usage":true + }, + "modalities":["text"], + "enable_thinking": true +}' +``` + +**返回结果** + +``` +data: {"choices":[{"delta":{"content":null,"role":"assistant","reasoning_content":""},"index":0,"logprobs":null,"finish_reason":null}],"object":"chat.completion.chunk","usage":null,"created":1757937336,"system_fingerprint":null,"model":"qwen3-omni-flash","id":"chatcmpl-ce3d6fe5-e717-4b7e-8b40-3aef12288d4c"} +data: {"choices":[{"finish_reason":null,"logprobs":null,"delta":{"content":null,"reasoning_content":"嗯"},"index":0}],"object":"chat.completion.chunk","usage":null,"reated":1757937336,"system_fingerprint":null,"model":"qwen3-omni-flash","id":"chatcmpl-ce3d6fe5-e717-4b7e-8b40-3aef12288d4c"} +data: {"choices":[{"delta":{"content":null,"reasoning_content":"，"},"finish_reason":null,"index":0,"logprobs":null}],"object":"chat.completion.chunk","usage":null,"reated":1757937336,"system_fingerprint":null,"model":"qwen3-omni-flash","id":"chatcmpl-ce3d6fe5-e717-4b7e-8b40-3aef12288d4c"} +...... +data: {"choices":[{"delta":{"content":"告诉我"},"finish_reason":null,"index":0,"logprobs":null}],"object":"chat.completion.chunk","usage":null,"created":1757937336,"tem_fingerprint":null,"model":"qwen3-omni-flash","id":"chatcmpl-ce3d6fe5-e717-4b7e-8b40-3aef12288d4c"} +data: {"choices":[{"delta":{"content":"！"},"finish_reason":null,"index":0,"logprobs":null}],"object":"chat.completion.chunk","usage":null,"created":1757937336,"systm_fingerprint":null,"model":"qwen3-omni-flash","id":"chatcmpl-ce3d6fe5-e717-4b7e-8b40-3aef12288d4c"} +data: {"choices":[{"finish_reason":"stop","delta":{"content":"","reasoning_content":null},"index":0,"logprobs":null}],"object":"chat.completion.chunk","usage":null,"created":1757937336,"system_fingerprint":null,"model":"qwen3-omni-flash","id":"chatcmpl-ce3d6fe5-e717-4b7e-8b40-3aef12288d4c"} +data: {"choices":[],"object":"chat.completion.chunk","usage":{"prompt_tokens":11,"completion_tokens":363,"total_tokens":374,"completion_tokens_details":{"reasoning_tokens":195,"text_tokens":168},"prompt_tokens_details":{"text_tokens":11}},"created":1757937336,"system_fingerprint":null,"model":"qwen3-omni-flash","id":"chatcmpl-ce3d6fe5-e717-4b7e-8b40-3aef12288d4c"} +``` + +## **多轮对话** + +您在使用 Qwen-Omni 模型的多轮对话功能时，需要注意： + +- Assistant Message + + 添加到 messages 数组中的 Assistant Message 只可以包含文本数据。 + +- User Message + + 一条 User Message 只可以包含文本和一种模态的数据，在多轮对话中您可以在不同的 User Message 中输入不同模态的数据。 + + +## OpenAI 兼容 + +Python + +``` +import os +from openai import OpenAI + +# 初始化OpenAI客户端 +client = OpenAI( + # 若没有配置环境变量，请用阿里云百炼API Key将下行替换为：api_key="sk-xxx", + # 新加坡和北京地域的API Key不同。获取API Key：https://help.aliyun.com/zh/model-studio/get-api-key + api_key=os.getenv("DASHSCOPE_API_KEY"), + # 以下是北京地域base_url，如果使用新加坡地域的模型，需要将base_url替换为：https://dashscope-intl.aliyuncs.com/compatible-mode/v1 + base_url="https://dashscope.aliyuncs.com/compatible-mode/v1", +) + +completion = client.chat.completions.create( + model="qwen3.5-omni-plus", # 模型为Qwen3-Omni-Flash时，请在非思考模式下运行 + messages=[ + { + "role": "user", + "content": [ + { + "type": "input_audio", + "input_audio": { + "data": "https://dashscope.oss-cn-beijing.aliyuncs.com/audios/welcome.mp3", + "format": "mp3", + }, + }, + {"type": "text", "text": "这段音频在说什么"}, + ], + }, + { + "role": "assistant", + "content": [{"type": "text", "text": "这段音频在说：欢迎使用阿里云"}], + }, + { + "role": "user", + "content": [{"type": "text", "text": "介绍一下这家公司？"}], + }, + ], + # 设置输出数据的模态，当前支持两种：["text","audio"]、["text"] + modalities=["text"], + # stream 必须设置为 True，否则会报错 + stream=True, + stream_options={"include_usage": True}, +) + +for chunk in completion: + if chunk.choices: + print(chunk.choices[0].delta) + else: + print(chunk.usage) +``` + +Node.js + +``` +import OpenAI from "openai"; + +const openai = new OpenAI({ + // 若没有配置环境变量，请用阿里云百炼API Key将下行替换为：apiKey: "sk-xxx", + // 新加坡和北京地域的API Key不同。获取API Key：https://help.aliyun.com/zh/model-studio/get-api-key + apiKey: process.env.DASHSCOPE_API_KEY, + // 以下是北京地域base_url，如果使用新加坡地域的模型，需要将base_url替换为：https://dashscope-intl.aliyuncs.com/compatible-mode/v1 + baseURL: 'https://dashscope.aliyuncs.com/compatible-mode/v1' +}); + +const completion = await openai.chat.completions.create({ + model: "qwen3.5-omni-plus", // 模型为Qwen3-Omni-Flash时，请在非思考模式下运行 + messages: [ + { + "role": "user", + "content": [ + { + "type": "input_audio", + "input_audio": { + "data": "https://dashscope.oss-cn-beijing.aliyuncs.com/audios/welcome.mp3", + "format": "mp3", + }, + }, + { "type": "text", "text": "这段音频在说什么" }, + ], + }, + { + "role": "assistant", + "content": [{ "type": "text", "text": "这段音频在说：欢迎使用阿里云" }], + }, + { + "role": "user", + "content": [{ "type": "text", "text": "介绍一下这家公司？" }] + }], + stream: true, + stream_options: { + include_usage: true + }, + modalities: ["text"] +}); + + +for await (const chunk of completion) { + if (Array.isArray(chunk.choices) && chunk.choices.length > 0) { + console.log(chunk.choices[0].delta); + } else { + console.log(chunk.usage); + } +} +``` + +curl + +``` +# ======= 重要提示 ======= +# 新加坡和北京地域的API Key不同。获取API Key：https://help.aliyun.com/zh/model-studio/get-api-key +# 以下是北京地域base_url，如果使用新加坡地域的模型，需要将base_url替换为：https://dashscope-intl.aliyuncs.com/compatible-mode/v1/chat/completions +# === 执行时请删除该注释 === + +curl -X POST https://dashscope.aliyuncs.com/compatible-mode/v1/chat/completions \ +-H "Authorization: Bearer $DASHSCOPE_API_KEY" \ +-H "Content-Type: application/json" \ +-d '{ + "model": "qwen3.5-omni-plus", + "messages": [ + { + "role": "user", + "content": [ + { + "type": "input_audio", + "input_audio": { + "data": "https://dashscope.oss-cn-beijing.aliyuncs.com/audios/welcome.mp3" + } + }, + { + "type": "text", + "text": "这段音频在说什么" + } + ] + }, + { + "role": "assistant", + "content": [ + { + "type": "text", + "text": "这段音频在说：欢迎使用阿里云" + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "介绍一下这家公司？" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "modalities": ["text"] +}' +``` + +## **解析输出的Base64 编码的音频数据** + +Qwen-Omni 模型输出的音频为流式输出的 Base64 编码数据。您可以在模型生成过程中维护一个字符串变量，将每个返回片段的 Base64 编码添加到字符串变量后，待生成结束后进行 Base64 解码，得到音频文件；也可以将每个返回片段的 Base64 编码数据实时解码并播放。 + +Python + +``` +# Installation instructions for pyaudio: +# APPLE Mac OS X +# brew install portaudio +# pip install pyaudio +# Debian/Ubuntu +# sudo apt-get install python-pyaudio python3-pyaudio +# or +# pip install pyaudio +# CentOS +# sudo yum install -y portaudio portaudio-devel && pip install pyaudio +# Microsoft Windows +# python -m pip install pyaudio + +import os +from openai import OpenAI +import base64 +import numpy as np +import soundfile as sf + +# 初始化OpenAI客户端 +client = OpenAI( + # 若没有配置环境变量，请用阿里云百炼API Key将下行替换为：api_key="sk-xxx", + # 新加坡和北京地域的API Key不同。获取API Key：https://help.aliyun.com/zh/model-studio/get-api-key + api_key=os.getenv("DASHSCOPE_API_KEY"), + # 以下是北京地域base_url，如果使用新加坡地域的模型，需要将base_url替换为：https://dashscope-intl.aliyuncs.com/compatible-mode/v1 + base_url="https://dashscope.aliyuncs.com/compatible-mode/v1", +) + +completion = client.chat.completions.create( + model="qwen3.5-omni-plus", # 模型为Qwen3-Omni-Flash时，请在非思考模式下运行 + messages=[{"role": "user", "content": "你是谁"}], + # 设置输出数据的模态，当前支持两种：["text","audio"]、["text"] + modalities=["text", "audio"], + audio={"voice": "Tina", "format": "wav"}, + # stream 必须设置为 True，否则会报错 + stream=True, + stream_options={"include_usage": True}, +) + +# 方式1: 待生成结束后再进行解码 +audio_string = "" +for chunk in completion: + if chunk.choices: + if hasattr(chunk.choices[0].delta, "audio"): + try: + audio_string += chunk.choices[0].delta.audio["data"] + except Exception as e: + print(chunk.choices[0].delta.content) + else: + print(chunk.usage) + +wav_bytes = base64.b64decode(audio_string) +audio_np = np.frombuffer(wav_bytes, dtype=np.int16) +sf.write("audio_assistant_py.wav", audio_np, samplerate=24000) + +# 方式2: 边生成边解码(使用方式2请将方式1的代码进行注释) +# # 初始化 PyAudio +# import pyaudio +# import time +# p = pyaudio.PyAudio() +# # 创建音频流 +# stream = p.open(format=pyaudio.paInt16, +# channels=1, +# rate=24000, +# output=True) + +# for chunk in completion: +# if chunk.choices: +# if hasattr(chunk.choices[0].delta, "audio"): +# try: +# audio_string = chunk.choices[0].delta.audio["data"] +# wav_bytes = base64.b64decode(audio_string) +# audio_np = np.frombuffer(wav_bytes, dtype=np.int16) +# # 直接播放音频数据 +# stream.write(audio_np.tobytes()) +# except Exception as e: +# print(chunk.choices[0].delta.content) + +# time.sleep(0.8) +# # 清理资源 +# stream.stop_stream() +# stream.close() +# p.terminate() +``` + +Node.js + +``` +// 运行前的准备工作: +// Windows/Mac/Linux 通用: +// 1. 确保已安装 Node.js (建议版本 >= 14) +// 2. 运行以下命令安装必要的依赖: +// npm install openai wav +// +// 如果要使用实时播放功能 (方式2), 还需要: +// Windows: +// npm install speaker +// Mac: +// brew install portaudio +// npm install speaker +// Linux (Ubuntu/Debian): +// sudo apt-get install libasound2-dev +// npm install speaker + +import OpenAI from "openai"; + +const openai = new OpenAI({ + // 若没有配置环境变量，请用阿里云百炼API Key将下行替换为：apiKey:"sk-xxx", + // 新加坡和北京地域的API Key不同。获取API Key：https://help.aliyun.com/zh/model-studio/get-api-key + apiKey: process.env.DASHSCOPE_API_KEY, + // 以下是北京地域base_url，如果使用新加坡地域的模型，需要将base_url替换为：https://dashscope-intl.aliyuncs.com/compatible-mode/v1 + baseURL: 'https://dashscope.aliyuncs.com/compatible-mode/v1' +}); + +const completion = await openai.chat.completions.create({ + model: "qwen3.5-omni-plus", //模型为Qwen3-Omni-Flash时，请在非思考模式下运行 + messages: [ + { + "role": "user", + "content": "你是谁？" + }], + stream: true, + stream_options: { + include_usage: true + }, + modalities: ["text", "audio"], + audio: { voice: "Tina", format: "wav" } +}); + +// 方式1: 待生成结束后再进行解码 +// 需要安装: npm install wav +import { createWriteStream } from 'node:fs'; // node:fs 是 Node.js 内置模块，无需安装 +import { Writer } from 'wav'; + +async function convertAudio(audioString, audioPath) { + try { + // 解码Base64字符串为Buffer + const wavBuffer = Buffer.from(audioString, 'base64'); + // 创建WAV文件写入流 + const writer = new Writer({ + sampleRate: 24000, // 采样率 + channels: 1, // 单声道 + bitDepth: 16 // 16位深度 + }); + // 创建输出文件流并建立管道连接 + const outputStream = createWriteStream(audioPath); + writer.pipe(outputStream); + + // 写入PCM数据并结束写入 + writer.write(wavBuffer); + writer.end(); + + // 使用Promise等待文件写入完成 + await new Promise((resolve, reject) => { + outputStream.on('finish', resolve); + outputStream.on('error', reject); + }); + + // 添加额外等待时间确保音频完整 + await new Promise(resolve => setTimeout(resolve, 800)); + + console.log(`音频文件已成功保存为 ${audioPath}`); + } catch (error) { + console.error('处理过程中发生错误:', error); + } +} + +let audioString = ""; +for await (const chunk of completion) { + if (Array.isArray(chunk.choices) && chunk.choices.length > 0) { + if (chunk.choices[0].delta.audio) { + if (chunk.choices[0].delta.audio["data"]) { + audioString += chunk.choices[0].delta.audio["data"]; + } + } + } else { + console.log(chunk.usage); + } +} +// 执行转换 +convertAudio(audioString, "audio_assistant_mjs.wav"); + + +// 方式2: 边生成边实时播放 +// 需要先按照上方系统对应的说明安装必要组件 +// import Speaker from 'speaker'; // 引入音频播放库 + +// // 创建扬声器实例（配置与 WAV 文件参数一致） +// const speaker = new Speaker({ +// sampleRate: 24000, // 采样率 +// channels: 1, // 声道数 +// bitDepth: 16, // 位深 +// signed: true // 有符号 PCM +// }); +// for await (const chunk of completion) { +// if (Array.isArray(chunk.choices) && chunk.choices.length > 0) { +// if (chunk.choices[0].delta.audio) { +// if (chunk.choices[0].delta.audio["data"]) { +// const pcmBuffer = Buffer.from(chunk.choices[0].delta.audio.data, 'base64'); +// // 直接写入扬声器播放 +// speaker.write(pcmBuffer); +// } +// } +// } else { +// console.log(chunk.usage); +// } +// } +// speaker.on('finish', () => console.log('播放完成')); +// speaker.end(); // 根据实际 API 流结束情况调用 +``` + +## **输入 Base64 编码的本地文件** + +> 使用 Base64 编码方式传入文件时，编码后的 Base64 字符串大小必须小于 10MB。 + +## 图片 + +以保存在本地的[eagle.png](https://help-static-aliyun-doc.aliyuncs.com/file-manage-files/zh-CN/20250326/nlgymo/eagle.png)为例。 + +Python + +``` +import os +from openai import OpenAI +import base64 + +client = OpenAI( + # 若没有配置环境变量，请用阿里云百炼API Key将下行替换为：api_key="sk-xxx", + # 新加坡和北京地域的API Key不同。获取API Key：https://help.aliyun.com/zh/model-studio/get-api-key + api_key=os.getenv("DASHSCOPE_API_KEY"), + # 以下是北京地域base_url，如果使用新加坡地域的模型，需要将base_url替换为：https://dashscope-intl.aliyuncs.com/compatible-mode/v1 + base_url="https://dashscope.aliyuncs.com/compatible-mode/v1", +) + + +# Base64 编码格式 +def encode_image(image_path): + with open(image_path, "rb") as image_file: + return base64.b64encode(image_file.read()).decode("utf-8") + + +base64_image = encode_image("eagle.png") + +completion = client.chat.completions.create( + model="qwen3.5-omni-plus",# 模型为Qwen3-Omni-Flash时，请在非思考模式下运行 + messages=[ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": {"url": f"data:image/png;base64,{base64_image}"}, + }, + {"type": "text", "text": "图中描绘的是什么景象？"}, + ], + }, + ], + # 设置输出数据的模态，当前支持两种：["text","audio"]、["text"] + modalities=["text", "audio"], + audio={"voice": "Tina", "format": "wav"}, + # stream 必须设置为 True，否则会报错 + stream=True, + stream_options={"include_usage": True}, +) + +for chunk in completion: + if chunk.choices: + print(chunk.choices[0].delta) + else: + print(chunk.usage) +``` + +Node.js + +``` +import OpenAI from "openai"; +import { readFileSync } from 'fs'; + +const openai = new OpenAI({ + // 若没有配置环境变量，请用阿里云百炼API Key将下行替换为：apiKey: "sk-xxx", + // 新加坡和北京地域的API Key不同。获取API Key：https://help.aliyun.com/zh/model-studio/get-api-key + apiKey: process.env.DASHSCOPE_API_KEY, + // 以下是北京地域base_url，如果使用新加坡地域的模型，需要将base_url替换为：https://dashscope-intl.aliyuncs.com/compatible-mode/v1 + baseURL: 'https://dashscope.aliyuncs.com/compatible-mode/v1' +}); + +const encodeImage = (imagePath) => { + const imageFile = readFileSync(imagePath); + return imageFile.toString('base64'); +}; +const base64Image = encodeImage("eagle.png") + +const completion = await openai.chat.completions.create({ + model: "qwen3.5-omni-plus", // 模型为Qwen3-Omni-Flash时，请在非思考模式下运行 + messages: [ + { + "role": "user", + "content": [{ + "type": "image_url", + "image_url": { "url": `data:image/png;base64,${base64Image}` }, + }, + { "type": "text", "text": "图中描绘的是什么景象？" }] + }], + stream: true, + stream_options: { + include_usage: true + }, + modalities: ["text", "audio"], + audio: { voice: "Tina", format: "wav" } +}); + +for await (const chunk of completion) { + if (Array.isArray(chunk.choices) && chunk.choices.length > 0) { + console.log(chunk.choices[0].delta); + } else { + console.log(chunk.usage); + } +} +``` + +## 音频 + +以保存在本地的[welcome.mp3](https://help-static-aliyun-doc.aliyuncs.com/file-manage-files/zh-CN/20250214/pijhos/welcome.mp3)为例。 + +Python + +``` +import os +from openai import OpenAI +import base64 +import numpy as np +import soundfile as sf +import requests + +client = OpenAI( + # 若没有配置环境变量，请用阿里云百炼API Key将下行替换为：api_key="sk-xxx", + # 新加坡和北京地域的API Key不同。获取API Key：https://help.aliyun.com/zh/model-studio/get-api-key + api_key=os.getenv("DASHSCOPE_API_KEY"), + # 以下是北京地域base_url，如果使用新加坡地域的模型，需要将base_url替换为：https://dashscope-intl.aliyuncs.com/compatible-mode/v1 + base_url="https://dashscope.aliyuncs.com/compatible-mode/v1", +) + + +def encode_audio(audio_path): + with open(audio_path, "rb") as audio_file: + return base64.b64encode(audio_file.read()).decode("utf-8") + + +base64_audio = encode_audio("welcome.mp3") + +completion = client.chat.completions.create( + model="qwen3.5-omni-plus", # 模型为Qwen3-Omni-Flash时，请在非思考模式下运行 + messages=[ + { + "role": "user", + "content": [ + { + "type": "input_audio", + "input_audio": { + "data": f"data:;base64,{base64_audio}", + "format": "mp3", + }, + }, + {"type": "text", "text": "这段音频在说什么"}, + ], + }, + ], + # 设置输出数据的模态，当前支持两种：["text","audio"]、["text"] + modalities=["text", "audio"], + audio={"voice": "Tina", "format": "wav"}, + # stream 必须设置为 True，否则会报错 + stream=True, + stream_options={"include_usage": True}, +) + +for chunk in completion: + if chunk.choices: + print(chunk.choices[0].delta) + else: + print(chunk.usage) +``` + +Node.js + +``` +import OpenAI from "openai"; +import { readFileSync } from 'fs'; + +const openai = new OpenAI({ + // 若没有配置环境变量，请用阿里云百炼API Key将下行替换为：apiKey: "sk-xxx", + // 新加坡和北京地域的API Key不同。获取API Key：https://help.aliyun.com/zh/model-studio/get-api-key + apiKey: process.env.DASHSCOPE_API_KEY, + // 以下是北京地域base_url，如果使用新加坡地域的模型，需要将base_url替换为：https://dashscope-intl.aliyuncs.com/compatible-mode/v1 + baseURL: 'https://dashscope.aliyuncs.com/compatible-mode/v1' +}); + +const encodeAudio = (audioPath) => { + const audioFile = readFileSync(audioPath); + return audioFile.toString('base64'); +}; +const base64Audio = encodeAudio("welcome.mp3") + +const completion = await openai.chat.completions.create({ + model: "qwen3.5-omni-plus", // 模型为Qwen3-Omni-Flash时，请在非思考模式下运行 + messages: [ + { + "role": "user", + "content": [{ + "type": "input_audio", + "input_audio": { "data": `data:;base64,${base64Audio}`, "format": "mp3" }, + }, + { "type": "text", "text": "这段音频在说什么" }] + }], + stream: true, + stream_options: { + include_usage: true + }, + modalities: ["text", "audio"], + audio: { voice: "Tina", format: "wav" } +}); + +for await (const chunk of completion) { + if (Array.isArray(chunk.choices) && chunk.choices.length > 0) { + console.log(chunk.choices[0].delta); + } else { + console.log(chunk.usage); + } +} +``` + +## 视频 + +## 视频文件 + +以保存在本地的[spring\_mountain.mp4](https://help-static-aliyun-doc.aliyuncs.com/file-manage-files/zh-CN/20250326/fqojlv/spring_mountain.mp4)为例。 + +Python + +``` +import os +from openai import OpenAI +import base64 +import numpy as np +import soundfile as sf + +client = OpenAI( + # 若没有配置环境变量，请用阿里云百炼API Key将下行替换为：api_key="sk-xxx", + api_key=os.getenv("DASHSCOPE_API_KEY"), + base_url="https://dashscope.aliyuncs.com/compatible-mode/v1", +) + + +# Base64 编码格式 +def encode_video(video_path): + with open(video_path, "rb") as video_file: + return base64.b64encode(video_file.read()).decode("utf-8") + + +base64_video = encode_video("spring_mountain.mp4") + +completion = client.chat.completions.create( + model="qwen3.5-omni-plus", # 模型为Qwen3-Omni-Flash时，请在非思考模式下运行 + messages=[ + { + "role": "user", + "content": [ + { + "type": "video_url", + "video_url": {"url": f"data:;base64,{base64_video}"}, + }, + {"type": "text", "text": "她在唱什么"}, + ], + }, + ], + # 设置输出数据的模态，当前支持两种：["text","audio"]、["text"] + modalities=["text", "audio"], + audio={"voice": "Tina", "format": "wav"}, + # stream 必须设置为 True，否则会报错 + stream=True, + stream_options={"include_usage": True}, +) + +for chunk in completion: + if chunk.choices: + print(chunk.choices[0].delta) + else: + print(chunk.usage) +``` + +Node.js + +``` +import OpenAI from "openai"; +import { readFileSync } from 'fs'; + +const openai = new OpenAI( + { + // 若没有配置环境变量，请用阿里云百炼API Key将下行替换为：apiKey: "sk-xxx", + apiKey: process.env.DASHSCOPE_API_KEY, + baseURL: "https://dashscope.aliyuncs.com/compatible-mode/v1" + } +); + +const encodeVideo = (videoPath) => { + const videoFile = readFileSync(videoPath); + return videoFile.toString('base64'); +}; +const base64Video = encodeVideo("spring_mountain.mp4") + +const completion = await openai.chat.completions.create({ + model: "qwen3.5-omni-plus", // 模型为Qwen3-Omni-Flash时，请在非思考模式下运行 + messages: [ + { + "role": "user", + "content": [{ + "type": "video_url", + "video_url": { "url": `data:;base64,${base64Video}` }, + }, + { "type": "text", "text": "她在唱什么" }] + }], + stream: true, + stream_options: { + include_usage: true + }, + modalities: ["text", "audio"], + audio: { voice: "Tina", format: "wav" } +}); + +for await (const chunk of completion) { + if (Array.isArray(chunk.choices) && chunk.choices.length > 0) { + console.log(chunk.choices[0].delta); + } else { + console.log(chunk.usage); + } +} +``` + +## 图片列表 + +以保存在本地的[football1.jpg](https://help-static-aliyun-doc.aliyuncs.com/file-manage-files/zh-CN/20250319/vzfwkh/football1.jpg)、[football2.jpg](https://help-static-aliyun-doc.aliyuncs.com/file-manage-files/zh-CN/20250319/vgkgqy/football2.jpg)、[football3.jpg](https://help-static-aliyun-doc.aliyuncs.com/file-manage-files/zh-CN/20250127/fytnla/football3.jpg)与[football4.jpg](https://help-static-aliyun-doc.aliyuncs.com/file-manage-files/zh-CN/20250127/ygitwp/football4.jpg)为例。 + +Python + +``` +import os +from openai import OpenAI +import base64 +import numpy as np +import soundfile as sf + +client = OpenAI( + # 若没有配置环境变量，请用阿里云百炼API Key将下行替换为：api_key="sk-xxx", + # 新加坡和北京地域的API Key不同。获取API Key：https://help.aliyun.com/zh/model-studio/get-api-key + api_key=os.getenv("DASHSCOPE_API_KEY"), + # 以下是北京地域base_url，如果使用新加坡地域的模型，需要将base_url替换为：https://dashscope-intl.aliyuncs.com/compatible-mode/v1 + base_url="https://dashscope.aliyuncs.com/compatible-mode/v1", +) + + +# Base64 编码格式 +def encode_image(image_path): + with open(image_path, "rb") as image_file: + return base64.b64encode(image_file.read()).decode("utf-8") + + +base64_image_1 = encode_image("football1.jpg") +base64_image_2 = encode_image("football2.jpg") +base64_image_3 = encode_image("football3.jpg") +base64_image_4 = encode_image("football4.jpg") + +completion = client.chat.completions.create( + model="qwen3.5-omni-plus", # 模型为Qwen3-Omni-Flash时，请在非思考模式下运行 + messages=[ + { + "role": "user", + "content": [ + { + "type": "video", + "video": [ + f"data:image/jpeg;base64,{base64_image_1}", + f"data:image/jpeg;base64,{base64_image_2}", + f"data:image/jpeg;base64,{base64_image_3}", + f"data:image/jpeg;base64,{base64_image_4}", + ], + }, + {"type": "text", "text": "描述这个视频的具体过程"}, + ], + } + ], + # 设置输出数据的模态，当前支持两种：["text","audio"]、["text"] + modalities=["text", "audio"], + audio={"voice": "Tina", "format": "wav"}, + # stream 必须设置为 True，否则会报错 + stream=True, + stream_options={"include_usage": True}, +) + +for chunk in completion: + if chunk.choices: + print(chunk.choices[0].delta) + else: + print(chunk.usage) +``` + +Node.js + +``` +import OpenAI from "openai"; +import { readFileSync } from 'fs'; + +const openai = new OpenAI({ + // 若没有配置环境变量，请用阿里云百炼API Key将下行替换为：apiKey: "sk-xxx", + // 新加坡和北京地域的API Key不同。获取API Key：https://help.aliyun.com/zh/model-studio/get-api-key + apiKey: process.env.DASHSCOPE_API_KEY, + // 以下是北京地域base_url，如果使用新加坡地域的模型，需要将base_url替换为：https://dashscope-intl.aliyuncs.com/compatible-mode/v1 + baseURL: 'https://dashscope.aliyuncs.com/compatible-mode/v1' +}); + +const encodeImage = (imagePath) => { + const imageFile = readFileSync(imagePath); + return imageFile.toString('base64'); + }; +const base64Image1 = encodeImage("football1.jpg") +const base64Image2 = encodeImage("football2.jpg") +const base64Image3 = encodeImage("football3.jpg") +const base64Image4 = encodeImage("football4.jpg") + +const completion = await openai.chat.completions.create({ + model: "qwen3.5-omni-plus", // 模型为Qwen3-Omni-Flash时，请在非思考模式下运行 + messages: [{ + role: "user", + content: [ + { + type: "video", + video: [ + `data:image/jpeg;base64,${base64Image1}`, + `data:image/jpeg;base64,${base64Image2}`, + `data:image/jpeg;base64,${base64Image3}`, + `data:image/jpeg;base64,${base64Image4}` + ] + }, + { + type: "text", + text: "描述这个视频的具体过程" + } + ] + }], + stream: true, + stream_options: { + include_usage: true + }, + modalities: ["text", "audio"], + audio: { voice: "Tina", format: "wav" } +}); + +for await (const chunk of completion) { + if (Array.isArray(chunk.choices) && chunk.choices.length > 0) { + console.log(chunk.choices[0].delta); + } else { + console.log(chunk.usage); + } +} +``` + +## API参考 + +关于千问Omni 模型的输入输出参数，请参见[OpenAI Chat](https://help.aliyun.com/zh/model-studio/qwen-api-via-openai-chat-completions)。 + +## 计费与限流 + +**计费规则** + +Qwen-Omni模型根据不同模态（音频、图像、视频）对应的Token数计费。计费详情请参见百炼控制台。 + +**音频、图片与视频转换为Token数的规则** + +## 音频 + +- `Qwen3.5-Omni系列`： + + - 输入音频计算公式：`总 Tokens 数 = 音频时长（单位：秒）* 7` + + - 输出音频计算公式：`总 Tokens 数 = 音频时长（单位：秒）* 12.5` + +- `Qwen3-Omni-Flash：输入与输出音频的计算公式均为总 Tokens 数 = 音频时长（单位：秒）* 12.5` + +- `Qwen-Omni-Turbo：输入与输出音频的计算公式均为总 Tokens 数 = 音频时长（单位：秒）* 25` + + +若音频时长不足1秒，则按 1 秒计算。 + +## 图片 + +- `Qwen3.5-Omni系列`、`Qwen3-Omni-Flash`模型**：**每`32x32`像素对应 1 个 Token + +- `Qwen-Omni-Turbo`模型：每`28x28`像素对应 1 个 Token + + +Qwen3.5-Omni 系列一张图最少需要 24 个 Token，其他模型最少需要 4 个 Token；默认最多支持 1280 个 Token。Qwen3.5-Omni 系列支持通过 `vl_high_resolution_images` 参数提升图片分辨率上限至 16384 个 Token（Qwen-Omni-Turbo、Qwen3-Omni-Flash 不支持该参数）。可使用以下代码，传入图像路径即可估算单张图片消耗的 Token 总量： + +``` +import math +from PIL import Image # pip install Pillow + +# ============ 模型参数配置（按需修改） ============ + +# 图像因子：Qwen3.5-Omni系列、Qwen3-Omni-Flash 为 32；Qwen-Omni-Turbo 为 28 +IMAGE_FACTOR = 32 + +# Token 下限：Qwen3.5-Omni系列为 24；Qwen-Omni-Turbo、Qwen3-Omni-Flash 为 4 +MIN_TOKENS = 24 + +# 高分辨率模式（仅 Qwen3.5-Omni 系列支持，Qwen-Omni-Turbo 和 Qwen3-Omni-Flash 不支持） +# True → Token 上限 16384 +# False → Token 上限 1280（默认） +VL_HIGH_RESOLUTION_IMAGES = False + +# ============ 像素范围（由上方参数自动计算） ============ + +MIN_PIXELS = MIN_TOKENS * IMAGE_FACTOR * IMAGE_FACTOR +MAX_PIXELS = (16384 if VL_HIGH_RESOLUTION_IMAGES else 1280) * IMAGE_FACTOR * IMAGE_FACTOR + + +def smart_resize(height, width, factor=IMAGE_FACTOR, + min_pixels=MIN_PIXELS, max_pixels=MAX_PIXELS): + """将图像宽高对齐到 factor 整数倍，并缩放到 [min_pixels, max_pixels] 范围内。""" + h_bar = max(factor, round(height / factor) * factor) + w_bar = max(factor, round(width / factor) * factor) + + if h_bar * w_bar > max_pixels: + beta = math.sqrt((height * width) / max_pixels) + h_bar = math.floor(height / beta / factor) * factor + w_bar = math.floor(width / beta / factor) * factor + elif h_bar * w_bar < min_pixels: + beta = math.sqrt(min_pixels / (height * width)) + h_bar = math.ceil(height * beta / factor) * factor + w_bar = math.ceil(width * beta / factor) * factor + + return h_bar, w_bar + + +if __name__ == "__main__": + image = Image.open("xxx/test.jpg") + print(f"原始尺寸：{image.width}x{image.height}") + + resized_h, resized_w = smart_resize(image.height, image.width) + token = int(resized_h * resized_w / (IMAGE_FACTOR * IMAGE_FACTOR)) + 2 + print(f"缩放后尺寸：{resized_w}x{resized_h}，Token 数：{token}") +``` + +## 视频 + +视频文件的 Token 分为 `video_tokens`（视觉）与 `audio_tokens`（音频）。 + +- `video_tokens` + + 计算过程较为复杂。请参见以下代码： + + ``` + # pip install opencv-python + import math + import cv2 + + # ============ 模型参数配置（按需修改） ============ + + # 图像因子：Qwen3.5-Omni系列、Qwen3-Omni-Flash 为 32；Qwen-Omni-Turbo 为 28 + IMAGE_FACTOR = 32 + + FRAME_FACTOR = 2 + FPS = 2 + MAX_RATIO = 200 + + # 视频帧的像素下限 + VIDEO_MIN_PIXELS = 64 * IMAGE_FACTOR * IMAGE_FACTOR + + # 视频帧的像素上限 + # Qwen3.5-Omni系列：640 * 32 * 32 + # Qwen3-Omni-Flash：768 * 32 * 32 + # Qwen-Omni-Turbo：768 * 28 * 28 + VIDEO_MAX_PIXELS = 640 * IMAGE_FACTOR * IMAGE_FACTOR + + # 最少抽取帧数：Qwen3.5-Omni系列、Qwen3-Omni-Flash 为 2；Qwen-Omni-Turbo 为 4 + FPS_MIN_FRAMES = 2 + + # 最大抽取帧数：Qwen3.5-Omni系列为 2048；Qwen3-Omni-Flash 为 128；Qwen-Omni-Turbo 为 80 + FPS_MAX_FRAMES = 2048 + + # 视频输入的最大像素值 + # Qwen3.5-Omni系列：180224 * 32 * 32 + # Qwen3-Omni-Flash：16384 * 32 * 32 + # Qwen-Omni-Turbo：16384 * 28 * 28 + VIDEO_TOTAL_PIXELS = 180224 * IMAGE_FACTOR * IMAGE_FACTOR + + + # ============ 核心函数 ============ + + def get_video_info(video_path): + """读取视频的基本信息：高度、宽度、总帧数、帧率。""" + cap = cv2.VideoCapture(video_path) + height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) + width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)) + total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) + fps = cap.get(cv2.CAP_PROP_FPS) + cap.release() + return height, width, total_frames, fps + + + def smart_nframes(total_frames, video_fps): + """根据视频时长和帧率，计算实际抽取的帧数。""" + min_frames = math.ceil(FPS_MIN_FRAMES / FRAME_FACTOR) * FRAME_FACTOR + max_frames = min(FPS_MAX_FRAMES, total_frames) // FRAME_FACTOR * FRAME_FACTOR + + duration = total_frames / video_fps if video_fps else 0 + if duration - int(duration) > (1 / FPS): + total_frames = math.ceil(duration * video_fps) + else: + total_frames = math.ceil(int(duration) * video_fps) + + nframes = total_frames / video_fps * FPS + nframes = int(min(max(nframes, min_frames), max_frames, total_frames)) + if not (FRAME_FACTOR <= nframes <= total_frames): + raise ValueError(f"nframes should in [{FRAME_FACTOR}, {total_frames}], got {nframes}") + return nframes + + + def smart_resize(height, width, nframes, factor=IMAGE_FACTOR): + """将视频帧缩放到合理的像素范围内，宽高对齐到 factor 整数倍。""" + max_pixels = max( + min(VIDEO_MAX_PIXELS, VIDEO_TOTAL_PIXELS / nframes * FRAME_FACTOR), + int(VIDEO_MIN_PIXELS * 1.05) + ) + if max(height, width) / min(height, width) > MAX_RATIO: + raise ValueError(f"aspect ratio exceeds {MAX_RATIO}") + + h_bar = max(factor, round(height / factor) * factor) + w_bar = max(factor, round(width / factor) * factor) + + if h_bar * w_bar > max_pixels: + beta = math.sqrt((height * width) / max_pixels) + h_bar = math.floor(height / beta / factor) * factor + w_bar = math.floor(width / beta / factor) * factor + elif h_bar * w_bar < VIDEO_MIN_PIXELS: + beta = math.sqrt(VIDEO_MIN_PIXELS / (height * width)) + h_bar = math.ceil(height * beta / factor) * factor + w_bar = math.ceil(width * beta / factor) * factor + + return h_bar, w_bar + + + # ============ 计算 Token ============ + + if __name__ == "__main__": + video_path = "spring_mountain.mp4" + + height, width, total_frames, video_fps = get_video_info(video_path) + print(f"视频信息：{width}x{height}，{total_frames} 帧，{video_fps:.1f} fps") + + nframes = smart_nframes(total_frames, video_fps) + resized_h, resized_w = smart_resize(height, width, nframes) + + video_tokens = int( + math.ceil(nframes / FPS) * resized_h / IMAGE_FACTOR * resized_w / IMAGE_FACTOR + ) + 2 + print(f"抽取帧数：{nframes}，缩放后尺寸：{resized_w}x{resized_h}，video_tokens：{video_tokens}") + ``` + +- `audio_tokens` + + - `Qwen3.5-Omni系列`： + + - 输入音频计算公式：`总 Tokens 数 = 音频时长（单位：秒）* 7` + + - 输出音频计算公式：`总 Tokens 数 = 音频时长（单位：秒）* 12.5` + + - `Qwen3-Omni-Flash：输入与输出音频的计算公式均为总 Tokens 数 = 音频时长（单位：秒）* 12.5` + + - `Qwen-Omni-Turbo：输入与输出音频的计算公式均为总 Tokens 数 = 音频时长（单位：秒）* 25` + + + 若音频时长不足1秒，则按 1 秒计算。 + + +**免费额度** + +关于免费额度的领取、查询、使用方法等详情，请参见[新人免费额度](https://help.aliyun.com/zh/model-studio/new-free-quota)。 + +**限流** + +模型限流规则及常见问题，请参见[限流](https://help.aliyun.com/zh/model-studio/rate-limit)。 + +## **常见问题** + +### **Q：如何给 Qwen-Omni-Turbo 模型设置角色？** + +A：Qwen-Omni-Turbo模型在输出模态包括音频时**不支持设定 System Message，**即使您在 System Message 中说明：“你是XXX...”等角色信息，Qwen-Omni 的自我认知依然会是千问。 + +- **方法1（推荐）：**Qwen3-Omni-Flash模型已支持设定**System Message，**建议切换至该系列模型。 + +- **方法2：**在messages 数组的开头手动添加用于角色设定的 User Message 和 Assistant Message，达到给 Qwen-Omni 模型设置角色的效果。 + + **用于角色设定的示例代码** + + ## OpenAI 兼容 + + Python + + ``` + import os + from openai import OpenAI + + client = OpenAI( + # 若没有配置环境变量，请用阿里云百炼API Key将下行替换为：api_key="sk-xxx", + api_key=os.getenv("DASHSCOPE_API_KEY"), + base_url="https://dashscope.aliyuncs.com/compatible-mode/v1", + ) + + completion = client.chat.completions.create( + model="qwen-omni-turbo", + messages=[ + {"role": "user", "content": "你是一个商场的导购员，你负责的商品有手机、电脑、冰箱"}, + {"role": "assistant", "content": "好的，我记住了你的设定。"}, + {"role": "user", "content": "你是谁"}, + ], + # 设置输出数据的模态，当前支持两种：["text","audio"]、["text"] + modalities=["text", "audio"], + audio={"voice": "Tina", "format": "wav"}, + # stream 必须设置为 True，否则会报错 + stream=True, + stream_options={"include_usage": True}, + ) + + for chunk in completion: + if chunk.choices: + print(chunk.choices[0].delta) + else: + print(chunk.usage) + ``` + + Node.js + + ``` + import OpenAI from "openai"; + + const openai = new OpenAI( + { + // 若没有配置环境变量，请用阿里云百炼API Key将下行替换为：apiKey: "sk-xxx", + apiKey: process.env.DASHSCOPE_API_KEY, + baseURL: "https://dashscope.aliyuncs.com/compatible-mode/v1" + } + ); + const completion = await openai.chat.completions.create({ + model: "qwen-omni-turbo", + messages: [ + { role: "user", content: "你是一个商场的导购员，你负责的商品有手机、电脑、冰箱" }, + { role: "assistant", content: "好的，我记住了你的设定。" }, + { role: "user", content: "你是谁？" } + ], + stream: true, + stream_options: { + include_usage: true + }, + modalities: ["text", "audio"], + audio: { voice: "Tina", format: "wav" } + }); + + for await (const chunk of completion) { + if (Array.isArray(chunk.choices) && chunk.choices.length > 0) { + console.log(chunk.choices[0].delta); + } else { + console.log(chunk.usage); + } + } + ``` + + curl + + ``` + curl -X POST https://dashscope.aliyuncs.com/compatible-mode/v1/chat/completions \ + -H "Authorization: Bearer $DASHSCOPE_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "qwen-omni-turbo", + "messages": [ + { + "role": "user", + "content": "你是一个商场的导购员，你负责的商品有手机、电脑、冰箱" + }, + { + "role": "assistant", + "content": "好的，我记住了你的设定。" + }, + { + "role": "user", + "content": "你是谁？" + } + ], + "stream":true, + "stream_options":{ + "include_usage":true + }, + "modalities":["text","audio"], + "audio":{"voice":"Tina","format":"wav"} + }' + ``` + + +## 错误码 + +如果模型调用失败并返回报错信息，请参见[错误信息](https://help.aliyun.com/zh/model-studio/error-code)进行解决。 + +## **音色列表** + +Qwen-Omni模型的音色列表可参见[音色列表](https://help.aliyun.com/zh/model-studio/omni-voice-list)。 \ No newline at end of file diff --git a/app/api/asr/prompt.md b/app/api/asr/prompt.md new file mode 100644 index 0000000..0074c13 --- /dev/null +++ b/app/api/asr/prompt.md @@ -0,0 +1,39 @@ +你将接收一段音频、视频中固定抽取的三帧画面，并可能收到一段视频描述作为辅助信息。请完成：语音活动检测（VAD）、声源分类、语音转文字，以及内容标签分类。 + +只能输出一个合法 JSON 对象，不要输出 Markdown、解释文字或代码块。 + +`content_tags` 必须从以下固定标签中选择 1 到 3 个，不能自造标签： + +美女、游戏、时政、美食、旅行、体育、科技、教育、宠物、音乐、影视、财经、生活、其他 + +**输出 JSON Schema（示例）** + +```json +{ + "speech_detected": true, + "language": "zh-CN", + "audio_type": null, + "transcript": ["大家好，我是xxx", "欢迎来到今天的视频", "今天我们来聊聊AI的未来"], + "non_speech_summary": null, + "content_tags": ["科技", "教育"] +} +``` + +**当无发言时返回：** + +```json +{ + "speech_detected": false, + "language": null, + "audio_type": "music | ambience | animal | mechanical | other", + "transcript": [], + "non_speech_summary": "示例：纯音乐-钢琴独奏，节奏舒缓；或 环境声-雨声伴随雷鸣。", + "content_tags": ["音乐"] +} +``` + +** 规则补充 ** + +* 允许`language` 为多标签（如 "zh-CN,en"）或为 `null`（无发言时）。 +* `transcript` 按自然语义分句，每段尽量保持完整短句。 +* 必须结合三帧画面、音频内容和辅助视频描述判断标签；信息不足时使用 `其他`。 diff --git a/app/api/asr/route.ts b/app/api/asr/route.ts new file mode 100644 index 0000000..5bc7f74 --- /dev/null +++ b/app/api/asr/route.ts @@ -0,0 +1,22 @@ +export const runtime = "nodejs"; + +import { type NextRequest, NextResponse } from "next/server"; +import { transcriptAweme } from "."; + +export async function GET(req: NextRequest) { + const { searchParams } = new URL(req.url); + const awemeId = searchParams.get("awemeId"); + if (!awemeId) { + return NextResponse.json( + { error: "Missing awemeId parameter" }, + { status: 400 }, + ); + } + try { + const result = await transcriptAweme(awemeId, /* {log: (msg) => console.log(`[ASR] ${msg}`)} */); + return NextResponse.json(result); + } catch (error) { + console.error("ASR failed:", error); + return NextResponse.json({ error: "ASR failed" }, { status: 500 }); + } +} diff --git a/app/api/feed/route.ts b/app/api/feed/route.ts index 1ef8ec4..301220c 100644 --- a/app/api/feed/route.ts +++ b/app/api/feed/route.ts @@ -1,7 +1,8 @@ -import { NextRequest, NextResponse } from "next/server"; -import { prisma } from "@/lib/prisma"; +import { type NextRequest, NextResponse } from "next/server"; import type { FeedItem, FeedResponse } from "@/app/types/feed"; +import { isAsrContentTag } from "@/lib/asr-tags"; import { getFileUrl } from "@/lib/minio"; +import { prisma } from "@/lib/prisma"; // Contract // Inputs: search params { before?: ISOString, limit?: number } @@ -11,24 +12,44 @@ export async function GET(req: NextRequest) { const { searchParams } = new URL(req.url); const limitParam = searchParams.get("limit"); const beforeParam = searchParams.get("before"); + const rawTag = (searchParams.get("tag") || "").trim(); + const selectedTag = rawTag && isAsrContentTag(rawTag) ? rawTag : null; + + if (rawTag && !selectedTag) { + return NextResponse.json({ error: "Invalid tag" }, { status: 400 }); + } const limit = Math.min(Math.max(Number(limitParam ?? "24"), 1), 60); // 1..60 const before = beforeParam ? new Date(beforeParam) : null; + const videoWhere = { + ...(before ? { created_at: { lt: before } } : {}), + ...(selectedTag + ? { transcript: { is: { content_tags: { has: selectedTag } } } } + : {}), + }; // fetch chunk from both tables const [videos, posts] = await Promise.all([ prisma.video.findMany({ - where: before ? { created_at: { lt: before } } : undefined, + where: videoWhere, orderBy: { created_at: "desc" }, take: limit, - include: { author: true }, - }), - prisma.imagePost.findMany({ - where: before ? { created_at: { lt: before } } : undefined, - orderBy: { created_at: "desc" }, - take: limit, - include: { author: true, images: { orderBy: { order: "asc" }, take: 1 } }, + include: { + author: true, + transcript: { select: { content_tags: true } }, + }, }), + selectedTag + ? Promise.resolve([]) + : prisma.imagePost.findMany({ + where: before ? { created_at: { lt: before } } : undefined, + orderBy: { created_at: "desc" }, + take: limit, + include: { + author: true, + images: { orderBy: { order: "asc" }, take: 1 }, + }, + }), ]); const merged: FeedItem[] = [ @@ -47,29 +68,33 @@ export async function GET(req: NextRequest) { sec_uid: v.author.sec_uid, }, likes: Number(v.digg_count), + content_tags: v.transcript?.content_tags ?? [], })), - ...posts.map((p) => ({ - type: "image" as const, - aweme_id: p.aweme_id, - created_at: p.created_at, - desc: p.desc, - cover_url: getFileUrl(p.images?.[0]?.url ?? null), - width: p.images?.[0]?.width ?? null, - height: p.images?.[0]?.height ?? null, - author: { - nickname: p.author.nickname, - avatar_url: getFileUrl(p.author.avatar_url ?? ""), - sec_uid: p.author.sec_uid, - }, - likes: Number(p.digg_count), - })), + ...posts.map((p) => { + const coverPath = p.images?.[0]?.url; + return { + type: "image" as const, + aweme_id: p.aweme_id, + created_at: p.created_at, + desc: p.desc, + cover_url: coverPath ? getFileUrl(coverPath) : null, + width: p.images?.[0]?.width ?? null, + height: p.images?.[0]?.height ?? null, + author: { + nickname: p.author.nickname, + avatar_url: getFileUrl(p.author.avatar_url ?? ""), + sec_uid: p.author.sec_uid, + }, + likes: Number(p.digg_count), + }; + }), ] .sort((a, b) => +new Date(b.created_at) - +new Date(a.created_at)) .slice(0, limit); const nextCursor = merged.length > 0 - ? new Date(merged[merged.length - 1].created_at as any).toISOString() + ? new Date(merged[merged.length - 1].created_at).toISOString() : null; const payload: FeedResponse = { items: merged, nextCursor }; return NextResponse.json(payload); diff --git a/app/api/fetcher/index.ts b/app/api/fetcher/index.ts index 069cf79..52f6e2d 100644 --- a/app/api/fetcher/index.ts +++ b/app/api/fetcher/index.ts @@ -21,7 +21,7 @@ import { releaseIsolatedContext, } from "@/app/api/fetcher/browser"; import { extractFirstFrame } from "@/app/api/media"; -import { transcriptAweme } from "../stt"; +import { transcriptAweme } from "../asr"; const DETAIL_PATH = "/aweme/v1/web/aweme/detail/"; const COMMENT_PATH = "/aweme/v1/web/comment/list/"; diff --git a/app/api/media.ts b/app/api/media.ts index a8786c9..972c0df 100644 --- a/app/api/media.ts +++ b/app/api/media.ts @@ -1,11 +1,28 @@ -import { execFile } from "child_process"; -import { promises as fs } from "fs"; -import os from "os"; -import path from "path"; -import { promisify } from "util"; +import { execFile } from "node:child_process"; +import { promises as fs } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { promisify } from "node:util"; const execFileAsync = promisify(execFile); +function getErrorMessage(error: unknown) { + return error instanceof Error ? error.message : String(error); +} + +function getErrorCode(error: unknown) { + return typeof error === "object" && error !== null && "code" in error + ? String((error as { code?: unknown }).code) + : null; +} + +function isMissingExecutableError(error: unknown) { + return ( + getErrorCode(error) === "ENOENT" || + /not found|is not recognized/i.test(getErrorMessage(error)) + ); +} + /** * 使用 ffmpeg 从视频二进制中提取第一帧，返回 JPEG buffer */ @@ -40,18 +57,14 @@ export async function extractFirstFrame( await execFileAsync(ffmpegCmd, args, { windowsHide: true }); const img = await fs.readFile(outPath); return { buffer: img, contentType: "image/jpeg", ext: "jpg" }; - } catch (e: any) { - if ( - e && - (e.code === "ENOENT" || - /not found|is not recognized/i.test(String(e.message))) - ) { + } catch (error) { + if (isMissingExecutableError(error)) { console.warn( "系统未检测到 ffmpeg，可安装并配置 PATH 或设置 FFMPEG_PATH 后启用封面提取。", ); return null; } - throw e; + throw error; } finally { try { await fs.unlink(inPath); @@ -62,6 +75,13 @@ export async function extractFirstFrame( } } +export type ExtractedVideoFrame = { + buffer: Buffer; + contentType: string; + ext: string; + timestampMs: number; +}; + /** * 使用 ffprobe 获取视频时长（毫秒） */ @@ -88,20 +108,16 @@ export async function getVideoDuration( windowsHide: true, }); const durationSeconds = parseFloat(stdout.trim()); - if (isNaN(durationSeconds)) return null; + if (Number.isNaN(durationSeconds)) return null; return Math.round(durationSeconds * 1000); // 转换为毫秒 - } catch (e: any) { - if ( - e && - (e.code === "ENOENT" || - /not found|is not recognized/i.test(String(e.message))) - ) { + } catch (error) { + if (isMissingExecutableError(error)) { console.warn( "系统未检测到 ffprobe，可安装并配置 PATH 或设置 FFPROBE_PATH 后启用时长提取。", ); return null; } - console.warn(`获取视频时长失败: ${e?.message || e}`); + console.warn(`获取视频时长失败: ${getErrorMessage(error)}`); return null; } finally { try { @@ -110,6 +126,102 @@ export async function getVideoDuration( } } +function getFrameTimestamps(durationMs: number | null, count: number) { + if (!durationMs || durationMs <= 0) { + return Array.from({ length: count }, (_, index) => index * 1000); + } + + if (count === 1) return [0]; + + const maxTimestamp = Math.max(0, durationMs - 200); + if (count === 3) { + return [ + Math.min(maxTimestamp, Math.max(0, Math.round(durationMs * 0.1))), + Math.min(maxTimestamp, Math.max(0, Math.round(durationMs * 0.5))), + Math.min(maxTimestamp, Math.max(0, Math.round(durationMs * 0.9))), + ]; + } + + return Array.from({ length: count }, (_, index) => { + const ratio = count === 1 ? 0 : index / (count - 1); + return Math.min(maxTimestamp, Math.max(0, Math.round(durationMs * ratio))); + }); +} + +/** + * 使用 ffmpeg 从视频中提取固定数量的 JPEG 帧。 + */ +export async function extractVideoFrames( + videoBuffer: Buffer, + opts?: { count?: number; durationMs?: number | null }, +): Promise { + const ffmpegCmd = process.env.FFMPEG_PATH || "ffmpeg"; + const tmpDir = os.tmpdir(); + const base = `dy_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`; + const inPath = path.join(tmpDir, `${base}.mp4`); + const count = Math.max(1, opts?.count ?? 3); + const outPaths = Array.from({ length: count }, (_, index) => + path.join(tmpDir, `${base}_${index}.jpg`), + ); + + try { + await fs.writeFile(inPath, videoBuffer); + const durationMs = + opts?.durationMs ?? (await getVideoDuration(videoBuffer)); + const timestamps = getFrameTimestamps(durationMs, count); + + const frames: ExtractedVideoFrame[] = []; + for (let index = 0; index < count; index++) { + const timestampSeconds = Math.max(0, timestamps[index] / 1000).toFixed(3); + const args = [ + "-hide_banner", + "-loglevel", + "error", + "-ss", + timestampSeconds, + "-i", + inPath, + "-frames:v", + "1", + "-q:v", + "3", + "-vf", + "scale=768:-2:force_original_aspect_ratio=decrease", + "-f", + "image2", + "-y", + outPaths[index], + ]; + await execFileAsync(ffmpegCmd, args, { windowsHide: true }); + frames.push({ + buffer: await fs.readFile(outPaths[index]), + contentType: "image/jpeg", + ext: "jpg", + timestampMs: timestamps[index], + }); + } + + return frames; + } catch (error) { + if (isMissingExecutableError(error)) { + console.warn( + "系统未检测到 ffmpeg，可安装并配置 PATH 或设置 FFMPEG_PATH 后启用视频帧提取。", + ); + return []; + } + throw error; + } finally { + try { + await fs.unlink(inPath); + } catch {} + for (const outPath of outPaths) { + try { + await fs.unlink(outPath); + } catch {} + } + } +} + /** * 使用 ffmpeg 将视频二进制转换为音频，返回音频 Buffer * @@ -189,20 +301,16 @@ export async function extractAudio( await execFileAsync(ffmpegCmd, args, { windowsHide: true }); const audio = await fs.readFile(outPath); return { buffer: audio, contentType, ext }; - } catch (e: any) { - if ( - e && - (e.code === "ENOENT" || - /not found|is not recognized/i.test(String(e.message))) - ) { + } catch (error) { + if (isMissingExecutableError(error)) { console.warn( "系统未检测到 ffmpeg，可安装并配置 PATH 或设置 FFMPEG_PATH 后启用音频提取。", ); return null; } // 一些环境可能缺少特定编码器（如 libmp3lame），提示并抛出原始错误 - console.warn(`提取音频失败: ${e?.message || e}`); - throw e; + console.warn(`提取音频失败: ${getErrorMessage(error)}`); + throw error; } finally { try { await fs.unlink(inPath); diff --git a/app/api/search/route.ts b/app/api/search/route.ts index 4bc728c..75668a0 100644 --- a/app/api/search/route.ts +++ b/app/api/search/route.ts @@ -1,11 +1,32 @@ +import { NextResponse } from "next/server"; +import { isAsrContentTag } from "@/lib/asr-tags"; import { json } from "@/lib/json"; import { getFileUrl } from "@/lib/minio"; -import { prisma } from "@/lib/prisma"; // 你的 Prisma 客户端实例 -import { NextResponse } from "next/server"; +import { prisma } from "@/lib/prisma"; + +type SearchRow = { + id: string; + awemeId: string; + type: "video" | "image"; + rank: number; + snippet: string; + contentTags: string[]; +}; + +function sanitizeSnippetHtml(snippet: string) { + return snippet + .replaceAll("&", "&") + .replaceAll("<", "<") + .replaceAll(">", ">") + .replaceAll("<mark>", "") + .replaceAll("</mark>", ""); +} export async function GET(req: Request) { const { searchParams } = new URL(req.url); const q = (searchParams.get("q") || "").trim(); + const rawTag = (searchParams.get("tag") || "").trim(); + const contentTag = rawTag && isAsrContentTag(rawTag) ? rawTag : null; const page = Math.max(1, Number(searchParams.get("page") || 1)); const limit = Math.min( 50, @@ -13,7 +34,11 @@ export async function GET(req: Request) { ); const offset = (page - 1) * limit; - if (!q) { + if (rawTag && !contentTag) { + return NextResponse.json({ error: "Invalid tag" }, { status: 400 }); + } + + if (!q && !contentTag) { return NextResponse.json({ results: [], total: 0 }); } @@ -23,90 +48,117 @@ export async function GET(req: Request) { // - 合并搜索来源：VideoTranscript.transcript_zh_tsv、Video.desc 与 ImagePost.desc // - 使用 UNION ALL 合并视频和图文的搜索结果 // - rank 取两者匹配得分的最大值，snippet 优先展示转写匹配，否则展示 desc 匹配 - const rows = await prisma.$queryRaw< - { - id: string; - awemeId: string; - type: "video" | "image"; - rank: number; - snippet: string; - }[] - >` - WITH tsq AS ( - SELECT websearch_to_tsquery('zhcfg', ${q}) AS query - ) - -- 视频搜索结果 - SELECT - COALESCE(vt.id, 'desc:' || v.aweme_id) AS id, - v.aweme_id AS "awemeId", - 'video'::text AS type, - GREATEST( - COALESCE(ts_rank(vt.transcript_zh_tsv, tsq.query), 0), - COALESCE(ts_rank(to_tsvector('zhcfg', v.desc), tsq.query), 0) - ) AS rank, - CASE - WHEN vt.transcript_zh_tsv @@ tsq.query THEN + const rows = q + ? await prisma.$queryRaw` + WITH tsq AS ( + SELECT websearch_to_tsquery('zhcfg', ${q}) AS query + ) + -- 视频搜索结果 + SELECT + COALESCE(vt.id, 'desc:' || v.aweme_id) AS id, + v.aweme_id AS "awemeId", + 'video'::text AS type, + GREATEST( + COALESCE(ts_rank(vt.transcript_zh_tsv, tsq.query), 0), + COALESCE(ts_rank(to_tsvector('zhcfg', v.desc), tsq.query), 0) + ) AS rank, + CASE + WHEN vt.transcript_zh_tsv @@ tsq.query THEN + ts_headline( + 'zhcfg', + array_to_string(vt.transcript, ' '), + tsq.query, + 'StartSel=,StopSel=,MaxFragments=2,MinWords=2,MaxWords=20' + ) + ELSE + ts_headline( + 'zhcfg', + v.desc, + tsq.query, + 'StartSel=,StopSel=,MaxFragments=2,MinWords=2,MaxWords=20' + ) + END AS snippet, + COALESCE(vt.content_tags, ARRAY[]::text[]) AS "contentTags" + FROM "Video" v + LEFT JOIN "VideoTranscript" vt ON vt."videoId" = v.aweme_id, + tsq + WHERE + (vt.transcript_zh_tsv @@ tsq.query OR to_tsvector('zhcfg', v.desc) @@ tsq.query) + AND CASE + WHEN ${contentTag}::text IS NULL THEN TRUE + ELSE vt.content_tags @> ARRAY[${contentTag}]::text[] + END + + UNION ALL + + -- 图文搜索结果 + SELECT + 'img_desc:' || ip.aweme_id AS id, + ip.aweme_id AS "awemeId", + 'image'::text AS type, + ts_rank(to_tsvector('zhcfg', ip.desc), tsq.query) AS rank, ts_headline( 'zhcfg', - array_to_string(vt.transcript, ' '), + ip.desc, tsq.query, 'StartSel=,StopSel=,MaxFragments=2,MinWords=2,MaxWords=20' - ) - ELSE - ts_headline( - 'zhcfg', + ) AS snippet, + ARRAY[]::text[] AS "contentTags" + FROM "ImagePost" ip, tsq + WHERE to_tsvector('zhcfg', ip.desc) @@ tsq.query + AND ${contentTag}::text IS NULL + + ORDER BY rank DESC + OFFSET ${offset} + LIMIT ${limit}; + ` + : await prisma.$queryRaw` + SELECT + vt.id AS id, + v.aweme_id AS "awemeId", + 'video'::text AS type, + 1.0::double precision AS rank, + COALESCE( + NULLIF(array_to_string(vt.transcript[1:2], ' '), ''), + vt.non_speech_summary, v.desc, - tsq.query, - 'StartSel=,StopSel=,MaxFragments=2,MinWords=2,MaxWords=20' - ) - END AS snippet - FROM "Video" v - LEFT JOIN "VideoTranscript" vt ON vt."videoId" = v.aweme_id, - tsq - WHERE - (vt.transcript_zh_tsv @@ tsq.query OR to_tsvector('zhcfg', v.desc) @@ tsq.query) - - UNION ALL - - -- 图文搜索结果 - SELECT - 'img_desc:' || ip.aweme_id AS id, - ip.aweme_id AS "awemeId", - 'image'::text AS type, - ts_rank(to_tsvector('zhcfg', ip.desc), tsq.query) AS rank, - ts_headline( - 'zhcfg', - ip.desc, - tsq.query, - 'StartSel=,StopSel=,MaxFragments=2,MinWords=2,MaxWords=20' - ) AS snippet - FROM "ImagePost" ip, tsq - WHERE to_tsvector('zhcfg', ip.desc) @@ tsq.query - - ORDER BY rank DESC - OFFSET ${offset} - LIMIT ${limit}; - `; + '标签匹配' + ) AS snippet, + vt.content_tags AS "contentTags" + FROM "VideoTranscript" vt + JOIN "Video" v ON v.aweme_id = vt."videoId" + WHERE vt.content_tags @> ARRAY[${contentTag}]::text[] + ORDER BY v.created_at DESC + OFFSET ${offset} + LIMIT ${limit}; + `; // 查询总数（参数化，避免注入） - const totalRows = await prisma.$queryRaw< - { - count: number; - }[] - >` - WITH tsq AS ( - SELECT websearch_to_tsquery('zhcfg', ${q}) AS query - ) - SELECT ( - (SELECT COUNT(*)::int FROM "Video" v - LEFT JOIN "VideoTranscript" vt ON vt."videoId" = v.aweme_id - WHERE (vt.transcript_zh_tsv @@ tsq.query OR to_tsvector('zhcfg', v.desc) @@ tsq.query)) - + - (SELECT COUNT(*)::int FROM "ImagePost" ip - WHERE to_tsvector('zhcfg', ip.desc) @@ tsq.query) - ) AS count - FROM tsq; - `; + const totalRows = q + ? await prisma.$queryRaw<{ count: number }[]>` + WITH tsq AS ( + SELECT websearch_to_tsquery('zhcfg', ${q}) AS query + ) + SELECT ( + (SELECT COUNT(*)::int FROM "Video" v + LEFT JOIN "VideoTranscript" vt ON vt."videoId" = v.aweme_id + WHERE (vt.transcript_zh_tsv @@ tsq.query OR to_tsvector('zhcfg', v.desc) @@ tsq.query) + AND CASE + WHEN ${contentTag}::text IS NULL THEN TRUE + ELSE vt.content_tags @> ARRAY[${contentTag}]::text[] + END) + + + (SELECT COUNT(*)::int FROM "ImagePost" ip + WHERE to_tsvector('zhcfg', ip.desc) @@ tsq.query + AND ${contentTag}::text IS NULL) + ) AS count + FROM tsq; + ` + : await prisma.$queryRaw<{ count: number }[]>` + SELECT COUNT(*)::int AS count + FROM "VideoTranscript" vt + WHERE vt.content_tags @> ARRAY[${contentTag}]::text[]; + `; // 分离视频和图文ID const videoIds = rows @@ -176,6 +228,7 @@ export async function GET(req: Request) { return json({ results: rows.map((r) => ({ ...r, + snippet: sanitizeSnippetHtml(r.snippet), video: r.type === "video" ? videos.find((v) => v.aweme_id === r.awemeId) diff --git a/app/api/stt/index.ts b/app/api/stt/index.ts deleted file mode 100644 index 4c05c7f..0000000 --- a/app/api/stt/index.ts +++ /dev/null @@ -1,113 +0,0 @@ -import fs from "fs"; -import OpenAI from "openai"; -import prompt from "./prompt.md"; -import { prisma } from "@/lib/prisma"; -import { downloadFile } from "@/lib/minio"; -import { extractAudio } from "../media"; -import { z } from "zod"; -import { zodResponseFormat } from "openai/helpers/zod"; -import { ProxyAgent, setGlobalDispatcher } from "undici"; - -const proxy = process.env.HTTPS_PROXY || process.env.HTTP_PROXY; -if (proxy) { - setGlobalDispatcher(new ProxyAgent(proxy)); -} - -const client = new OpenAI({ - apiKey: process.env.OPENAI_API_KEY, - baseURL: process.env.OPENAI_API_BASE_URL, -}); - -// Zod schema aligned with prompt.md -export const SttSchema = z - .object({ - speech_detected: z.boolean(), - language: z.string().min(1).nullable(), - audio_type: z.string().nullable(), - transcript: z.array(z.string()), - non_speech_summary: z.string().nullable(), - }) - .strict(); - -export type SttResult = z.infer; - -async function transcriptAudio(audio: Buffer | string) { - if (typeof audio === "string") { - audio = fs.readFileSync(audio); - } - const base64Audio = Buffer.from(audio).toString("base64"); - - const completion = await client.chat.completions.create({ - model: "gemini-2.5-flash", - messages: [ - { - role: "user", - content: [ - { type: "text", text: prompt }, - { - type: "input_audio", - input_audio: { data: base64Audio, format: "mp3" }, - }, - ], - }, - ], - response_format: zodResponseFormat(SttSchema, "stt_result"), - }); - - const data = completion.choices?.[0]?.message; - console.log("转写结果", data.content); - - if (!data || !data.content) { - throw new Error("No STT result returned from model"); - } - try { - return JSON.parse(data.content) as SttResult; - } catch (e) { - throw new Error("Failed to parse STT result JSON"); - } -} - -export async function transcriptAweme(awemeId: string): Promise { - const aweme = await prisma.video.findUnique({ - where: { aweme_id: awemeId }, - }); - if (!aweme) { - throw new Error("Aweme not found or aweme is not a video post"); - } - const vPath = aweme.video_url; - const buffer = await downloadFile(vPath); - - const audioDat = await extractAudio(buffer, { - format: "mp3", - bitrateKbps: 128, - }); - - if (!audioDat || !audioDat.buffer) { - throw new Error("Failed to extract audio from video"); - } - - // 调用大模型生成转写结果 - const result = await transcriptAudio(audioDat.buffer); - - // 将转写结果持久化到数据库 - await prisma.videoTranscript.upsert({ - where: { videoId: awemeId }, - update: { - speech_detected: result.speech_detected, - language: result.language, - audio_type: result.audio_type, - transcript: result.transcript, - non_speech_summary: result.non_speech_summary, - }, - create: { - videoId: awemeId, - speech_detected: result.speech_detected, - language: result.language, - audio_type: result.audio_type, - transcript: result.transcript, - non_speech_summary: result.non_speech_summary, - }, - }); - - return result; -} diff --git a/app/api/stt/prompt.md b/app/api/stt/prompt.md deleted file mode 100644 index ef32b13..0000000 --- a/app/api/stt/prompt.md +++ /dev/null @@ -1,29 +0,0 @@ -你将接收一段音频。请完成：你将接收一段音频。语音活动检测（VAD）与声源分类。 - -**输出 JSON Schema（示例）** - -```json -{ - "speech_detected": true, - "language": "zh-CN", - "audio_type": null, - "transcript": ["大家好，我是xxx", "欢迎来到今天的视频", "今天我们来聊聊AI的未来"], - "non_speech_summary": null, -} -``` - -**当无发言时返回：** - -```json -{ - "speech_detected": false, - "language": null, - "audio_type": "music | ambience | animal | mechanical | other", - "transcript": [], - "non_speech_summary": "示例：纯音乐-钢琴独奏，节奏舒缓；或 环境声-雨声伴随雷鸣。", -} -``` - -** 规则补充 ** - -* 允许`language` 为多标签（如 "zh-CN,en"）或为 `null`（无发言时）。 diff --git a/app/api/stt/route.ts b/app/api/stt/route.ts deleted file mode 100644 index 40ca475..0000000 --- a/app/api/stt/route.ts +++ /dev/null @@ -1,23 +0,0 @@ -export const runtime = "nodejs"; -import { NextRequest, NextResponse } from "next/server"; -import { prisma } from "@/lib/prisma"; -import type { FeedItem, FeedResponse } from "@/app/types/feed"; -import { getFileUrl } from "@/lib/minio"; -import { transcriptAweme } from "."; - -// Contract -// Inputs: search params { before?: ISOString, limit?: number } -// Output: { items: FeedItem[], nextCursor: ISOString | null } - -export async function GET(req: NextRequest) { - const { searchParams } = new URL(req.url); - const awemeId = searchParams.get("awemeId"); - if (!awemeId) { - return NextResponse.json( - { error: "Missing awemeId parameter" }, - { status: 400 }, - ); - } - const script = await transcriptAweme(awemeId); - return NextResponse.json(script); -} diff --git a/app/aweme/[awemeId]/components/TranscriptPanel.tsx b/app/aweme/[awemeId]/components/TranscriptPanel.tsx index e2cb36b..bde578b 100644 --- a/app/aweme/[awemeId]/components/TranscriptPanel.tsx +++ b/app/aweme/[awemeId]/components/TranscriptPanel.tsx @@ -1,6 +1,6 @@ "use client"; -import { X, Copy, CheckCheck, Check } from "lucide-react"; +import { Check, CheckCheck, Copy, X } from "lucide-react"; import { useState } from "react"; import type { VideoTranscript } from "../types"; @@ -38,18 +38,23 @@ export function TranscriptPanel({ if (!transcript?.speech_detected || !transcript.transcript?.length) { return ( -
+
+
+
+
+ {ASR_CONTENT_TAGS.map((tag) => ( + + ))} +
+
{/* 结果区域 */} @@ -165,7 +229,7 @@ export default function SearchClient({
)} - {!loading && !searched && !query && ( + {!loading && !searched && !query && !selectedTag && (

输入关键词开始搜索

@@ -207,11 +271,18 @@ export default function SearchClient({ target="_blank" className="flex-shrink-0 w-64 h-80 sm:w-70 sm:h-96 rounded-xl overflow-hidden bg-zinc-900 group" > - {content.desc} + {content.cover_url ? ( + {content.desc} + ) : ( +
+ )} {/* 右侧：信息区 */} @@ -231,6 +302,14 @@ export default function SearchClient({ 匹配度: {(item.rank * 100).toFixed(1)}% + {item.contentTags.map((tag) => ( + + {tag} + + ))}
{/* 描述 */} @@ -245,11 +324,18 @@ export default function SearchClient({ {/* 作者信息 */}
- {content.author.nickname} + {content.author.avatar_url ? ( + {content.author.nickname} + ) : ( +
+ )} {content.author.nickname} @@ -264,11 +350,12 @@ export default function SearchClient({
+ > + {renderSnippet(item.snippet)} +
diff --git a/app/types/feed.ts b/app/types/feed.ts index 2483c31..179ae4e 100644 --- a/app/types/feed.ts +++ b/app/types/feed.ts @@ -15,6 +15,7 @@ export type FeedItem = ( cover_url: string | null; width?: number | null; height?: number | null; + content_tags?: string[]; }; export interface FeedResponse { diff --git a/lib/asr-tags.ts b/lib/asr-tags.ts new file mode 100644 index 0000000..f5484f3 --- /dev/null +++ b/lib/asr-tags.ts @@ -0,0 +1,40 @@ +export const ASR_CONTENT_TAGS = [ + "美女", + "游戏", + "时政", + "美食", + "旅行", + "体育", + "科技", + "教育", + "宠物", + "音乐", + "影视", + "财经", + "生活", + "其他", +] as const; + +export type AsrContentTag = (typeof ASR_CONTENT_TAGS)[number]; + +const asrContentTagSet = new Set(ASR_CONTENT_TAGS); + +export function isAsrContentTag(value: string): value is AsrContentTag { + return asrContentTagSet.has(value); +} + +export function normalizeAsrContentTags(tags: unknown): AsrContentTag[] { + const rawTags = Array.isArray(tags) + ? tags + : typeof tags === "string" + ? tags.split(/[、,，\s]+/).filter(Boolean) + : []; + + const normalized = rawTags.filter( + (tag): tag is AsrContentTag => + typeof tag === "string" && isAsrContentTag(tag), + ); + + const uniqueTags = [...new Set(normalized)].slice(0, 3); + return uniqueTags.length > 0 ? uniqueTags : ["其他"]; +} diff --git a/package.json b/package.json index bab48d4..683d6ee 100644 --- a/package.json +++ b/package.json @@ -6,6 +6,7 @@ "dev": "next dev --turbopack", "build": "next build --turbopack", "start": "next start", + "asr": "node scripts/asr-cli.cjs", "lint": "biome check", "format": "biome format --write" }, @@ -21,7 +22,6 @@ "puppeteer-extra-plugin-stealth": "^2.11.2", "react": "19.1.0", "react-dom": "19.1.0", - "undici": "^7.16.0", "zod": "^4.1.12" }, "devDependencies": { diff --git a/prisma/migrations/20260429_add_asr_content_tags/migration.sql b/prisma/migrations/20260429_add_asr_content_tags/migration.sql new file mode 100644 index 0000000..dd2b163 --- /dev/null +++ b/prisma/migrations/20260429_add_asr_content_tags/migration.sql @@ -0,0 +1,5 @@ +ALTER TABLE "VideoTranscript" +ADD COLUMN "content_tags" TEXT[] NOT NULL DEFAULT ARRAY[]::TEXT[]; + +CREATE INDEX "video_transcript_content_tags_idx" +ON "VideoTranscript" USING GIN ("content_tags"); \ No newline at end of file diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 20c3e17..43dda07 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -145,11 +145,13 @@ model VideoTranscript { audio_type String? transcript String[] non_speech_summary String? + content_tags String[] @default([]) createdAt DateTime @default(now()) updatedAt DateTime @updatedAt transcript_zh_tsv Unsupported("tsvector")? video Video @relation(fields: [videoId], references: [aweme_id], onDelete: Cascade) @@index([videoId]) + @@index([content_tags], map: "video_transcript_content_tags_idx", type: Gin) @@index([transcript_zh_tsv], map: "video_transcript_zh_idx", type: Gin) } diff --git a/scripts/asr-cli.cjs b/scripts/asr-cli.cjs new file mode 100755 index 0000000..374391d --- /dev/null +++ b/scripts/asr-cli.cjs @@ -0,0 +1,135 @@ +#!/usr/bin/env node +const fs = require("node:fs"); +const Module = require("node:module"); +const path = require("node:path"); + +const rootDir = path.resolve(__dirname, ".."); + +function loadEnvFile(filePath, override = false) { + if (!fs.existsSync(filePath)) return; + + const lines = fs.readFileSync(filePath, "utf8").split(/\r?\n/); + for (const line of lines) { + const trimmed = line.trim(); + if (!trimmed || trimmed.startsWith("#")) continue; + + const normalized = trimmed.startsWith("export ") + ? trimmed.slice("export ".length).trim() + : trimmed; + const match = normalized.match(/^([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*)$/); + if (!match) continue; + + const key = match[1]; + let value = match[2].trim(); + if ( + (value.startsWith('"') && value.endsWith('"')) || + (value.startsWith("'") && value.endsWith("'")) + ) { + value = value.slice(1, -1); + } + value = value.replace(/\\n/g, "\n"); + + if (override || process.env[key] === undefined) { + process.env[key] = value; + } + } +} + +function registerTypeScriptRuntime() { + const ts = require("typescript"); + const originalResolveFilename = Module._resolveFilename; + + Module._resolveFilename = function resolveFilename( + request, + parent, + isMain, + options, + ) { + if (request.startsWith("@/")) { + return originalResolveFilename.call( + this, + path.join(rootDir, request.slice(2)), + parent, + isMain, + options, + ); + } + return originalResolveFilename.call(this, request, parent, isMain, options); + }; + + const compile = (module, filename) => { + const source = fs.readFileSync(filename, "utf8"); + const output = ts.transpileModule(source, { + fileName: filename, + compilerOptions: { + module: ts.ModuleKind.CommonJS, + target: ts.ScriptTarget.ES2022, + esModuleInterop: true, + resolveJsonModule: true, + jsx: ts.JsxEmit.ReactJSX, + moduleResolution: ts.ModuleResolutionKind.Node10, + }, + }).outputText; + module._compile(output, filename); + }; + + require.extensions[".ts"] = compile; + require.extensions[".tsx"] = compile; + require.extensions[".md"] = (module, filename) => { + module.exports = fs.readFileSync(filename, "utf8"); + }; +} + +function getAwemeId() { + const args = process.argv.slice(2); + const fromFlag = args.find((arg) => arg.startsWith("--aweme-id=")); + if (fromFlag) return fromFlag.slice("--aweme-id=".length).trim(); + + const flagIndex = args.findIndex( + (arg) => arg === "--aweme-id" || arg === "-a", + ); + if (flagIndex >= 0) return (args[flagIndex + 1] || "").trim(); + + return (args.find((arg) => !arg.startsWith("-")) || "").trim(); +} + +function usage() { + console.log("用法: npm run asr -- "); + console.log("或: node scripts/asr-cli.cjs --aweme-id "); +} + +function logStep(message) { + const now = new Date().toISOString(); + console.log(`[asr ${now}] ${message}`); +} + +async function main() { + loadEnvFile(path.join(rootDir, ".env")); + loadEnvFile(path.join(rootDir, ".env.local"), true); + registerTypeScriptRuntime(); + + const awemeId = getAwemeId(); + if (!awemeId) { + usage(); + process.exitCode = 1; + return; + } + + logStep(`开始执行 aweme_id=${awemeId}`); + + const { transcriptAweme } = require(path.join(rootDir, "app/api/asr")); + const { prisma } = require(path.join(rootDir, "lib/prisma")); + + try { + const result = await transcriptAweme(awemeId, { log: logStep }); + logStep("最终结果:"); + console.log(JSON.stringify(result, null, 2)); + } finally { + await prisma.$disconnect(); + } +} + +main().catch((error) => { + console.error("[asr] 执行失败:", error); + process.exitCode = 1; +});