From 61183e0c9aa0f250bf66ce2da16c1700bf3bb50d Mon Sep 17 00:00:00 2001 From: feie9456 Date: Fri, 17 Apr 2026 11:32:01 +0800 Subject: [PATCH] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E5=AE=B6=E4=BA=BA=E7=BB=91?= =?UTF-8?q?=E5=AE=9A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- build.gradle | 2 +- duix-sdk/build.gradle | 15 +- gradle/wrapper/gradle-wrapper.properties | 2 +- qwen-docs/function_calling.md | 4554 +++++++++++++++++ qwen-docs/main.md | 2374 +++++++++ test/build.gradle | 2 + test/src/main/AndroidManifest.xml | 12 + .../test/realtime/OmniRealtimeClient.java | 118 +- .../duix/test/service/DashScopeConfig.java | 1 + .../duix/test/service/DeviceIdentityStore.kt | 44 + .../duix/test/service/FamilyBridgeService.kt | 218 + .../duix/test/ui/activity/CallActivity.kt | 317 +- .../ui/activity/DeviceSettingsActivity.kt | 80 + .../duix/test/ui/activity/MainActivity.kt | 20 + .../duix/test/ui/activity/SettingsActivity.kt | 41 + .../duix/test/util/QrCodeBitmapFactory.kt | 29 + test/src/main/res/drawable/ic_arrow_back.xml | 9 + .../main/res/drawable/ic_chevron_right.xml | 9 + test/src/main/res/drawable/ic_settings.xml | 9 + .../main/res/drawable/shape_qr_surface.xml | 7 + .../res/drawable/shape_warm_icon_button.xml | 7 + .../res/layout/activity_device_settings.xml | 140 + test/src/main/res/layout/activity_main.xml | 60 +- .../src/main/res/layout/activity_settings.xml | 159 + test/src/main/res/values-zh-rCN/strings.xml | 23 + test/src/main/res/values/strings.xml | 23 + 26 files changed, 8238 insertions(+), 37 deletions(-) create mode 100644 qwen-docs/function_calling.md create mode 100644 qwen-docs/main.md create mode 100644 test/src/main/java/ai/guiji/duix/test/service/DeviceIdentityStore.kt create mode 100644 test/src/main/java/ai/guiji/duix/test/service/FamilyBridgeService.kt create mode 100644 test/src/main/java/ai/guiji/duix/test/ui/activity/DeviceSettingsActivity.kt create mode 100644 test/src/main/java/ai/guiji/duix/test/ui/activity/SettingsActivity.kt create mode 100644 test/src/main/java/ai/guiji/duix/test/util/QrCodeBitmapFactory.kt create mode 100644 test/src/main/res/drawable/ic_arrow_back.xml create mode 100644 test/src/main/res/drawable/ic_chevron_right.xml create mode 100644 test/src/main/res/drawable/ic_settings.xml create mode 100644 test/src/main/res/drawable/shape_qr_surface.xml create mode 100644 test/src/main/res/drawable/shape_warm_icon_button.xml create mode 100644 test/src/main/res/layout/activity_device_settings.xml create mode 100644 test/src/main/res/layout/activity_settings.xml diff --git a/build.gradle b/build.gradle index d22ff0c..b481f00 100644 --- a/build.gradle +++ b/build.gradle @@ -11,7 +11,7 @@ buildscript { google() } dependencies { - classpath 'com.android.tools.build:gradle:8.1.2' + classpath 'com.android.tools.build:gradle:8.13.2' classpath 'org.jetbrains.kotlin:kotlin-gradle-plugin:1.8.10' } } diff --git a/duix-sdk/build.gradle b/duix-sdk/build.gradle index 608ea47..4f231d9 100644 --- a/duix-sdk/build.gradle +++ b/duix-sdk/build.gradle @@ -1,15 +1,16 @@ plugins { id 'com.android.library' } +def sdkVersionCode = (rootProject.ext.has('versionCode') ? rootProject.ext.versionCode : 1) as Integer +def sdkVersionName = (rootProject.ext.has('versionName') ? rootProject.ext.versionName : '1.0.0').toString() android { namespace 'ai.guiji.duix.sdk.client' compileSdk 33 + ndkVersion '25.1.8937393' defaultConfig { minSdk 24 - versionCode 13 - versionName '4.1.1' externalNativeBuild { cmake { @@ -25,20 +26,20 @@ android { minifyEnabled false proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules.pro' - buildConfigField("String", "VERSION_NAME", "\"${defaultConfig.versionName}\"") - buildConfigField('int', 'VERSION_CODE', "${defaultConfig.versionCode}") + buildConfigField("String", "VERSION_NAME", "\"${sdkVersionName}\"") + buildConfigField('int', 'VERSION_CODE', "${sdkVersionCode}") } release { minifyEnabled false proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules.pro' - buildConfigField("String", "VERSION_NAME", "\"${defaultConfig.versionName}\"") - buildConfigField('int', 'VERSION_CODE', "${defaultConfig.versionCode}") + buildConfigField("String", "VERSION_NAME", "\"${sdkVersionName}\"") + buildConfigField('int', 'VERSION_CODE', "${sdkVersionCode}") android.libraryVariants.all { variant -> variant.outputs.all { - outputFileName = "duix_client_sdk_${buildType.name}_${defaultConfig.versionName}.aar" + outputFileName = "duix_client_sdk_${buildType.name}_${sdkVersionName}.aar" } } } diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 34e4582..55bee6b 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -3,4 +3,4 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.0-all.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.13-all.zip diff --git a/qwen-docs/function_calling.md b/qwen-docs/function_calling.md new file mode 100644 index 0000000..683670d --- /dev/null +++ b/qwen-docs/function_calling.md @@ -0,0 +1,4554 @@ +大模型在面对实时性问题、数学计算等问题时可能效果不佳。Function Calling 通过引入外部工具，让大模型可以回答原本无法解决的问题。 + +## **工作原理** + +Function Calling 通过在应用程序和大模型之间的多步骤交互，使大模型可以参考外部工具信息进行回答。 + +1. **发起第一次模型调用** + + 应用程序首先向大模型发起一个包含用户问题与模型可调用工具清单的请求。 + +2. **接收模型的工具调用指令（工具名称与入参）** + + 若模型判断需要调用外部工具，会返回一个JSON格式的指令，用于告知应用程序需要执行的函数与入参。 + + > 若模型判断无需调用工具，会返回自然语言格式的回复。 + +3. **在应用端运行工具** + + 应用程序接收到工具指令后，需要运行工具，获得工具输出结果。 + +4. **发起第二次模型调用** + + 获取到工具输出结果后，需添加至模型的上下文（messages），再次发起模型调用。 + +5. **接收来自模型的最终响应** + + 模型将工具输出结果和用户问题信息整合，生成自然语言格式的回复。 + + +工作流程示意图如下所示： + +![image](https://help-static-aliyun-doc.aliyuncs.com/assets/img/zh-CN/4604236771/CAEQYhiBgIDE9unxyRkiIDVjNDgwMDBjYTE0NzQ5MTBiMzBiNjJkNTk5ZjE2YTIz4358988_20240409160230.951.svg) + +## **支持的模型** + +## 千问 + +- **文本生成模型** + + - 千问Max：Qwen3-Max系列、Qwen-Max系列 + + - 千问Plus：Qwen3.6-Plus系列、Qwen3.5-Plus系列、Qwen-Plus系列 + + - 千问Flash：Qwen3.5-Flash系列、Qwen-Flash系列 + + - 千问Coder：Qwen3-Coder系列、Qwen2.5-Coder系列、Qwen-Coder系列 + + - 千问Turbo：Qwen-Turbo系列 + + - Qwen3.5开源系列 + + - Qwen3开源系列 + + - Qwen2.5开源系列 + +- **多模态模型** + + - 千问VL： Qwen3-VL-Plus系列、 Qwen3-VL-Flash系列 + + - 千问Omni：Qwen3.5-Omni-Plus系列、Qwen3.5-Omni-Flash系列、Qwen3-Omni-Flash系列 + + - 千问Omni-Realtime：Qwen3.5-Omni-Plus-Realtime系列、Qwen3.5-Omni-Flash-Realtime系列 + + - Qwen3-VL 开源系列 + + +## DeepSeek + +- 阿里云百炼部署 + + - deepseek-v3.2 + + - deepseek-v3.2-exp（非思考模式） + + - deepseek-v3.1（非思考模式） + + - deepseek-r1 + + - deepseek-r1-0528 + + - deepseek-v3 + +- 硅基流动部署 + + - siliconflow/deepseek-v3.2 + + - siliconflow/deepseek-v3.1-terminus + + - siliconflow/deepseek-r1-0528 + + - siliconflow/deepseek-v3-0324 + +- 快手万擎部署 + + - vanchin/deepseek-v3.2-think + + - vanchin/deepseek-v3.1-terminus + + - vanchin/deepseek-r1 + + - vanchin/deepseek-v3 + + +## GLM + +- glm-5.1 + +- glm-5 + +- glm-4.7 + +- glm-4.6 + +- glm-4.5 + +- glm-4.5-air + + +## Kimi + +- 阿里云百炼部署 + + - kimi-k2.5 + + - kimi-k2-thinking + + - Moonshot-Kimi-K2-Instruct + +- 月之暗面部署 + + - kimi/kimi-k2.5 + + +> 使用 kimi/kimi-k2.5 进行多轮 Function Calling 时，必须在每轮 assistant 消息中保留 `reasoning_content` 字段，否则会报错。 + +## MiniMax + +- 阿里云百炼部署 + + - MiniMax-M2.5 + + - MiniMax-M2.1 + +- 稀宇科技部署 + + - MiniMax/MiniMax-M2.7 + + - MiniMax/MiniMax-M2.5 + + - MiniMax/MiniMax-M2.1 + + +## **快速开始** + +您需要已[获取API Key](https://help.aliyun.com/zh/model-studio/get-api-key)并[配置API Key到环境变量](https://help.aliyun.com/zh/model-studio/configure-api-key-through-environment-variables)。如果通过 OpenAI SDK 或 DashScope SDK 进行调用，需要[安装SDK](https://help.aliyun.com/zh/model-studio/install-sdk#210ee28162bs7)。 + +以天气查询场景为例，介绍快速使用 Function Calling 的方法。 + +## OpenAI 兼容 + +Python + +``` +from openai import OpenAI +from datetime import datetime +import json +import os +import random + + +# 初始化客户端 +client = OpenAI( + # 若没有配置环境变量，请用阿里云百炼API Key将下行替换为：api_key="sk-xxx", + # 各地域的API Key不同。获取API Key：https://help.aliyun.com/zh/model-studio/get-api-key + api_key=os.getenv("DASHSCOPE_API_KEY"), + # 以下是北京地域base_url，如果使用新加坡地域的模型，需要将base_url替换为：https://dashscope-intl.aliyuncs.com/compatible-mode/v1 + base_url="https://dashscope.aliyuncs.com/compatible-mode/v1", +) +# 模拟用户问题 +USER_QUESTION = "北京天气咋样" +# 定义工具列表 +tools = [ + { + "type": "function", + "function": { + "name": "get_current_weather", + "description": "当你想查询指定城市的天气时非常有用。", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "城市或县区，比如北京市、杭州市、余杭区等。", + } + }, + "required": ["location"], + }, + }, + }, +] + + +# 模拟天气查询工具 +def get_current_weather(arguments): + weather_conditions = ["晴天", "多云", "雨天"] + random_weather = random.choice(weather_conditions) + location = arguments["location"] + return f"{location}今天是{random_weather}。" + + +# 封装模型响应函数 +def get_response(messages): + completion = client.chat.completions.create( + model="qwen3.6-plus", + extra_body={"enable_thinking": False}, + messages=messages, + tools=tools, + ) + return completion + + +messages = [{"role": "user", "content": USER_QUESTION}] +response = get_response(messages) +assistant_output = response.choices[0].message +if assistant_output.content is None: + assistant_output.content = "" +messages.append(assistant_output) +# 如果不需要调用工具，直接输出内容 +if assistant_output.tool_calls is None: + print(f"无需调用天气查询工具，直接回复：{assistant_output.content}") +else: + # 进入工具调用循环 + while assistant_output.tool_calls is not None: + tool_call = assistant_output.tool_calls[0] + tool_call_id = tool_call.id + func_name = tool_call.function.name + arguments = json.loads(tool_call.function.arguments) + print(f"正在调用工具 [{func_name}]，参数：{arguments}") + # 执行工具 + tool_result = get_current_weather(arguments) + # 构造工具返回信息 + tool_message = { + "role": "tool", + "tool_call_id": tool_call_id, + "content": tool_result, # 保持原始工具输出 + } + print(f"工具返回：{tool_message['content']}") + messages.append(tool_message) + # 再次调用模型，获取总结后的自然语言回复 + response = get_response(messages) + assistant_output = response.choices[0].message + if assistant_output.content is None: + assistant_output.content = "" + messages.append(assistant_output) + print(f"助手最终回复：{assistant_output.content}") +``` + +Node.js + +``` +import OpenAI from 'openai'; + +// 初始化客户端 +const openai = new OpenAI({ + // 若没有配置环境变量，请用阿里云百炼API Key将下行替换为：apiKey: "sk-xxx", + // 各地域的API Key不同。获取API Key：https://help.aliyun.com/zh/model-studio/get-api-key + apiKey: process.env.DASHSCOPE_API_KEY, + // 以下是北京地域base_url，如果使用新加坡地域的模型，需要将base_url替换为：https://dashscope-intl.aliyuncs.com/compatible-mode/v1 + baseURL: "https://dashscope.aliyuncs.com/compatible-mode/v1", +}); + +// 定义工具列表 +const tools = [ + { + type: "function", + function: { + name: "get_current_weather", + description: "当你想查询指定城市的天气时非常有用。", + parameters: { + type: "object", + properties: { + location: { + type: "string", + description: "城市或县区，比如北京市、杭州市、余杭区等。", + }, + }, + required: ["location"], + }, + }, + }, +]; + +// 模拟天气查询工具 +const getCurrentWeather = (args) => { + const weatherConditions = ["晴天", "多云", "雨天"]; + const randomWeather = weatherConditions[Math.floor(Math.random() * weatherConditions.length)]; + const location = args.location; + return `${location}今天是${randomWeather}。`; +}; + +// 封装模型响应函数 +const getResponse = async (messages) => { + const response = await openai.chat.completions.create({ + model: "qwen3.6-plus", + enable_thinking: false, + messages: messages, + tools: tools, + }); + return response; +}; + +const main = async () => { + const input = "北京天气咋样"; + + let messages = [ + { + role: "user", + content: input, + } + ]; + let response = await getResponse(messages); + let assistantOutput = response.choices[0].message; + // 确保 content 不是 null + if (!assistantOutput.content) assistantOutput.content = ""; + messages.push(assistantOutput); + // 判断是否需要调用工具 + if (!assistantOutput.tool_calls) { + console.log(`无需调用天气查询工具，直接回复：${assistantOutput.content}`); + } else { + // 进入工具调用循环 + while (assistantOutput.tool_calls) { + const toolCall = assistantOutput.tool_calls[0]; + const toolCallId = toolCall.id; + const funcName = toolCall.function.name; + const funcArgs = JSON.parse(toolCall.function.arguments); + console.log(`正在调用工具 [${funcName}]，参数：`, funcArgs); + // 执行工具 + const toolResult = getCurrentWeather(funcArgs); + // 构造工具返回信息 + const toolMessage = { + role: "tool", + tool_call_id: toolCallId, + content: toolResult, + }; + console.log(`工具返回：${toolMessage.content}`); + messages.push(toolMessage); + // 再次调用模型获取自然语言总结 + response = await getResponse(messages); + assistantOutput = response.choices[0].message; + if (!assistantOutput.content) assistantOutput.content = ""; + messages.push(assistantOutput); + } + console.log(`助手最终回复：${assistantOutput.content}`); + } +}; + +// 启动程序 +main().catch(console.error); +``` + +## DashScope + +Python + +``` +import os +from dashscope import MultiModalConversation +import json +import random +import dashscope + +# 若使用新加坡地域的模型，请释放下列注释 +# dashscope.base_http_api_url = "https://dashscope-intl.aliyuncs.com/api/v1" + +# 1. 定义工具列表 +tools = [ + { + "type": "function", + "function": { + "name": "get_current_weather", + "description": "当你想查询指定城市的天气时非常有用。", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "城市或县区，比如北京市、杭州市、余杭区等。", + } + }, + "required": ["location"], + }, + }, + } +] + +# 2. 模拟天气查询工具 +def get_current_weather(arguments): + weather_conditions = ["晴天", "多云", "雨天"] + random_weather = random.choice(weather_conditions) + location = arguments["location"] + return f"{location}今天是{random_weather}。" + +# 3. 封装模型响应函数 +def get_response(messages): + response = MultiModalConversation.call( + # 如果没有配置环境变量，请将下行替换为api_key="sk-xxx" + api_key=os.getenv("DASHSCOPE_API_KEY"), + # 此处以多模态模型 qwen3.6-plus 为例。若调用 qwen3-max、qwen-plus 等纯文本模型需使用纯文本模型接口，参见https://help.aliyun.com/model-studio/qwen-api-via-dashscope + model="qwen3.6-plus", + enable_thinking=False, + messages=messages, + tools=tools, + result_format="message", + ) + return response + +# 4. 初始化对话历史 +messages = [ + { + "role": "user", + "content": [{"text": "北京天气咋样"}] + } +] + +# 5. 第一次调用模型 +response = get_response(messages) +assistant_output = response.output.choices[0].message +messages.append(assistant_output) + +# 6. 判断是否需要调用工具 +if "tool_calls" not in assistant_output or not assistant_output["tool_calls"]: + print(f"无需调用工具，直接回复：{assistant_output['content']}") +else: + # 7. 进入工具调用循环 + # 循环条件：只要最新的模型回复中包含工具调用请求 + while "tool_calls" in assistant_output and assistant_output["tool_calls"]: + tool_call = assistant_output["tool_calls"][0] + # 解析工具调用的信息 + func_name = tool_call["function"]["name"] + arguments = json.loads(tool_call["function"]["arguments"]) + tool_call_id = tool_call.get("id") # 获取 tool_call_id + print(f"正在调用工具 [{func_name}]，参数：{arguments}") + # 执行对应的工具函数 + tool_result = get_current_weather(arguments) + # 构造工具返回信息 + tool_message = { + "role": "tool", + "content": tool_result, + "tool_call_id": tool_call_id + } + print(f"工具返回：{tool_message['content']}") + messages.append(tool_message) + # 再次调用模型，让模型基于工具结果进行回复 + response = get_response(messages) + assistant_output = response.output.choices[0].message + messages.append(assistant_output) + # 8. 输出最终的自然语言回复 + content = assistant_output["content"] + if isinstance(content, list) and content: + content = content[0].get("text", "") if isinstance(content[0], dict) else str(content[0]) + print(f"助手最终回复：{content}") +``` + +Java + +``` +import com.alibaba.dashscope.aigc.multimodalconversation.MultiModalConversation; +import com.alibaba.dashscope.aigc.multimodalconversation.MultiModalConversationParam; +import com.alibaba.dashscope.aigc.multimodalconversation.MultiModalConversationResult; +import com.alibaba.dashscope.common.MultiModalMessage; +import com.alibaba.dashscope.common.Role; +import com.alibaba.dashscope.exception.NoApiKeyException; +import com.alibaba.dashscope.exception.UploadFileException; +import com.alibaba.dashscope.tools.FunctionDefinition; +import com.alibaba.dashscope.tools.ToolCallBase; +import com.alibaba.dashscope.tools.ToolCallFunction; +import com.alibaba.dashscope.tools.ToolFunction; +import com.alibaba.dashscope.utils.Constants; +import com.alibaba.dashscope.utils.JsonUtils; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Random; + +public class Main { + // 若使用新加坡地域的模型，请释放下列注释 + // static {Constants.baseHttpApiUrl="https://dashscope-intl.aliyuncs.com/api/v1";} + + /** + * 从 MultiModalMessage 的 content 中提取纯文本。 + * content 格式为 List>，例如：[{text=天气是晴天}] + */ + @SuppressWarnings("unchecked") + public static String getTextContent(Object content) { + if (content instanceof List) { + for (Object item : (List) content) { + if (item instanceof Map) { + Object text = ((Map) item).get("text"); + if (text != null) return text.toString(); + } + } + } + return content != null ? content.toString() : ""; + } + + /** + * 第一步：定义工具的本地实现。 + * @param arguments 模型传入的、包含工具所需参数的JSON字符串。 + * @return 工具执行后的结果字符串。 + */ + public static String getCurrentWeather(String arguments) { + try { + // 模型提供的参数是JSON格式的，需要我们手动解析。 + ObjectMapper objectMapper = new ObjectMapper(); + JsonNode argsNode = objectMapper.readTree(arguments); + String location = argsNode.get("location").asText(); + + // 用随机结果来模拟真实的API调用或业务逻辑。 + List weatherConditions = Arrays.asList("晴天", "多云", "雨天"); + String randomWeather = weatherConditions.get(new Random().nextInt(weatherConditions.size())); + + return location + "今天是" + randomWeather + "。"; + } catch (Exception e) { + // 异常处理，确保程序健壮性。 + return "无法解析地点参数。"; + } + } + + public static void main(String[] args) { + try { + // 第二步：向模型描述（注册）我们的工具。 + String weatherParamsSchema = + "{\"type\":\"object\",\"properties\":{\"location\":{\"type\":\"string\",\"description\":\"城市或县区，比如北京市、杭州市、余杭区等。\"}},\"required\":[\"location\"]}"; + + FunctionDefinition weatherFunction = FunctionDefinition.builder() + .name("get_current_weather") // 工具的唯一标识名，必须与本地实现对应。 + .description("当你想查询指定城市的天气时非常有用。") // 清晰的描述能帮助模型更好地决定何时使用该工具。 + .parameters(JsonUtils.parseString(weatherParamsSchema).getAsJsonObject()) + .build(); + MultiModalConversation conv = new MultiModalConversation(); + String userInput = "北京天气咋样"; + + List messages = new ArrayList<>(); + messages.add(MultiModalMessage.builder().role(Role.USER.getValue()) + .content(Arrays.asList(Collections.singletonMap("text", userInput))).build()); + + // 第四步：首次调用模型。将用户的请求和我们定义好的工具列表一同发送给模型。 + MultiModalConversationParam param = MultiModalConversationParam.builder() + .model("qwen3.6-plus") //此处以多模态模型 qwen3.6-plus 为例。若调用 qwen3-max、qwen-plus 等纯文本模型需使用纯文本模型接口，参见https://help.aliyun.com/model-studio/qwen-api-via-dashscope + .enableThinking(false) + .apiKey(System.getenv("DASHSCOPE_API_KEY")) // 从环境变量中获取API Key。 + .messages(messages) // 传入当前的对话历史。 + .tools(Arrays.asList(ToolFunction.builder().function(weatherFunction).build())) // 传入可用的工具列表。 + .build(); + + MultiModalConversationResult result = conv.call(param); + MultiModalMessage assistantOutput = result.getOutput().getChoices().get(0).getMessage(); + messages.add(assistantOutput); // 将模型的首次回复也加入到对话历史中。 + + // 第五步：检查模型的回复，判断它是否请求调用工具。 + if (assistantOutput.getToolCalls() == null || assistantOutput.getToolCalls().isEmpty()) { + // 情况A：模型没有调用工具，而是直接给出了回答。 + System.out.println("无需调用天气查询工具，直接回复：" + getTextContent(assistantOutput.getContent())); + } else { + // 情况B：模型决定调用工具。 + // 使用 while 循环可以处理模型连续调用多次工具的场景。 + while (assistantOutput.getToolCalls() != null && !assistantOutput.getToolCalls().isEmpty()) { + ToolCallBase toolCall = assistantOutput.getToolCalls().get(0); + + // 从模型的回复中解析出工具调用的具体信息（要调用的函数名、参数）。 + ToolCallFunction functionCall = (ToolCallFunction) toolCall; + String funcName = functionCall.getFunction().getName(); + String funcArguments = functionCall.getFunction().getArguments(); + System.out.println("正在调用工具 [" + funcName + "]，参数：" + funcArguments); + + // 根据工具名，在本地执行对应的Java方法。 + String toolResult = getCurrentWeather(funcArguments); + + // 构造一个 role 为 "tool" 的消息，其中包含工具的执行结果。 + MultiModalMessage toolMessage = MultiModalMessage.builder() + .role("tool") + .toolCallId(toolCall.getId()) + .content(Arrays.asList(Collections.singletonMap("text", toolResult))) + .build(); + System.out.println("工具返回：" + toolResult); + messages.add(toolMessage); // 将工具的返回结果也加入到对话历史中。 + + // 第六步：再次调用模型。 + param.setMessages((List) messages); + result = conv.call(param); + assistantOutput = result.getOutput().getChoices().get(0).getMessage(); + messages.add(assistantOutput); + } + + // 第七步：打印模型经过总结后，生成的最终回复。 + System.out.println("助手最终回复：" + getTextContent(assistantOutput.getContent())); + } + + } catch (NoApiKeyException | UploadFileException e) { + System.err.println("错误: " + e.getMessage()); + } catch (Exception e) { + e.printStackTrace(); + } + } +} +``` + +运行后得到如下输出： + +``` +正在调用工具 [get_current_weather]，参数：{'location': '北京'} +工具返回：北京今天是多云。 +助手最终回复：北京今天是多云的天气。 +``` + +## **如何使用** + +Function Calling 支持两种传入工具信息的方式： + +- 方式一：通过 tools 参数传入（推荐） + + 请参见[如何使用](#038e24005c1vt)，按照**定义工具**、**创建messages数组**、**发起 Function Calling**、**运行工具函数**、**大模型总结工具函数输出**的步骤进行调用。 + +- 方式二：通过 System Message 传入 + + 通过 tools 参数传入时，服务端会根据模型自动适配合适的 prompt 模板并组装，因此推荐您优先使用 tools 参数。 如果您在使用 Qwen 模型时不期望使用 tools 参数，请参见[通过 System Message 传入工具信息](#afa52dfc56tqq)。 + + +以下内容以 OpenAI 兼容调用方式为例，通过 tools 参数传入工具信息，向您分步骤介绍 Function Calling 的详细用法。 + +假设业务场景中会收到天气查询与时间查询两类问题。 + +### **1\. 定义工具** + +工具是连接大模型与外部世界的桥梁，首先需要定义工具。 + +#### **1.1. 创建工具函数** + +创建两个工具函数：天气查询工具与时间查询工具。 + +- **天气查询工具** + + 接收`arguments`参数，`arguments`格式为`{"location": "查询的地点"}`。工具的输出为字符串，格式为：`“{位置}今天是{天气}”`。 + + > 为了便于演示，此处定义的天气查询工具并不真正查询天气，会从晴天、多云、雨天随机选择。在实际业务中可使用如[高德天气查询](https://lbs.amap.com/api/webservice/guide/api/weatherinfo)等工具进行替换。 + +- **时间查询工具** + + 时间查询工具不需要输入参数。工具的输出为字符串，格式为：`“当前时间：{查询到的时间}。”`。 + + > 如果使用 Node.js，请运行`npm install date-fns`安装获取时间的工具包 date-fns： + + +Python + +``` +## 步骤1:定义工具函数 + +# 添加导入random模块 +import random +from datetime import datetime + +# 模拟天气查询工具。返回结果示例：“北京今天是雨天。” +def get_current_weather(arguments): + # 定义备选的天气条件列表 + weather_conditions = ["晴天", "多云", "雨天"] + # 随机选择一个天气条件 + random_weather = random.choice(weather_conditions) + # 从 JSON 中提取位置信息 + location = arguments["location"] + # 返回格式化的天气信息 + return f"{location}今天是{random_weather}。" + +# 查询当前时间的工具。返回结果示例：“当前时间：2024-04-15 17:15:18。“ +def get_current_time(): + # 获取当前日期和时间 + current_datetime = datetime.now() + # 格式化当前日期和时间 + formatted_time = current_datetime.strftime('%Y-%m-%d %H:%M:%S') + # 返回格式化后的当前时间 + return f"当前时间：{formatted_time}。" + +# 测试工具函数并输出结果，运行后续步骤时可以去掉以下四句测试代码 +print("测试工具输出：") +print(get_current_weather({"location": "上海"})) +print(get_current_time()) +print("\n") +``` + +Node.js + +``` +// 步骤1:定义工具函数 + +// 导入时间查询工具 +import { format } from 'date-fns'; + +function getCurrentWeather(args) { + // 定义备选的天气条件列表 + const weatherConditions = ["晴天", "多云", "雨天"]; + // 随机选择一个天气条件 + const randomWeather = weatherConditions[Math.floor(Math.random() * weatherConditions.length)]; + // 从 JSON 中提取位置信息 + const location = args.location; + // 返回格式化的天气信息 + return `${location}今天是${randomWeather}。`; +} + +function getCurrentTime() { + // 获取当前日期和时间 + const currentDatetime = new Date(); + // 格式化当前日期和时间 + const formattedTime = format(currentDatetime, 'yyyy-MM-dd HH:mm:ss'); + // 返回格式化后的当前时间 + return `当前时间：${formattedTime}。`; +} + +// 测试工具函数并输出结果，运行后续步骤时可以去掉以下四句测试代码 +console.log("测试工具输出：") +console.log(getCurrentWeather({location:"上海"})); +console.log(getCurrentTime()); +console.log("\n") +``` + +运行工具后，得到输出： + +``` +测试工具输出： +上海今天是多云。 +当前时间：2025-01-08 20:21:45。 +``` + +#### **1.2 创建 tools 数组** + +人类在选择工具之前，需要对工具有全面的了解，包括工具的功能、何时使用以及输入参数等。大模型也需要这些信息才能更准确地选择工具。请根据以下 JSON 格式提供工具信息。 + +| - `type`字段固定为`"function"`； - `function`字段为 Object 类型； - `name`字段为自定义的工具函数名称，建议使用与函数相同的名称，如`get_current_weather`或`get_current_time`； - `description`字段是对工具函数功能的描述，大模型会参考该字段来选择是否使用该工具函数。 - `parameters`字段是对工具函数入参的描述，类型是 Object ，大模型会参考该字段来进行入参的提取。如果工具函数不需要输入参数，则无需指定`parameters`参数。 - `type`字段固定为`"object"`； - `properties`字段描述了入参的名称、数据类型与描述，为 Object 类型，Key 值为入参的名称，Value 值为入参的数据类型与描述； - `required`字段指定哪些参数为必填项，为 Array 类型。 | 对于天气查询工具来说，工具描述信息的格式如下： ``` { "type": "function", "function": { "name": "get_current_weather", "description": "当你想查询指定城市的天气时非常有用。", "parameters": { "type": "object", "properties": { "location": { "type": "string", "description": "城市或县区，比如北京市、杭州市、余杭区等。" } }, "required": ["location"] } } } ``` | +| --- | --- | + +在发起 Function Calling 前，需在代码中定义工具信息数组（tools），包含每个工具的函数名、描述和参数定义。这个数组在后续发起Function Calling请求时作为参数传入。 + +Python + +``` +# 请将以下代码粘贴到步骤1代码后 + +## 步骤2:创建 tools 数组 + +tools = [ + { + "type": "function", + "function": { + "name": "get_current_time", + "description": "当你想知道现在的时间时非常有用。", + "parameters": {} + } + }, + { + "type": "function", + "function": { + "name": "get_current_weather", + "description": "当你想查询指定城市的天气时非常有用。", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "城市或县区，比如北京市、杭州市、余杭区等。", + } + }, + "required": ["location"] + } + } + } +] +tool_name = [tool["function"]["name"] for tool in tools] +print(f"创建了{len(tools)}个工具，为：{tool_name}\n") +``` + +Node.js + +``` +// 请将以下代码粘贴到步骤1代码后 + +// 步骤2:创建 tools 数组 + +const tools = [ + { + type: "function", + function: { + name: "get_current_time", + description: "当你想知道现在的时间时非常有用。", + parameters: {} + } + }, + { + type: "function", + function: { + name: "get_current_weather", + description: "当你想查询指定城市的天气时非常有用。", + parameters: { + type: "object", + properties: { + location: { + type: "string", + description: "城市或县区，比如北京市、杭州市、余杭区等。", + } + }, + required: ["location"] + } + } + } + ]; + +const toolNames = tools.map(tool => tool.function.name); +console.log(`创建了${tools.length}个工具，为：${toolNames.join(', ')}\n`); +``` + +### **2\. 创建messages数组** + +Function Calling 通过 messages 数组向大模型传入指令与上下文信息。发起 Function Calling 前，messages 数组需要包含 System Message 与 User Message。 + +#### **System Message** + +尽管在[创建 tools 数组](#b7c8a0e72a9d0)时已经对工具的作用与何时使用工具进行了描述，但在 System Message 中强调何时调用工具通常会提高工具调用的准确率。在当前场景下，可以将 System Prompt 设置为： + +``` +你是一个很有帮助的助手。如果用户提问关于天气的问题，请调用 ‘get_current_weather’ 函数; +如果用户提问关于时间的问题，请调用‘get_current_time’函数。 +请以友好的语气回答问题。 +``` + +#### **User Message** + +User Message 用于传入用户提问的问题。假设用户提问“上海天气”，此时的 messages 数组为： + +Python + +``` +# 步骤3:创建messages数组 +# 请将以下代码粘贴到步骤2 代码后 +# 文本生成模型的 User Message示例 +messages = [ + { + "role": "system", + "content": """你是一个很有帮助的助手。如果用户提问关于天气的问题，请调用 ‘get_current_weather’ 函数; + 如果用户提问关于时间的问题，请调用‘get_current_time’函数。 + 请以友好的语气回答问题。""", + }, + { + "role": "user", + "content": "上海天气" + } +] + +# 多模态模型的 User Message示例 +# messages=[ +# { +# "role": "system", +# "content": """你是一个很有帮助的助手。如果用户提问关于天气的问题，请调用 ‘get_current_weather’ 函数; +# 如果用户提问关于时间的问题，请调用‘get_current_time’函数。 +# 请以友好的语气回答问题。""", +# }, +# {"role": "user", +# "content": [{"type": "image_url","image_url": {"url": "https://img.alicdn.com/imgextra/i2/O1CN01FbTJon1ErXVGMRdsN_!!6000000000405-0-tps-1024-683.jpg"}}, +# {"type": "text", "text": "根据图像上的地点，查询该地点当前天气"}]}, +# ] + +print("messages 数组创建完成\n") +``` + +Node.js + +``` +// 步骤3:创建messages数组 +// 请将以下代码粘贴到步骤2 代码后 +const messages = [ + { + role: "system", + content: "你是一个很有帮助的助手。如果用户提问关于天气的问题，请调用 ‘get_current_weather’ 函数; 如果用户提问关于时间的问题，请调用‘get_current_time’函数。请以友好的语气回答问题。", + }, + { + role: "user", + content: "上海天气" + } +]; +// 多模态模型的 User Message示例， +// const messages: [{ +// role: "user", +// content: [{type: "image_url", image_url: {"url": "https://help-static-aliyun-doc.aliyuncs.com/file-manage-files/zh-CN/20241022/emyrja/dog_and_girl.jpeg"}}, +// {type: "text", text: "图中描绘的是什么景象?"}] +// }]; + +console.log("messages 数组创建完成\n"); +``` + +> 由于备选工具包含天气查询与时间查询，也可提问关于当前时间的问题。 + +### **3\. 发起 Function Calling** + +将创建好的 `tools` 与 `messages` 传入大模型，即可发起一次 Function Calling。大模型会判断是否调用工具。若调用，则返回该工具的函数名与参数。 + +> 支持的模型参见[支持的模型](#48631a8d2emg8)。 + +Python + +``` +# 步骤4:发起 function calling +# 请将以下代码粘贴到步骤3 代码后 + +from openai import OpenAI +import os + +client = OpenAI( + # 若没有配置环境变量，请用阿里云百炼API Key将下行替换为：api_key="sk-xxx", + # 各地域的API Key不同。获取API Key：https://help.aliyun.com/zh/model-studio/get-api-key + api_key=os.getenv("DASHSCOPE_API_KEY"), + # 以下是北京地域base_url，如果使用新加坡地域的模型，需要将base_url替换为：https://dashscope-intl.aliyuncs.com/compatible-mode/v1 + base_url="https://dashscope.aliyuncs.com/compatible-mode/v1", +) + +def function_calling(): + completion = client.chat.completions.create( + model="qwen3.6-plus", # 此处以qwen3.6-plus为例，可按需更换模型名称。模型列表：https://help.aliyun.com/zh/model-studio/getting-started/models + extra_body={"enable_thinking": False}, + messages=messages, + tools=tools + ) + print("返回对象：") + print(completion.choices[0].message.model_dump_json()) + print("\n") + return completion + +print("正在发起function calling...") +completion = function_calling() +``` + +Node.js + +``` +// 步骤4:发起 function calling +// 请将以下代码粘贴到步骤3 代码后 + +import OpenAI from "openai"; +const openai = new OpenAI( + { + // 若没有配置环境变量，请用百炼API Key将下行替换为：apiKey: "sk-xxx", + // 各地域的API Key不同。获取API Key：https://help.aliyun.com/zh/model-studio/get-api-key + apiKey: process.env.DASHSCOPE_API_KEY, + // 以下是北京地域base_url，如果使用新加坡地域的模型，需要将base_url替换为：https://dashscope-intl.aliyuncs.com/compatible-mode/v1 + baseURL: "https://dashscope.aliyuncs.com/compatible-mode/v1" + } +); + +async function functionCalling() { + const completion = await openai.chat.completions.create({ + model: "qwen3.6-plus", // 此处以qwen3.6-plus为例，可按需更换模型名称。模型列表：https://help.aliyun.com/zh/model-studio/getting-started/models + enable_thinking: false, + messages: messages, + tools: tools + }); + console.log("返回对象："); + console.log(JSON.stringify(completion.choices[0].message)); + console.log("\n"); + return completion; +} + +const completion = await functionCalling(); +``` + +由于用户提问为上海天气，大模型指定需要使用的工具函数名称为：`"get_current_weather"`，函数的入参为：`"{\"location\": \"上海\"}"`。 + +``` +{ + "content": "", + "refusal": null, + "role": "assistant", + "audio": null, + "function_call": null, + "tool_calls": [ + { + "id": "call_6596dafa2a6a46f7a217da", + "function": { + "arguments": "{\"location\": \"上海\"}", + "name": "get_current_weather" + }, + "type": "function", + "index": 0 + } + ] +} +``` + +需要注意，如果问题被大模型判断为无需使用工具，会通过`content`参数直接回复。在输入“你好”时，`tool_calls`参数为空，返回对象格式为： + +``` +{ + "content": "你好！有什么可以帮助你的吗？如果你有关于天气或者时间的问题，我特别擅长回答。", + "refusal": null, + "role": "assistant", + "audio": null, + "function_call": null, + "tool_calls": null +} +``` + +> 如果`tool_calls` 参数为空，可使程序直接返回 `content`，无需运行以下步骤。 + +> 若希望每次发起 Function Calling 后大模型都可以选择指定工具，请参见[强制工具调用](#b0910669927j7)。 + +### **4\. 运行工具函数** + +运行工具函数是将大模型的决策转化为实际操作的关键步骤。 + +> 运行工具函数的过程由您的计算环境而非大模型来完成。 + +由于大模型只可以输出字符串格式的内容，因此在运行工具函数前，需要对字符串格式的工具函数与入参分别解析。 + +- 工具函数 + + 建立一个**工具函数名称**到**工具函数实体**的映射`function_mapper`，将返回的工具函数字符串映射到工具函数实体； + +- 入参 + + Function Calling 返回的入参为 JSON 字符串，使用工具将其解析为 JSON 对象，提取入参信息。 + + +完成解析后，将参数传入工具函数并执行，获取输出结果。 + +Python + +``` +# 步骤5:运行工具函数 +# 请将以下代码粘贴到步骤4 代码后 +import json + +print("正在执行工具函数...") +# 从返回的结果中获取函数名称和入参 +function_name = completion.choices[0].message.tool_calls[0].function.name +arguments_string = completion.choices[0].message.tool_calls[0].function.arguments + +# 使用json模块解析参数字符串 +arguments = json.loads(arguments_string) +# 创建一个函数映射表 +function_mapper = { + "get_current_weather": get_current_weather, + "get_current_time": get_current_time +} +# 获取函数实体 +function = function_mapper[function_name] +# 如果入参为空，则直接调用函数 +if arguments == {}: + function_output = function() +# 否则，传入参数后调用函数 +else: + function_output = function(arguments) +# 打印工具的输出 +print(f"工具函数输出：{function_output}\n") +``` + +Node.js + +``` +// 步骤5:运行工具函数 +// 请将以下代码粘贴到步骤4 代码后 + +console.log("正在执行工具函数..."); +const function_name = completion.choices[0].message.tool_calls[0].function.name; +const arguments_string = completion.choices[0].message.tool_calls[0].function.arguments; + +// 使用JSON模块解析参数字符串 +const args = JSON.parse(arguments_string); + +// 创建一个函数映射表 +const functionMapper = { + "get_current_weather": getCurrentWeather, + "get_current_time": getCurrentTime +}; + +// 获取函数实体 +const func = functionMapper[function_name]; + +// 如果入参为空，则直接调用函数 +let functionOutput; +if (Object.keys(args).length === 0) { + functionOutput = func(); +} else { + // 否则，传入参数后调用函数 + functionOutput = func(args); +} + +// 打印工具的输出 +console.log(`工具函数输出：${functionOutput}\n`); +``` + +运行后得到如下输出： + +``` +上海今天是多云。 +``` + +**说明** + +在实际业务场景中，许多工具的核心功能是执行具体操作（如邮件发送、文件上传等），而非数据查询，这类工具执行后不会输出字符串。为帮助大模型了解工具运行状态，建议在设计此类工具时添加如“邮件发送完成”、"操作执行失败"等状态描述信息。 + +### **5\. 大模型总结工具函数输出** + +工具函数的输出格式较为固定，如果直接返回给用户，可能会有语气生硬、不够灵活等问题。如果您希望大模型能够综合用户输入以及工具输出结果，生成自然语言风格的回复，可以将工具输出提交到模型上下文并再次向模型发出请求。 + +1. 添加 Assistant Message + + [发起 Function Calling](#9f478b2e76wrk)后，通过`completion.choices[0].message`得到 Assistant Message，首先将它添加到 messages 数组中； + +2. 添加 Tool Message + + 将工具的输出通过`{"role": "tool", "content": "工具的输出","tool_call_id": completion.choices[0].message.tool_calls[0].id}`形式添加到 messages 数组。 + + **说明** + + - 请确保工具的输出为字符串格式。 + + - `tool_call_id` 是系统为每一次的工具调用请求生成的**唯一标识符**。模型可能一次性要求调用多个工具，将多个工具结果返回给模型时，`tool_call_id`可确保工具的输出结果能够与它的调用意图对应。 + + + +Python + +``` +# 步骤6:向大模型提交工具输出 +# 请将以下代码粘贴到步骤5 代码后 + +messages.append(completion.choices[0].message) +print("已添加assistant message") +messages.append({"role": "tool", "content": function_output, "tool_call_id": completion.choices[0].message.tool_calls[0].id}) +print("已添加tool message\n") +``` + +Node.js + +``` +// 步骤6:向大模型提交工具输出 +// 请将以下代码粘贴到步骤5 代码后 + +messages.push(completion.choices[0].message); +console.log("已添加 assistant message") +messages.push({ + "role": "tool", + "content": functionOutput, + "tool_call_id": completion.choices[0].message.tool_calls[0].id +}); +console.log("已添加 tool message\n"); +``` + +此时的 messages 数组为： + +``` +[ + System Message -- 指引模型调用工具的策略 + User Message -- 用户的问题 + Assistant Message -- 模型返回的工具调用信息 + Tool Message -- 工具的输出信息（如果采用下文介绍的并行工具调用，可能有多个 Tool Message） +] +``` + +更新 messages 数组后，运行以下代码。 + +Python + +``` +# 步骤7:大模型总结工具输出 +# 请将以下代码粘贴到步骤6 代码后 +print("正在总结工具输出...") +completion = function_calling() +``` + +Node.js + +``` +// 步骤7:大模型总结工具输出 +// 请将以下代码粘贴到步骤6 代码后 + +console.log("正在总结工具输出..."); +const completion_1 = await functionCalling(); +``` + +可从`content`得到回复内容：“上海今天的天气是多云。如果您有其他问题，欢迎继续提问。” + +``` +{ + "content": "上海今天的天气是多云。如果您有其他问题，欢迎继续提问。", + "refusal": null, + "role": "assistant", + "audio": null, + "function_call": null, + "tool_calls": null +} +``` + +至此，您已完成了一次完整的 Function Calling 流程。 + +## **进阶用法** + +### **指定工具调用方式** + +#### **并行工具调用** + +单一城市的天气查询经过一次工具调用即可。如果输入问题需要调用多次工具，如“北京上海的天气如何”或“杭州天气，以及现在几点了”，[发起 Function Calling](#9f478b2e76wrk) 后只会返回一个工具调用信息，以提问“北京上海的天气如何”为例： + +``` +{ + "content": "", + "refusal": null, + "role": "assistant", + "audio": null, + "function_call": null, + "tool_calls": [ + { + "id": "call_61a2bbd82a8042289f1ff2", + "function": { + "arguments": "{\"location\": \"北京市\"}", + "name": "get_current_weather" + }, + "type": "function", + "index": 0 + } + ] +} +``` + +返回结果中只有北京市的入参信息。为了解决这一问题，在[发起 Function Calling](#9f478b2e76wrk)时，可设置请求参数`parallel_tool_calls`为`true`，这样返回对象中将包含所有需要调用的工具函数与入参信息。 + +**说明** + +并行工具调用适合任务之间无依赖的情况。若任务之间有依赖关系（工具A的输入与工具B的输出结果有关），请参见[快速开始](#dd5a3dca390k9)，通过while循环实现串行工具调用（一次调用一个工具）。 + +Python + +``` +def function_calling(): + completion = client.chat.completions.create( + model="qwen3.6-plus", # 此处以qwen3.6-plus为例，可按需更换模型名称 + extra_body={"enable_thinking": False}, + messages=messages, + tools=tools, + # 新增参数 + parallel_tool_calls=True + ) + print("返回对象：") + print(completion.choices[0].message.model_dump_json()) + print("\n") + return completion + +print("正在发起function calling...") +completion = function_calling() +``` + +Node.js + +``` +async function functionCalling() { + const completion = await openai.chat.completions.create({ + model: "qwen3.6-plus", // 此处以qwen3.6-plus为例，可按需更换模型名称 + enable_thinking: false, + messages: messages, + tools: tools, + parallel_tool_calls: true + }); + console.log("返回对象："); + console.log(JSON.stringify(completion.choices[0].message)); + console.log("\n"); + return completion; +} + +const completion = await functionCalling(); +``` + +在返回对象的`tool_calls`数组中包含了北京上海的入参信息： + +``` +{ + "content": "", + "role": "assistant", + "tool_calls": [ + { + "function": { + "name": "get_current_weather", + "arguments": "{\"location\": \"北京市\"}" + }, + "index": 0, + "id": "call_c2d8a3a24c4d4929b26ae2", + "type": "function" + }, + { + "function": { + "name": "get_current_weather", + "arguments": "{\"location\": \"上海市\"}" + }, + "index": 1, + "id": "call_dc7f2f678f1944da9194cd", + "type": "function" + } + ] +} +``` + +#### **强制工具调用** + +大模型生成内容具有不确定性，有时会选择错误的工具进行调用。如果您希望对于某一类问题，大模型能够采取人为设定的策略（如强制使用某个工具、强制不使用工具），可修改`tool_choice`参数。`tool_choice`参数的默认值为`"auto"`，表示由大模型自主判断如何进行工具调用。 + +> 大模型总结工具函数输出时，请将`tool_choice`参数去除，否则 API 仍会返回工具调用信息。 + +- **强制使用某个工具** + + 如果您希望对于某一类问题，Function Calling 能强制调用某个工具，可设定`tool_choice`参数为`{"type": "function", "function": {"name": "the_function_to_call"}}`，大模型将不参与工具的选择，只输出入参信息。 + + 假设当前场景中只包含天气查询的问题，可修改 function\_calling 代码为： + + Python + + ``` + def function_calling(): + completion = client.chat.completions.create( + model="qwen3.6-plus", + extra_body={"enable_thinking": False}, + messages=messages, + tools=tools, + tool_choice={"type": "function", "function": {"name": "get_current_weather"}} + ) + print(completion.model_dump_json()) + + function_calling() + ``` + + Node.js + + ``` + async function functionCalling() { + const response = await openai.chat.completions.create({ + model: "qwen3.6-plus", + enable_thinking: false, + messages: messages, + tools: tools, + tool_choice: {"type": "function", "function": {"name": "get_current_weather"}} + }); + console.log("返回对象："); + console.log(JSON.stringify(response.choices[0].message)); + console.log("\n"); + return response; + } + + const response = await functionCalling(); + ``` + + 无论输入什么问题，返回对象的工具函数都会是`get_current_weather`。 + + > 使用该策略前请确保问题与选择的工具相关，否则可能返回不符合预期的结果。 + +- **强制不使用工具** + + 如果您希望无论输入什么问题，Function Calling 都不会进行工具调用（返回对象中包含回复内容`content`而`tool_calls`参数为空），可设定`tool_choice`参数为`"none"`，或不传入`tools`参数，Function Calling 返回的`tool_calls`参数将始终为空。 + + 假设当前场景中的问题均无需调用工具，可修改 function\_calling 代码为： + + Python + + ``` + def function_calling(): + completion = client.chat.completions.create( + model="qwen3.6-plus", + extra_body={"enable_thinking": False}, + messages=messages, + tools=tools, + tool_choice="none" + ) + print(completion.model_dump_json()) + + function_calling() + ``` + + Node.js + + ``` + async function functionCalling() { + const completion = await openai.chat.completions.create({ + model: "qwen3.6-plus", + enable_thinking: false, + messages: messages, + tools: tools, + tool_choice: "none" + }); + console.log("返回对象："); + console.log(JSON.stringify(completion.choices[0].message)); + console.log("\n"); + return completion; + } + + const completion = await functionCalling(); + ``` + + +### **多轮对话** + +用户可能在第一轮提问“北京天气”，第二轮提问“上海的呢？”若模型上下文中没有第一轮的信息，模型无法判断需要调用哪个工具。建议在多轮对话场景中，每轮结束后维持 messages 数组，在此基础上添加 User Message并[发起 Function Calling](#9f478b2e76wrk)以及后续步骤。messages 结构如下所示： + +``` +[ + System Message -- 指引模型调用工具的策略 + User Message -- 用户的问题 + Assistant Message -- 模型返回的工具调用信息 + Tool Message -- 工具的输出信息 + Assistant Message -- 模型总结的工具调用信息 + User Message -- 用户第二轮的问题 +] +``` + +### **流式输出** + +为了提升用户体验和减少等待时间，可使用流式输出实时获取工具函数名称与入参信息。其中： + +- 工具调用的参数信息：以数据流的形式分块返回。 + +- 工具函数名称：在流式响应的第一个数据块中返回。 + + +Python + +``` +from openai import OpenAI +import os + +client = OpenAI( + # 若没有配置环境变量，请用阿里云百炼API Key将下行替换为：api_key="sk-xxx", + # 各地域的API Key不同。获取API Key：https://help.aliyun.com/zh/model-studio/get-api-key + api_key=os.getenv("DASHSCOPE_API_KEY"), + # 以下是北京地域base_url，如果使用新加坡地域的模型，需要将base_url替换为：https://dashscope-intl.aliyuncs.com/compatible-mode/v1 + base_url="https://dashscope.aliyuncs.com/compatible-mode/v1", +) +tools = [ + { + "type": "function", + "function": { + "name": "get_current_weather", + "description": "当你想查询指定城市的天气时非常有用。", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "城市或县区，比如北京市、杭州市、余杭区等。", + } + }, + "required": ["location"], + }, + }, + }, +] + +stream = client.chat.completions.create( + model="qwen3.6-plus", + extra_body={"enable_thinking": False}, + messages=[{"role": "user", "content": "杭州天气?"}], + tools=tools, + stream=True +) + +for chunk in stream: + delta = chunk.choices[0].delta + print(delta.tool_calls) +``` + +Node.js + +``` +import { OpenAI } from "openai"; + +const openai = new OpenAI( + { + // 若没有配置环境变量，请用百炼API Key将下行替换为：apiKey: "sk-xxx", + // 各地域的API Key不同。获取API Key：https://help.aliyun.com/zh/model-studio/get-api-key + apiKey: process.env.DASHSCOPE_API_KEY, + // 以下是北京地域base_url，如果使用新加坡地域的模型，需要将base_url替换为：https://dashscope-intl.aliyuncs.com/compatible-mode/v1 + baseURL: "https://dashscope.aliyuncs.com/compatible-mode/v1" + } +); +const tools = [ + { + "type": "function", + "function": { + "name": "getCurrentWeather", + "description": "当你想查询指定城市的天气时非常有用。", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "城市或县区，比如北京市、杭州市、余杭区等。" + } + }, + "required": ["location"] + } + } + } +]; + +const stream = await openai.chat.completions.create({ + model: "qwen3.6-plus", + enable_thinking: false, + messages: [{ role: "user", content: "北京天气" }], + tools: tools, + stream: true, +}); + +for await (const chunk of stream) { + const delta = chunk.choices[0].delta; + console.log(delta.tool_calls); +} +``` + +运行后得到如下输出： + +``` +[ChoiceDeltaToolCall(index=0, id='call_8f08d2b0fc0c4d8fab7123', function=ChoiceDeltaToolCallFunction(arguments='{"location":', name='get_current_weather'), type='function')] +[ChoiceDeltaToolCall(index=0, id='', function=ChoiceDeltaToolCallFunction(arguments=' "杭州"}', name=None), type='function')] +None +``` + +运行以下代码拼接入参信息（`arguments`）： + +Python + +``` +tool_calls = {} +for response_chunk in stream: + delta_tool_calls = response_chunk.choices[0].delta.tool_calls + if delta_tool_calls: + for tool_call_chunk in delta_tool_calls: + call_index = tool_call_chunk.index + tool_call_chunk.function.arguments = tool_call_chunk.function.arguments or "" + if call_index not in tool_calls: + tool_calls[call_index] = tool_call_chunk + else: + tool_calls[call_index].function.arguments += tool_call_chunk.function.arguments +print(tool_calls[0].model_dump_json()) +``` + +Node.js + +``` +const toolCalls = {}; +for await (const responseChunk of stream) { + const deltaToolCalls = responseChunk.choices[0]?.delta?.tool_calls; + if (deltaToolCalls) { + for (const toolCallChunk of deltaToolCalls) { + const index = toolCallChunk.index; + toolCallChunk.function.arguments = toolCallChunk.function.arguments || ""; + if (!toolCalls[index]) { + toolCalls[index] = { ...toolCallChunk }; + if (!toolCalls[index].function) { + toolCalls[index].function = { name: '', arguments: '' }; + } + } + else if (toolCallChunk.function?.arguments) { + toolCalls[index].function.arguments += toolCallChunk.function.arguments; + } + } + } +} +console.log(JSON.stringify(toolCalls[0])); +``` + +获得如下输出： + +``` +{"index":0,"id":"call_16c72bef988a4c6c8cc662","function":{"arguments":"{\"location\": \"杭州\"}","name":"get_current_weather"},"type":"function"} +``` + +在使用大模型总结工具函数输出步骤，添加的 Assistant Message 需要符合下方格式。仅需将下方的`tool_calls`中的元素替换为以上内容即可。 + +``` +{ + "content": "", + "refusal": None, + "role": "assistant", + "audio": None, + "function_call": None, + "tool_calls": [ + { + "id": "call_xxx", + "function": { + "arguments": '{"location": "xx"}', + "name": "get_current_weather", + }, + "type": "function", + "index": 0, + } + ], +} +``` + +### **Responses API 的工具调用** + +上述示例均基于 OpenAI Chat Completions 与DashScope API实现工具调用。若使用 OpenAI Responses API，工具调用的整体流程相同，但在接口格式上有以下区别： + +| **维度** | **Chat Completions** | **Responses API** | +| --- | --- | --- | +| 工具定义格式 | ``` { "type": "function", "function": { "name":..., "parameters":... } } ``` | ``` { "type": "function", "name":..., "parameters":... } ``` | +| 工具调用输出 | response.choices\\[0\\].message.tool\\_calls | response.output 中 type 为 function\\_call 的项 | +| 工具结果回传 | ``` { "role": "tool", "tool_call_id":..., "content":... } ``` | ``` { "type": "function_call_output", "call_id":..., "output":... } ``` | +| 最终回复 | response.choices\\[0\\].message.content | response.output\\_text | + +Python + +``` +from openai import OpenAI +import json +import os +import random + +# 初始化客户端 +client = OpenAI( + # 若没有配置环境变量，请用阿里云百炼API Key将下行替换为：api_key="sk-xxx", + # 各地域的API Key不同。获取API Key：https://help.aliyun.com/zh/model-studio/get-api-key + api_key=os.getenv("DASHSCOPE_API_KEY"), + base_url="https://dashscope.aliyuncs.com/compatible-mode/v1", +) +# 模拟用户问题 +USER_QUESTION = "北京天气咋样" +# 定义工具列表 +tools = [ + { + "type": "function", + "name": "get_current_weather", + "description": "当你想查询指定城市的天气时非常有用。", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "城市或县区，比如北京市、杭州市、余杭区等。", + } + }, + "required": ["location"], + }, + } +] + + +# 模拟天气查询工具 +def get_current_weather(arguments): + weather_conditions = ["晴天", "多云", "雨天"] + random_weather = random.choice(weather_conditions) + location = arguments["location"] + return f"{location}今天是{random_weather}。" + + +# 封装模型响应函数 +def get_response(input_data): + response = client.responses.create( + model="qwen3.6-plus", + extra_body={"enable_thinking": False}, + input=input_data, + tools=tools, + ) + return response + + +# 维护对话上下文 +conversation = [{"role": "user", "content": USER_QUESTION}] + +response = get_response(conversation) +function_calls = [item for item in response.output if item.type == "function_call"] +# 如果不需要调用工具，直接输出内容 +if not function_calls: + print(f"助手最终回复：{response.output_text}") +else: + # 进入工具调用循环 + while function_calls: + for fc in function_calls: + func_name = fc.name + arguments = json.loads(fc.arguments) + print(f"正在调用工具 [{func_name}]，参数：{arguments}") + # 执行工具 + tool_result = get_current_weather(arguments) + print(f"工具返回：{tool_result}") + # 将工具调用和结果成对追加到上下文中 + conversation.append( + { + "type": "function_call", + "name": fc.name, + "arguments": fc.arguments, + "call_id": fc.call_id, + } + ) + conversation.append( + { + "type": "function_call_output", + "call_id": fc.call_id, + "output": tool_result, + } + ) + # 携带完整上下文再次调用模型 + response = get_response(conversation) + function_calls = [ + item for item in response.output if item.type == "function_call" + ] + print(f"助手最终回复：{response.output_text}") +``` + +Node.js + +``` +import OpenAI from "openai"; + +// 初始化客户端 +const openai = new OpenAI({ + // 若没有配置环境变量，请用阿里云百炼API Key将下行替换为：apiKey: "sk-xxx", + // 各地域的API Key不同。获取API Key：https://help.aliyun.com/zh/model-studio/get-api-key + apiKey: process.env.DASHSCOPE_API_KEY, + baseURL: + "https://dashscope.aliyuncs.com/compatible-mode/v1", +}); + +// 定义工具列表 +const tools = [ + { + type: "function", + name: "get_current_weather", + description: "当你想查询指定城市的天气时非常有用。", + parameters: { + type: "object", + properties: { + location: { + type: "string", + description: "城市或县区，比如北京市、杭州市、余杭区等。", + }, + }, + required: ["location"], + }, + }, +]; + +// 模拟天气查询工具 +const getCurrentWeather = (args) => { + const weatherConditions = ["晴天", "多云", "雨天"]; + const randomWeather = + weatherConditions[Math.floor(Math.random() * weatherConditions.length)]; + const location = args.location; + return `${location}今天是${randomWeather}。`; +}; + +// 封装模型响应函数 +const getResponse = async (inputData) => { + const response = await openai.responses.create({ + model: "qwen3.6-plus", + enable_thinking: false, + input: inputData, + tools: tools, + }); + return response; +}; + +const main = async () => { + const userQuestion = "北京天气"; + + // 维护对话上下文 + const conversation = [{ role: "user", content: userQuestion }]; + + let response = await getResponse(conversation); + let functionCalls = response.output.filter( + (item) => item.type === "function_call" + ); + // 如果不需要调用工具，直接输出内容 + if (functionCalls.length === 0) { + console.log(`助手最终回复：${response.output_text}`); + } else { + // 进入工具调用循环 + while (functionCalls.length > 0) { + for (const fc of functionCalls) { + const funcName = fc.name; + const args = JSON.parse(fc.arguments); + console.log(`正在调用工具 [${funcName}]，参数：`, args); + // 执行工具 + const toolResult = getCurrentWeather(args); + console.log(`工具返回：${toolResult}`); + // 将工具调用和结果成对追加到上下文中 + conversation.push({ + type: "function_call", + name: fc.name, + arguments: fc.arguments, + call_id: fc.call_id, + }); + conversation.push({ + type: "function_call_output", + call_id: fc.call_id, + output: toolResult, + }); + } + // 携带完整上下文再次调用模型 + response = await getResponse(conversation); + functionCalls = response.output.filter( + (item) => item.type === "function_call" + ); + } + console.log(`助手最终回复：${response.output_text}`); + } +}; + +// 启动程序 +main().catch(console.error); +``` + +### **全模态模型的工具调用** + +全模态模型支持工具调用，包括 Qwen-Omni 系列和 Qwen-Omni-Realtime 系列，两者的调用方式不同。 + +#### Qwen-Omni 系列 + +Qwen3.5-Omni-Plus、Qwen3.5-Omni-Flash、Qwen3-Omni-Flash 系列支持工具调用，通过 OpenAI 兼容接口调用。在**获取工具信息阶段**，与其他模型有以下不同： + +- **必须使用流式输出：**千问Omni仅支持流式输出，在获取工具信息时也必须设置 `stream=True`。 + +- **建议仅输出文本：**模型在获取工具信息（函数的名称和参数）时仅需文本信息，为避免生成不必要的音频，建议设置 `modalities=["text"]`。当输出包含文本和音频两种模态时，获取工具信息时需要跳过音频数据块。 + + +> 千问Omni详情参见：[非实时（Qwen-Omni）](https://help.aliyun.com/zh/model-studio/qwen-omni)。 + +Python + +``` +from openai import OpenAI +import os + +client = OpenAI( + # 若没有配置环境变量，请用阿里云百炼API Key将下行替换为：api_key="sk-xxx", + # 各地域的API Key不同。获取API Key：https://help.aliyun.com/zh/model-studio/get-api-key + api_key=os.getenv("DASHSCOPE_API_KEY"), + # 以下是北京地域base_url，如果使用新加坡地域的模型，需要将base_url替换为：https://dashscope-intl.aliyuncs.com/compatible-mode/v1 + base_url="https://dashscope.aliyuncs.com/compatible-mode/v1", +) + +tools = [ + { + "type": "function", + "function": { + "name": "get_current_weather", + "description": "当你想查询指定城市的天气时非常有用。", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "城市或县区，比如北京市、杭州市、余杭区等。", + } + }, + "required": ["location"], + }, + }, + }, +] + +completion = client.chat.completions.create( + model="qwen3.5-omni-plus", + messages=[{"role": "user", "content": "杭州天气?"}], + + # 设置输出数据的模态，可取值：["text"]、["text","audio"]，建议设置为["text"] + modalities=["text"], + + # stream 必须设置为 True，否则会报错 + stream=True, + tools=tools +) + +for chunk in completion: + # 如果输出包含音频模态，请将下列条件改为：if chunk.choices and not hasattr(chunk.choices[0].delta, "audio"): + if chunk.choices: + delta = chunk.choices[0].delta + print(delta.tool_calls) +``` + +Node.js + +``` +import { OpenAI } from "openai"; + +const openai = new OpenAI( + { + // 若没有配置环境变量，请用百炼API Key将下行替换为：apiKey: "sk-xxx", + // 各地域的API Key不同。获取API Key：https://help.aliyun.com/zh/model-studio/get-api-key + apiKey: process.env.DASHSCOPE_API_KEY, + // 以下是北京地域base_url，如果使用新加坡地域的模型，需要将base_url替换为：https://dashscope-intl.aliyuncs.com/compatible-mode/v1 + baseURL: "https://dashscope.aliyuncs.com/compatible-mode/v1" + } +); + +const tools = [ + { + "type": "function", + "function": { + "name": "getCurrentWeather", + "description": "当你想查询指定城市的天气时非常有用。", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "城市或县区，比如北京市、杭州市、余杭区等。" + } + }, + "required": ["location"] + } + } + } +]; + +const stream = await openai.chat.completions.create({ + model: "qwen3-omni-flash", + messages: [ + { + "role": "user", + "content": "杭州天气" + }], + stream: true, + // 设置输出数据的模态，可取值：["text"]、["text","audio"]，建议设置为["text"] + modalities: ["text"], + tools:tools +}); + +for await (const chunk of stream) { + // 如果输出包含音频，请将条件语句替换为：if (chunk.choices?.length && chunk.choices[0].delta && !('audio' in chunk.choices[0].delta)) + const delta = chunk.choices[0].delta; + console.log(delta.tool_calls); +} +``` + +运行后得到如下输出： + +``` +[ChoiceDeltaToolCall(index=0, id='call_391c8e5787bc4972a388aa', function=ChoiceDeltaToolCallFunction(arguments=None, name='get_current_weather'), type='function')] +[ChoiceDeltaToolCall(index=0, id='call_391c8e5787bc4972a388aa', function=ChoiceDeltaToolCallFunction(arguments=' {"location": "杭州市"}', name=None), type='function')] +None +``` + +拼接入参信息（`arguments`）的代码请参见[流式输出](#ee24b4ca07c53)。 + +#### Qwen-Omni-Realtime 系列 + +Qwen3.5-Omni-Plus-Realtime、Qwen3.5-Omni-Flash-Realtime 系列支持工具调用，适用于语音对话场景。可通过 DashScope SDK 或 WebSocket 原生协议调用。 + +**工作流程**： + +建立 WebSocket 连接后，通过 `session.update` 传入工具定义，即可进入以下交互流程： + +**阶段一：语音输入与工具调用** + +1. 用户发起语音提问，客户端采集音频并发送至服务端（对应 `append_audio()` 方法），服务端 VAD 检测语音结束后进行模型推理，判断需要调用工具。 + +2. 服务端将工具调用信息返回给客户端（对应 `response.function_call_arguments.done` 事件），包含函数名（`name`）、函数入参（`arguments`）和调用标识（`call_id`），示例如下： + + ``` + { + "type": "response.function_call_arguments.done", + "response_id": "resp_JnTOsWXlFhKcFohZbtfz6", + "item_id": "item_Rhcms7CauTNsQprV5S4Hr", + "output_index": 0, + "name": "get_current_weather", + "call_id": "call_2be200f4cafe419b9530dd", + "arguments": "{\"location\": \"杭州\"}" + } + ``` + +3. 客户端根据函数名和入参，在本地执行对应的工具函数，获得执行结果。 + + +**阶段二：客户端回传工具结果并触发最终响应** + +4. 客户端将工具执行结果发回服务端（对应 `conversation.item.create` 事件），包含调用标识（`call_id`）和执行结果（`output`），示例如下： + + ``` + { + "type": "conversation.item.create", + "item": { + "type": "function_call_output", + "call_id": "call_2be200f4cafe419b9530dd", + "output": "杭州今天天气为晴，气温25℃，微风" + } + } + ``` + +5. 客户端继续发送 `response.create` 事件，触发服务端基于工具执行结果生成最终语音回答。 + +6. 客户端接收服务端返回的语音和文本（对应 `response.audio.delta` 和 `response.audio_transcript.delta` 事件），播放语音回复给用户。 + + +> Qwen-Omni-Realtime 系列不支持 `tool_choice` 和 `parallel_tool_calls` 参数。 + +> 千问Omni-Realtime详情请参见：[实时（Qwen-Omni-Realtime）](https://help.aliyun.com/zh/model-studio/realtime)、[客户端事件](https://help.aliyun.com/zh/model-studio/client-events)、[服务端事件](https://help.aliyun.com/zh/model-studio/server-events)。 + +## DashScope Python SDK + +``` +# DashScope Python SDK v1.25.17 +import os +import uuid +import threading +import traceback +import json +import base64 +import signal +import sys +import time +from typing import Dict, Any, Optional, List +import pyaudio +import queue +import contextlib +import dashscope +from dashscope.audio.qwen_omni import * + +# ==================== 常量定义 ==================== +VOICE = 'Tina' +MODEL = "qwen3.5-omni-plus-realtime" +# 如果需要访问新加坡地域，请WS_URL将替换为：wss://dashscope-intl.aliyuncs.com/api-ws/v1/realtime +WS_URL = "wss://dashscope.aliyuncs.com/api-ws/v1/realtime" +# 配置 API Key，若没有设置环境变量，请用 API Key 将下行替换为 dashscope.api_key = "sk-xxx" +dashscope.api_key = os.getenv('DASHSCOPE_API_KEY') +AUDIO_SAMPLE_RATE = 16000 +AUDIO_CHUNK_SIZE = 3200 +OUTPUT_AUDIO_SAMPLE_RATE = 24000 + + +# ==================== 工具定义 ==================== +def get_train_price(src: str, dst: str) -> str: + """查询火车票价格""" + return f"{src}到{dst}的火车票价格为100~200元。" + + +def get_flight_price(src: str, dst: str) -> str: + """查询飞机票价格""" + return f"{src}到{dst}的机票价格为200~300美元。" + + +def get_current_weather(location: str) -> str: + """查询指定城市天气""" + return f"{location}今天天气为霾转晴，气温4/-4℃，微风" + + +# 统一的 OpenAI 格式工具定义 +TOOLS = [ + { + "type": "function", + "function": { + "name": "get_current_weather", + "description": "当你想查询指定城市的天气时非常有用。", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "城市或县区，比如北京市、杭州市、余杭区等。", + } + }, + "required": ["location"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "get_flight_price", + "description": "当你想查询飞机票价格时非常有用。", + "parameters": { + "type": "object", + "properties": { + "src": { + "type": "string", + "description": "飞机起飞的城市，比如北京市、杭州市等。", + }, + "dst": { + "type": "string", + "description": "飞机降落的城市，比如北京市、杭州市区等。", + }, + }, + "required": ["src", "dst"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "get_train_price", + "description": "当你想查询火车票价格时非常有用。", + "parameters": { + "type": "object", + "properties": { + "src": { + "type": "string", + "description": "火车出发的城市，比如北京市、杭州市等。", + }, + "dst": { + "type": "string", + "description": "火车到达的城市，比如北京市、杭州市区等。", + }, + }, + "required": ["src", "dst"], + }, + }, + }, +] + +# 工具名称到函数的映射 +TOOL_FUNCTIONS = { + "get_current_weather": get_current_weather, + "get_flight_price": get_flight_price, + "get_train_price": get_train_price, +} + + +# ==================== 工具调用处理 ==================== +def handle_tool_call(tool_call_response: Dict[str, Any]) -> Dict[str, Any]: + """ + 处理工具调用请求 + + Args: + tool_call_response: 包含 name, arguments, call_id 的工具调用信息 + + Returns: + 更新后的工具调用响应，包含 output 字段 + """ + try: + function_name = tool_call_response['name'] + tool_call_arguments = json.loads(tool_call_response['arguments']) + + print(f'[Tool Call] 开始处理: name={function_name}, args={tool_call_arguments}') + + # 查找对应的函数 + if function_name not in TOOL_FUNCTIONS: + tool_call_response['output'] = f"客户端未找到工具: {function_name}" + print(f'[Tool Call] 错误: 未找到工具 {function_name}') + return tool_call_response + + # 调用函数 + func = TOOL_FUNCTIONS[function_name] + result = func(**tool_call_arguments) + tool_call_response['output'] = result + + print(f'[Tool Call] 完成: {result}') + return tool_call_response + + except Exception as e: + error_msg = f"工具调用失败: {str(e)}" + tool_call_response['output'] = error_msg + print(f'[Tool Call] 异常: {error_msg}') + traceback.print_exc() + return tool_call_response + + +def send_tool_call_response(conversation: OmniRealtimeConversation, response: Dict[str, Any]) -> None: + """发送工具调用结果到服务端""" + conversation.create_item({ + "id": 'item_' + uuid.uuid4().hex, + "type": "function_call_output", + "call_id": response['call_id'], + "output": response["output"], + }) + +# ==================== PCM 音频播放器 ==================== +class PCMPlayer: + """ + PCM 音频播放器 + + 使用双线程架构实现实时音频播放： + - 解码线程：将 base64 编码的音频数据解码为原始 PCM 数据 + - 播放线程：将 PCM 数据写入音频输出设备 + + 支持动态添加音频数据、取消播放、保存音频文件等功能。 + """ + + def __init__(self, pya: pyaudio.PyAudio, sample_rate=24000, chunk_size_ms=100, save_file=False): + """ + 初始化 PCM 播放器 + + Args: + pya: pyaudio.PyAudio 实例 + sample_rate: 音频采样率（Hz），默认 24000 + chunk_size_ms: 音频块大小（毫秒），影响取消播放的延迟，默认 100ms + save_file: 是否保存播放的音频到文件（result.pcm），默认 False + """ + + self.pya = pya + self.sample_rate = sample_rate + self.chunk_size_bytes = chunk_size_ms * sample_rate * 2 // 1000 + self.player_stream = pya.open(format=pyaudio.paInt16, + channels=1, + rate=sample_rate, + output=True) + + self.raw_audio_buffer: queue.Queue = queue.Queue() + self.b64_audio_buffer: queue.Queue = queue.Queue() + self.status_lock = threading.Lock() + self.status = 'playing' + self.decoder_thread = threading.Thread(target=self.decoder_loop) + self.player_thread = threading.Thread(target=self.player_loop) + self.decoder_thread.start() + self.player_thread.start() + self.complete_event: threading.Event = None + self.save_file = save_file + if self.save_file: + self.out_file = open('result.pcm', 'wb') + + def decoder_loop(self): + """解码线程：将 base64 音频数据解码为 PCM 原始数据""" + while self.status != 'stop': + recv_audio_b64 = None + with contextlib.suppress(queue.Empty): + recv_audio_b64 = self.b64_audio_buffer.get(timeout=0.1) + if recv_audio_b64 is None: + continue + recv_audio_raw = base64.b64decode(recv_audio_b64) + # push raw audio data into queue by chunk + for i in range(0, len(recv_audio_raw), self.chunk_size_bytes): + chunk = recv_audio_raw[i:i + self.chunk_size_bytes] + self.raw_audio_buffer.put(chunk) + if self.save_file: + self.out_file.write(chunk) + + def player_loop(self): + """播放线程：将 PCM 数据写入音频输出设备""" + while self.status != 'stop': + recv_audio_raw = None + with contextlib.suppress(queue.Empty): + recv_audio_raw = self.raw_audio_buffer.get(timeout=0.1) + if recv_audio_raw is None: + if self.complete_event: + self.complete_event.set() + continue + # write chunk to pyaudio audio player, wait until finish playing this chunk. + self.player_stream.write(recv_audio_raw) + + def cancel_playing(self): + """取消播放：清空所有缓冲队列""" + self.b64_audio_buffer.queue.clear() + self.raw_audio_buffer.queue.clear() + + def add_data(self, data): + """添加 base64 编码的音频数据到播放队列""" + self.b64_audio_buffer.put(data) + + def wait_for_complete(self): + """等待播放完成""" + self.complete_event = threading.Event() + self.complete_event.wait() + self.complete_event = None + + def shutdown(self): + """关闭播放器并释放资源""" + self.status = 'stop' + self.decoder_thread.join() + self.player_thread.join() + self.player_stream.close() + if self.save_file: + self.out_file.close() + +# ==================== 音频管理器 ==================== +class AudioManager: + """管理音频输入输出资源""" + + def __init__(self): + self.pya: Optional[pyaudio.PyAudio] = None + self.mic_stream: Optional[pyaudio.Stream] = None + self.player: Optional[PCMPlayer] = None + + def initialize(self) -> None: + """初始化音频设备""" + print('初始化音频设备...') + self.pya = pyaudio.PyAudio() + self.mic_stream = self.pya.open( + format=pyaudio.paInt16, + channels=1, + rate=AUDIO_SAMPLE_RATE, + input=True + ) + self.player = PCMPlayer(self.pya, sample_rate=OUTPUT_AUDIO_SAMPLE_RATE) + print('音频设备初始化完成') + + def read_audio_chunk(self) -> Optional[bytes]: + """读取音频数据块""" + if not self.mic_stream: + return None + try: + return self.mic_stream.read(AUDIO_CHUNK_SIZE, exception_on_overflow=False) + except Exception as e: + print(f'[Error] 读取音频数据失败: {e}') + return None + + def cleanup(self) -> None: + """清理音频资源""" + print('清理音频资源...') + if self.player: + self.player.shutdown() + if self.mic_stream: + self.mic_stream.close() + if self.pya: + self.pya.terminate() + print('音频资源清理完成') + + +# ==================== 回调处理器 ==================== +class OmniCallback(OmniRealtimeCallback): + """Omni 实时对话回调处理器""" + + def __init__(self, audio_manager: AudioManager): + self.audio_manager = audio_manager + self.tool_calls: Dict[str, Dict[str, Any]] = {} + self.all_response_text: str = '' + self.last_package_time: float = 0 + self.is_first_text: bool = True + self.is_first_audio: bool = True + self.conversation: Optional[OmniRealtimeConversation] = None + + def set_conversation(self, conversation: OmniRealtimeConversation) -> None: + """设置对话实例引用""" + self.conversation = conversation + + def on_open(self) -> None: + """连接建立时的回调""" + print('连接已建立') + self.audio_manager.initialize() + self.last_package_time = time.time() * 1000 + self.is_first_text = True + self.is_first_audio = True + self.tool_calls = {} + self.all_response_text = '' + + def on_close(self, close_status_code: int, close_msg: str) -> None: + """连接关闭时的回调""" + print(f'连接已关闭: code={close_status_code}, msg={close_msg}') + self.audio_manager.cleanup() + sys.exit(0) + + def on_event(self, response: Dict[str, Any]) -> None: + """处理事件回调""" + try: + event_type = response.get('type', '') + + # 会话创建 + if event_type == 'session.created': + print(f'会话已启动: {response["session"]["id"]}') + + # 语音转文本完成 + elif event_type == 'conversation.item.input_audio_transcription.completed': + print(f'用户问题: {response.get("transcript", "")}') + + # 文本增量响应 + elif event_type in ('response.audio_transcript.delta', 'response.text.delta'): + if self.is_first_text: + self.is_first_text = False + latency = time.time() * 1000 - self.last_package_time + print(f'首字延迟 (VAD结束): {latency:.0f} ms') + + text = response.get('delta', '') + self.all_response_text += text + + # 音频增量响应 + elif event_type == 'response.audio.delta': + if self.is_first_audio: + self.is_first_audio = False + latency = time.time() * 1000 - self.last_package_time + print(f'首音延迟 (VAD结束): {latency:.0f} ms') + + audio_interval = time.time() * 1000 - self.last_package_time + print(f'音频间隔: {audio_interval:.0f} ms') + self.last_package_time = time.time() * 1000 + + recv_audio_b64 = response.get('delta', '') + if self.audio_manager.player: + self.audio_manager.player.add_data(recv_audio_b64) + + # VAD 检测到语音开始 + elif event_type == 'input_audio_buffer.speech_started': + print('====== VAD 检测到语音开始 ======') + if self.audio_manager.player: + self.audio_manager.player.cancel_playing() + + # VAD 检测到语音结束 + elif event_type == 'input_audio_buffer.speech_stopped': + print('====== VAD 检测到语音结束 ======') + self.last_package_time = time.time() * 1000 + self.is_first_text = True + self.is_first_audio = True + self.tool_calls = {} + + # 函数调用参数完成 + elif event_type == 'response.function_call_arguments.done': + print('====== 收到工具调用请求 ======') + call_id = response.get('call_id', '') + self.tool_calls[call_id] = response.copy() + self.tool_calls[call_id]['processed'] = False + + # 响应完成 + elif event_type == 'response.done': + print('====== 响应完成 ======') + print(f'完整回复: {self.all_response_text}') + + if self.conversation: + response_id = self.conversation.get_last_response_id() + text_delay = self.conversation.get_last_first_text_delay() + audio_delay = self.conversation.get_last_first_audio_delay() + + # 只有当所有指标都可用时才打印详细指标 + if response_id is not None and text_delay is not None and audio_delay is not None: + print(f'[Metric] 响应ID: {response_id}, ' + f'首字延迟: {text_delay:.0f}ms, ' + f'首音延迟: {audio_delay:.0f}ms') + else: + print('[Metric] 指标信息暂不可用（可能是工具调用后的响应）') + + self.all_response_text = '' + + except Exception as e: + print(f'[Error] 处理事件异常: {e}') + traceback.print_exc() + + def process_pending_tool_calls(self) -> bool: + """ + 处理待处理的工具调用 + + Returns: + 是否有新的工具调用需要响应 + """ + has_pending = False + + for call_id, tool_call in self.tool_calls.items(): + if not tool_call.get('processed', False): + has_pending = True + tool_call['processed'] = True + + # 处理工具调用 + result = handle_tool_call(tool_call) + + # 发送结果到服务端 + if self.conversation: + send_tool_call_response(self.conversation, result) + + return has_pending + + +# ==================== 主程序 ==================== +def main(): + """主函数""" + print('正在初始化 Omni 实时对话...') + + # 创建音频管理器 + audio_manager = AudioManager() + + # 创建回调处理器 + callback = OmniCallback(audio_manager) + + # 创建对话实例 + conversation = OmniRealtimeConversation( + api_key=dashscope.api_key, + url=WS_URL, + model=MODEL, + callback=callback, + ) + + # 设置回调中的对话引用 + callback.set_conversation(conversation) + + # 建立连接 + conversation.connect() + + # 配置会话参数 + omni_output_modalities = [MultiModality.AUDIO, MultiModality.TEXT] + + conversation.update_session( + output_modalities=omni_output_modalities, + voice=VOICE, + input_audio_format=AudioFormat.PCM_16000HZ_MONO_16BIT, + output_audio_format=AudioFormat.PCM_24000HZ_MONO_16BIT, + enable_input_audio_transcription=True, + enable_turn_detection=True, + turn_detection_type='server_vad', + tools=TOOLS, + ) + + # 设置信号处理 + def signal_handler(sig, frame): + print('\n接收到 Ctrl+C，正在停止...') + conversation.close() + audio_manager.cleanup() + print('Omni 实时对话已停止') + sys.exit(0) + + signal.signal(signal.SIGINT, signal_handler) + print("按 Ctrl+C 停止对话...\n") + + # 主循环：持续发送音频并检查工具调用 + try: + while True: + # 处理待处理的工具调用 + has_tool_calls = callback.process_pending_tool_calls() + + if has_tool_calls: + print("*** 工具调用完成，创建新响应 ***") + conversation.create_response( + instructions=None, + output_modalities=omni_output_modalities + ) + print('====== 工具调用处理完成 ======\n') + + # 读取并发送音频数据 + audio_data = audio_manager.read_audio_chunk() + if audio_data: + audio_b64 = base64.b64encode(audio_data).decode('ascii') + conversation.append_audio(audio_b64) + else: + break + + except KeyboardInterrupt: + signal_handler(signal.SIGINT, None) + except Exception as e: + print(f'[Error] 主循环异常: {e}') + traceback.print_exc() + finally: + conversation.close() + audio_manager.cleanup() + + +if __name__ == '__main__': + main() +``` + +## DashScope Java SDK + +``` +// DashScope Java SDK v2.22.15 +import com.alibaba.dashscope.audio.omni.*; +import com.alibaba.dashscope.exception.NoApiKeyException; +import com.google.gson.Gson; +import com.google.gson.JsonObject; +import javax.sound.sampled.*; +import java.nio.ByteBuffer; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Function; + +public class Main { + + public static void main(String[] args) { + try { + // 初始化组件 + AudioPlayer audioPlayer = new AudioPlayer(); + ToolRegistry toolRegistry = new ToolRegistry(); + ConversationHandler handler = new ConversationHandler(audioPlayer, toolRegistry); + + // 创建并配置会话 + OmniRealtimeParam param = OmniRealtimeParam.builder() + .model("qwen3.5-omni-plus-realtime") + .apikey(System.getenv("DASHSCOPE_API_KEY")) + // 如果需要访问新加坡地域，请url将替换为：wss://dashscope-intl.aliyuncs.com/api-ws/v1/realtime + .url("wss://dashscope.aliyuncs.com/api-ws/v1/realtime") + .build(); + + OmniRealtimeConversation conversation = new OmniRealtimeConversation(param, handler); + conversation.connect(); + + // 配置会话参数 + configureSession(conversation, toolRegistry); + + // 启动音频采集 + startAudioCapture(conversation, handler); + + // 清理资源 + cleanup(conversation, audioPlayer); + + } catch (NoApiKeyException e) { + System.err.println("未找到API KEY: 请设置环境变量 DASHSCOPE_API_KEY"); + } catch (Exception e) { + e.printStackTrace(); + } + } + + private static void configureSession(OmniRealtimeConversation conversation, ToolRegistry toolRegistry) { + HashMap additionalConfig = new HashMap<>(); + additionalConfig.put("tools", toolRegistry.buildToolsDefinition()); + + conversation.updateSession(OmniRealtimeConfig.builder() + .modalities(Arrays.asList(OmniRealtimeModality.AUDIO, OmniRealtimeModality.TEXT)) + .voice("Tina") + .enableTurnDetection(true) + .enableInputAudioTranscription(true) + .parameters(additionalConfig) + .build()); + + System.out.println("工具调用已开启，请开始说话（按Ctrl+C退出）..."); + } + + private static void startAudioCapture(OmniRealtimeConversation conversation, ConversationHandler handler) + throws LineUnavailableException { + AudioFormat format = new AudioFormat(16000, 16, 1, true, false); + TargetDataLine mic = AudioSystem.getTargetDataLine(format); + mic.open(format); + mic.start(); + + ByteBuffer buffer = ByteBuffer.allocate(3200); + while (!handler.getShouldStop().get()) { + int bytesRead = mic.read(buffer.array(), 0, buffer.capacity()); + if (bytesRead > 0) { + conversation.appendAudio(Base64.getEncoder().encodeToString(buffer.array())); + + // 检查并处理待处理的工具调用 + if (handler.hasPendingToolCalls()) { + System.out.println("*** create response after call tools"); + handler.processPendingToolCalls(conversation); + conversation.createResponse(null, Arrays.asList(OmniRealtimeModality.AUDIO, OmniRealtimeModality.TEXT)); + System.out.println("======TOOL CALL END======"); + } + } + try { + Thread.sleep(20); + } catch (InterruptedException ignored) {} + } + + mic.close(); + } + + private static void cleanup(OmniRealtimeConversation conversation, AudioPlayer audioPlayer) { + try { + conversation.close(1000, "正常结束"); + audioPlayer.close(); + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** + * 音频播放器 - 负责音频数据的顺序播放 + */ + static class AudioPlayer { + private final SourceDataLine line; + private final Queue audioQueue = new ConcurrentLinkedQueue<>(); + private final Thread playerThread; + private final AtomicBoolean shouldStop = new AtomicBoolean(false); + + public AudioPlayer() throws LineUnavailableException { + AudioFormat format = new AudioFormat(24000, 16, 1, true, false); + line = AudioSystem.getSourceDataLine(format); + line.open(format); + line.start(); + + playerThread = new Thread(this::playLoop, "AudioPlayer"); + playerThread.start(); + } + + private void playLoop() { + while (!shouldStop.get()) { + byte[] audio = audioQueue.poll(); + if (audio != null) { + line.write(audio, 0, audio.length); + } else { + try { + Thread.sleep(10); + } catch (InterruptedException ignored) {} + } + } + } + + public void play(String base64Audio) { + audioQueue.add(Base64.getDecoder().decode(base64Audio)); + } + + public void close() { + shouldStop.set(true); + try { + playerThread.join(1000); + } catch (InterruptedException ignored) {} + line.drain(); + line.close(); + } + } + + /** + * 工具注册表 - 管理可用的工具及其实现 + */ + static class ToolRegistry { + private final Map> tools = new ConcurrentHashMap<>(); + private final Map pendingToolCalls = new ConcurrentHashMap<>(); + + public ToolRegistry() { + registerDefaultTools(); + } + + private void registerDefaultTools() { + registerTool("get_current_weather", this::getCurrentWeather); + registerTool("get_flight_price", this::getFlightPrice); + registerTool("get_train_price", this::getTrainPrice); + } + + public void registerTool(String name, Function handler) { + tools.put(name, handler); + } + + /** + * 构建工具定义（OpenAI 格式） + */ + public List> buildToolsDefinition() { + List> definitions = new ArrayList<>(); + + definitions.add(createFunctionDefinition( + "get_current_weather", + "当你想查询指定城市的天气时非常有用。", + createParamsSchema( + Collections.singletonMap("location", + createProperty("string", "城市或县区，比如北京市、杭州市、余杭区等。")), + Collections.singletonList("location") + ) + )); + + Map flightProps = new HashMap<>(); + flightProps.put("src", createProperty("string", "飞机起飞的城市，比如北京市、杭州市等。")); + flightProps.put("dst", createProperty("string", "飞机降落的城市，比如北京市、杭州市区等。")); + definitions.add(createFunctionDefinition( + "get_flight_price", + "当你想查询飞机票价格时非常有用。", + createParamsSchema(flightProps, Arrays.asList("src", "dst")) + )); + + Map trainProps = new HashMap<>(); + trainProps.put("src", createProperty("string", "火车出发的城市，比如北京市、杭州市等。")); + trainProps.put("dst", createProperty("string", "火车到达的城市，比如北京市、杭州市区等。")); + definitions.add(createFunctionDefinition( + "get_train_price", + "当你想查询火车票价格时非常有用。", + createParamsSchema(trainProps, Arrays.asList("src", "dst")) + )); + + return definitions; + } + + private Map createFunctionDefinition(String name, String description, Map parameters) { + Map function = new HashMap<>(); + function.put("name", name); + function.put("description", description); + function.put("parameters", parameters); + + Map tool = new HashMap<>(); + tool.put("type", "function"); + tool.put("function", function); + return tool; + } + + private Map createParamsSchema(Map properties, List required) { + Map schema = new HashMap<>(); + schema.put("type", "object"); + schema.put("properties", properties); + schema.put("required", required); + return schema; + } + + private Map createProperty(String type, String description) { + Map prop = new HashMap<>(); + prop.put("type", type); + prop.put("description", description); + return prop; + } + + /** + * 添加工具调用到待处理队列 + */ + public void addPendingToolCall(String callId, JsonObject toolCall) { + pendingToolCalls.put(callId, toolCall); + } + + /** + * 检查是否有待处理的工具调用 + */ + public boolean hasPendingToolCalls() { + return !pendingToolCalls.isEmpty(); + } + + /** + * 处理所有待处理的工具调用 + */ + public void processPendingToolCalls(OmniRealtimeConversation conversation) { + if (pendingToolCalls.isEmpty()) { + return; + } + + for (Map.Entry entry : pendingToolCalls.entrySet()) { + String callId = entry.getKey(); + JsonObject toolCall = entry.getValue(); + + String result = executeTool(toolCall); + sendToolResult(conversation, callId, result); + } + + pendingToolCalls.clear(); + } + + private String executeTool(JsonObject toolCall) { + String functionName = toolCall.get("name").getAsString(); + JsonObject arguments = new Gson().fromJson( + toolCall.get("arguments").getAsString(), + JsonObject.class + ); + + System.out.println("[Tool Call] start handling: " + functionName + ", args: " + arguments); + + Function handler = tools.get(functionName); + if (handler == null) { + return "client没有找到这个工具，调用失败。"; + } + + String result = handler.apply(arguments); + System.out.println("[Tool Call] response: " + result); + return result; + } + + private void sendToolResult(OmniRealtimeConversation conversation, String callId, String output) { + JsonObject item = new JsonObject(); + item.addProperty("id", "item_" + UUID.randomUUID().toString().replace("-", "")); + item.addProperty("type", "function_call_output"); + item.addProperty("call_id", callId); + item.addProperty("output", output); + + conversation.createItem(item); + } + + // ===== 工具实现 ===== + + private String getCurrentWeather(JsonObject args) { + String location = args.get("location").getAsString(); + return location + "今天天气为霾转晴，气温4/-4℃，微风"; + } + + private String getFlightPrice(JsonObject args) { + String src = args.get("src").getAsString(); + String dst = args.get("dst").getAsString(); + return src + "到" + dst + "的机票价格为200~300美元。"; + } + + private String getTrainPrice(JsonObject args) { + String src = args.get("src").getAsString(); + String dst = args.get("dst").getAsString(); + return "invalid apikey error"; + } + } + + /** + * 会话处理器 - 处理 WebSocket 事件 + */ + static class ConversationHandler extends OmniRealtimeCallback { + private final AudioPlayer audioPlayer; + private final ToolRegistry toolRegistry; + private final AtomicBoolean shouldStop = new AtomicBoolean(false); + private final AtomicReference responseTextRef = new AtomicReference<>(new StringBuilder()); + + private long lastPackageTime = 0; + private boolean isFirstText = true; + private boolean isFirstAudio = true; + + public ConversationHandler(AudioPlayer audioPlayer, ToolRegistry toolRegistry) { + this.audioPlayer = audioPlayer; + this.toolRegistry = toolRegistry; + } + + public AtomicBoolean getShouldStop() { + return shouldStop; + } + + @Override + public void onOpen() { + System.out.println("连接已建立"); + } + + @Override + public void onClose(int code, String reason) { + System.out.println("连接已关闭"); + shouldStop.set(true); + } + + @Override + public void onEvent(JsonObject message) { + String type = message.get("type").getAsString(); + + switch (type) { + case "session.created": + handleSessionCreated(message); + break; + case "conversation.item.input_audio_transcription.completed": + handleTranscriptionCompleted(message); + break; + case "response.audio_transcript.delta": + case "response.text.delta": + handleTextDelta(message); + break; + case "response.audio.delta": + handleAudioDelta(message); + break; + case "input_audio_buffer.speech_started": + handleSpeechStarted(); + break; + case "input_audio_buffer.speech_stopped": + handleSpeechStopped(); + break; + case "response.function_call_arguments.done": + handleFunctionCall(message); + break; + case "response.done": + handleResponseDone(); + break; + default: + break; + } + } + + private void handleSessionCreated(JsonObject message) { + String sessionId = message.get("session").getAsJsonObject().get("id").getAsString(); + System.out.println("start session: " + sessionId); + } + + private void handleTranscriptionCompleted(JsonObject message) { + System.out.println("question: " + message.get("transcript").getAsString()); + } + + private void handleTextDelta(JsonObject message) { + if (isFirstText) { + isFirstText = false; + System.out.println("first text latency from vad end: " + + (System.currentTimeMillis() - lastPackageTime) + " ms"); + } + String text = message.get("delta").getAsString(); + responseTextRef.get().append(text); + } + + private void handleAudioDelta(JsonObject message) { + if (isFirstAudio) { + isFirstAudio = false; + System.out.println("first audio latency from vad end: " + + (System.currentTimeMillis() - lastPackageTime) + " ms"); + } + System.out.println("audio interval: " + (System.currentTimeMillis() - lastPackageTime) + " ms"); + lastPackageTime = System.currentTimeMillis(); + audioPlayer.play(message.get("delta").getAsString()); + } + + private void handleSpeechStarted() { + System.out.println("======VAD Speech Start======"); + } + + private void handleSpeechStopped() { + System.out.println("======VAD Speech End======"); + lastPackageTime = System.currentTimeMillis(); + isFirstText = true; + isFirstAudio = true; + } + + private void handleFunctionCall(JsonObject message) { + System.out.println("======TOOL CALL======"); + String callId = message.get("call_id").getAsString(); + toolRegistry.addPendingToolCall(callId, message); + } + + private void handleResponseDone() { + System.out.println("======RESPONSE DONE======"); + System.out.println("all response text: " + responseTextRef.get()); + responseTextRef.set(new StringBuilder()); + } + + /** + * 检查是否有待处理的工具调用 + */ + public boolean hasPendingToolCalls() { + return toolRegistry.hasPendingToolCalls(); + } + + /** + * 处理所有待处理的工具调用 + */ + public void processPendingToolCalls(OmniRealtimeConversation conversation) { + toolRegistry.processPendingToolCalls(conversation); + } + } +} +``` + +## WebSocket(Python) + +``` +import asyncio +import json +import base64 +import os +import pyaudio +import websockets + +# ==================== 常量定义 ==================== +API_KEY = os.getenv("DASHSCOPE_API_KEY") +# 如果需要访问新加坡地域，请替换为： +# wss://dashscope-intl.aliyuncs.com/api-ws/v1/realtime +URL = "wss://dashscope.aliyuncs.com/api-ws/v1/realtime" +MODEL = "qwen3.5-omni-plus-realtime" +VOICE = "Ethan" + + +# ==================== 工具定义 ==================== +def get_current_weather(location): + """查询指定城市天气""" + return f"{location}今天天气为霾转晴，气温4/-4℃，微风" + + +def get_flight_price(src, dst): + """查询飞机票价格""" + return f"{src}到{dst}的机票价格为200~300美元。" + + +def get_train_price(src, dst): + """查询火车票价格""" + return f"{src}到{dst}的火车票价格为100~200元。" + + +# 工具名称到函数的映射 +TOOL_FUNCTIONS = { + "get_current_weather": get_current_weather, + "get_flight_price": get_flight_price, + "get_train_price": get_train_price, +} + +TOOLS = [ + { + "type": "function", + "function": { + "name": "get_current_weather", + "description": "当你想查询指定城市的天气时非常有用。", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "城市或县区，比如北京市、杭州市、余杭区等。", + } + }, + "required": ["location"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "get_flight_price", + "description": "当你想查询飞机票价格时非常有用。", + "parameters": { + "type": "object", + "properties": { + "src": { + "type": "string", + "description": "飞机起飞的城市，比如北京市、杭州市等。", + }, + "dst": { + "type": "string", + "description": "飞机降落的城市，比如北京市、杭州市区等。", + }, + }, + "required": ["src", "dst"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "get_train_price", + "description": "当你想查询火车票价格时非常有用。", + "parameters": { + "type": "object", + "properties": { + "src": { + "type": "string", + "description": "火车出发的城市，比如北京市、杭州市等。", + }, + "dst": { + "type": "string", + "description": "火车到达的城市，比如北京市、杭州市区等。", + }, + }, + "required": ["src", "dst"], + }, + }, + }, +] + + +# ==================== 工具调用处理 ==================== +def handle_tool_call(name, arguments_str): + """ + 处理工具调用请求 + + Args: + name: 工具函数名称 + arguments_str: JSON 格式的入参字符串 + + Returns: + 工具执行结果字符串 + """ + try: + arguments = json.loads(arguments_str) + print(f'[工具调用] 开始处理: name={name}, args={arguments}') + + func = TOOL_FUNCTIONS.get(name) + if func is None: + result = f"客户端未找到工具: {name}" + print(f'[工具调用] 错误: {result}') + return result + + result = func(**arguments) + print(f'[工具调用] 完成: {result}') + return result + + except Exception as e: + error_msg = f"工具调用失败: {str(e)}" + print(f'[工具调用] 异常: {error_msg}') + return error_msg + + +# ==================== 主程序 ==================== +async def main(): + """主函数：建立 WebSocket 连接并进行语音对话""" + pya = pyaudio.PyAudio() + speaker = pya.open(format=pyaudio.paInt16, channels=1, rate=24000, output=True) + + # 建立 WebSocket 连接 + headers = { + "Authorization": f"bearer {API_KEY}", + "X-DashScope-OmniRealtime": "true", + } + async with websockets.connect( + f"{URL}?model={MODEL}", additional_headers=headers, + ) as ws: + await ws.recv() + + # 配置会话参数 + await ws.send(json.dumps({ + "type": "session.update", + "session": { + "modalities": ["text", "audio"], + "voice": VOICE, + "input_audio_format": "pcm16", + "output_audio_format": "pcm16", + "instructions": "你是个人助理小云", + "turn_detection": {"type": "server_vad"}, + "input_audio_transcription": {"model": "gummy-realtime-v1"}, + "tools": TOOLS, + }, + })) + await ws.recv() + + # 音频采集协程 + async def send_audio(): + mic = pya.open(format=pyaudio.paInt16, channels=1, rate=16000, input=True) + try: + while True: + data = mic.read(3200, exception_on_overflow=False) + await ws.send(json.dumps({ + "type": "input_audio_buffer.append", + "audio": base64.b64encode(data).decode(), + })) + await asyncio.sleep(0.01) + except asyncio.CancelledError: + mic.close() + + pending = {} + all_response_text = "" + send_task = asyncio.create_task(send_audio()) + print("工具调用已启用，对着麦克风说话 (Ctrl+C 退出)...") + + # 事件处理循环 + async for raw in ws: + msg = json.loads(raw) + t = msg["type"] + + # 会话创建 + if t == "session.created": + print(f"会话已启动: {msg['session']['id']}") + + # 播放音频 + elif t == "response.audio.delta": + speaker.write(base64.b64decode(msg["delta"])) + + # 文本增量响应 + elif t in ("response.audio_transcript.delta", "response.text.delta"): + all_response_text += msg.get("delta", "") + + # 用户语音转文本 + elif t == "conversation.item.input_audio_transcription.completed": + print(f"[用户] {msg['transcript']}") + + # VAD 检测到语音开始 + elif t == "input_audio_buffer.speech_started": + print("====== VAD 检测到语音开始 ======") + + # VAD 检测到语音结束 + elif t == "input_audio_buffer.speech_stopped": + print("====== VAD 检测到语音结束 ======") + + # 收到工具调用请求 + elif t == "response.function_call_arguments.done": + print("====== 收到工具调用请求 ======") + pending[msg["call_id"]] = { + "name": msg["name"], + "arguments": msg["arguments"], + } + + # 响应完成 + elif t == "response.done": + if pending: + # 执行待处理的工具调用 + for cid, info in pending.items(): + result = handle_tool_call(info["name"], info["arguments"]) + # 发送工具执行结果 + await ws.send(json.dumps({ + "type": "conversation.item.create", + "item": { + "type": "function_call_output", + "call_id": cid, + "output": result, + }, + })) + pending.clear() + # 触发服务端继续生成回复 + await ws.send(json.dumps({ + "type": "response.create", + "response": {"modalities": ["text", "audio"]}, + })) + print("====== 工具调用处理完成 ======") + else: + # 普通响应完成，打印完整回复 + if all_response_text: + print(f"[模型] {all_response_text}") + all_response_text = "" + + send_task.cancel() + speaker.close() + pya.terminate() + + +asyncio.run(main()) +``` + +### **深度思考模型的工具调用** + +深度思考模型在输出工具调用信息前会进行思考，可以提升决策的可解释性与可靠性。 + +1. **思考过程** + + 模型逐步分析用户意图、识别所需工具、验证参数合法性，并规划调用策略； + +2. **工具调用** + + 模型以结构化格式输出一个或多个函数调用请求。 + + > 支持并行工具调用。 + + +以下展示流式调用深度思考模型的工具调用示例。 + +> 文本生成思考模型请参见：[深度思考](https://help.aliyun.com/zh/model-studio/deep-thinking)；多模态思考模型请参见：[图像与视频理解](https://help.aliyun.com/zh/model-studio/vision)、[非实时（Qwen-Omni）](https://help.aliyun.com/zh/model-studio/qwen-omni)。 + +> `tool_choice`参数只支持设置为`"auto"`（默认值，表示由模型自主选择工具）或`"none"`（强制模型不选择工具）。 + +## OpenAI兼容 + +## Python + +### **示例代码** + +``` +import os +from openai import OpenAI + +# 初始化OpenAI客户端，配置阿里云DashScope服务 +client = OpenAI( + # 若没有配置环境变量，请用阿里云百炼API Key将下行替换为：api_key="sk-xxx", + # 各地域的API Key不同。获取API Key：https://help.aliyun.com/zh/model-studio/get-api-key + api_key=os.getenv("DASHSCOPE_API_KEY"), # 从环境变量读取API密钥 + # 以下是北京地域base_url，如果使用新加坡地域的模型，需要将base_url替换为：https://dashscope-intl.aliyuncs.com/compatible-mode/v1 + base_url="https://dashscope.aliyuncs.com/compatible-mode/v1", +) + +# 定义可用工具列表 +tools = [ + # 工具1 获取当前时刻的时间 + { + "type": "function", + "function": { + "name": "get_current_time", + "description": "当你想知道现在的时间时非常有用。", + "parameters": {} # 无需参数 + } + }, + # 工具2 获取指定城市的天气 + { + "type": "function", + "function": { + "name": "get_current_weather", + "description": "当你想查询指定城市的天气时非常有用。", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "城市或县区，比如北京市、杭州市、余杭区等。" + } + }, + "required": ["location"] # 必填参数 + } + } + } +] + +messages = [{"role": "user", "content": input("请输入问题：")}] + +# 多模态模型的 message示例 +# messages = [{ +# "role": "user", +# "content": [ +# {"type": "image_url","image_url": {"url": "https://img.alicdn.com/imgextra/i4/O1CN014CJhzi20NOzo7atOC_!!6000000006837-2-tps-2048-1365.png"}}, +# {"type": "text", "text": "根据图像上的地点，请问该地点当前的天气"}] +# }] + +completion = client.chat.completions.create( + # 此处以qwen3.6-plus为例，可更换为其它深度思考模型 + model="qwen3.6-plus", + messages=messages, + extra_body={ + # 开启深度思考 + "enable_thinking": True + }, + tools=tools, + parallel_tool_calls=True, + stream=True, + # 解除注释后，可以获取到token消耗信息 + # stream_options={ + # "include_usage": True + # } +) + +reasoning_content = "" # 定义完整思考过程 +answer_content = "" # 定义完整回复 +tool_info = [] # 存储工具调用信息 +is_answering = False # 判断是否结束思考过程并开始回复 +print("="*20+"思考过程"+"="*20) +for chunk in completion: + if not chunk.choices: + # 处理用量统计信息 + print("\n"+"="*20+"Usage"+"="*20) + print(chunk.usage) + else: + delta = chunk.choices[0].delta + # 处理AI的思考过程（链式推理） + if hasattr(delta, 'reasoning_content') and delta.reasoning_content is not None: + reasoning_content += delta.reasoning_content + print(delta.reasoning_content,end="",flush=True) # 实时输出思考过程 + + # 处理最终回复内容 + else: + if not is_answering: # 首次进入回复阶段时打印标题 + is_answering = True + print("\n"+"="*20+"回复内容"+"="*20) + if delta.content is not None: + answer_content += delta.content + print(delta.content,end="",flush=True) # 流式输出回复内容 + + # 处理工具调用信息（支持并行工具调用） + if delta.tool_calls is not None: + for tool_call in delta.tool_calls: + index = tool_call.index # 工具调用索引，用于并行调用 + + # 动态扩展工具信息存储列表 + while len(tool_info) <= index: + tool_info.append({}) + + # 收集工具调用ID（用于后续函数调用） + if tool_call.id: + tool_info[index]['id'] = tool_info[index].get('id', '') + tool_call.id + + # 收集函数名称（用于后续路由到具体函数） + if tool_call.function and tool_call.function.name: + tool_info[index]['name'] = tool_info[index].get('name', '') + tool_call.function.name + + # 收集函数参数（JSON字符串格式，需要后续解析） + if tool_call.function and tool_call.function.arguments: + tool_info[index]['arguments'] = tool_info[index].get('arguments', '') + tool_call.function.arguments + +print(f"\n"+"="*19+"工具调用信息"+"="*19) +if not tool_info: + print("没有工具调用") +else: + print(tool_info) +``` + +### **返回结果** + +输入“四个直辖市的天气”，得到以下返回结果： + +``` +====================思考过程==================== +好的，用户问的是“四个直辖市的天气”。首先，我需要明确四个直辖市是哪几个。根据中国的行政区划，直辖市包括北京、上海、天津和重庆。所以用户想知道这四个城市的天气情况。 + +接下来，我需要检查可用的工具。提供的工具中有get_current_weather函数，参数是location，类型字符串。每个城市需要单独查询，因为函数一次只能查一个地点。因此，我需要为每个直辖市调用一次这个函数。 + +然后，我需要考虑如何生成正确的工具调用。每个调用应该包含城市名称作为参数。比如，第一个调用是北京，第二个是上海，依此类推。确保参数名称是location，值是正确的城市名。 + +另外，用户可能希望得到每个城市的天气信息，所以需要确保每个函数调用都正确无误。可能需要连续调用四次，每次对应一个城市。不过，根据工具的使用规则，可能需要分多次处理，或者一次生成多个调用。但根据示例，可能每次只调用一个函数，所以可能需要逐步进行。 + +最后，确认是否有其他需要考虑的因素，比如参数是否正确，城市名称是否准确，以及是否需要处理可能的错误情况，比如城市不存在或API不可用。但目前看来，四个直辖市都是明确的，应该没问题。 +====================回复内容==================== + +===================工具调用信息=================== +[{'id': 'call_767af2834c12488a8fe6e3', 'name': 'get_current_weather', 'arguments': '{"location": "北京市"}'}, {'id': 'call_2cb05a349c89437a947ada', 'name': 'get_current_weather', 'arguments': '{"location": "上海市"}'}, {'id': 'call_988dd180b2ca4b0a864ea7', 'name': 'get_current_weather', 'arguments': '{"location": "天津市"}'}, {'id': 'call_4e98c57ea96a40dba26d12', 'name': 'get_current_weather', 'arguments': '{"location": "重庆市"}'}] +``` + +## Node.js + +### **示例代码** + +``` +import OpenAI from "openai"; +import readline from 'node:readline/promises'; +import { stdin as input, stdout as output } from 'node:process'; + +const openai = new OpenAI({ + // 各地域的API Key不同。获取API Key：https://help.aliyun.com/zh/model-studio/get-api-key + apiKey: process.env.DASHSCOPE_API_KEY, + // 以下是北京地域base_url，如果使用新加坡地域的模型，需要将base_url替换为：https://dashscope-intl.aliyuncs.com/compatible-mode/v1 + baseURL: "https://dashscope.aliyuncs.com/compatible-mode/v1" +}); + +const tools = [ + { + type: "function", + function: { + name: "get_current_time", + description: "当你想知道现在的时间时非常有用。", + parameters: {} + } + }, + { + type: "function", + function: { + name: "get_current_weather", + description: "当你想查询指定城市的天气时非常有用。", + parameters: { + type: "object", + properties: { + location: { + type: "string", + description: "城市或县区，比如北京市、杭州市、余杭区等。" + } + }, + required: ["location"] + } + } + } +]; + +async function main() { + const rl = readline.createInterface({ input, output }); + const question = await rl.question("请输入您的问题："); + rl.close(); + + const messages = [{ role: "user", content: question }]; + // 多模态模型的 message示例 + // const messages= [{ + // role: "user", + // content: [{type: "image_url", image_url: {url: "https://img.alicdn.com/imgextra/i2/O1CN01FbTJon1ErXVGMRdsN_!!6000000000405-0-tps-1024-683.jpg"}}, + // {type: "text", text: "图中地点的天气？"}] + // }]; + let reasoningContent = ""; + let answerContent = ""; + const toolInfo = []; + let isAnswering = false; + + console.log("=".repeat(20) + "思考过程" + "=".repeat(20)); + + try { + const stream = await openai.chat.completions.create({ + // 此处以qwen3.6-plus为例，可更换为其它深度思考模型 + model: "qwen3.6-plus", + messages, + // 开启深度思考 + enable_thinking: true, + tools, + stream: true, + parallel_tool_calls: true + }); + + for await (const chunk of stream) { + if (!chunk.choices?.length) { + console.log("\n" + "=".repeat(20) + "Usage" + "=".repeat(20)); + console.log(chunk.usage); + continue; + } + + const delta = chunk.choices[0]?.delta; + if (!delta) continue; + + // 处理思考过程 + if (delta.reasoning_content) { + reasoningContent += delta.reasoning_content; + process.stdout.write(delta.reasoning_content); + } + // 处理回复内容 + else { + if (!isAnswering) { + isAnswering = true; + console.log("\n" + "=".repeat(20) + "回复内容" + "=".repeat(20)); + } + if (delta.content) { + answerContent += delta.content; + process.stdout.write(delta.content); + } + // 处理工具调用 + if (delta.tool_calls) { + for (const toolCall of delta.tool_calls) { + const index = toolCall.index; + + // 确保数组长度足够 + while (toolInfo.length <= index) { + toolInfo.push({}); + } + + // 更新工具ID + if (toolCall.id) { + toolInfo[index].id = (toolInfo[index].id || "") + toolCall.id; + } + + // 更新函数名称 + if (toolCall.function?.name) { + toolInfo[index].name = (toolInfo[index].name || "") + toolCall.function.name; + } + + // 更新参数 + if (toolCall.function?.arguments) { + toolInfo[index].arguments = (toolInfo[index].arguments || "") + toolCall.function.arguments; + } + } + } + } + } + + console.log("\n" + "=".repeat(19) + "工具调用信息" + "=".repeat(19)); + console.log(toolInfo.length ? toolInfo : "没有工具调用"); + + } catch (error) { + console.error("发生错误:", error); + } +} + +main(); +``` + +### **返回结果** + +输入“四个直辖市的天气”，得到以下返回结果： + +``` +请输入您的问题：四个直辖市的天气 +====================思考过程==================== +好的，用户问的是四个直辖市的天气。首先，我需要明确中国的四个直辖市分别是哪几个。北京、上海、天津和重庆，对吧？接下来，我需要为每个城市调用天气查询功能。 + +但是用户的问题可能需要我分别获取这四个城市的天气情况。每个城市都需要调用一次get_current_weather函数，参数是各自的城市名称。我需要确保参数正确，比如直辖市的全名，比如“北京市”、“上海市”、“天津市”和“重庆市”。 + +然后，我需要按照顺序依次调用这四个城市的天气接口。每个调用都需要单独的tool_call。用户可能希望得到每个城市的当前天气信息，所以需要确保每个调用都正确无误。可能需要注意每个城市的正确拼写和名称，避免出现错误。例如，重庆有时可能被简称为“重庆市”，所以参数里应该用全称。 + +现在，我需要生成四个tool_call，每个对应一个直辖市。检查每个参数是否正确，然后按顺序排列。这样用户就能得到四个直辖市的天气数据了。 +====================回复内容==================== + +===================工具调用信息=================== +[ + { + id: 'call_21dc802e717f491298d1b2', + name: 'get_current_weather', + arguments: '{"location": "北京市"}' + }, + { + id: 'call_2cd3be1d2f694c4eafd4e5', + name: 'get_current_weather', + arguments: '{"location": "上海市"}' + }, + { + id: 'call_48cf3f78e02940bd9085e4', + name: 'get_current_weather', + arguments: '{"location": "天津市"}' + }, + { + id: 'call_e230a2b4c64f4e658d223e', + name: 'get_current_weather', + arguments: '{"location": "重庆市"}' + } +] +``` + +## HTTP + +### **示例代码** + +## curl + +``` +# ======= 重要提示 ======= +# 各地域的API Key不同。获取API Key：https://help.aliyun.com/zh/model-studio/get-api-key +# 以下是北京地域base_url，如果使用新加坡地域的模型，需要将base_url替换为：https://dashscope-intl.aliyuncs.com/compatible-mode/v1/chat/completions +# === 执行时请删除该注释 === +curl -X POST https://dashscope.aliyuncs.com/compatible-mode/v1/chat/completions \ +-H "Authorization: Bearer $DASHSCOPE_API_KEY" \ +-H "Content-Type: application/json" \ +-d '{ + "model": "qwen3.6-plus", + "messages": [ + { + "role": "user", + "content": "杭州天气怎么样" + } + ], + "tools": [ + { + "type": "function", + "function": { + "name": "get_current_time", + "description": "当你想知道现在的时间时非常有用。", + "parameters": {} + } + }, + { + "type": "function", + "function": { + "name": "get_current_weather", + "description": "当你想查询指定城市的天气时非常有用。", + "parameters": { + "type": "object", + "properties": { + "location":{ + "type": "string", + "description": "城市或县区，比如北京市、杭州市、余杭区等。" + } + }, + "required": ["location"] + } + } + } + ], + "enable_thinking": true, + "stream": true +}' +``` + +## DashScope + +## Python + +### **示例代码** + +``` +import dashscope +from dashscope import MultiModalConversation + +# 若使用新加坡地域的模型，请释放下列注释 +# dashscope.base_http_api_url = "https://dashscope-intl.aliyuncs.com/api/v1" + +tools = [ + # 工具1 获取当前时刻的时间 + { + "type": "function", + "function": { + "name": "get_current_time", + "description": "当你想知道现在的时间时非常有用。", + "parameters": {} # 因为获取当前时间无需输入参数，因此parameters为空字典 + } + }, + # 工具2 获取指定城市的天气 + { + "type": "function", + "function": { + "name": "get_current_weather", + "description": "当你想查询指定城市的天气时非常有用。", + "parameters": { + "type": "object", + "properties": { + # 查询天气时需要提供位置，因此参数设置为location + "location": { + "type": "string", + "description": "城市或县区，比如北京市、杭州市、余杭区等。" + } + }, + "required": ["location"] + } + } + } +] + +# 定义问题 +messages = [{"role": "user", "content": [{"text": input("请输入问题：")}]}] + +# 多模态模型的 message示例 +# messages = [ +# { +# "role": "user", +# "content": [ +# {"image": "https://img.alicdn.com/imgextra/i2/O1CN01FbTJon1ErXVGMRdsN_!!6000000000405-0-tps-1024-683.jpg"}, +# {"text": "图像地点天气？"}] +# }] + +completion = MultiModalConversation.call( + # 此处以qwen3.6-plus为例，可更换为其它深度思考模型 + model="qwen3.6-plus", + messages=messages, + enable_thinking=True, + tools=tools, + parallel_tool_calls=True, + stream=True, + incremental_output=True, + result_format="message" +) + +reasoning_content = "" +answer_content = "" +tool_info = [] +is_answering = False +print("="*20+"思考过程"+"="*20) + +for chunk in completion: + if chunk.status_code == 200: + msg = chunk.output.choices[0].message + + # 处理思考过程 + if 'reasoning_content' in msg and msg.reasoning_content: + reasoning_content += msg.reasoning_content + print(msg.reasoning_content, end="", flush=True) + + # 处理回复内容 + if 'content' in msg and msg.content: + if not is_answering: + is_answering = True + print("\n"+"="*20+"回复内容"+"="*20) + answer_content += msg.content + print(msg.content, end="", flush=True) + + # 处理工具调用 + if 'tool_calls' in msg and msg.tool_calls: + for tool_call in msg.tool_calls: + index = tool_call['index'] + + while len(tool_info) <= index: + tool_info.append({'id': '', 'name': '', 'arguments': ''}) # 初始化所有字段 + + # 增量更新工具ID + if 'id' in tool_call: + tool_info[index]['id'] += tool_call.get('id', '') + + # 增量更新函数信息 + if 'function' in tool_call: + func = tool_call['function'] + # 增量更新函数名称 + if 'name' in func: + tool_info[index]['name'] += func.get('name', '') + # 增量更新参数 + if 'arguments' in func: + tool_info[index]['arguments'] += func.get('arguments', '') + +print(f"\n"+"="*19+"工具调用信息"+"="*19) +if not tool_info: + print("没有工具调用") +else: + print(tool_info) +``` + +### **返回结果** + +输入“四个直辖市的天气”，得到以下返回结果： + +``` +请输入问题：四个直辖市的天气 +====================思考过程==================== +好的，用户问的是四个直辖市的天气。首先，我需要确认中国的四个直辖市分别是哪些。北京、上海、天津和重庆，对吧？接下来，用户需要每个城市的天气情况，所以我需要调用天气查询功能。 + +不过，问题来了，用户没有指定具体的城市名称，只是说四个直辖市。可能需要我明确每个直辖市的名称，然后分别查询。比如，北京、上海、天津、重庆这四个直辖市。我需要确保每个城市都正确无误。 + +然后，我要检查可用的工具，用户提供的函数是get_current_weather，参数是location。因此，我需要为每个直辖市调用这个函数，传入对应的城市名称作为参数。比如，第一次调用location是北京市，第二次是上海市，第三次是天津市，第四次是重庆市。 + +不过，可能需要注意，像重庆这样的直辖市，有时候可能需要更具体的区，但用户可能只需要市级的天气。所以直接使用直辖市名称应该没问题。接下来，我需要生成四个独立的函数调用，每个对应一个直辖市。这样用户就能得到四个城市的天气情况了。 + +最后，确保每个调用的参数正确，并且没有遗漏。这样用户的问题就能得到完整的回答了。 +===================工具调用信息=================== +[{'id': 'call_2f774ed97b0e4b24ab10ec', 'name': 'get_current_weather', 'arguments': '{"location": "北京市"}'}, {'id': 'call_dc3b05b88baa48c58bc33a', 'name': 'get_current_weather', 'arguments': '{"location": "上海市"}}'}, {'id': 'call_249b2de2f73340cdb46cbc', 'name': 'get_current_weather', 'arguments': '{"location": "天津市"}'}, {'id': 'call_833333634fda49d1b39e87', 'name': 'get_current_weather', 'arguments': '{"location": "重庆市"}}'}] +``` + +## Java + +### **示例代码** + +``` +// dashscope SDK的版本 >= 2.19.4 +import java.util.Arrays; + +import com.alibaba.dashscope.exception.UploadFileException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import com.alibaba.dashscope.aigc.multimodalconversation.MultiModalConversation; +import com.alibaba.dashscope.aigc.multimodalconversation.MultiModalConversationParam; +import com.alibaba.dashscope.aigc.multimodalconversation.MultiModalConversationResult; +import com.alibaba.dashscope.common.MultiModalMessage; +import com.alibaba.dashscope.aigc.generation.Generation; +import com.alibaba.dashscope.aigc.generation.GenerationParam; +import com.alibaba.dashscope.aigc.generation.GenerationResult; +import com.alibaba.dashscope.common.Message; +import com.alibaba.dashscope.common.Role; +import com.alibaba.dashscope.exception.ApiException; +import com.alibaba.dashscope.exception.InputRequiredException; +import com.alibaba.dashscope.exception.NoApiKeyException; +import com.alibaba.dashscope.utils.Constants; +import com.alibaba.dashscope.utils.JsonUtils; +import com.alibaba.dashscope.tools.ToolFunction; +import com.alibaba.dashscope.tools.FunctionDefinition; +import io.reactivex.Flowable; +import com.fasterxml.jackson.databind.node.ObjectNode; +import java.lang.System; +import com.github.victools.jsonschema.generator.Option; +import com.github.victools.jsonschema.generator.OptionPreset; +import com.github.victools.jsonschema.generator.SchemaGenerator; +import com.github.victools.jsonschema.generator.SchemaGeneratorConfig; +import com.github.victools.jsonschema.generator.SchemaGeneratorConfigBuilder; +import com.github.victools.jsonschema.generator.SchemaVersion; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.Collections; + +public class Main { + private static final Logger logger = LoggerFactory.getLogger(Main.class); + private static ObjectNode jsonSchemaWeather; + private static ObjectNode jsonSchemaTime; + // 若使用新加坡地域的模型，请释放下列注释 + // static {Constants.baseHttpApiUrl="https://dashscope-intl.aliyuncs.com/api/v1";} + + static class TimeTool { + public String call() { + LocalDateTime now = LocalDateTime.now(); + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); + return "当前时间：" + now.format(formatter) + "。"; + } + } + + static class WeatherTool { + private String location; + + public WeatherTool(String location) { + this.location = location; + } + + public String call() { + return location + "今天是晴天"; + } + } + + static { + SchemaGeneratorConfigBuilder configBuilder = new SchemaGeneratorConfigBuilder( + SchemaVersion.DRAFT_2020_12, OptionPreset.PLAIN_JSON); + SchemaGeneratorConfig config = configBuilder + .with(Option.EXTRA_OPEN_API_FORMAT_VALUES) + .without(Option.FLATTENED_ENUMS_FROM_TOSTRING) + .build(); + SchemaGenerator generator = new SchemaGenerator(config); + jsonSchemaWeather = generator.generateSchema(WeatherTool.class); + jsonSchemaTime = generator.generateSchema(TimeTool.class); + } + private static void handleGenerationResult(GenerationResult message) { + System.out.println(JsonUtils.toJson(message)); + } + + // 为文本生成模型创建工具调用方法 + public static void streamCallWithMessage(Generation gen, Message userMsg) + throws NoApiKeyException, ApiException, InputRequiredException { + GenerationParam param = buildGenerationParam(userMsg); + Flowable result = gen.streamCall(param); + result.blockingForEach(message -> handleGenerationResult(message)); + } + // 构建文本生成模型参数，支持工具调用 + private static GenerationParam buildGenerationParam(Message userMsg) { + FunctionDefinition fdWeather = buildFunctionDefinition( + "get_current_weather", "获取指定地区的天气", jsonSchemaWeather); + FunctionDefinition fdTime = buildFunctionDefinition( + "get_current_time", "获取当前时刻的时间", jsonSchemaTime); + + return GenerationParam.builder() + // 各地域的API Key不同。获取API Key：https://www.alibabacloud.com/help/zh/model-studio/get-api-key + .apiKey(System.getenv("DASHSCOPE_API_KEY")) + .model("qwen3.6-plus") + .enableThinking(true) + .messages(Arrays.asList(userMsg)) + .resultFormat(GenerationParam.ResultFormat.MESSAGE) + .incrementalOutput(true) + .tools(Arrays.asList( + ToolFunction.builder().function(fdWeather).build(), + ToolFunction.builder().function(fdTime).build())) + .build(); + } + + // 为多模态模型创建工具调用方法 + public static void streamCallWithMultiModalMessage(MultiModalConversation conv, MultiModalMessage userMsg) + throws NoApiKeyException, ApiException, UploadFileException { + MultiModalConversationParam param = buildMultiModalConversationParam(userMsg); + Flowable result = conv.streamCall(param); + result.blockingForEach(message -> System.out.println(JsonUtils.toJson(message))); + } + + // 构建多模态模型参数，支持工具调用 + private static MultiModalConversationParam buildMultiModalConversationParam(MultiModalMessage userMsg) { + FunctionDefinition fdWeather = buildFunctionDefinition( + "get_current_weather", "获取指定地区的天气", jsonSchemaWeather); + FunctionDefinition fdTime = buildFunctionDefinition( + "get_current_time", "获取当前时刻的时间", jsonSchemaTime); + + return MultiModalConversationParam.builder() + .apiKey(System.getenv("DASHSCOPE_API_KEY")) + .model("qwen3-vl-plus") // 使用多模态模型 Qwen3-VL + .enableThinking(true) + .messages(Arrays.asList(userMsg)) + .tools(Arrays.asList( // 配置工具列表 + ToolFunction.builder().function(fdWeather).build(), + ToolFunction.builder().function(fdTime).build())) + .build(); + } + + private static FunctionDefinition buildFunctionDefinition( + String name, String description, ObjectNode schema) { + return FunctionDefinition.builder() + .name(name) + .description(description) + .parameters(JsonUtils.parseString(schema.toString()).getAsJsonObject()) + .build(); + } + + public static void main(String[] args) { + try { + MultiModalConversation conv = new MultiModalConversation(); + MultiModalMessage userMsg = MultiModalMessage.builder().role(Role.USER.getValue()) + .content(Arrays.asList(Collections.singletonMap("text", "请告诉我杭州的天气"))).build(); + try { + streamCallWithMultiModalMessage(conv, userMsg); + } catch (UploadFileException e) { + throw new RuntimeException(e); + } +// 使用文本生成模型进行工具调用时，请解除下列注释 +// Generation gen = new Generation(); +// Message userMessage = Message.builder() +// .role(Role.USER.getValue()) +// .content("请告诉我杭州的天气") +// .build(); +// try { +// streamCallWithMessage(gen, userMessage); +// } catch (InputRequiredException e) { +// throw new RuntimeException(e); +// } + } catch (ApiException | NoApiKeyException e) { + logger.error("An exception occurred: {}", e.getMessage()); + } + System.exit(0); + } +} +``` + +### **返回结果** + +``` +{"requestId":"4edb81cd-4647-9d5d-88f9-a4f30bc6d8dd","usage":{"input_tokens":238,"output_tokens":6,"total_tokens":244},"output":{"choices":[{"finish_reason":"null","message":{"role":"assistant","content":"","reasoning_content":"好的，用户让我"}}]}} +{"requestId":"4edb81cd-4647-9d5d-88f9-a4f30bc6d8dd","usage":{"input_tokens":238,"output_tokens":12,"total_tokens":250},"output":{"choices":[{"finish_reason":"null","message":{"role":"assistant","content":"","reasoning_content":"告诉杭州的天气。我"}}]}} +{"requestId":"4edb81cd-4647-9d5d-88f9-a4f30bc6d8dd","usage":{"input_tokens":238,"output_tokens":16,"total_tokens":254},"output":{"choices":[{"finish_reason":"null","message":{"role":"assistant","content":"","reasoning_content":"需要先确定是否有"}}]}} +{"requestId":"4edb81cd-4647-9d5d-88f9-a4f30bc6d8dd","usage":{"input_tokens":238,"output_tokens":22,"total_tokens":260},"output":{"choices":[{"finish_reason":"null","message":{"role":"assistant","content":"","reasoning_content":"相关的工具可用。查看提供的"}}]}} +{"requestId":"4edb81cd-4647-9d5d-88f9-a4f30bc6d8dd","usage":{"input_tokens":238,"output_tokens":28,"total_tokens":266},"output":{"choices":[{"finish_reason":"null","message":{"role":"assistant","content":"","reasoning_content":"工具，发现有一个get_current"}}]}} +{"requestId":"4edb81cd-4647-9d5d-88f9-a4f30bc6d8dd","usage":{"input_tokens":238,"output_tokens":34,"total_tokens":272},"output":{"choices":[{"finish_reason":"null","message":{"role":"assistant","content":"","reasoning_content":"_weather函数，参数是location"}}]}} +{"requestId":"4edb81cd-4647-9d5d-88f9-a4f30bc6d8dd","usage":{"input_tokens":238,"output_tokens":38,"total_tokens":276},"output":{"choices":[{"finish_reason":"null","message":{"role":"assistant","content":"","reasoning_content":"。所以应该调"}}]}} +{"requestId":"4edb81cd-4647-9d5d-88f9-a4f30bc6d8dd","usage":{"input_tokens":238,"output_tokens":43,"total_tokens":281},"output":{"choices":[{"finish_reason":"null","message":{"role":"assistant","content":"","reasoning_content":"用这个函数，参数"}}]}} +{"requestId":"4edb81cd-4647-9d5d-88f9-a4f30bc6d8dd","usage":{"input_tokens":238,"output_tokens":48,"total_tokens":286},"output":{"choices":[{"finish_reason":"null","message":{"role":"assistant","content":"","reasoning_content":"设为杭州。不需要"}}]}} +{"requestId":"4edb81cd-4647-9d5d-88f9-a4f30bc6d8dd","usage":{"input_tokens":238,"output_tokens":52,"total_tokens":290},"output":{"choices":[{"finish_reason":"null","message":{"role":"assistant","content":"","reasoning_content":"其他工具，因为"}}]}} +{"requestId":"4edb81cd-4647-9d5d-88f9-a4f30bc6d8dd","usage":{"input_tokens":238,"output_tokens":56,"total_tokens":294},"output":{"choices":[{"finish_reason":"null","message":{"role":"assistant","content":"","reasoning_content":"用户只问了"}}]}} +{"requestId":"4edb81cd-4647-9d5d-88f9-a4f30bc6d8dd","usage":{"input_tokens":238,"output_tokens":60,"total_tokens":298},"output":{"choices":[{"finish_reason":"null","message":{"role":"assistant","content":"","reasoning_content":"天气。接下来构造"}}]}} +{"requestId":"4edb81cd-4647-9d5d-88f9-a4f30bc6d8dd","usage":{"input_tokens":238,"output_tokens":64,"total_tokens":302},"output":{"choices":[{"finish_reason":"null","message":{"role":"assistant","content":"","reasoning_content":"tool_call，把"}}]}} +{"requestId":"4edb81cd-4647-9d5d-88f9-a4f30bc6d8dd","usage":{"input_tokens":238,"output_tokens":68,"total_tokens":306},"output":{"choices":[{"finish_reason":"null","message":{"role":"assistant","content":"","reasoning_content":"名称和参数填"}}]}} +{"requestId":"4edb81cd-4647-9d5d-88f9-a4f30bc6d8dd","usage":{"input_tokens":238,"output_tokens":73,"total_tokens":311},"output":{"choices":[{"finish_reason":"null","message":{"role":"assistant","content":"","reasoning_content":"进去。确保参数是"}}]}} +{"requestId":"4edb81cd-4647-9d5d-88f9-a4f30bc6d8dd","usage":{"input_tokens":238,"output_tokens":78,"total_tokens":316},"output":{"choices":[{"finish_reason":"null","message":{"role":"assistant","content":"","reasoning_content":"JSON对象，location是"}}]}} +{"requestId":"4edb81cd-4647-9d5d-88f9-a4f30bc6d8dd","usage":{"input_tokens":238,"output_tokens":82,"total_tokens":320},"output":{"choices":[{"finish_reason":"null","message":{"role":"assistant","content":"","reasoning_content":"字符串。检查无"}}]}} +{"requestId":"4edb81cd-4647-9d5d-88f9-a4f30bc6d8dd","usage":{"input_tokens":238,"output_tokens":88,"total_tokens":326},"output":{"choices":[{"finish_reason":"null","message":{"role":"assistant","content":"","reasoning_content":"误后返回。"}}]}} +{"requestId":"4edb81cd-4647-9d5d-88f9-a4f30bc6d8dd","usage":{"input_tokens":238,"output_tokens":106,"total_tokens":344},"output":{"choices":[{"finish_reason":"null","message":{"role":"assistant","content":"","reasoning_content":"","tool_calls":[{"type":"function","id":"call_ecc41296dccc47baa01567","function":{"name":"get_current_weather","arguments":"{\"location\": \"杭州"}}]}}]}} +{"requestId":"4edb81cd-4647-9d5d-88f9-a4f30bc6d8dd","usage":{"input_tokens":238,"output_tokens":108,"total_tokens":346},"output":{"choices":[{"finish_reason":"tool_calls","message":{"role":"assistant","content":"","reasoning_content":"","tool_calls":[{"type":"function","id":"","function":{"arguments":"\"}"}}]}}]}} +``` + +## HTTP + +### **示例代码** + +## curl + +``` +# ======= 重要提示 ======= +# 如果使用纯文本生成模型，请将url替换为 https://dashscope.aliyuncs.com/api/v1/services/aigc/text-generation/generation +# 各地域的API Key不同。获取API Key：https://help.aliyun.com/zh/model-studio/get-api-key +# 以下为北京地域url，若使用新加坡地域的模型，需将url替换为：https://dashscope-intl.aliyuncs.com/api/v1/services/aigc/multimodal-generation/generation +# === 执行时请删除该注释 === +curl -X POST "https://dashscope.aliyuncs.com/api/v1/services/aigc/multimodal-generation/generation" \ +-H "Authorization: Bearer $DASHSCOPE_API_KEY" \ +-H "Content-Type: application/json" \ +-H "X-DashScope-SSE: enable" \ +-d '{ + "model": "qwen3.6-plus", + "input":{ + "messages":[ + { + "role": "user", + "content": [{"text": "杭州天气"}] + } + ] + }, + "parameters": { + "enable_thinking": true, + "incremental_output": true, + "result_format": "message", + "tools": [{ + "type": "function", + "function": { + "name": "get_current_time", + "description": "当你想知道现在的时间时非常有用。", + "parameters": {} + } + },{ + "type": "function", + "function": { + "name": "get_current_weather", + "description": "当你想查询指定城市的天气时非常有用。", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "城市或县区，比如北京市、杭州市、余杭区等。" + } + }, + "required": ["location"] + } + } + }] + } +}' +``` + +## **应用于生产环境** + +### **测试工具调用准确率** + +- **建立评估体系**： + + 构建贴近真实业务的测试数据集，并定义清晰的评估指标，如：工具选择准确率、参数提取准确率、端到端成功率等。 + +- **优化提示词** + + 根据测试暴露出的具体问题（选错工具、参数错误），针对性地优化系统提示词、工具描述和参数描述，是核心的调优手段。 + +- **升级模型** + + 当提示词工程调优无法提升性能时，升级到能力更强的模型版本（如 `qwen3.6-plus`）是提升指标最直接有效的方法。 + + +### **动态控制工具数量** + +当应用集成的工具数量超过几十甚至上百个时，将整个工具库全部提供给模型会带来以下问题： + +- **性能下降**：模型在庞大的工具集中选择正确工具的难度剧增； + +- **成本与延迟**：大量的工具描述会消耗巨量的输入 Token，导致费用上升和响应变慢； + + +解决方案是**在调用模型前，增加一个工具路由/检索层**，根据用户当前查询，从完整的工具库中快速、精准地筛选出一个小而相关的工具子集，再将其提供给模型。 + +**实现工具路由的几种主流方法：** + +- **语义检索** + + 预先将所有工具的描述信息（`description`）通过 Embedding 模型转化为向量，并存入向量数据库。当用户查询时，将查询向量通过向量相似度搜索，召回最相关的 Top-K 个工具。 + +- **混合检索** + + 将语义检索的“模糊匹配”能力与传统关键词或元数据标签的“精确匹配”能力相结合。为工具添加 `tags` 或 `keywords` 字段，检索时同时进行向量搜索和关键词过滤，可以大幅提升高频或特定场景下的召回精准度。 + +- **轻量级 LLM 路由器** + + 对于更复杂的路由逻辑，可以使用一个更小、更快、更便宜的模型（如 Qwen-Flash）作为前置“路由模型”。它的任务是根据用户问题输出相关的工具名称列表。 + + +**实践建议** + +- **保持候选集精简**：无论使用何种方法，最终提供给主模型的工具数量建议**不超过 20 个**。这是在模型认知负荷、成本、延迟和准确率之间的最佳平衡点。 + +- **分层过滤策略**：可以构建一个漏斗式的路由策略。例如，先用成本极低的关键词/规则匹配进行第一轮筛选，过滤掉明显不相关的工具，再对剩余的工具进行语义检索，从而提高效率和质量。 + + +### **工具安全性原则** + +将工具执行能力开放给大模型时，必须将安全置于首位。核心原则是“最小权限”和“人类确认”。 + +- **最小权限原则**：为模型提供的工具集应严格遵守最小权限原则。默认情况下，工具应是只读的（如查询天气、搜索文档），避免直接提供任何涉及状态变更或资源操作的“写”权限。 + +- **危险工具隔离**：请勿向大模型直接提供危险工具，例如执行任意代码（`code interpreter`）、操作文件系统（`fs.delete`）、执行数据库删除或更新操作（`db.drop_table`）或涉及资金流转的工具（`payment.transfer`）。 + +- **人类参与**：对于所有高权限或不可逆的操作，必须引入人工审核和确认环节。模型可以生成操作请求，但最终的执行“按钮”必须由人类用户点击。例如，模型可以准备好一封邮件，但发送操作需要用户确认。 + + +### **用户体验优化** + +Function Calling 链路较长，任何一个环节出问题都可能导致用户体验下降。 + +#### **处理工具运行失败** + +工具运行失败是常见情况。可采取以下策略： + +- **最大重试次数**：设置合理的重试上限（例如 3 次），避免因连续失败导致用户长时间等待或系统资源浪费。 + +- **提供兜底话术**：当重试耗尽或遇到无法解决的错误时，应向用户返回清晰、友好的提示信息，例如：“抱歉，我暂时无法查询到相关信息，可能是服务有些繁忙，请您稍后再试。” + + +#### **应对处理延迟** + +较高的延迟会降低用户满意度，需要通过前端交互和后端优化来改善。 + +- **设置超时时间**：为 Function Calling 的每一步设置独立且合理的超时时间。一旦超时，应立即中断操作并给出反馈。 + +- **提供即时反馈**：开始执行 Function Calling 时，建议在界面上给出提示，如“正在为您查询天气...”、“正在搜索相关信息...”，向用户实时反馈处理进度。 + + +## **计费说明** + +除了 messages 数组中的 Token 外，工具描述信息也会作为输入 Token 拼接到提示词中进行计费。 + +## **通过 System Message 传入工具信息** + +推荐您参考[如何使用](#038e24005c1vt)部分，通过 tools 参数向大模型传入工具信息。如果您需要通过 System Message 传入工具信息，为了模型的最佳效果，请参考以下代码中的提示词模板： + +## OpenAI兼容 + +## Python + +### **示例代码** + +``` +import os +from openai import OpenAI +import json + +client = OpenAI( + # 若没有配置环境变量，请用百炼API Key将下行替换为：api_key="sk-xxx", + # 各地域的API Key不同。获取API Key：https://help.aliyun.com/zh/model-studio/get-api-key + api_key=os.getenv("DASHSCOPE_API_KEY"), + # 以下是北京地域base_url，如果使用新加坡地域的模型，需要将base_url替换为：https://dashscope-intl.aliyuncs.com/compatible-mode/v1 + base_url="https://dashscope.aliyuncs.com/compatible-mode/v1", +) + +# 自定义 System prompt，可根据您的需求修改 +custom_prompt = "你是一个智能助手，专门负责调用各种工具来帮助用户解决问题。你可以根据用户的需求选择合适的工具并正确调用它们。" + +tools = [ + # 工具1 获取当前时刻的时间 + { + "type": "function", + "function": { + "name": "get_current_time", + "description": "当你想知道现在的时间时非常有用。", + "parameters": {} + } + }, + # 工具2 获取指定城市的天气 + { + "type": "function", + "function": { + "name": "get_current_weather", + "description": "当你想查询指定城市的天气时非常有用。", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "城市或县区，比如北京市、杭州市、余杭区等。" + } + }, + "required": ["location"] + } + } + } +] + +# 遍历tools列表，为每个工具构建描述 +tools_descriptions = [] +for tool in tools: + tool_json = json.dumps(tool, ensure_ascii=False) + tools_descriptions.append(tool_json) + +# 将所有工具描述组合成一个字符串 +tools_content = "\n".join(tools_descriptions) + +system_prompt = f"""{custom_prompt} + +# Tools + +You may call one or more functions to assist with the user query. + +You are provided with function signatures within XML tags: + +{tools_content} + + +For each function call, return a json object with function name and arguments within XML tags: + +{{"name": , "arguments": }} +""" + +messages = [ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": "几点了"} +] + +completion = client.chat.completions.create( + model="qwen3.6-plus", + extra_body={"enable_thinking": False}, + messages=messages, +) +print(completion.model_dump_json()) +``` + +## Node.js + +### 示例代码 + +``` +import OpenAI from "openai"; + +const client = new OpenAI({ + // 若没有配置环境变量，请用百炼API Key将下行替换为：apiKey: "sk-xxx", + // 各地域的API Key不同。获取API Key：https://help.aliyun.com/zh/model-studio/get-api-key + apiKey: process.env.DASHSCOPE_API_KEY, + // 以下是北京地域base_url，如果使用新加坡地域的模型，需要将base_url替换为：https://dashscope-intl.aliyuncs.com/compatible-mode/v1 + baseURL: "https://dashscope.aliyuncs.com/compatible-mode/v1", +}); + +// 自定义 System prompt +const customPrompt = "你是一个智能助手，专门负责调用各种工具来帮助用户解决问题。你可以根据用户的需求选择合适的工具并正确调用它们。"; + +const tools = [ + // 工具1 获取当前时刻的时间 + { + "type": "function", + "function": { + "name": "get_current_time", + "description": "当你想知道现在的时间时非常有用。", + "parameters": {} + } + }, + // 工具2 获取指定城市的天气 + { + "type": "function", + "function": { + "name": "get_current_weather", + "description": "当你想查询指定城市的天气时非常有用。", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "城市或县区，比如北京市、杭州市、余杭区等。" + } + }, + "required": ["location"] + } + } + } +]; + +// 遍历tools列表，为每个工具构建描述 +const toolsDescriptions = []; +for (const tool of tools) { + const toolJson = JSON.stringify(tool, null, 2); + toolsDescriptions.push(toolJson); +} + +// 将所有工具描述组合成一个字符串 +const toolsContent = toolsDescriptions.join("\n"); + +const systemPrompt = `${customPrompt} + +# Tools + +You may call one or more functions to assist with the user query. + +You are provided with function signatures within XML tags: + +${toolsContent} + + +For each function call, return a json object with function name and arguments within XML tags: + +{"name": , "arguments": } +`; + +const messages = [ + {"role": "system", "content": systemPrompt}, + {"role": "user", "content": "几点了"} +]; + +async function main() { + try { + const completion = await client.chat.completions.create({ + model: "qwen3.6-plus", + enable_thinking: false, + messages: messages, + }); + + console.log(JSON.stringify(completion, null, 2)); + } catch (error) { + console.error("Error:", error); + } +} + +main(); +``` + +## DashScope + +## Python + +### **示例代码** + +``` +import os +from dashscope import MultiModalConversation +import json +import dashscope + +# 若使用新加坡地域的模型，请释放下列注释 +# dashscope.base_http_api_url = "https://dashscope-intl.aliyuncs.com/api/v1" + +# 自定义 System prompt +custom_prompt = "你是一个智能助手，专门负责调用各种工具来帮助用户解决问题。你可以根据用户的需求选择合适的工具并正确调用它们。" + +tools = [ + # 工具1 获取当前时刻的时间 + { + "type": "function", + "function": { + "name": "get_current_time", + "description": "当你想知道现在的时间时非常有用。", + "parameters": {} + } + }, + # 工具2 获取指定城市的天气 + { + "type": "function", + "function": { + "name": "get_current_weather", + "description": "当你想查询指定城市的天气时非常有用。", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "城市或县区，比如北京市、杭州市、余杭区等。" + } + }, + "required": ["location"] + } + } + } +] + +# 遍历tools列表，为每个工具构建描述 +tools_descriptions = [] +for tool in tools: + tool_json = json.dumps(tool, ensure_ascii=False) + tools_descriptions.append(tool_json) + +# 将所有工具描述组合成一个字符串 +tools_content = "\n".join(tools_descriptions) + +system_prompt = f"""{custom_prompt} + +# Tools + +You may call one or more functions to assist with the user query. + +You are provided with function signatures within XML tags: + +{tools_content} + + +For each function call, return a json object with function name and arguments within XML tags: + +{{"name": , "arguments": }} +""" + +messages = [ + {"role": "system", "content": [{"text": system_prompt}]}, + {"role": "user", "content": [{"text": "几点了"}]} +] + +response = MultiModalConversation.call( + # 若没有配置环境变量，请用百炼API Key将下行替换为：api_key="sk-xxx", + api_key=os.getenv("DASHSCOPE_API_KEY"), + model="qwen3.6-plus", + enable_thinking=False, + messages=messages, + result_format="message", # 将输出设置为message形式 +) + +print(response) +``` + +## Java + +### 示例代码 + +``` +// Copyright (c) Alibaba, Inc. and its affiliates. +// version >= 2.12.0 + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import com.alibaba.dashscope.aigc.multimodalconversation.MultiModalConversation; +import com.alibaba.dashscope.aigc.multimodalconversation.MultiModalConversationParam; +import com.alibaba.dashscope.aigc.multimodalconversation.MultiModalConversationResult; +import com.alibaba.dashscope.common.MultiModalMessage; +import com.alibaba.dashscope.common.Role; +import com.alibaba.dashscope.exception.ApiException; +import com.alibaba.dashscope.exception.UploadFileException; +import com.alibaba.dashscope.exception.NoApiKeyException; +import com.alibaba.dashscope.utils.Constants; +import com.alibaba.dashscope.utils.JsonUtils; + +public class Main { + // 若使用新加坡地域的模型，请释放下列注释 + // static {Constants.baseHttpApiUrl="https://dashscope-intl.aliyuncs.com/api/v1";} + public static void main(String[] args) { + try { + callToolWithCustomPrompt(); + } catch (ApiException | NoApiKeyException | UploadFileException e) { + System.out.println(String.format("Exception: %s", e.getMessage())); + } catch (Exception e) { + System.out.println(String.format("Exception: %s", e.getMessage())); + } + System.exit(0); + } + + public static void callToolWithCustomPrompt() + throws NoApiKeyException, ApiException, UploadFileException { + + // 自定义 System prompt + String customPrompt = "你是一个智能助手，专门负责调用各种工具来帮助用户解决问题。你可以根据用户的需求选择合适的工具并正确调用它们。"; + + // 构建工具描述 + String[] toolsDescriptions = { + // 工具1 获取当前时刻的时间 + "{\n" + + " \"type\": \"function\",\n" + + " \"function\": {\n" + + " \"name\": \"get_current_time\",\n" + + " \"description\": \"当你想知道现在的时间时非常有用。\",\n" + + " \"parameters\": {}\n" + + " }\n" + + "}", + // 工具2 获取指定城市的天气 + "{\n" + + " \"type\": \"function\",\n" + + " \"function\": {\n" + + " \"name\": \"get_current_weather\",\n" + + " \"description\": \"当你想查询指定城市的天气时非常有用。\",\n" + + " \"parameters\": {\n" + + " \"type\": \"object\",\n" + + " \"properties\": {\n" + + " \"location\": {\n" + + " \"type\": \"string\",\n" + + " \"description\": \"城市或县区，比如北京市、杭州市、余杭区等。\"\n" + + " }\n" + + " },\n" + + " \"required\": [\"location\"]\n" + + " }\n" + + " }\n" + + "}" + }; + + // 将所有工具描述组合成一个字符串 + String toolsContent = String.join("\n", toolsDescriptions); + + // 构建系统提示词 + String systemPrompt = String.format("%s\n\n" + + "# Tools\n\n" + + "You may call one or more functions to assist with the user query.\n\n" + + "You are provided with function signatures within XML tags:\n" + + "\n%s\n\n\n" + + "For each function call, return a json object with function name and arguments within XML tags:\n" + + + "\n" + + "{\"name\": , \"arguments\": }\n" + + "", + customPrompt, toolsContent); + + // 构建消息列表 + MultiModalMessage systemMsg = MultiModalMessage.builder() + .role(Role.SYSTEM.getValue()) + .content(Arrays.asList(Collections.singletonMap("text", systemPrompt))) + .build(); + + MultiModalMessage userMsg = MultiModalMessage.builder() + .role(Role.USER.getValue()) + .content(Arrays.asList(Collections.singletonMap("text", "几点了"))) + .build(); + + List messages = new ArrayList<>(Arrays.asList(systemMsg, userMsg)); + + // 构建请求参数 + MultiModalConversationParam param = MultiModalConversationParam.builder() + // 模型列表：https://help.aliyun.com/zh/model-studio/getting-started/models + .model("qwen3.6-plus") + .enableThinking(false) + // 若没有配置环境变量，请用百炼API Key将下行替换为：.apiKey("sk-xxx") + .apiKey(System.getenv("DASHSCOPE_API_KEY")) + .messages(messages) + .build(); + + // 调用多模态对话接口 + MultiModalConversation conv = new MultiModalConversation(); + MultiModalConversationResult result = conv.call(param); + + // 输出结果 + System.out.println(JsonUtils.toJson(result)); + } +} +``` + +> 运行以上代码后，可以使用 XML 解析器提取 和 之间的工具调用信息，包括函数名和入参。 + +## 错误码 + +如果模型调用失败并返回报错信息，请参见[错误信息](https://help.aliyun.com/zh/model-studio/error-code)进行解决。 \ No newline at end of file diff --git a/qwen-docs/main.md b/qwen-docs/main.md new file mode 100644 index 0000000..1604710 --- /dev/null +++ b/qwen-docs/main.md @@ -0,0 +1,2374 @@ +Qwen-Omni-Realtime 是千问推出的实时音视频聊天模型。能同时理解流式的音频与图像输入（例如从视频流中实时抽取的连续图像帧），并实时输出高质量的文本与音频。 + +**支持的地域：**北京、新加坡，需使用各地域的 [API Key](https://help.aliyun.com/zh/model-studio/get-api-key)。 + +**在线体验：**请参见[如何在线体验 Qwen-Omni-Realtime 模型？](#14a119f447ehf) + +## **如何使用** + +### **1\. 建立连接** + +Qwen-Omni-Realtime 模型通过 WebSocket 协议接入，可通过以下 Python 示例代码建立连接。也可通过DashScope SDK 建立连接。 + +**说明** + +请注意，Qwen-Omni-Realtime 的单次 WebSocket 会话最长可持续 **120 分钟**。达到此上限后，服务将主动关闭连接。 + +## WebSocket 原生连接 + +连接时需要以下配置项： + +| **配置项** | **说明** | +| --- | --- | +| 调用地址 | 中国内地（北京）：wss://dashscope.aliyuncs.com/api-ws/v1/realtime 国际（新加坡）：wss://dashscope-intl.aliyuncs.com/api-ws/v1/realtime | +| 查询参数 | 查询参数为model，需指定为访问的模型名。示例：`?model=qwen3.5-omni-plus-realtime` | +| 请求头 | 使用 Bearer Token 鉴权：Authorization: Bearer DASHSCOPE\\_API\\_KEY > DASHSCOPE\\_API\\_KEY 是您在百炼上申请的[API Key](https://help.aliyun.com/zh/model-studio/get-api-key)。 | + +``` +# pip install websocket-client +import json +import websocket +import os + +API_KEY=os.getenv("DASHSCOPE_API_KEY") +API_URL = "wss://dashscope.aliyuncs.com/api-ws/v1/realtime?model=qwen3.5-omni-plus-realtime" + +headers = [ + "Authorization: Bearer " + API_KEY +] + +def on_open(ws): + print(f"Connected to server: {API_URL}") +def on_message(ws, message): + data = json.loads(message) + print("Received event:", json.dumps(data, indent=2)) +def on_error(ws, error): + print("Error:", error) + +ws = websocket.WebSocketApp( + API_URL, + header=headers, + on_open=on_open, + on_message=on_message, + on_error=on_error +) + +ws.run_forever() +``` + +## DashScope SDK + +Python + +``` +# SDK 版本不低于1.23.9 +import os +import json +from dashscope.audio.qwen_omni import OmniRealtimeConversation,OmniRealtimeCallback +import dashscope +# 若没有配置 API Key，请将下行改为 dashscope.api_key = "sk-xxx" +dashscope.api_key = os.getenv("DASHSCOPE_API_KEY") + +class PrintCallback(OmniRealtimeCallback): + def on_open(self) -> None: + print("Connected Successfully") + def on_event(self, response: dict) -> None: + print("Received event:") + print(json.dumps(response, indent=2, ensure_ascii=False)) + def on_close(self, close_status_code: int, close_msg: str) -> None: + print(f"Connection closed (code={close_status_code}, msg={close_msg}).") + +callback = PrintCallback() +conversation = OmniRealtimeConversation( + model="qwen3.5-omni-plus-realtime", + callback=callback, + # 以下为北京地域url，若使用新加坡地域的模型，需将url替换为：wss://dashscope-intl.aliyuncs.com/api-ws/v1/realtime + url="wss://dashscope.aliyuncs.com/api-ws/v1/realtime" +) +try: + conversation.connect() + print("Conversation started. Press Ctrl+C to exit.") + conversation.thread.join() +except KeyboardInterrupt: + conversation.close() +``` + +Java + +``` +// SDK 版本不低于 2.20.9 +import com.alibaba.dashscope.audio.omni.*; +import com.alibaba.dashscope.exception.NoApiKeyException; +import com.google.gson.JsonObject; +import java.util.concurrent.CountDownLatch; + +public class Main { + public static void main(String[] args) throws InterruptedException, NoApiKeyException { + CountDownLatch latch = new CountDownLatch(1); + OmniRealtimeParam param = OmniRealtimeParam.builder() + .model("qwen3.5-omni-plus-realtime") + .apikey(System.getenv("DASHSCOPE_API_KEY")) + // 以下为北京地域url，若使用新加坡地域的模型，需将url替换为：wss://dashscope-intl.aliyuncs.com/api-ws/v1/realtime + .url("wss://dashscope.aliyuncs.com/api-ws/v1/realtime") + .build(); + + OmniRealtimeConversation conversation = new OmniRealtimeConversation(param, new OmniRealtimeCallback() { + @Override + public void onOpen() { + System.out.println("Connected Successfully"); + } + @Override + public void onEvent(JsonObject message) { + System.out.println(message); + } + @Override + public void onClose(int code, String reason) { + System.out.println("connection closed code: " + code + ", reason: " + reason); + latch.countDown(); + } + }); + conversation.connect(); + latch.await(); + conversation.close(1000, "bye"); + System.exit(0); + } +} +``` + +### **2\. 配置会话** + +发送客户端事件[session.update](https://help.aliyun.com/zh/model-studio/client-events#26a8302028sjm)： + +``` +{ + // 该事件的id，由客户端生成 + "event_id": "event_ToPZqeobitzUJnt3QqtWg", + // 事件类型，固定为session.update + "type": "session.update", + // 会话配置 + "session": { + // 输出模态，支持设置为["text"]（仅输出文本）或["text","audio"]（输出文本与音频）。 + "modalities": [ + "text", + "audio" + ], + // 输出音频的音色 + "voice": "Ethan", + // 输入音频格式，当前仅支持设置为pcm。 + "input_audio_format": "pcm", + // 输出音频格式，当前仅支持设置为pcm。 + "output_audio_format": "pcm", + // 系统消息，用于设定模型的目标或角色。 + "instructions": "你是某五星级酒店的AI客服专员，请准确且友好地解答客户关于房型、设施、价格、预订政策的咨询。请始终以专业和乐于助人的态度回应，杜绝提供未经证实或超出酒店服务范围的信息。", + // 是否开启语音活动检测。若需启用，需传入一个配置对象，服务端将据此自动检测语音起止。 + // 设置为null表示由客户端决定何时发起模型响应。 + "turn_detection": { + // VAD类型，取值为server_vad或semantic_vad。使用qwen3.5-omni-realtime模型时推荐设为semantic_vad。 + "type": "semantic_vad", + // VAD检测阈值。建议在嘈杂的环境中增加，在安静的环境中降低。 + "threshold": 0.5, + // 检测语音停止的静音持续时间，超过此值后会触发模型响应 + "silence_duration_ms": 800 + } + } +} +``` + +### **3\. 输入音频与图片** + +客户端通过[input\_audio\_buffer.append](https://help.aliyun.com/zh/model-studio/client-events#2ea4b5e41fhjd)和 [input\_image\_buffer.append](https://help.aliyun.com/zh/model-studio/client-events#c28ed38410nfw) 事件发送 Base64 编码的音频和图片数据到服务端缓冲区。音频输入是必需的；图片输入是可选的。 + +> 图片可以来自本地文件，或从视频流中实时采集。 + +> 启用服务端VAD时，服务端会在检测到语音结束时自动提交数据并触发响应。禁用VAD时（手动模式），客户端必须在发送完数据后，主动调用[input\_audio\_buffer.commit](https://help.aliyun.com/zh/model-studio/client-events#1cbea5fa7fkfl)事件来提交。 + +### **4\. 接收模型响应** + +模型的响应格式取决于配置的输出模态。 + +- **仅输出文本** + + 通过[response.text.delta](https://help.aliyun.com/zh/model-studio/server-events#0c54be63e0c3w)事件接收流式文本，[response.text.done](https://help.aliyun.com/zh/model-studio/server-events#d675635a94jfb)事件获取完整文本。 + +- **输出文本+音频** + + - **文本**：通过[response.audio\_transcript.delta](https://help.aliyun.com/zh/model-studio/server-events#35396453cfood)事件接收流式文本，[response.audio\_transcript.done](https://help.aliyun.com/zh/model-studio/server-events#f4d1698567bsm)事件获取完整文本。 + + - **音频**：通过[response.audio.delta](https://help.aliyun.com/zh/model-studio/server-events#a25cc50a15car)事件获取 Base64 编码的流式输出音频数据。[response.audio.done](https://help.aliyun.com/zh/model-studio/server-events#9e8eb59c67qnt)事件标志音频数据生成完成。 + + +## **模型选型** + +Qwen3.5-Omni-Realtime 是千问最新推出的实时多模态模型，相比于上一代的 Qwen3-Omni-Flash-Realtime： + +- **智能水平** + + 模型智力大幅提升，与 Qwen3.5-Plus 智能水平相当。 + +- **联网搜索** + + 原生支持联网搜索（WebSearch），模型可自主判断是否需要搜索来回应即时问题。详见[联网搜索](#a8b2c3d4e5s6h)。 + +- **工具调用** + + 支持 Function Calling，模型可自主判断是否需要调用外部工具，实现与外部系统的交互。详见[Qwen-Omni-Realtime 系列](https://help.aliyun.com/zh/model-studio/qwen-function-calling#rt02realtime01)。 + +- **语义打断** + + 自动识别对话意图，避免附和声和无意义背景音触发打断。 + +- **语音控制** + + 通过语音指令控制声音大小、语速和情绪，如“语速快一些”、“声音大一些”、“用开心的语气”等。 + +- **支持的语言** + + 支持 [113 种语种和方言](https://help.aliyun.com/zh/model-studio/qwen-omni#model-comparison-table)的语音识别，以及 [36 种语种和方言](https://help.aliyun.com/zh/model-studio/qwen-omni#model-comparison-table)的语音生成。 + +- **支持的音色** + + 支持 55 种音色（47 种多语言 + 8 种方言），具体可查看[音色列表](#f4c9fd97f221z)。 + + +> 联网搜索和工具调用不兼容，不可同时开启。 + +> 模型的名称、上下文、价格、快照版本等信息请参见百炼控制台；并发限流条件请参考[限流](https://help.aliyun.com/zh/model-studio/rate-limit)。 + +## **快速开始** + +您需要[获取API Key](https://help.aliyun.com/zh/model-studio/get-api-key)并[配置API Key到环境变量](https://help.aliyun.com/zh/model-studio/configure-api-key-through-environment-variables)。 + +请选择您熟悉的编程语言，通过以下步骤快速体验与 Realtime 模型实时对话的功能。 + +## DashScope Python SDK + +- **准备运行环境** + + +您的 Python 版本需要不低于 3.10。 + +首先根据您的操作系统安装 pyaudio。 + +## macOS + +``` +brew install portaudio && pip install pyaudio +``` + +## Debian/Ubuntu + +- **若未使用虚拟环境**，可直接通过系统包管理器安装： + + ``` + sudo apt-get install python3-pyaudio + ``` + +- **若使用虚拟环境**，需先安装编译依赖： + + ``` + sudo apt update + sudo apt install -y python3-dev portaudio19-dev + ``` + + 然后在已激活的虚拟环境中使用 pip 安装： + + ``` + pip install pyaudio + ``` + + +## CentOS + +``` +sudo yum install -y portaudio portaudio-devel && pip install pyaudio +``` + +## Windows + +``` +pip install pyaudio +``` + +安装完成后，通过 pip 安装依赖： + +``` +pip install websocket-client dashscope +``` + +- **选择交互模式** + + - VAD 模式（Voice Activity Detection，自动检测语音起止） + + 服务端自动判断用户何时开始与停止说话并作出回应。 + + - Manual 模式（按下即说，松开即发送） + + 客户端控制语音起止。用户说话结束后，客户端需主动发送消息至服务端。 + + + ## VAD 模式 + + 新建一个 python 文件，命名为vad\_dash.py，并将以下代码复制到文件中： + + vad\_dash.py + + ``` + # 依赖：dashscope >= 1.23.9，pyaudio + import os + import base64 + import time + import pyaudio + from dashscope.audio.qwen_omni import MultiModality, AudioFormat,OmniRealtimeCallback,OmniRealtimeConversation + import dashscope + + # 配置参数：地址、API Key、音色、模型、模型角色 + # 指定地域，设为cn表示中国内地（北京），设为intl表示国际（新加坡） + region = 'cn' + base_domain = 'dashscope.aliyuncs.com' if region == 'cn' else 'dashscope-intl.aliyuncs.com' + url = f'wss://{base_domain}/api-ws/v1/realtime' + # 配置 API Key，若没有设置环境变量，请用 API Key 将下行替换为 dashscope.api_key = "sk-xxx" + dashscope.api_key = os.getenv('DASHSCOPE_API_KEY') + # 指定音色 + voice = 'Ethan' + # 指定模型 + model = 'qwen3.5-omni-plus-realtime' + # 指定模型角色 + instructions = "你是个人助理小云，请用幽默风趣的方式回答用户的问题" + class SimpleCallback(OmniRealtimeCallback): + def __init__(self, pya): + self.pya = pya + self.out = None + def on_open(self): + # 初始化音频输出流 + self.out = self.pya.open( + format=pyaudio.paInt16, + channels=1, + rate=24000, + output=True + ) + def on_event(self, response): + if response['type'] == 'response.audio.delta': + # 播放音频 + self.out.write(base64.b64decode(response['delta'])) + elif response['type'] == 'conversation.item.input_audio_transcription.completed': + # 打印转录文本 + print(f"[User] {response['transcript']}") + elif response['type'] == 'response.audio_transcript.done': + # 打印助手回复文本 + print(f"[LLM] {response['transcript']}") + + # 1. 初始化音频设备 + pya = pyaudio.PyAudio() + # 2. 创建回调函数和会话 + callback = SimpleCallback(pya) + conv = OmniRealtimeConversation(model=model, callback=callback, url=url) + # 3. 建立连接并配置会话 + conv.connect() + conv.update_session(output_modalities=[MultiModality.AUDIO, MultiModality.TEXT], voice=voice, instructions=instructions) + # 4. 初始化音频输入流 + mic = pya.open(format=pyaudio.paInt16, channels=1, rate=16000, input=True) + # 5. 主循环处理音频输入 + print("对话已开始，对着麦克风说话 (Ctrl+C 退出)...") + try: + while True: + audio_data = mic.read(3200, exception_on_overflow=False) + conv.append_audio(base64.b64encode(audio_data).decode()) + time.sleep(0.01) + except KeyboardInterrupt: + # 清理资源 + conv.close() + mic.close() + callback.out.close() + pya.terminate() + print("\n对话结束") + ``` + + 运行`vad_dash.py`，通过麦克风即可与 Qwen-Omni-Realtime 模型实时对话，系统会检测您的音频起始位置并自动发送到服务器，无需您手动发送。 + + ## Manual 模式 + + 新建一个 python 文件，命名为`manual_dash.py`，并将以下代码复制进文件中： + + manual\_dash.py + + ``` + # 依赖：dashscope >= 1.23.9，pyaudio。 + import os + import base64 + import sys + import threading + import pyaudio + from dashscope.audio.qwen_omni import * + import dashscope + + # 如果没有设置环境变量，请用您的 API Key 将下行替换为 dashscope.api_key = "sk-xxx" + dashscope.api_key = os.getenv('DASHSCOPE_API_KEY') + voice = 'Ethan' + + class MyCallback(OmniRealtimeCallback): + """最简回调：建立连接时初始化扬声器，事件中直接播放返回音频。""" + def __init__(self, ctx): + super().__init__() + self.ctx = ctx + + def on_open(self) -> None: + # 连接建立后初始化 PyAudio 与扬声器(24k/mono/16bit) + print('connection opened') + try: + self.ctx['pya'] = pyaudio.PyAudio() + self.ctx['out'] = self.ctx['pya'].open( + format=pyaudio.paInt16, + channels=1, + rate=24000, + output=True + ) + print('audio output initialized') + except Exception as e: + print('[Error] audio init failed: {}'.format(e)) + + def on_close(self, close_status_code, close_msg) -> None: + print('connection closed with code: {}, msg: {}'.format(close_status_code, close_msg)) + sys.exit(0) + + def on_event(self, response: str) -> None: + try: + t = response['type'] + handlers = { + 'session.created': lambda r: print('start session: {}'.format(r['session']['id'])), + 'conversation.item.input_audio_transcription.completed': lambda r: print('question: {}'.format(r['transcript'])), + 'response.audio_transcript.delta': lambda r: print('llm text: {}'.format(r['delta'])), + 'response.audio.delta': self._play_audio, + 'response.done': self._response_done, + } + h = handlers.get(t) + if h: + h(response) + except Exception as e: + print('[Error] {}'.format(e)) + + def _play_audio(self, response): + # 直接解码base64并写入输出流进行播放 + if self.ctx['out'] is None: + return + try: + data = base64.b64decode(response['delta']) + self.ctx['out'].write(data) + except Exception as e: + print('[Error] audio playback failed: {}'.format(e)) + + def _response_done(self, response): + # 标记本轮对话完成，用于主循环等待 + if self.ctx['conv'] is not None: + print('[Metric] response: {}, first text delay: {}, first audio delay: {}'.format( + self.ctx['conv'].get_last_response_id(), + self.ctx['conv'].get_last_first_text_delay(), + self.ctx['conv'].get_last_first_audio_delay(), + )) + if self.ctx['resp_done'] is not None: + self.ctx['resp_done'].set() + + def shutdown_ctx(ctx): + """安全释放音频与PyAudio资源。""" + try: + if ctx['out'] is not None: + ctx['out'].close() + ctx['out'] = None + except Exception: + pass + try: + if ctx['pya'] is not None: + ctx['pya'].terminate() + ctx['pya'] = None + except Exception: + pass + + + def record_until_enter(pya_inst: pyaudio.PyAudio, sample_rate=16000, chunk_size=3200): + """按 Enter 停止录音，返回PCM字节。""" + frames = [] + stop_evt = threading.Event() + + stream = pya_inst.open( + format=pyaudio.paInt16, + channels=1, + rate=sample_rate, + input=True, + frames_per_buffer=chunk_size + ) + + def _reader(): + while not stop_evt.is_set(): + try: + frames.append(stream.read(chunk_size, exception_on_overflow=False)) + except Exception: + break + + t = threading.Thread(target=_reader, daemon=True) + t.start() + input() # 用户再次按 Enter 停止录音 + stop_evt.set() + t.join(timeout=1.0) + try: + stream.close() + except Exception: + pass + return b''.join(frames) + + + if __name__ == '__main__': + print('Initializing ...') + # 运行时上下文：存放音频与会话句柄 + ctx = {'pya': None, 'out': None, 'conv': None, 'resp_done': threading.Event()} + callback = MyCallback(ctx) + conversation = OmniRealtimeConversation( + model='qwen3.5-omni-plus-realtime', + callback=callback, + # 以下为北京地域url，若使用新加坡地域的模型，需将url替换为：wss://dashscope-intl.aliyuncs.com/api-ws/v1/realtime + url="wss://dashscope.aliyuncs.com/api-ws/v1/realtime", + ) + try: + conversation.connect() + except Exception as e: + print('[Error] connect failed: {}'.format(e)) + sys.exit(1) + + ctx['conv'] = conversation + # 会话配置：启用文本+音频输出（禁用服务端VAD，改为手动录音） + conversation.update_session( + output_modalities=[MultiModality.AUDIO, MultiModality.TEXT], + voice=voice, + enable_input_audio_transcription=True, + # 对输入音频做语音转录的模型，仅支持gummy-realtime-v1 + input_audio_transcription_model='gummy-realtime-v1', + enable_turn_detection=False, + instructions="你是个人助理小云，请你准确且友好地解答用户的问题，始终以乐于助人的态度回应。" + ) + + try: + turn = 1 + while True: + print(f"\n--- 第 {turn} 轮对话 ---") + print("按 Enter 开始录音（输入 q 回车退出）...") + user_input = input() + if user_input.strip().lower() in ['q', 'quit']: + print("用户请求退出...") + break + print("录音中... 再次按 Enter 停止录音。") + if ctx['pya'] is None: + ctx['pya'] = pyaudio.PyAudio() + recorded = record_until_enter(ctx['pya']) + if not recorded: + print("未录制到有效音频，请重试。") + continue + print(f"成功录制音频: {len(recorded)} 字节，发送中...") + + # 以3200字节为块发送（对应16k/16bit/100ms） + chunk_size = 3200 + for i in range(0, len(recorded), chunk_size): + chunk = recorded[i:i+chunk_size] + conversation.append_audio(base64.b64encode(chunk).decode('ascii')) + + print("发送完成，等待模型响应...") + ctx['resp_done'].clear() + conversation.commit() + conversation.create_response() + ctx['resp_done'].wait() + print('播放音频完成') + turn += 1 + except KeyboardInterrupt: + print("\n程序被用户中断") + finally: + shutdown_ctx(ctx) + print("程序退出") + ``` + + 运行`manual_dash.py`，按 Enter 键开始说话，再按一次获取模型响应的音频。 + + +## DashScope Java SDK + +**选择交互模式** + +- VAD 模式（Voice Activity Detection，自动检测语音起止） + + Realtime API 自动判断用户何时开始与停止说话并作出回应。 + +- Manual 模式（按下即说，松开即发送） + + 客户端控制语音起止。用户说话结束后，客户端需主动发送消息至服务端。 + + +## VAD 模式 + +OmniServerVad.java + +``` +import com.alibaba.dashscope.audio.omni.*; +import com.alibaba.dashscope.exception.NoApiKeyException; +import com.google.gson.JsonObject; +import javax.sound.sampled.*; +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.Base64; +import java.util.Map; +import java.util.Queue; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.atomic.AtomicBoolean; + +public class OmniServerVad { + static class SequentialAudioPlayer { + private final SourceDataLine line; + private final Queue audioQueue = new ConcurrentLinkedQueue<>(); + private final Thread playerThread; + private final AtomicBoolean shouldStop = new AtomicBoolean(false); + + public SequentialAudioPlayer() throws LineUnavailableException { + AudioFormat format = new AudioFormat(24000, 16, 1, true, false); + line = AudioSystem.getSourceDataLine(format); + line.open(format); + line.start(); + + playerThread = new Thread(() -> { + while (!shouldStop.get()) { + byte[] audio = audioQueue.poll(); + if (audio != null) { + line.write(audio, 0, audio.length); + } else { + try { Thread.sleep(10); } catch (InterruptedException ignored) {} + } + } + }, "AudioPlayer"); + playerThread.start(); + } + + public void play(String base64Audio) { + try { + byte[] audio = Base64.getDecoder().decode(base64Audio); + audioQueue.add(audio); + } catch (Exception e) { + System.err.println("音频解码失败: " + e.getMessage()); + } + } + + public void cancel() { + audioQueue.clear(); + line.flush(); + } + + public void close() { + shouldStop.set(true); + try { playerThread.join(1000); } catch (InterruptedException ignored) {} + line.drain(); + line.close(); + } + } + + public static void main(String[] args) { + try { + SequentialAudioPlayer player = new SequentialAudioPlayer(); + AtomicBoolean userIsSpeaking = new AtomicBoolean(false); + AtomicBoolean shouldStop = new AtomicBoolean(false); + + OmniRealtimeParam param = OmniRealtimeParam.builder() + .model("qwen3.5-omni-plus-realtime") + .apikey(System.getenv("DASHSCOPE_API_KEY")) + // 以下为中国内地（北京）地域url，若使用国际（新加坡）地域的模型，需将url替换为：wss://dashscope-intl.aliyuncs.com/api-ws/v1/realtime + .url("wss://dashscope.aliyuncs.com/api-ws/v1/realtime") + .build(); + + OmniRealtimeConversation conversation = new OmniRealtimeConversation(param, new OmniRealtimeCallback() { + @Override public void onOpen() { + System.out.println("连接已建立"); + } + @Override public void onClose(int code, String reason) { + System.out.println("连接已关闭 (" + code + "): " + reason); + shouldStop.set(true); + } + @Override public void onEvent(JsonObject event) { + handleEvent(event, player, userIsSpeaking); + } + }); + + conversation.connect(); + conversation.updateSession(OmniRealtimeConfig.builder() + .modalities(Arrays.asList(OmniRealtimeModality.AUDIO, OmniRealtimeModality.TEXT)) + .voice("Ethan") + .enableTurnDetection(true) + .enableInputAudioTranscription(true) + .parameters(Map.of("instructions", + "你是五星级酒店的AI客服专员，请准确且友好地解答客户关于房型、设施、价格、预订政策的咨询。请始终以专业和乐于助人的态度回应，杜绝提供未经证实或超出酒店服务范围的信息。")) + .build() + ); + + System.out.println("请开始说话（自动检测语音开始/结束，按Ctrl+C退出）..."); + AudioFormat format = new AudioFormat(16000, 16, 1, true, false); + TargetDataLine mic = AudioSystem.getTargetDataLine(format); + mic.open(format); + mic.start(); + + ByteBuffer buffer = ByteBuffer.allocate(3200); + while (!shouldStop.get()) { + int bytesRead = mic.read(buffer.array(), 0, buffer.capacity()); + if (bytesRead > 0) { + try { + conversation.appendAudio(Base64.getEncoder().encodeToString(buffer.array())); + } catch (Exception e) { + if (e.getMessage() != null && e.getMessage().contains("closed")) { + System.out.println("对话已关闭，停止录音"); + break; + } + } + } + Thread.sleep(20); + } + + conversation.close(1000, "正常结束"); + player.close(); + mic.close(); + System.out.println("\n程序已退出"); + + } catch (NoApiKeyException e) { + System.err.println("未找到API KEY: 请设置环境变量 DASHSCOPE_API_KEY"); + System.exit(1); + } catch (Exception e) { + e.printStackTrace(); + } + } + + private static void handleEvent(JsonObject event, SequentialAudioPlayer player, AtomicBoolean userIsSpeaking) { + String type = event.get("type").getAsString(); + switch (type) { + case "input_audio_buffer.speech_started": + System.out.println("\n[用户开始说话]"); + player.cancel(); + userIsSpeaking.set(true); + break; + case "input_audio_buffer.speech_stopped": + System.out.println("[用户停止说话]"); + userIsSpeaking.set(false); + break; + case "response.audio.delta": + if (!userIsSpeaking.get()) { + player.play(event.get("delta").getAsString()); + } + break; + case "conversation.item.input_audio_transcription.completed": + System.out.println("用户: " + event.get("transcript").getAsString()); + break; + case "response.audio_transcript.delta": + System.out.print(event.get("delta").getAsString()); + break; + case "response.done": + System.out.println("回复完成"); + break; + } + } +} +``` + +运行`OmniServerVad.main()`方法，通过麦克风即可与 Realtime 模型实时对话，系统会检测您的音频起始位置并自动发送到服务器，无需您手动发送。 + +## Manual 模式 + +OmniWithoutServerVad.java + +``` +// DashScope Java SDK 版本不低于2.20.9 + +import com.alibaba.dashscope.audio.omni.*; +import com.alibaba.dashscope.exception.NoApiKeyException; +import com.google.gson.JsonObject; +import javax.sound.sampled.*; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.Arrays; +import java.util.Base64; +import java.util.HashMap; +import java.util.Queue; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; + +public class OmniWithoutServerVad { + // RealtimePcmPlayer 类定义开始 + public static class RealtimePcmPlayer { + private int sampleRate; + private SourceDataLine line; + private AudioFormat audioFormat; + private Thread decoderThread; + private Thread playerThread; + private AtomicBoolean stopped = new AtomicBoolean(false); + private Queue b64AudioBuffer = new ConcurrentLinkedQueue<>(); + private Queue RawAudioBuffer = new ConcurrentLinkedQueue<>(); + + // 构造函数初始化音频格式和音频线路 + public RealtimePcmPlayer(int sampleRate) throws LineUnavailableException { + this.sampleRate = sampleRate; + this.audioFormat = new AudioFormat(this.sampleRate, 16, 1, true, false); + DataLine.Info info = new DataLine.Info(SourceDataLine.class, audioFormat); + line = (SourceDataLine) AudioSystem.getLine(info); + line.open(audioFormat); + line.start(); + decoderThread = new Thread(new Runnable() { + @Override + public void run() { + while (!stopped.get()) { + String b64Audio = b64AudioBuffer.poll(); + if (b64Audio != null) { + byte[] rawAudio = Base64.getDecoder().decode(b64Audio); + RawAudioBuffer.add(rawAudio); + } else { + try { + Thread.sleep(100); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + } + } + }); + playerThread = new Thread(new Runnable() { + @Override + public void run() { + while (!stopped.get()) { + byte[] rawAudio = RawAudioBuffer.poll(); + if (rawAudio != null) { + try { + playChunk(rawAudio); + } catch (IOException e) { + throw new RuntimeException(e); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } else { + try { + Thread.sleep(100); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + } + } + }); + decoderThread.start(); + playerThread.start(); + } + + // 播放一个音频块并阻塞直到播放完成 + private void playChunk(byte[] chunk) throws IOException, InterruptedException { + if (chunk == null || chunk.length == 0) return; + + int bytesWritten = 0; + while (bytesWritten < chunk.length) { + bytesWritten += line.write(chunk, bytesWritten, chunk.length - bytesWritten); + } + int audioLength = chunk.length / (this.sampleRate*2/1000); + // 等待缓冲区中的音频播放完成 + Thread.sleep(audioLength - 10); + } + + public void write(String b64Audio) { + b64AudioBuffer.add(b64Audio); + } + + public void cancel() { + b64AudioBuffer.clear(); + RawAudioBuffer.clear(); + } + + public void waitForComplete() throws InterruptedException { + while (!b64AudioBuffer.isEmpty() || !RawAudioBuffer.isEmpty()) { + Thread.sleep(100); + } + line.drain(); + } + + public void shutdown() throws InterruptedException { + stopped.set(true); + decoderThread.join(); + playerThread.join(); + if (line != null && line.isRunning()) { + line.drain(); + line.close(); + } + } + } // RealtimePcmPlayer 类定义结束 + // 新增录音方法 + private static void recordAndSend(TargetDataLine line, OmniRealtimeConversation conversation) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + byte[] buffer = new byte[3200]; + AtomicBoolean stopRecording = new AtomicBoolean(false); + + // 启动监听Enter键的线程 + Thread enterKeyListener = new Thread(() -> { + try { + System.in.read(); + stopRecording.set(true); + } catch (IOException e) { + e.printStackTrace(); + } + }); + enterKeyListener.start(); + + // 录音循环 + while (!stopRecording.get()) { + int count = line.read(buffer, 0, buffer.length); + if (count > 0) { + out.write(buffer, 0, count); + } + } + + // 发送录音数据 + byte[] audioData = out.toByteArray(); + String audioB64 = Base64.getEncoder().encodeToString(audioData); + conversation.appendAudio(audioB64); + out.close(); + } + + public static void main(String[] args) throws InterruptedException, LineUnavailableException { + OmniRealtimeParam param = OmniRealtimeParam.builder() + .model("qwen3.5-omni-plus-realtime") + // .apikey("your-dashscope-api-key") + .build(); + AtomicReference responseDoneLatch = new AtomicReference<>(null); + responseDoneLatch.set(new CountDownLatch(1)); + + RealtimePcmPlayer audioPlayer = new RealtimePcmPlayer(24000); + final AtomicReference conversationRef = new AtomicReference<>(null); + OmniRealtimeConversation conversation = new OmniRealtimeConversation(param, new OmniRealtimeCallback() { + @Override + public void onOpen() { + System.out.println("connection opened"); + } + @Override + public void onEvent(JsonObject message) { + String type = message.get("type").getAsString(); + switch(type) { + case "session.created": + System.out.println("start session: " + message.get("session").getAsJsonObject().get("id").getAsString()); + break; + case "conversation.item.input_audio_transcription.completed": + System.out.println("question: " + message.get("transcript").getAsString()); + break; + case "response.audio_transcript.delta": + System.out.println("got llm response delta: " + message.get("delta").getAsString()); + break; + case "response.audio.delta": + String recvAudioB64 = message.get("delta").getAsString(); + audioPlayer.write(recvAudioB64); + break; + case "response.done": + System.out.println("======RESPONSE DONE======"); + if (conversationRef.get() != null) { + System.out.println("[Metric] response: " + conversationRef.get().getResponseId() + + ", first text delay: " + conversationRef.get().getFirstTextDelay() + + " ms, first audio delay: " + conversationRef.get().getFirstAudioDelay() + " ms"); + } + responseDoneLatch.get().countDown(); + break; + default: + break; + } + } + @Override + public void onClose(int code, String reason) { + System.out.println("connection closed code: " + code + ", reason: " + reason); + } + }); + conversationRef.set(conversation); + try { + conversation.connect(); + } catch (NoApiKeyException e) { + throw new RuntimeException(e); + } + OmniRealtimeConfig config = OmniRealtimeConfig.builder() + .modalities(Arrays.asList(OmniRealtimeModality.AUDIO, OmniRealtimeModality.TEXT)) + .voice("Ethan") + .enableTurnDetection(false) + // 设定模型角色 + .parameters(new HashMap() {{ + put("instructions","你是个人助理小云，请你准确且友好地解答用户的问题，始终以乐于助人的态度回应。"); + }}) + .build(); + conversation.updateSession(config); + + // 新增麦克风录音功能 + AudioFormat format = new AudioFormat(16000, 16, 1, true, false); + DataLine.Info info = new DataLine.Info(TargetDataLine.class, format); + + if (!AudioSystem.isLineSupported(info)) { + System.out.println("Line not supported"); + return; + } + + TargetDataLine line = null; + try { + line = (TargetDataLine) AudioSystem.getLine(info); + line.open(format); + line.start(); + + while (true) { + System.out.println("按Enter开始录音..."); + try { + System.in.read(); + } catch (IOException e) { + System.err.println("读取输入时发生错误: " + e.getMessage()); + break; // 发生错误时退出循环 + } + System.out.println("开始录音，请说话...再次按Enter停止录音并发送"); + recordAndSend(line, conversation); + conversation.commit(); + conversation.createResponse(null, null); + // 重置latch以便下次等待 + responseDoneLatch.set(new CountDownLatch(1)); + } + } catch (LineUnavailableException | IOException e) { + e.printStackTrace(); + } finally { + if (line != null) { + line.stop(); + line.close(); + } + } + }} +``` + +运行`OmniWithoutServerVad.main()`方法，按 Enter 键开始录音，录音过程中再次按 Enter 键停止录音并发送，随后将接收并播放模型响应。 + +## WebSocket（Python） + +- **准备运行环境** + + 您的 Python 版本需要不低于 3.10。 + + 首先根据您的操作系统来安装 pyaudio。 + + ## macOS + + ``` + brew install portaudio && pip install pyaudio + ``` + + ## Debian/Ubuntu + + ``` + sudo apt-get install python3-pyaudio + + 或者 + + pip install pyaudio + ``` + + > 推荐使用`pip install pyaudio`。如果安装失败，请先根据您的操作系统安装`portaudio`依赖。 + + ## CentOS + + ``` + sudo yum install -y portaudio portaudio-devel && pip install pyaudio + ``` + + ## Windows + + ``` + pip install pyaudio + ``` + + 安装完成后，通过 pip 安装 websocket 相关的依赖： + + ``` + pip install websockets==15.0.1 + ``` + +- **创建客户端** + + 在本地新建一个 python 文件，命名为`omni_realtime_client.py`，并将以下代码复制进文件中： + + omni\_realtime\_client.py + + ``` + import asyncio + import websockets + import json + import base64 + import time + from typing import Optional, Callable, List, Dict, Any + from enum import Enum + + class TurnDetectionMode(Enum): + SERVER_VAD = "server_vad" + SEMANTIC_VAD = "semantic_vad" # 使用qwen3.5-omni-realtime模型时推荐 + MANUAL = "manual" + + class OmniRealtimeClient: + + def __init__( + self, + base_url, + api_key: str, + model: str = "", + voice: str = "Ethan", + instructions: str = "You are a helpful assistant.", + turn_detection_mode: TurnDetectionMode = TurnDetectionMode.SERVER_VAD, + on_text_delta: Optional[Callable[[str], None]] = None, + on_audio_delta: Optional[Callable[[bytes], None]] = None, + on_input_transcript: Optional[Callable[[str], None]] = None, + on_output_transcript: Optional[Callable[[str], None]] = None, + extra_event_handlers: Optional[Dict[str, Callable[[Dict[str, Any]], None]]] = None + ): + self.base_url = base_url + self.api_key = api_key + self.model = model + self.voice = voice + self.instructions = instructions + self.ws = None + self.on_text_delta = on_text_delta + self.on_audio_delta = on_audio_delta + self.on_input_transcript = on_input_transcript + self.on_output_transcript = on_output_transcript + self.turn_detection_mode = turn_detection_mode + self.extra_event_handlers = extra_event_handlers or {} + + # 当前回复状态 + self._current_response_id = None + self._current_item_id = None + self._is_responding = False + # 输入/输出转录打印状态 + self._print_input_transcript = True + self._output_transcript_buffer = "" + + async def connect(self) -> None: + """与 Realtime API 建立 WebSocket 连接。""" + url = f"{self.base_url}?model={self.model}" + headers = { + "Authorization": f"Bearer {self.api_key}" + } + self.ws = await websockets.connect(url, additional_headers=headers) + + # 会话配置 + session_config = { + "modalities": ["text", "audio"], + "voice": self.voice, + "instructions": self.instructions, + "input_audio_format": "pcm", + "output_audio_format": "pcm", + "input_audio_transcription": { + "model": "gummy-realtime-v1" + } + } + + if self.turn_detection_mode == TurnDetectionMode.MANUAL: + session_config['turn_detection'] = None + await self.update_session(session_config) + elif self.turn_detection_mode == TurnDetectionMode.SERVER_VAD: + session_config['turn_detection'] = { + "type": "server_vad", + "threshold": 0.1, + "prefix_padding_ms": 500, + "silence_duration_ms": 900 + } + await self.update_session(session_config) + elif self.turn_detection_mode == TurnDetectionMode.SEMANTIC_VAD: + session_config['turn_detection'] = { + "type": "semantic_vad", + "threshold": 0.1, + "prefix_padding_ms": 500, + "silence_duration_ms": 900 + } + await self.update_session(session_config) + else: + raise ValueError(f"Invalid turn detection mode: {self.turn_detection_mode}") + + async def send_event(self, event) -> None: + event['event_id'] = "event_" + str(int(time.time() * 1000)) + await self.ws.send(json.dumps(event)) + + async def update_session(self, config: Dict[str, Any]) -> None: + """更新会话配置。""" + event = { + "type": "session.update", + "session": config + } + await self.send_event(event) + + async def stream_audio(self, audio_chunk: bytes) -> None: + """向 API 流式发送原始音频数据。""" + # 仅支持 16bit 16kHz 单声道 PCM + audio_b64 = base64.b64encode(audio_chunk).decode() + append_event = { + "type": "input_audio_buffer.append", + "audio": audio_b64 + } + await self.send_event(append_event) + + async def commit_audio_buffer(self) -> None: + """提交音频缓冲区以触发处理。""" + event = { + "type": "input_audio_buffer.commit" + } + await self.send_event(event) + + async def append_image(self, image_chunk: bytes) -> None: + """向图像缓冲区追加图像数据。 + 图像数据可以来自本地文件，也可以来自实时视频流。 + 注意: + - 图像格式必须为 JPG 或 JPEG。推荐分辨率为 480P 或 720P，最高支持 1080P。 + - 单张图片大小不应超过 500KB。 + - 将图像数据编码为 Base64 后再发送。 + - 建议以 1张/秒 的频率向服务端发送图像。 + - 在发送图像数据之前，需要至少发送过一次音频数据。 + """ + image_b64 = base64.b64encode(image_chunk).decode() + event = { + "type": "input_image_buffer.append", + "image": image_b64 + } + await self.send_event(event) + + async def create_response(self) -> None: + """向 API 请求生成回复（仅在手动模式下需要调用）。""" + event = { + "type": "response.create" + } + await self.send_event(event) + + async def cancel_response(self) -> None: + """取消当前回复。""" + event = { + "type": "response.cancel" + } + await self.send_event(event) + + async def handle_interruption(self): + """处理用户对当前回复的打断。""" + if not self._is_responding: + return + # 1. 取消当前回复 + if self._current_response_id: + await self.cancel_response() + + self._is_responding = False + self._current_response_id = None + self._current_item_id = None + + async def handle_messages(self) -> None: + try: + async for message in self.ws: + event = json.loads(message) + event_type = event.get("type") + if event_type == "error": + print(" Error: ", event['error']) + continue + elif event_type == "response.created": + self._current_response_id = event.get("response", {}).get("id") + self._is_responding = True + elif event_type == "response.output_item.added": + self._current_item_id = event.get("item", {}).get("id") + elif event_type == "response.done": + self._is_responding = False + self._current_response_id = None + self._current_item_id = None + elif event_type == "input_audio_buffer.speech_started": + print("检测到语音开始") + if self._is_responding: + print("处理打断") + await self.handle_interruption() + elif event_type == "input_audio_buffer.speech_stopped": + print("检测到语音结束") + elif event_type == "response.text.delta": + if self.on_text_delta: + self.on_text_delta(event["delta"]) + elif event_type == "response.audio.delta": + if self.on_audio_delta: + audio_bytes = base64.b64decode(event["delta"]) + self.on_audio_delta(audio_bytes) + elif event_type == "conversation.item.input_audio_transcription.completed": + transcript = event.get("transcript", "") + print(f"用户: {transcript}") + if self.on_input_transcript: + await asyncio.to_thread(self.on_input_transcript, transcript) + self._print_input_transcript = True + elif event_type == "response.audio_transcript.delta": + if self.on_output_transcript: + delta = event.get("delta", "") + if not self._print_input_transcript: + self._output_transcript_buffer += delta + else: + if self._output_transcript_buffer: + await asyncio.to_thread(self.on_output_transcript, self._output_transcript_buffer) + self._output_transcript_buffer = "" + await asyncio.to_thread(self.on_output_transcript, delta) + elif event_type == "response.audio_transcript.done": + print(f"大模型: {event.get('transcript', '')}") + self._print_input_transcript = False + elif event_type in self.extra_event_handlers: + self.extra_event_handlers[event_type](event) + except websockets.exceptions.ConnectionClosed: + print(" Connection closed") + except Exception as e: + print(" Error in message handling: ", str(e)) + async def close(self) -> None: + """关闭 WebSocket 连接。""" + if self.ws: + await self.ws.close() + ``` + +- **选择交互模式** + + - VAD 模式（Voice Activity Detection，自动检测语音起止） + + Realtime API 自动判断用户何时开始与停止说话并作出回应。 + + - Manual 模式（按下即说，松开即发送） + + 客户端控制语音起止。用户说话结束后，客户端需主动发送消息至服务端。 + + + ## VAD 模式 + + 在`omni_realtime_client.py`的同级目录下新建另一个 python 文件，命名为`vad_mode.py`，并将以下代码复制进文件中： + + vad\_mode.py + + ``` + # -- coding: utf-8 -- + import os, asyncio, pyaudio, queue, threading + from omni_realtime_client import OmniRealtimeClient, TurnDetectionMode + + # 音频播放器类（处理中断） + class AudioPlayer: + def __init__(self, pyaudio_instance, rate=24000): + self.stream = pyaudio_instance.open(format=pyaudio.paInt16, channels=1, rate=rate, output=True) + self.queue = queue.Queue() + self.stop_evt = threading.Event() + self.interrupt_evt = threading.Event() + threading.Thread(target=self._run, daemon=True).start() + + def _run(self): + while not self.stop_evt.is_set(): + try: + data = self.queue.get(timeout=0.5) + if data is None: break + if not self.interrupt_evt.is_set(): self.stream.write(data) + self.queue.task_done() + except queue.Empty: continue + + def add_audio(self, data): self.queue.put(data) + def handle_interrupt(self): self.interrupt_evt.set(); self.queue.queue.clear() + def stop(self): self.stop_evt.set(); self.queue.put(None); self.stream.stop_stream(); self.stream.close() + + # 麦克风录音并发送 + async def record_and_send(client): + p = pyaudio.PyAudio() + stream = p.open(format=pyaudio.paInt16, channels=1, rate=16000, input=True, frames_per_buffer=3200) + print("开始录音，请讲话...") + try: + while True: + audio_data = stream.read(3200) + await client.stream_audio(audio_data) + await asyncio.sleep(0.02) + finally: + stream.stop_stream(); stream.close(); p.terminate() + + async def main(): + p = pyaudio.PyAudio() + player = AudioPlayer(pyaudio_instance=p) + + client = OmniRealtimeClient( + # 以下是中国内地（北京）地域 base_url，国际（新加坡）地域base_url为wss://dashscope-intl.aliyuncs.com/api-ws/v1/realtime + base_url="wss://dashscope.aliyuncs.com/api-ws/v1/realtime", + api_key=os.environ.get("DASHSCOPE_API_KEY"), + model="qwen3.5-omni-plus-realtime", + voice="Ethan", + instructions="你是小云，风趣幽默的好助手", + # 使用qwen3.5-omni-realtime模型时推荐设为SEMANTIC_VAD + turn_detection_mode=TurnDetectionMode.SEMANTIC_VAD, + on_text_delta=lambda t: print(f"\nAssistant: {t}", end="", flush=True), + on_audio_delta=player.add_audio, + ) + + await client.connect() + print("连接成功，开始实时对话...") + + # 并发运行 + await asyncio.gather(client.handle_messages(), record_and_send(client)) + + if __name__ == "__main__": + try: + asyncio.run(main()) + except KeyboardInterrupt: + print("\n程序已退出。") + ``` + + 运行`vad_mode.py`，通过麦克风即可与 Realtime 模型实时对话，系统会检测您的音频起始位置并自动发送到服务器，无需您手动发送。 + + ## Manual 模式 + + 在`omni_realtime_client.py`的同级目录下新建另一个 python 文件，命名为`manual_mode.py`，并将以下代码复制进文件中： + + manual\_mode.py + + ``` + # -- coding: utf-8 -- + import os + import asyncio + import time + import threading + import queue + import pyaudio + from omni_realtime_client import OmniRealtimeClient, TurnDetectionMode + + class AudioPlayer: + """实时音频播放器类""" + def __init__(self, sample_rate=24000, channels=1, sample_width=2): + self.sample_rate = sample_rate + self.channels = channels + self.sample_width = sample_width # 2 bytes for 16-bit + self.audio_queue = queue.Queue() + self.is_playing = False + self.play_thread = None + self.pyaudio_instance = None + self.stream = None + self._lock = threading.Lock() # 添加锁来同步访问 + self._last_data_time = time.time() # 记录最后接收数据的时间 + self._response_done = False # 添加响应完成标志 + self._waiting_for_response = False # 标记是否正在等待服务器响应 + # 记录最后一次向音频流写入数据的时间及最近一次音频块的时长，用于更精确地判断播放结束 + self._last_play_time = time.time() + self._last_chunk_duration = 0.0 + + def start(self): + """启动音频播放器""" + with self._lock: + if self.is_playing: + return + + self.is_playing = True + + try: + self.pyaudio_instance = pyaudio.PyAudio() + + # 创建音频输出流 + self.stream = self.pyaudio_instance.open( + format=pyaudio.paInt16, # 16-bit + channels=self.channels, + rate=self.sample_rate, + output=True, + frames_per_buffer=1024 + ) + + # 启动播放线程 + self.play_thread = threading.Thread(target=self._play_audio) + self.play_thread.daemon = True + self.play_thread.start() + + print("音频播放器已启动") + except Exception as e: + print(f"启动音频播放器失败: {e}") + self._cleanup_resources() + raise + + def stop(self): + """停止音频播放器""" + with self._lock: + if not self.is_playing: + return + + self.is_playing = False + + # 清空队列 + while not self.audio_queue.empty(): + try: + self.audio_queue.get_nowait() + except queue.Empty: + break + + # 等待播放线程结束（在锁外面等待，避免死锁） + if self.play_thread and self.play_thread.is_alive(): + self.play_thread.join(timeout=2.0) + + # 再次获取锁来清理资源 + with self._lock: + self._cleanup_resources() + + print("音频播放器已停止") + + def _cleanup_resources(self): + """清理音频资源（必须在锁内调用）""" + try: + # 关闭音频流 + if self.stream: + if not self.stream.is_stopped(): + self.stream.stop_stream() + self.stream.close() + self.stream = None + except Exception as e: + print(f"关闭音频流时出错: {e}") + + try: + if self.pyaudio_instance: + self.pyaudio_instance.terminate() + self.pyaudio_instance = None + except Exception as e: + print(f"终止PyAudio时出错: {e}") + + def add_audio_data(self, audio_data): + """添加音频数据到播放队列""" + if self.is_playing and audio_data: + self.audio_queue.put(audio_data) + with self._lock: + self._last_data_time = time.time() # 更新最后接收数据的时间 + self._waiting_for_response = False # 收到数据，不再等待 + + def stop_receiving_data(self): + """标记不再接收新的音频数据""" + with self._lock: + self._response_done = True + self._waiting_for_response = False # 响应结束，不再等待 + + def prepare_for_next_turn(self): + """为下一轮对话重置播放器状态。""" + with self._lock: + self._response_done = False + self._last_data_time = time.time() + self._last_play_time = time.time() + self._last_chunk_duration = 0.0 + self._waiting_for_response = True # 开始等待下一轮响应 + + # 清空上一轮可能残留的音频数据 + while not self.audio_queue.empty(): + try: + self.audio_queue.get_nowait() + except queue.Empty: + break + + def is_finished_playing(self): + """检查是否已经播放完所有音频数据""" + with self._lock: + queue_size = self.audio_queue.qsize() + time_since_last_data = time.time() - self._last_data_time + time_since_last_play = time.time() - self._last_play_time + + # ---------------------- 智能结束判定 ---------------------- + # 1. 首选：如果服务器已标记完成且播放队列为空 + # 进一步等待最近一块音频播放完毕（音频块时长 + 0.1s 容错）。 + if self._response_done and queue_size == 0: + min_wait = max(self._last_chunk_duration + 0.1, 0.5) # 至少等待 0.5s + if time_since_last_play >= min_wait: + return True + + # 2. 备用：如果长时间没有新数据且播放队列为空 + # 当服务器没有明确发出 `response.done` 时，此逻辑作为保障 + if not self._waiting_for_response and queue_size == 0 and time_since_last_data > 1.0: + print("\n(超时未收到新音频，判定播放结束)") + return True + + return False + + def _play_audio(self): + """播放音频数据的工作线程""" + while True: + # 检查是否应该停止 + with self._lock: + if not self.is_playing: + break + stream_ref = self.stream # 获取流的引用 + + try: + # 从队列中获取音频数据，超时0.1秒 + audio_data = self.audio_queue.get(timeout=0.1) + + # 再次检查状态和流的有效性 + with self._lock: + if self.is_playing and stream_ref and not stream_ref.is_stopped(): + try: + # 播放音频数据 + stream_ref.write(audio_data) + # 更新最近播放信息 + self._last_play_time = time.time() + self._last_chunk_duration = len(audio_data) / (self.channels * self.sample_width) / self.sample_rate + except Exception as e: + print(f"写入音频流时出错: {e}") + break + + # 标记该数据块已处理完成 + self.audio_queue.task_done() + + except queue.Empty: + # 队列为空时继续等待 + continue + except Exception as e: + print(f"播放音频时出错: {e}") + break + + class MicrophoneRecorder: + """实时麦克风录音器""" + def __init__(self, sample_rate=16000, channels=1, chunk_size=3200): + self.sample_rate = sample_rate + self.channels = channels + self.chunk_size = chunk_size + self.pyaudio_instance = None + self.stream = None + self.frames = [] + self._is_recording = False + self._record_thread = None + + def _recording_thread(self): + """录音工作线程""" + # 在 _is_recording 为 True 期间，持续从音频流中读取数据 + while self._is_recording: + try: + # 使用 exception_on_overflow=False 避免因缓冲区溢出而崩溃 + data = self.stream.read(self.chunk_size, exception_on_overflow=False) + self.frames.append(data) + except (IOError, OSError) as e: + # 当流被关闭时，读取操作可能会引发错误 + print(f"录音流读取错误，可能已关闭: {e}") + break + + def start(self): + """开始录音""" + if self._is_recording: + print("录音已在进行中。") + return + + self.frames = [] + self._is_recording = True + + try: + self.pyaudio_instance = pyaudio.PyAudio() + self.stream = self.pyaudio_instance.open( + format=pyaudio.paInt16, + channels=self.channels, + rate=self.sample_rate, + input=True, + frames_per_buffer=self.chunk_size + ) + + self._record_thread = threading.Thread(target=self._recording_thread) + self._record_thread.daemon = True + self._record_thread.start() + print("麦克风录音已开始...") + except Exception as e: + print(f"启动麦克风失败: {e}") + self._is_recording = False + self._cleanup() + raise + + def stop(self): + """停止录音并返回音频数据""" + if not self._is_recording: + return None + + self._is_recording = False + + # 等待录音线程安全退出 + if self._record_thread: + self._record_thread.join(timeout=1.0) + + self._cleanup() + + print("麦克风录音已停止。") + return b''.join(self.frames) + + def _cleanup(self): + """安全地清理 PyAudio 资源""" + if self.stream: + try: + if self.stream.is_active(): + self.stream.stop_stream() + self.stream.close() + except Exception as e: + print(f"关闭音频流时出错: {e}") + + if self.pyaudio_instance: + try: + self.pyaudio_instance.terminate() + except Exception as e: + print(f"终止 PyAudio 实例时出错: {e}") + + self.stream = None + self.pyaudio_instance = None + + async def interactive_test(): + """ + 交互式测试脚本：允许多轮连续对话，每轮可以发送音频和图片。 + """ + # ------------------- 1. 初始化和连接 (一次性) ------------------- + api_key = os.environ.get("DASHSCOPE_API_KEY") + if not api_key: + print("请设置DASHSCOPE_API_KEY环境变量") + return + + print("--- 实时多轮音视频对话客户端 ---") + print("正在初始化音频播放器和客户端...") + + audio_player = AudioPlayer() + audio_player.start() + + def on_audio_received(audio_data): + audio_player.add_audio_data(audio_data) + + def on_response_done(event): + print("\n(收到响应结束标记)") + audio_player.stop_receiving_data() + + realtime_client = OmniRealtimeClient( + base_url="wss://dashscope.aliyuncs.com/api-ws/v1/realtime", + api_key=api_key, + model="qwen3.5-omni-plus-realtime", + voice="Ethan", + instructions="你是个人助理小云，请你准确且友好地解答用户的问题，始终以乐于助人的态度回应。", # 设定模型角色 + on_text_delta=lambda text: print(f"助手回复: {text}", end="", flush=True), + on_audio_delta=on_audio_received, + turn_detection_mode=TurnDetectionMode.MANUAL, + extra_event_handlers={"response.done": on_response_done} + ) + + message_handler_task = None + try: + await realtime_client.connect() + print("已连接到服务器。输入 'q' 或 'quit' 可随时退出程序。") + message_handler_task = asyncio.create_task(realtime_client.handle_messages()) + await asyncio.sleep(0.5) + + turn_counter = 1 + # ------------------- 2. 多轮对话循环 ------------------- + while True: + print(f"\n--- 第 {turn_counter} 轮对话 ---") + audio_player.prepare_for_next_turn() + + recorded_audio = None + image_paths = [] + + # --- 获取用户输入：从麦克风录音 --- + loop = asyncio.get_event_loop() + recorder = MicrophoneRecorder(sample_rate=16000) # 推荐使用16k采样率进行语音识别 + + print("准备录音。按 Enter 键开始录音 (或输入 'q' 退出)...") + user_input = await loop.run_in_executor(None, input) + if user_input.strip().lower() in ['q', 'quit']: + print("用户请求退出...") + return + + try: + recorder.start() + except Exception: + print("无法启动录音，请检查您的麦克风权限和设备。跳过本轮。") + continue + + print("录音中... 再次按 Enter 键停止录音。") + await loop.run_in_executor(None, input) + + recorded_audio = recorder.stop() + + if not recorded_audio or len(recorded_audio) == 0: + print("未录制到有效音频，请重新开始本轮对话。") + continue + + # --- 获取图片输入 (可选) --- + # 以下图片输入功能已被注释，暂时禁用。若需启用请取消下方代码注释。 + # print("\n请逐行输入【图片文件】的绝对路径 (可选)。完成后，输入 's' 或按 Enter 发送请求。") + # while True: + # path = input("图片路径: ").strip() + # if path.lower() == 's' or path == '': + # break + # if path.lower() in ['q', 'quit']: + # print("用户请求退出...") + # return + # + # if not os.path.isabs(path): + # print("错误: 请输入绝对路径。") + # continue + # if not os.path.exists(path): + # print(f"错误: 文件不存在 -> {path}") + # continue + # image_paths.append(path) + # print(f"已添加图片: {os.path.basename(path)}") + + # --- 3. 发送数据并获取响应 --- + print("\n--- 输入确认 ---") + print(f"待处理音频: 1个 (来自麦克风), 图片: {len(image_paths)}个") + print("------------------") + + # 3.1 发送录制的音频 + try: + print(f"发送麦克风录音 ({len(recorded_audio)}字节)") + await realtime_client.stream_audio(recorded_audio) + await asyncio.sleep(0.1) + except Exception as e: + print(f"发送麦克风录音失败: {e}") + continue + + # 3.2 发送所有图片文件 + # 以下图片发送代码已被注释，暂时禁用。 + # for i, path in enumerate(image_paths): + # try: + # with open(path, "rb") as f: + # data = f.read() + # print(f"发送图片 {i+1}: {os.path.basename(path)} ({len(data)}字节)") + # await realtime_client.append_image(data) + # await asyncio.sleep(0.1) + # except Exception as e: + # print(f"发送图片 {os.path.basename(path)} 失败: {e}") + + # 3.3 提交并等待响应 + print("提交所有输入，请求服务器响应...") + await realtime_client.commit_audio_buffer() + await realtime_client.create_response() + + print("等待并播放服务器响应音频...") + start_time = time.time() + max_wait_time = 60 + while not audio_player.is_finished_playing(): + if time.time() - start_time > max_wait_time: + print(f"\n等待超时 ({max_wait_time}秒), 进入下一轮。") + break + await asyncio.sleep(0.2) + + print("\n本轮音频播放完成！") + turn_counter += 1 + + except (asyncio.CancelledError, KeyboardInterrupt): + print("\n程序被中断。") + except Exception as e: + print(f"发生未处理的错误: {e}") + finally: + # ------------------- 4. 清理资源 ------------------- + print("\n正在关闭连接并清理资源...") + if message_handler_task and not message_handler_task.done(): + message_handler_task.cancel() + + if 'realtime_client' in locals() and realtime_client.ws and not realtime_client.ws.close: + await realtime_client.close() + print("连接已关闭。") + + audio_player.stop() + print("程序退出。") + + if __name__ == "__main__": + try: + asyncio.run(interactive_test()) + except KeyboardInterrupt: + print("\n程序被用户强制退出。") + ``` + + 运行`manual_mode.py`，按 Enter 键开始说话，再按一次获取模型响应的音频。 + + +## **交互流程** + +## VAD 模式 + +将[session.update](https://help.aliyun.com/zh/model-studio/client-events#af43722339yva)事件的`session.turn_detection.type` 设为`"server_vad"`或`"semantic_vad"`以[启用 VAD 模式](https://help.aliyun.com/zh/model-studio/client-events#6218707b80vlt)。适用于语音通话场景。 + +交互流程如下： + +1. 服务端检测到语音开始，发送[input\_audio\_buffer.speech\_started](https://help.aliyun.com/zh/model-studio/server-events#b99f92d872n5c) 事件。 + +2. 客户端随时发送 [input\_audio\_buffer.append](https://help.aliyun.com/zh/model-studio/client-events#8d11313f2198k)与[input\_image\_buffer.append](https://help.aliyun.com/zh/model-studio/client-events#e27908854eaht) 事件追加音频与图片至缓冲区。 + + > 发送 input\_image\_buffer.append 事件前，至少发送过一次 input\_audio\_buffer.append 事件。 + +3. 服务端检测到语音结束，发送[input\_audio\_buffer.speech\_stopped](https://help.aliyun.com/zh/model-studio/server-events#fd08ffdf0a2mt) 事件。 + +4. 服务端发送[input\_audio\_buffer.committed](https://help.aliyun.com/zh/model-studio/server-events#bd9bfdc258afy) 事件提交音频缓冲区。 + +5. 服务端发送 [conversation.item.created](https://help.aliyun.com/zh/model-studio/server-events#bb4547ed5b5ht) 事件，包含从缓冲区创建的用户消息项。 + + +| **生命周期** | **客户端事件** | **服务端事件** | +| --- | --- | --- | +| 会话初始化 | [session.update](https://help.aliyun.com/zh/model-studio/client-events#af43722339yva) > 会话配置 | [session.created](https://help.aliyun.com/zh/model-studio/server-events#39689ed6e90ag) > 会话已创建 [session.updated](https://help.aliyun.com/zh/model-studio/server-events#424ef2e774q9p) > 会话配置已更新 | +| 用户音频输入 | [input\\_audio\\_buffer.append](https://help.aliyun.com/zh/model-studio/client-events#8d11313f2198k) > 添加音频到缓冲区 [input\\_image\\_buffer.append](https://help.aliyun.com/zh/model-studio/client-events#e27908854eaht) > 添加图片到缓冲区 | [input\\_audio\\_buffer.speech\\_started](https://help.aliyun.com/zh/model-studio/server-events#b99f92d872n5c) > 检测到语音开始 [input\\_audio\\_buffer.speech\\_stopped](https://help.aliyun.com/zh/model-studio/server-events#fd08ffdf0a2mt) > 检测到语音结束 [input\\_audio\\_buffer.committed](https://help.aliyun.com/zh/model-studio/server-events#bd9bfdc258afy) > 服务器收到提交的音频 | +| 服务器音频输出 | 无 | [response.created](https://help.aliyun.com/zh/model-studio/server-events#38033afc582r1) > 服务端开始生成响应 [response.output\\_item.added](https://help.aliyun.com/zh/model-studio/server-events#dae2260d40qtu) > 响应时有新的输出内容 [conversation.item.created](https://help.aliyun.com/zh/model-studio/server-events#bb4547ed5b5ht) > 对话项被创建 [response.content\\_part.added](https://help.aliyun.com/zh/model-studio/server-events#de7fa0b877j25) > 新的输出内容添加到assistant message [response.audio\\_transcript.delta](https://help.aliyun.com/zh/model-studio/server-events#35396453cfood) > 增量生成的转录文字 [response.audio.delta](https://help.aliyun.com/zh/model-studio/server-events#a25cc50a15car) > 模型增量生成的音频 [response.audio\\_transcript.done](https://help.aliyun.com/zh/model-studio/server-events#f4d1698567bsm) > 文本转录完成 [response.audio.done](https://help.aliyun.com/zh/model-studio/server-events#9e8eb59c67qnt) > 音频生成完成 [response.content\\_part.done](https://help.aliyun.com/zh/model-studio/server-events#011ad54242wft) > Assistant message 的文本或音频内容流式输出完成 [response.output\\_item.done](https://help.aliyun.com/zh/model-studio/server-events#f580421f45w3h) > Assistant message 的整个输出项流式传输完成 [response.done](https://help.aliyun.com/zh/model-studio/server-events#f2333c777d9s4) > 响应完成 | + +## Manual 模式 + +将[session.update](https://help.aliyun.com/zh/model-studio/client-events#af43722339yva)事件的`session.turn_detection` 设为 `null` 以启用 Manual 模式。此模式下，客户端通过显式发送`input_audio_buffer.commit` 和`response.create`事件请求服务器响应。适用于按下即说场景，如聊天软件中的发送语音。 + +交互流程如下： + +1. 客户端随时发送 [input\_audio\_buffer.append](https://help.aliyun.com/zh/model-studio/client-events#8d11313f2198k)与[input\_image\_buffer.append](https://help.aliyun.com/zh/model-studio/client-events#e27908854eaht)事件追加音频与图片至缓冲区。 + + > 发送 input\_image\_buffer.append 事件前，至少发送过一次 input\_audio\_buffer.append 事件。 + +2. 客户端发送[input\_audio\_buffer.commit](https://help.aliyun.com/zh/model-studio/client-events#1cbea5fa7fkfl)事件提交音频缓冲区与图像缓冲区，告知服务端本轮的用户输入（音频及图片）已全部发送完毕。 + +3. 服务端响应 [input\_audio\_buffer.committed](https://help.aliyun.com/zh/model-studio/server-events#bd9bfdc258afy)事件。 + +4. 客户端发送[response.create](https://help.aliyun.com/zh/model-studio/client-events#a42f8e9111n72)事件，等待服务端返回模型的输出。 + +5. 服务端响应[conversation.item.created](https://help.aliyun.com/zh/model-studio/server-events#bb4547ed5b5ht)事件。 + + +| **生命周期** | **客户端事件** | **服务端事件** | +| --- | --- | --- | +| 会话初始化 | [session.update](https://help.aliyun.com/zh/model-studio/client-events#af43722339yva) > 会话配置 | [session.created](https://help.aliyun.com/zh/model-studio/server-events#39689ed6e90ag) > 会话已创建 [session.updated](https://help.aliyun.com/zh/model-studio/server-events#424ef2e774q9p) > 会话配置已更新 | +| 用户音频输入 | [input\\_audio\\_buffer.append](https://help.aliyun.com/zh/model-studio/client-events#8d11313f2198k) > 添加音频到缓冲区 [input\\_image\\_buffer.append](https://help.aliyun.com/zh/model-studio/client-events#e27908854eaht) > 添加图片到缓冲区 [input\\_audio\\_buffer.commit](https://help.aliyun.com/zh/model-studio/client-events#1cbea5fa7fkfl) > 提交音频与图片到服务器 [response.create](https://help.aliyun.com/zh/model-studio/client-events#a42f8e9111n72) > 创建模型响应 | [input\\_audio\\_buffer.committed](https://help.aliyun.com/zh/model-studio/server-events#bd9bfdc258afy) > 服务器收到提交的音频 | +| 服务器音频输出 | [input\\_audio\\_buffer.clear](https://help.aliyun.com/zh/model-studio/client-events#3b44f2f2abakw) > 清除缓冲区的音频 | [response.created](https://help.aliyun.com/zh/model-studio/server-events#38033afc582r1) > 服务端开始生成响应 [response.output\\_item.added](https://help.aliyun.com/zh/model-studio/server-events#dae2260d40qtu) > 响应时有新的输出内容 [conversation.item.created](https://help.aliyun.com/zh/model-studio/server-events#bb4547ed5b5ht) > 对话项被创建 [response.content\\_part.added](https://help.aliyun.com/zh/model-studio/server-events#de7fa0b877j25) > 新的输出内容添加到assistant message 项 [response.audio\\_transcript.delta](https://help.aliyun.com/zh/model-studio/server-events#35396453cfood) > 增量生成的转录文字 [response.audio.delta](https://help.aliyun.com/zh/model-studio/server-events#a25cc50a15car) > 模型增量生成的音频 [response.audio\\_transcript.done](https://help.aliyun.com/zh/model-studio/server-events#f4d1698567bsm) > 完成文本转录 [response.audio.done](https://help.aliyun.com/zh/model-studio/server-events#9e8eb59c67qnt) > 完成音频生成 [response.content\\_part.done](https://help.aliyun.com/zh/model-studio/server-events#011ad54242wft) > Assistant message 的文本或音频内容流式输出完成 [response.output\\_item.done](https://help.aliyun.com/zh/model-studio/server-events#f580421f45w3h) > Assistant message 的整个输出项流式传输完成 [response.done](https://help.aliyun.com/zh/model-studio/server-events#f2333c777d9s4) > 响应完成 | + +## **联网搜索** + +联网搜索功能使模型能够基于实时检索数据进行回复，适用于股票价格、天气预报等需要即时信息的场景。模型可自主判断是否需要搜索来回应用户的即时问题。 + +> 联网搜索仅 `Qwen3.5-Omni-Realtime` 模型支持，且默认关闭，需通过 `session.update` 事件启用。 + +> 计费请参考[计费说明](https://help.aliyun.com/zh/model-studio/web-search#92ce83df3a599)中的`agent`策略。 + +### **启用方式** + +在 `session.update` 事件中添加以下参数： + +- `enable_search`：设置为 `true` 启用联网搜索功能。 + +- `search_options.enable_source`：设置为 `true` 返回搜索结果来源列表。 + + +参数详情请参见[session.update](https://help.aliyun.com/zh/model-studio/client-events#7692180895hvk)。 + +### **响应格式** + +启用联网搜索后，`response.done` 事件中的 `usage` 会新增 `plugins` 字段，用于记录搜索计量信息： + +``` +{ + "usage": { + "total_tokens": 2937, + "input_tokens": 2554, + "output_tokens": 383, + "input_tokens_details": { + "text_tokens": 2512, + "audio_tokens": 42 + }, + "output_tokens_details": { + "text_tokens": 90, + "audio_tokens": 293 + }, + "plugins": { + "search": { + "count": 1, + "strategy": "agent" + } + } + } +} +``` + +### **代码示例** + +以下示例展示如何在实时对话中启用联网搜索功能。 + +## DashScope Python SDK + +在 `update_session` 调用中传入 `enable_search` 和 `search_options` 参数： + +``` +import os +import base64 +import time +import json +import pyaudio +from dashscope.audio.qwen_omni import MultiModality, AudioFormat, OmniRealtimeCallback, OmniRealtimeConversation +import dashscope + +dashscope.api_key = os.getenv('DASHSCOPE_API_KEY') +url = 'wss://dashscope.aliyuncs.com/api-ws/v1/realtime' +model = 'qwen3.5-omni-plus-realtime' +voice = 'Tina' + +class SearchCallback(OmniRealtimeCallback): + def __init__(self, pya): + self.pya = pya + self.out = None + def on_open(self): + self.out = self.pya.open(format=pyaudio.paInt16, channels=1, rate=24000, output=True) + def on_event(self, response): + if response['type'] == 'response.audio.delta': + self.out.write(base64.b64decode(response['delta'])) + elif response['type'] == 'conversation.item.input_audio_transcription.completed': + print(f"[User] {response['transcript']}") + elif response['type'] == 'response.audio_transcript.done': + print(f"[LLM] {response['transcript']}") + elif response['type'] == 'response.done': + usage = response.get('response', {}).get('usage', {}) + plugins = usage.get('plugins', {}) + if plugins.get('search'): + print(f"[Search] count={plugins['search']['count']}, strategy={plugins['search']['strategy']}") + +pya = pyaudio.PyAudio() +callback = SearchCallback(pya) +conv = OmniRealtimeConversation(model=model, callback=callback, url=url) +conv.connect() +conv.update_session( + output_modalities=[MultiModality.AUDIO, MultiModality.TEXT], + voice=voice, + instructions="你是个人助理小云", + enable_search=True, + search_options={'enable_source': True} +) +mic = pya.open(format=pyaudio.paInt16, channels=1, rate=16000, input=True) +print("联网搜索已启用，对着麦克风说话 (Ctrl+C 退出)...") +try: + while True: + audio_data = mic.read(3200, exception_on_overflow=False) + conv.append_audio(base64.b64encode(audio_data).decode()) + time.sleep(0.01) +except KeyboardInterrupt: + conv.close() + mic.close() + callback.out.close() + pya.terminate() + print("\n对话结束") +``` + +## DashScope Java SDK + +在 `updateSession` 中通过 `parameters` 传入联网搜索配置： + +``` +import com.alibaba.dashscope.audio.omni.*; +import com.alibaba.dashscope.exception.NoApiKeyException; +import com.google.gson.JsonObject; +import javax.sound.sampled.*; +import java.nio.ByteBuffer; +import java.util.*; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.atomic.AtomicBoolean; + +public class OmniSearch { + static class SequentialAudioPlayer { + private final SourceDataLine line; + private final Queue audioQueue = new ConcurrentLinkedQueue<>(); + private final Thread playerThread; + private final AtomicBoolean shouldStop = new AtomicBoolean(false); + + public SequentialAudioPlayer() throws LineUnavailableException { + AudioFormat format = new AudioFormat(24000, 16, 1, true, false); + line = AudioSystem.getSourceDataLine(format); + line.open(format); + line.start(); + playerThread = new Thread(() -> { + while (!shouldStop.get()) { + byte[] audio = audioQueue.poll(); + if (audio != null) { + line.write(audio, 0, audio.length); + } else { + try { Thread.sleep(10); } catch (InterruptedException ignored) {} + } + } + }, "AudioPlayer"); + playerThread.start(); + } + + public void play(String base64Audio) { + audioQueue.add(Base64.getDecoder().decode(base64Audio)); + } + public void close() { + shouldStop.set(true); + try { playerThread.join(1000); } catch (InterruptedException ignored) {} + line.drain(); + line.close(); + } + } + + public static void main(String[] args) { + try { + SequentialAudioPlayer player = new SequentialAudioPlayer(); + AtomicBoolean shouldStop = new AtomicBoolean(false); + + OmniRealtimeParam param = OmniRealtimeParam.builder() + .model("qwen3.5-omni-plus-realtime") + .apikey(System.getenv("DASHSCOPE_API_KEY")) + .url("wss://dashscope.aliyuncs.com/api-ws/v1/realtime") + .build(); + + OmniRealtimeConversation conversation = new OmniRealtimeConversation(param, new OmniRealtimeCallback() { + @Override public void onOpen() { + System.out.println("连接已建立"); + } + @Override public void onClose(int code, String reason) { + System.out.println("连接已关闭"); + shouldStop.set(true); + } + @Override public void onEvent(JsonObject event) { + String type = event.get("type").getAsString(); + if ("response.audio.delta".equals(type)) { + player.play(event.get("delta").getAsString()); + } else if ("response.audio_transcript.done".equals(type)) { + System.out.println("[LLM] " + event.get("transcript").getAsString()); + } else if ("response.done".equals(type)) { + JsonObject response = event.getAsJsonObject("response"); + if (response != null && response.has("usage")) { + JsonObject usage = response.getAsJsonObject("usage"); + if (usage.has("plugins")) { + JsonObject plugins = usage.getAsJsonObject("plugins"); + if (plugins.has("search")) { + JsonObject search = plugins.getAsJsonObject("search"); + System.out.println("[Search] count=" + search.get("count").getAsInt() + + ", strategy=" + search.get("strategy").getAsString()); + } + } + } + } + } + }); + + conversation.connect(); + conversation.updateSession(OmniRealtimeConfig.builder() + .modalities(Arrays.asList(OmniRealtimeModality.AUDIO, OmniRealtimeModality.TEXT)) + .voice("Tina") + .enableTurnDetection(true) + .enableInputAudioTranscription(true) + .parameters(Map.of( + "instructions", "你是个人助理小云", + "enable_search", true, + "search_options", Map.of("enable_source", true) + )) + .build() + ); + + System.out.println("联网搜索已启用，请开始说话（按Ctrl+C退出）..."); + AudioFormat format = new AudioFormat(16000, 16, 1, true, false); + TargetDataLine mic = AudioSystem.getTargetDataLine(format); + mic.open(format); + mic.start(); + + ByteBuffer buffer = ByteBuffer.allocate(3200); + while (!shouldStop.get()) { + int bytesRead = mic.read(buffer.array(), 0, buffer.capacity()); + if (bytesRead > 0) { + conversation.appendAudio(Base64.getEncoder().encodeToString(buffer.array())); + } + Thread.sleep(20); + } + + conversation.close(1000, "正常结束"); + player.close(); + mic.close(); + } catch (NoApiKeyException e) { + System.err.println("未找到API KEY: 请设置环境变量 DASHSCOPE_API_KEY"); + } catch (Exception e) { + e.printStackTrace(); + } + } +} +``` + +## WebSocket（Python） + +在 `session.update` 的 JSON 中添加 `enable_search` 和 `search_options` 字段： + +``` +import json +import os +import websocket +import base64 +import pyaudio +import threading + +API_KEY = os.getenv("DASHSCOPE_API_KEY") +API_URL = "wss://dashscope.aliyuncs.com/api-ws/v1/realtime?model=qwen3.5-omni-plus-realtime" + +pya = pyaudio.PyAudio() +out_stream = pya.open(format=pyaudio.paInt16, channels=1, rate=24000, output=True) + +def on_open(ws): + ws.send(json.dumps({ + "type": "session.update", + "session": { + "modalities": ["text", "audio"], + "voice": "Tina", + "instructions": "你是个人助理小云", + "input_audio_format": "pcm", + "output_audio_format": "pcm", + "enable_search": True, + "search_options": { + "enable_source": True + } + } + })) + print("联网搜索已启用，对着麦克风说话...") + def send_audio(): + mic = pya.open(format=pyaudio.paInt16, channels=1, rate=16000, input=True) + try: + while True: + audio = mic.read(3200, exception_on_overflow=False) + ws.send(json.dumps({ + "type": "input_audio_buffer.append", + "audio": base64.b64encode(audio).decode() + })) + except Exception: + mic.close() + threading.Thread(target=send_audio, daemon=True).start() + +def on_message(ws, message): + event = json.loads(message) + if event["type"] == "response.audio.delta": + out_stream.write(base64.b64decode(event["delta"])) + elif event["type"] == "response.audio_transcript.done": + print(f"[LLM] {event['transcript']}") + elif event["type"] == "response.done": + usage = event.get("response", {}).get("usage", {}) + plugins = usage.get("plugins", {}) + if plugins.get("search"): + print(f"[Search] count={plugins['search']['count']}, strategy={plugins['search']['strategy']}") + +def on_error(ws, error): + print(f"Error: {error}") + +headers = ["Authorization: Bearer " + API_KEY] +ws = websocket.WebSocketApp(API_URL, header=headers, on_open=on_open, on_message=on_message, on_error=on_error) +ws.run_forever() +``` + +## **API 参考** + +- [客户端事件](https://help.aliyun.com/zh/model-studio/client-events) + +- [服务端事件](https://help.aliyun.com/zh/model-studio/server-events) + + +## **计费与限流** + +### **计费规则** + +Qwen-Omni-Realtime 模型根据不同模态（音频、图像）对应的Token数计费。计费详情请参见百炼控制台。 + +**音频、图片转换为Token数的规则** + +## 音频 + +- `Qwen3.5-Omni-Realtime：` + + - 输入音频计算公式：`总 Token 数 = 音频时长（单位：秒）* 7` + + - 输出音频计算公式：`总 Tokens 数 = 音频时长（单位：秒）* 12.5` + +- `Qwen3-Omni-Flash-Realtime：`输入与输出音频的计算公式均为`总 Token 数 = 音频时长（单位：秒）* 12.5` + +- `Qwen-Omni-Turbo-Realtime：`输入与输出音频的计算公式均为`总 Token 数 = 音频时长（单位：秒）* 25` + + 若音频时长不足1秒，则按 1 秒计算。 + + +## 图片 + +- `Qwen3.5-Omni-Plus-Realtime`模型**：**每`32x32`像素对应 1 个 Token + +- `Qwen3-Omni-Flash-Realtime`模型**：**每`32x32`像素对应 1 个 Token + +- `Qwen-Omni-Turbo-Realtime`模型：每`28x28`像素对应 1 个 Token + + +一张图最少需要 4 个 Token，最多支持 1280 个 Token；可使用以下代码，传入图像路径和会话时长即可估算图片消耗的 Token 总量： + +``` +# 使用以下命令安装Pillow库：pip install Pillow +from PIL import Image +import math + +# Qwen-Omni-Turbo-Realtime模型，缩放因子为28 +# factor = 28 +# Qwen3-Omni-Flash-Realtime、Qwen3.5-Omni-Realtime模型，缩放因子为32 +factor = 32 + +def token_calculate(image_path='', duration=10): + """ + :param image_path: 图像路径 + :param duration: 会话连接时长 + :return: 图像的Token数 + """ + if len(image_path) > 0: + # 打开指定的PNG图片文件 + image = Image.open(image_path) + # 获取图片的原始尺寸 + height = image.height + width = image.width + print(f"缩放前的图像尺寸为：高度为{height}，宽度为{width}") + # 将高度调整为factor的整数倍 + h_bar = round(height / factor) * factor + # 将宽度调整为factor的整数倍 + w_bar = round(width / factor) * factor + # 图像的Token下限：4个Token + min_pixels = factor * factor * 4 + # 图像的Token上限：1280个Token + max_pixels = 1280 * factor * factor + # 对图像进行缩放处理，调整像素的总数在范围[min_pixels,max_pixels]内 + if h_bar * w_bar > max_pixels: + # 计算缩放因子beta，使得缩放后的图像总像素数不超过max_pixels + beta = math.sqrt((height * width) / max_pixels) + # 重新计算调整后的高度，确保为factor的整数倍 + h_bar = math.floor(height / beta / factor) * factor + # 重新计算调整后的宽度，确保为factor的整数倍 + w_bar = math.floor(width / beta / factor) * factor + elif h_bar * w_bar < min_pixels: + # 计算缩放因子beta，使得缩放后的图像总像素数不低于min_pixels + beta = math.sqrt(min_pixels / (height * width)) + # 重新计算调整后的高度，确保为factor的整数倍 + h_bar = math.ceil(height * beta / factor) * factor + # 重新计算调整后的宽度，确保为factor的整数倍 + w_bar = math.ceil(width * beta / factor) * factor + print(f"缩放后的图像尺寸为：高度为{h_bar}，宽度为{w_bar}") + # 计算图像的Token数：总像素除以factor * factor + token = int((h_bar * w_bar) / (factor * factor)) + print(f"缩放后的token数量为：{token}") + total_token = token * math.ceil(duration / 2) + print(f"总Token数为：{total_token}") + return total_token + else: + print("错误：image_path参数为空，无法计算Token数") + return 0 + +if __name__ == "__main__": + total_token = token_calculate(image_path="xxx/test.jpg", duration=10) + +``` + +### **限流** + +模型限流规则请参见[限流](https://help.aliyun.com/zh/model-studio/rate-limit)。 + +## **常见问题** + +### **如何在线体验 Qwen-Omni-Realtime 模型？** + +A：您可以通过以下方式一键部署： + +1. 访问[函数计算模板](https://fcnext.console.aliyun.com/applications/create?template=omni-realtime@dev)，**部署类型**选择**直接部署**，**百炼 API-KEY** 填入您的 API Key；单击**创建并部署默认环境**。 + +2. 等待约一分钟，在**环境详情**的**环境信息**中获取**访问域名**，**将访问域名的**`**http**`**改成**`**https**`（示例：https://omni-realtime.fcv3.xxxx.cn-hangzhou.fc.devsapp.net），修改后的 HTTPS 链接指向一个可在线体验的 Web 应用，可通过它与模型进行实时视频或语音通话。 + + +**重要** + +此链接使用自签名证书，仅用于临时测试。首次访问时，浏览器会显示安全警告，这是预期行为，**请勿在生产环境使用**。如需继续，请按浏览器提示操作（如点击“高级” → “继续前往（不安全）”）。 + +> 通过**资源信息**\-**函数资源**查看项目源代码。 + +> [函数计算](https://help.aliyun.com/zh/functioncompute/fc/product-overview/trial-quota-1)与[阿里云百炼](https://help.aliyun.com/zh/model-studio/new-free-quota)均为新用户提供免费额度，可以覆盖简单调试所需成本，额度耗尽后按量计费。只有在访问的情况下会产生费用。 + +### **怎么向模型输入图片？** + +A：通过客户端发送[input\_image\_buffer.append](https://help.aliyun.com/zh/model-studio/client-events#c28ed38410nfw)事件。 + +- VAD 模式 + + 该模式会根据语音检测情况自动提交音频与图片，请在服务端响应[input\_audio\_buffer.speech\_stopped](https://help.aliyun.com/zh/model-studio/server-events#fd08ffdf0a2mt)前发送[input\_image\_buffer.append](https://help.aliyun.com/zh/model-studio/client-events#c28ed38410nfw)事件。 + +- Manual 模式 + + 参见[Manual 模式](#b3fed8b9161vd)代码，将图片输入与提交的两部分代码取消注释，即可传入本地图片。 + + +若用于视频通话场景，可以对视频抽帧，建议以 1张/秒 的频率向服务端发送图像。[input\_image\_buffer.append](https://help.aliyun.com/zh/model-studio/client-events#c28ed38410nfw) 事件。 DashScope SDK 代码请参见[Omni-Realtime 示例代码](https://github.com/aliyun/alibabacloud-bailian-speech-demo/tree/master/samples/conversation/omni)。 + +## **错误码** + +如果模型调用失败并返回报错信息，请参见[错误信息](https://help.aliyun.com/zh/model-studio/error-code)进行解决。 + +## **音色列表** + +Qwen-Omni-Realtime模型的音色列表可参见[音色列表](https://help.aliyun.com/zh/model-studio/omni-voice-list)。 + +/\* 让引用上下间距调小，避免内容显示过于稀疏 \*/ .unionContainer .markdown-body blockquote { margin: 4px 0; } .aliyun-docs-content table.qwen blockquote { border-left: none; /\* 添加这一行来移除表格里的引用文字的左侧边框 \*/ padding-left: 5px; /\* 左侧内边距 \*/ margin: 4px 0; } \ No newline at end of file diff --git a/test/build.gradle b/test/build.gradle index f5e9fab..d54fb16 100644 --- a/test/build.gradle +++ b/test/build.gradle @@ -10,6 +10,7 @@ static def getCurrentTime() { android { namespace 'ai.guiji.duix.test' compileSdk 34 + ndkVersion '25.1.8937393' defaultConfig { applicationId "ai.guiji.duix.test" @@ -87,4 +88,5 @@ dependencies { implementation project(":duix-sdk") implementation 'com.squareup.okhttp3:okhttp:4.10.0' + implementation 'com.google.zxing:core:3.5.3' } \ No newline at end of file diff --git a/test/src/main/AndroidManifest.xml b/test/src/main/AndroidManifest.xml index 2f84f62..69b61b1 100644 --- a/test/src/main/AndroidManifest.xml +++ b/test/src/main/AndroidManifest.xml @@ -37,6 +37,18 @@ android:exported="false" android:theme="@style/Theme.DUIX.Call"> + + + + + + \ No newline at end of file diff --git a/test/src/main/java/ai/guiji/duix/test/realtime/OmniRealtimeClient.java b/test/src/main/java/ai/guiji/duix/test/realtime/OmniRealtimeClient.java index d23ffc1..e75c3e0 100644 --- a/test/src/main/java/ai/guiji/duix/test/realtime/OmniRealtimeClient.java +++ b/test/src/main/java/ai/guiji/duix/test/realtime/OmniRealtimeClient.java @@ -9,6 +9,10 @@ import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.UUID; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; @@ -71,6 +75,10 @@ public class OmniRealtimeClient { private final AtomicBoolean assistantTextFinalized = new AtomicBoolean(false); private final StringBuilder assistantTextBuffer = new StringBuilder(); + public static String getDefaultInstructions() { + return DEFAULT_INSTRUCTIONS; + } + public OmniRealtimeClient(SessionOptions sessionOptions, Callback callback) { this.sessionOptions = sessionOptions; this.callback = callback; @@ -170,7 +178,15 @@ public class OmniRealtimeClient { } session.put("input_audio_format", "pcm"); session.put("instructions", options.getInstructions()); - session.put("enable_search", options.isEnableSearch()); + session.put("enable_search", options.isEnableSearch() && !options.hasTools()); + + if (options.hasTools()) { + JSONArray tools = new JSONArray(); + for (ToolDefinition toolDefinition : options.getTools()) { + tools.put(toolDefinition.toJson()); + } + session.put("tools", tools); + } if (options.isEnableInputAudioTranscription()) { JSONObject inputAudioTranscription = new JSONObject(); @@ -263,6 +279,16 @@ public class OmniRealtimeClient { dispatchAssistantFinalText(event.optString("text", event.optString("transcript", ""))); break; + case "response.function_call_arguments.done": + String toolName = event.optString("name", ""); + String callId = event.optString("call_id", ""); + String arguments = event.optString("arguments", "{}"); + Log.d(TAG, "Tool call: " + toolName + ", callId=" + callId + ", arguments=" + arguments); + if (callback != null) { + callback.onToolCallRequested(toolName, callId, arguments); + } + break; + case "response.output_item.done": dispatchAssistantFinalTextIfNeeded(extractTextFromItem(event.optJSONObject("item"))); break; @@ -391,6 +417,35 @@ public class OmniRealtimeClient { return isConnected.get(); } + public boolean sendFunctionCallOutput(String callId, String output) { + if (callId == null || callId.trim().isEmpty()) { + return false; + } + try { + JSONObject item = new JSONObject(); + item.put("type", "function_call_output"); + item.put("call_id", callId); + item.put("output", output == null ? "" : output); + + JSONObject event = new JSONObject(); + event.put("event_id", nextEventId()); + event.put("type", "conversation.item.create"); + event.put("item", item); + + if (!sendEvent(event)) { + return false; + } + + JSONObject responseCreate = new JSONObject(); + responseCreate.put("event_id", nextEventId()); + responseCreate.put("type", "response.create"); + return sendEvent(responseCreate); + } catch (JSONException e) { + Log.e(TAG, "Failed to send function_call_output: " + e.getMessage()); + return false; + } + } + private void handleServerError(JSONObject event) { String errMsg = event.optJSONObject("error") != null ? event.optJSONObject("error").optString("message", "Unknown error") @@ -510,7 +565,7 @@ public class OmniRealtimeClient { } private String nextEventId() { - return "event_" + System.currentTimeMillis(); + return "event_" + UUID.randomUUID().toString().replace("-", ""); } public static class SessionOptions { @@ -528,29 +583,48 @@ public class OmniRealtimeClient { private final boolean enableTurnDetection; private final boolean enableInputAudioTranscription; private final OutputMode outputMode; + private final List tools; public SessionOptions(String voice, boolean enableSearch) { this(voice, enableSearch, OutputMode.AUDIO_AND_TEXT); } public SessionOptions(String voice, boolean enableSearch, OutputMode outputMode) { + this(voice, enableSearch, outputMode, null, null); + } + + public SessionOptions(String voice, boolean enableSearch, OutputMode outputMode, + String instructions, List tools) { this(DEFAULT_API_KEY, DEFAULT_BASE_URL, DEFAULT_MODEL, voice, DEFAULT_INSTRUCTIONS, - enableSearch, true, true, outputMode); + enableSearch, true, true, outputMode, tools, instructions); } public SessionOptions(String apiKey, String baseUrl, String model, String voice, String instructions, boolean enableSearch, boolean enableTurnDetection, boolean enableInputAudioTranscription, OutputMode outputMode) { + this(apiKey, baseUrl, model, voice, instructions, enableSearch, + enableTurnDetection, enableInputAudioTranscription, outputMode, null, null); + } + + public SessionOptions(String apiKey, String baseUrl, String model, String voice, + String instructions, boolean enableSearch, boolean enableTurnDetection, + boolean enableInputAudioTranscription, OutputMode outputMode, + List tools, String ignoredInstructionsOverride) { this.apiKey = apiKey == null ? "" : apiKey; this.baseUrl = baseUrl == null || baseUrl.trim().isEmpty() ? DEFAULT_BASE_URL : baseUrl; this.model = model == null || model.trim().isEmpty() ? DEFAULT_MODEL : model; this.voice = voice == null || voice.trim().isEmpty() ? "Ethan" : voice; - this.instructions = instructions == null || instructions.trim().isEmpty() - ? DEFAULT_INSTRUCTIONS : instructions; + String resolvedInstructions = ignoredInstructionsOverride; + if (resolvedInstructions == null || resolvedInstructions.trim().isEmpty()) { + resolvedInstructions = instructions; + } + this.instructions = resolvedInstructions == null || resolvedInstructions.trim().isEmpty() + ? DEFAULT_INSTRUCTIONS : resolvedInstructions; this.enableSearch = enableSearch; this.enableTurnDetection = enableTurnDetection; this.enableInputAudioTranscription = enableInputAudioTranscription; this.outputMode = outputMode == null ? OutputMode.AUDIO_AND_TEXT : outputMode; + this.tools = tools == null ? Collections.emptyList() : new ArrayList<>(tools); } public String getApiKey() { @@ -584,6 +658,38 @@ public class OmniRealtimeClient { public boolean isAudioOutputEnabled() { return outputMode == OutputMode.AUDIO_AND_TEXT; } + + public boolean hasTools() { + return !tools.isEmpty(); + } + + public List getTools() { + return tools; + } + } + + public static class ToolDefinition { + private final String name; + private final String description; + private final JSONObject parameters; + + public ToolDefinition(String name, String description, JSONObject parameters) { + this.name = name == null ? "" : name; + this.description = description == null ? "" : description; + this.parameters = parameters == null ? new JSONObject() : parameters; + } + + public JSONObject toJson() throws JSONException { + JSONObject function = new JSONObject(); + function.put("name", name); + function.put("description", description); + function.put("parameters", new JSONObject(parameters.toString())); + + JSONObject tool = new JSONObject(); + tool.put("type", "function"); + tool.put("function", function); + return tool; + } } public interface Callback { @@ -603,6 +709,8 @@ public class OmniRealtimeClient { void onAssistantTranscriptDone(String transcript); + void onToolCallRequested(String toolName, String callId, String argumentsJson); + void onAssistantAudioDelta(byte[] pcmData); void onAssistantAudioDone(); diff --git a/test/src/main/java/ai/guiji/duix/test/service/DashScopeConfig.java b/test/src/main/java/ai/guiji/duix/test/service/DashScopeConfig.java index 152d178..b009df5 100644 --- a/test/src/main/java/ai/guiji/duix/test/service/DashScopeConfig.java +++ b/test/src/main/java/ai/guiji/duix/test/service/DashScopeConfig.java @@ -7,6 +7,7 @@ public final class DashScopeConfig { public static final String VOICE_CLONE_URL = "https://dashscope.aliyuncs.com/api/v1/services/audio/tts/customization"; public static final String VOICE_CLONE_MODEL = "qwen-voice-enrollment"; public static final String VOICE_CLONE_TTS_MODEL = "qwen3-tts-vc-realtime-2026-01-15"; + public static final String MONITOR_BASE_URL = "https://digital-human.xn--876a.net"; public static final int AUDIO_SAMPLE_RATE = 24000; private DashScopeConfig() { diff --git a/test/src/main/java/ai/guiji/duix/test/service/DeviceIdentityStore.kt b/test/src/main/java/ai/guiji/duix/test/service/DeviceIdentityStore.kt new file mode 100644 index 0000000..65e59b9 --- /dev/null +++ b/test/src/main/java/ai/guiji/duix/test/service/DeviceIdentityStore.kt @@ -0,0 +1,44 @@ +package ai.guiji.duix.test.service + +import android.content.Context +import android.net.Uri +import java.util.Locale +import java.util.UUID + +class DeviceIdentityStore(context: Context) { + + private val prefs = context.getSharedPreferences(PREFS_NAME, Context.MODE_PRIVATE) + + fun getOrCreateDeviceUuid(): String { + val current = prefs.getString(KEY_DEVICE_UUID, null)?.trim()?.lowercase(Locale.US) + if (!current.isNullOrBlank()) { + return current + } + + val deviceUuid = UUID.randomUUID().toString().lowercase(Locale.US) + prefs.edit().putString(KEY_DEVICE_UUID, deviceUuid).apply() + return deviceUuid + } + + fun getBindUrl(): String { + val deviceUuid = getOrCreateDeviceUuid() + return prefs.getString(KEY_BIND_URL, null)?.takeIf { it.isNotBlank() } + ?: buildFallbackBindUrl(deviceUuid) + } + + fun saveBindUrl(bindUrl: String) { + prefs.edit().putString(KEY_BIND_URL, bindUrl.trim()).apply() + } + + fun buildFallbackBindUrl(deviceUuid: String = getOrCreateDeviceUuid()): String { + val normalized = deviceUuid.trim().lowercase(Locale.US) + return DashScopeConfig.MONITOR_BASE_URL.trimEnd('/') + + "/bind?deviceUuid=" + Uri.encode(normalized) + } + + companion object { + private const val PREFS_NAME = "device_identity_store" + private const val KEY_DEVICE_UUID = "device_uuid" + private const val KEY_BIND_URL = "bind_url" + } +} \ No newline at end of file diff --git a/test/src/main/java/ai/guiji/duix/test/service/FamilyBridgeService.kt b/test/src/main/java/ai/guiji/duix/test/service/FamilyBridgeService.kt new file mode 100644 index 0000000..62c7b70 --- /dev/null +++ b/test/src/main/java/ai/guiji/duix/test/service/FamilyBridgeService.kt @@ -0,0 +1,218 @@ +package ai.guiji.duix.test.service + +import ai.guiji.duix.test.App +import ai.guiji.duix.test.BuildConfig +import android.content.Context +import android.os.Build +import okhttp3.MediaType.Companion.toMediaType +import okhttp3.Request +import okhttp3.RequestBody.Companion.toRequestBody +import org.json.JSONArray +import org.json.JSONObject + +class FamilyBridgeService(context: Context) { + + private val appContext = context.applicationContext + private val identityStore = DeviceIdentityStore(appContext) + private val jsonMediaType = "application/json; charset=utf-8".toMediaType() + + fun getDeviceUuid(): String = identityStore.getOrCreateDeviceUuid() + + fun getBindUrl(): String = identityStore.getBindUrl() + + fun registerCurrentDevice(): DeviceRegistrationResult { + val payload = JSONObject().apply { + put("deviceUuid", getDeviceUuid()) + put("displayName", buildDisplayName()) + put("appVersion", BuildConfig.VERSION_NAME) + } + val response = postJson("/api/device/register", payload) + val bindUrl = response.optString("bindUrl").ifBlank { + identityStore.buildFallbackBindUrl(getDeviceUuid()) + } + identityStore.saveBindUrl(bindUrl) + + return DeviceRegistrationResult( + deviceUuid = response.optString("deviceUuid", getDeviceUuid()), + bindUrl = bindUrl, + ) + } + + fun fetchFamilyMessages(take: Int = 5): FamilyMessageFetchResult { + val normalizedTake = take.coerceIn(1, 8) + val url = DashScopeConfig.MONITOR_BASE_URL.trimEnd('/') + + "/api/device/messages?deviceUuid=${getDeviceUuid()}&take=$normalizedTake" + val response = executeJsonRequest( + Request.Builder() + .url(url) + .get() + .build() + ) + + val messagesJson = response.optJSONArray("messages") ?: JSONArray() + val messages = buildList { + for (index in 0 until messagesJson.length()) { + val item = messagesJson.optJSONObject(index) ?: continue + add( + FamilyMessageItem( + id = item.optString("id"), + content = item.optString("content"), + importance = item.optString("importance"), + createdAt = item.optString("createdAt"), + ) + ) + } + } + + return FamilyMessageFetchResult( + summaryText = response.optString("summaryText"), + unreadCount = response.optInt("unreadCount"), + messages = messages, + ) + } + + fun markFamilyMessagesRead(messageIds: List) { + val payload = JSONObject().apply { + put("deviceUuid", getDeviceUuid()) + put("messageIds", JSONArray(messageIds)) + } + postJson("/api/device/messages/read", payload) + } + + fun recordUsageEvent(eventType: String, detailJson: JSONObject? = null) { + val payload = JSONObject().apply { + put("deviceUuid", getDeviceUuid()) + put("eventType", eventType) + if (detailJson != null) { + put("detailJson", detailJson.toString()) + } + } + postJson("/api/device/usage", payload) + } + + fun recordToolCall( + toolName: String, + arguments: JSONObject, + outputText: String, + status: String, + callId: String? = null, + sessionId: String? = null, + ) { + val payload = JSONObject().apply { + put("deviceUuid", getDeviceUuid()) + put("toolName", toolName) + put("arguments", arguments) + put("outputText", outputText) + put("status", status) + if (!callId.isNullOrBlank()) { + put("callId", callId) + } + if (!sessionId.isNullOrBlank()) { + put("sessionId", sessionId) + } + } + postJson("/api/device/tool-call", payload) + } + + fun uploadConversationTurns(sessionId: String?, turns: List) { + if (turns.isEmpty()) { + return + } + val payload = JSONObject().apply { + put("deviceUuid", getDeviceUuid()) + if (!sessionId.isNullOrBlank()) { + put("sessionId", sessionId) + } + put( + "turns", + JSONArray().apply { + turns.forEach { turn -> + put( + JSONObject().apply { + put("role", turn.role) + put("content", turn.content) + if (!turn.metaJson.isNullOrBlank()) { + put("metaJson", turn.metaJson) + } + } + ) + } + } + ) + } + postJson("/api/device/conversations", payload) + } + + private fun postJson(path: String, payload: JSONObject): JSONObject { + return executeJsonRequest( + Request.Builder() + .url(DashScopeConfig.MONITOR_BASE_URL.trimEnd('/') + path) + .post(payload.toString().toRequestBody(jsonMediaType)) + .build() + ) + } + + private fun executeJsonRequest(request: Request): JSONObject { + App.getOkHttpClient().newCall(request).execute().use { response -> + val responseText = response.body?.string().orEmpty() + val json = responseText.takeIf { it.isNotBlank() }?.let(::JSONObject) ?: JSONObject() + if (!response.isSuccessful) { + val message = json.optString("error").ifBlank { + response.message.ifBlank { "请求失败" } + } + throw IllegalStateException(message) + } + return json + } + } + + private fun buildDisplayName(): String { + val manufacturer = Build.MANUFACTURER?.trim().orEmpty() + val model = Build.MODEL?.trim().orEmpty() + return listOf(manufacturer, model) + .filter { it.isNotBlank() } + .distinct() + .joinToString(" ") + .ifBlank { "长辈陪伴终端" } + } + + data class DeviceRegistrationResult( + val deviceUuid: String, + val bindUrl: String, + ) + + data class FamilyMessageItem( + val id: String, + val content: String, + val importance: String, + val createdAt: String, + ) + + data class FamilyMessageFetchResult( + val summaryText: String, + val unreadCount: Int, + val messages: List, + ) + + data class ConversationTurnPayload( + val role: String, + val content: String, + val metaJson: String? = null, + ) + + companion object { + const val EVENT_APP_OPEN = "APP_OPEN" + const val EVENT_SETTINGS_OPENED = "SETTINGS_OPENED" + const val EVENT_AI_SESSION_STARTED = "AI_SESSION_STARTED" + const val EVENT_AI_SESSION_ENDED = "AI_SESSION_ENDED" + const val EVENT_CAMERA_ENABLED = "CAMERA_ENABLED" + const val EVENT_CAMERA_DISABLED = "CAMERA_DISABLED" + + const val ROLE_USER = "USER" + const val ROLE_ASSISTANT = "ASSISTANT" + const val ROLE_TOOL = "TOOL" + + const val STATUS_SUCCESS = "SUCCESS" + const val STATUS_FAILURE = "FAILURE" + } +} \ No newline at end of file diff --git a/test/src/main/java/ai/guiji/duix/test/ui/activity/CallActivity.kt b/test/src/main/java/ai/guiji/duix/test/ui/activity/CallActivity.kt index a315850..962ed40 100644 --- a/test/src/main/java/ai/guiji/duix/test/ui/activity/CallActivity.kt +++ b/test/src/main/java/ai/guiji/duix/test/ui/activity/CallActivity.kt @@ -13,6 +13,7 @@ import ai.guiji.duix.test.databinding.ActivityCallBinding import ai.guiji.duix.test.realtime.OmniRealtimeClient import ai.guiji.duix.test.realtime.QwenTtsRealtimeClient import ai.guiji.duix.test.service.DashScopeConfig +import ai.guiji.duix.test.service.FamilyBridgeService import ai.guiji.duix.test.ui.adapter.MotionAdapter import ai.guiji.duix.test.ui.dialog.AudioRecordDialog import ai.guiji.duix.test.ui.widget.AudioSpectrumView @@ -29,9 +30,14 @@ import android.view.View import android.widget.CompoundButton import android.widget.Toast import com.bumptech.glide.Glide +import org.json.JSONArray +import org.json.JSONObject import java.io.File import java.io.FileInputStream import java.io.FileOutputStream +import java.util.UUID +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors class CallActivity : BaseActivity() { @@ -74,11 +80,15 @@ class CallActivity : BaseActivity() { private val mainHandler = Handler(Looper.getMainLooper()) private var assistantTextBuffer = StringBuilder() private var hasStartedPushForResponse = false - private val enableWebSearch = true + private val enableWebSearch = false + private val enableToolCalling = true private var isCloneMode = false private var clonedVoice = "" private var cloneTtsModel = DashScopeConfig.VOICE_CLONE_TTS_MODEL private var ttsRealtimeClient: QwenTtsRealtimeClient? = null + private val familyBridgeService by lazy { FamilyBridgeService(applicationContext) } + private val bridgeExecutor: ExecutorService = Executors.newSingleThreadExecutor() + private var currentSessionId: String? = null override fun onCreate(savedInstanceState: Bundle?) { super.onCreate(savedInstanceState) @@ -96,6 +106,10 @@ class CallActivity : BaseActivity() { clonedVoice = intent.getStringExtra("clonedVoice") ?: "" cloneTtsModel = intent.getStringExtra("cloneTtsModel") ?: DashScopeConfig.VOICE_CLONE_TTS_MODEL + enqueueBridgeTask("设备注册失败") { + familyBridgeService.registerCurrentDevice() + } + Glide.with(mContext).load("file:///android_asset/bg/bg1.png").into(binding.ivBg) binding.glTextureView.setEGLContextClientVersion(GL_CONTEXT_VERSION) @@ -263,7 +277,75 @@ class CallActivity : BaseActivity() { } else { OmniRealtimeClient.SessionOptions.OutputMode.AUDIO_AND_TEXT } - return OmniRealtimeClient.SessionOptions(voice, enableWebSearch, outputMode) + return OmniRealtimeClient.SessionOptions( + voice, + enableWebSearch, + outputMode, + buildRealtimeInstructions(), + buildRealtimeTools(), + ) + } + + private fun buildRealtimeInstructions(): String { + if (!enableToolCalling) { + return OmniRealtimeClient.getDefaultInstructions() + } + return buildString { + append(OmniRealtimeClient.getDefaultInstructions()) + append("\n\n") + append("当用户明确要给儿子、女儿、老伴、家里人或亲友带话、留言、报平安时，必须调用 leave_message_for_family 工具，不要假装已经转达。\n") + append("当用户问家里人有没有留言、想听家人说了什么、想看看新消息时，必须调用 fetch_family_messages 工具，不要自己编造留言。\n") + append("工具执行结果返回后，再用温和、简短、生活化的中文向老人说明情况。") + } + } + + private fun buildRealtimeTools(): List { + if (!enableToolCalling) { + return emptyList() + } + + val leaveMessageParameters = JSONObject().apply { + put("type", "object") + put("properties", JSONObject().apply { + put("recipientRelation", JSONObject().apply { + put("type", "string") + put("description", "留言对象与老人的关系，例如儿子、女儿、老伴、孙女。") + }) + put("message", JSONObject().apply { + put("type", "string") + put("description", "老人想让家里人知道的完整留言内容。") + }) + put("importance", JSONObject().apply { + put("type", "string") + put("description", "留言紧急程度。") + put("enum", JSONArray().put("LOW").put("NORMAL").put("HIGH").put("URGENT")) + }) + }) + put("required", JSONArray().put("message")) + } + + val fetchFamilyMessagesParameters = JSONObject().apply { + put("type", "object") + put("properties", JSONObject().apply { + put("count", JSONObject().apply { + put("type", "integer") + put("description", "想读取几条家人留言，通常 1 到 5 条。") + }) + }) + } + + return listOf( + OmniRealtimeClient.ToolDefinition( + "leave_message_for_family", + "当老人明确要求给家里人带话、留言、报平安或请转告时使用。", + leaveMessageParameters, + ), + OmniRealtimeClient.ToolDefinition( + "fetch_family_messages", + "当老人想听家人最新留言，或询问有没有新消息时使用。", + fetchFamilyMessagesParameters, + ), + ) } private fun isImageAppendError(message: String): Boolean { @@ -284,6 +366,7 @@ class CallActivity : BaseActivity() { super.onDestroy() stopAIConversation() duix?.release() + bridgeExecutor.shutdownNow() } private fun playPCMStream(){ @@ -359,6 +442,7 @@ class CallActivity : BaseActivity() { binding.audioSpectrum.clear() assistantTextBuffer.clear() hasStartedPushForResponse = false + currentSessionId = UUID.randomUUID().toString() binding.btnAIConversation.setImageResource(R.drawable.ic_mic_off) binding.btnAIConversation.setBackgroundResource(R.drawable.shape_fab_ai_active) @@ -370,7 +454,7 @@ class CallActivity : BaseActivity() { binding.btnCameraToggle.setBackgroundResource(R.drawable.shape_fab_camera_off) binding.btnCameraToggle.contentDescription = getString(R.string.camera_off) isCameraEnabled = false - applyMessage("AI对话: 已连接, model=${OmniRealtimeClient.DEFAULT_MODEL}, search=on, cloneMode=$isCloneMode") + applyMessage("AI对话: 已连接, model=${OmniRealtimeClient.DEFAULT_MODEL}, search=${if (enableWebSearch) "on" else "off"}, tools=${if (enableToolCalling) "on" else "off"}, cloneMode=$isCloneMode") streamingRecorder = StreamingAudioRecorder(mContext, object : StreamingAudioRecorder.StreamingCallback { override fun onAudioData(pcmData: ByteArray) { omniClient?.appendAudio(pcmData) @@ -384,6 +468,15 @@ class CallActivity : BaseActivity() { }) streamingRecorder?.start() } + enqueueBridgeTask("AI会话启动记录失败") { + familyBridgeService.registerCurrentDevice() + val detail = JSONObject().apply { + put("sessionId", currentSessionId) + put("cloneMode", isCloneMode) + put("voice", voice) + } + familyBridgeService.recordUsageEvent(FamilyBridgeService.EVENT_AI_SESSION_STARTED, detail) + } } override fun onDisconnected() { @@ -422,6 +515,7 @@ class CallActivity : BaseActivity() { mainHandler.post { binding.tvSubtitle.text = getString(R.string.ai_call_user_said, transcript) } + uploadConversationTurnAsync(FamilyBridgeService.ROLE_USER, transcript) } override fun onAssistantTextDelta(delta: String) { @@ -437,6 +531,19 @@ class CallActivity : BaseActivity() { assistantTextBuffer.append(transcript) binding.tvSubtitle.text = transcript } + uploadConversationTurnAsync(FamilyBridgeService.ROLE_ASSISTANT, transcript) + } + + override fun onToolCallRequested(toolName: String, callId: String, argumentsJson: String) { + mainHandler.post { + binding.tvSubtitle.text = when (toolName) { + "leave_message_for_family" -> "我来帮您把这句话记给家里人。" + "fetch_family_messages" -> "我来帮您看看家里人的新留言。" + else -> getString(R.string.ai_call_thinking) + } + } + applyMessage("AI工具调用: $toolName, callId=$callId, arguments=$argumentsJson") + handleRealtimeToolCall(toolName, callId, argumentsJson) } override fun onAssistantAudioDelta(pcmData: ByteArray) { @@ -484,6 +591,8 @@ class CallActivity : BaseActivity() { } private fun stopAIConversation() { + val endedSessionId = currentSessionId + currentSessionId = null isAIConversationActive = false stopCamera() stopCloneSpeech() @@ -507,6 +616,16 @@ class CallActivity : BaseActivity() { applyMessage("AI对话: 已结束") } } + + if (!endedSessionId.isNullOrBlank()) { + enqueueBridgeTask("AI会话结束记录失败") { + val detail = JSONObject().apply { + put("sessionId", endedSessionId) + put("cloneMode", isCloneMode) + } + familyBridgeService.recordUsageEvent(FamilyBridgeService.EVENT_AI_SESSION_ENDED, detail) + } + } } private fun speakWithClonedVoice(text: String) { @@ -602,6 +721,11 @@ class CallActivity : BaseActivity() { cameraFrameCapture?.start() binding.btnCameraToggle.setBackgroundResource(R.drawable.shape_fab_camera_on) binding.btnCameraToggle.contentDescription = getString(R.string.camera_on) + enqueueBridgeTask("镜头开启记录失败") { + val detail = JSONObject() + currentSessionId?.let { detail.put("sessionId", it) } + familyBridgeService.recordUsageEvent(FamilyBridgeService.EVENT_CAMERA_ENABLED, detail) + } } else { stopCamera() binding.btnCameraToggle.setBackgroundResource(R.drawable.shape_fab_camera_off) @@ -610,12 +734,199 @@ class CallActivity : BaseActivity() { } private fun stopCamera() { + val wasEnabled = isCameraEnabled || cameraFrameCapture != null cameraFrameCapture?.stop() cameraFrameCapture = null isCameraEnabled = false binding.cameraPreview.visibility = View.GONE + if (wasEnabled) { + enqueueBridgeTask("镜头关闭记录失败") { + val detail = JSONObject() + currentSessionId?.let { detail.put("sessionId", it) } + familyBridgeService.recordUsageEvent(FamilyBridgeService.EVENT_CAMERA_DISABLED, detail) + } + } } + private fun enqueueBridgeTask(label: String, task: () -> Unit) { + if (bridgeExecutor.isShutdown) { + return + } + bridgeExecutor.execute { + try { + task() + } catch (throwable: Throwable) { + applyMessage("$label: ${throwable.message}") + } + } + } + + private fun uploadConversationTurnAsync(role: String, content: String, metaJson: JSONObject? = null) { + enqueueBridgeTask("对话记录上传失败") { + uploadConversationTurnInternal(role, content, metaJson) + } + } + + private fun uploadConversationTurnInternal(role: String, content: String, metaJson: JSONObject? = null) { + val sessionId = currentSessionId + val normalizedContent = content.trim() + if (sessionId.isNullOrBlank() || normalizedContent.isEmpty()) { + return + } + familyBridgeService.uploadConversationTurns( + sessionId, + listOf( + FamilyBridgeService.ConversationTurnPayload( + role = role, + content = normalizedContent, + metaJson = metaJson?.toString(), + ) + ) + ) + } + + private fun handleRealtimeToolCall(toolName: String, callId: String, argumentsJson: String) { + enqueueBridgeTask("工具调用处理失败") { + val arguments = runCatching { JSONObject(argumentsJson) }.getOrElse { JSONObject() } + val result = executeRealtimeTool(toolName, callId, arguments) + uploadConversationTurnInternal( + FamilyBridgeService.ROLE_TOOL, + result.outputText, + JSONObject().apply { + put("toolName", toolName) + put("callId", callId) + put("status", result.status) + put("arguments", arguments) + } + ) + val sent = omniClient?.sendFunctionCallOutput(callId, result.outputText) == true + if (!sent) { + applyMessage("工具结果回传失败: $toolName") + } + } + } + + private fun executeRealtimeTool( + toolName: String, + callId: String, + arguments: JSONObject, + ): ToolExecutionResult { + return when (toolName) { + "leave_message_for_family" -> executeLeaveMessageTool(callId, arguments) + "fetch_family_messages" -> executeFetchFamilyMessagesTool(callId, arguments) + else -> { + val outputText = "这个请求我还不能直接处理，您可以换一种说法再告诉我。" + runCatching { + familyBridgeService.recordToolCall( + toolName = toolName, + arguments = arguments, + outputText = outputText, + status = FamilyBridgeService.STATUS_FAILURE, + callId = callId, + sessionId = currentSessionId, + ) + } + ToolExecutionResult(outputText, FamilyBridgeService.STATUS_FAILURE) + } + } + } + + private fun executeLeaveMessageTool(callId: String, arguments: JSONObject): ToolExecutionResult { + val message = arguments.optString("message").trim() + val relation = arguments.optString("recipientRelation").trim().ifBlank { "家里人" } + + if (message.isEmpty()) { + val outputText = "我还没听清您想转告的话，您再慢慢说一遍，我帮您记下来。" + runCatching { + familyBridgeService.recordToolCall( + toolName = "leave_message_for_family", + arguments = arguments, + outputText = outputText, + status = FamilyBridgeService.STATUS_FAILURE, + callId = callId, + sessionId = currentSessionId, + ) + } + return ToolExecutionResult(outputText, FamilyBridgeService.STATUS_FAILURE) + } + + return runCatching { + val outputText = "已记下要转告给${relation}的话：$message" + familyBridgeService.recordToolCall( + toolName = "leave_message_for_family", + arguments = arguments, + outputText = outputText, + status = FamilyBridgeService.STATUS_SUCCESS, + callId = callId, + sessionId = currentSessionId, + ) + ToolExecutionResult(outputText, FamilyBridgeService.STATUS_SUCCESS) + }.getOrElse { error -> + val outputText = "暂时还没连上家人的留言通道，请稍后再试一次。" + runCatching { + familyBridgeService.recordToolCall( + toolName = "leave_message_for_family", + arguments = arguments, + outputText = outputText, + status = FamilyBridgeService.STATUS_FAILURE, + callId = callId, + sessionId = currentSessionId, + ) + } + applyMessage("留言工具失败: ${error.message}") + ToolExecutionResult(outputText, FamilyBridgeService.STATUS_FAILURE) + } + } + + private fun executeFetchFamilyMessagesTool(callId: String, arguments: JSONObject): ToolExecutionResult { + val requestedCount = arguments.optInt("count", 3).coerceIn(1, 5) + + return runCatching { + val result = familyBridgeService.fetchFamilyMessages(requestedCount) + if (result.messages.isNotEmpty()) { + familyBridgeService.markFamilyMessagesRead(result.messages.map { it.id }) + } + val outputText = if (result.messages.isEmpty()) { + "家里人暂时还没有新的留言。" + } else { + buildString { + append(result.summaryText.ifBlank { "我帮您看了看，家里人有新的留言。" }) + result.messages.take(requestedCount).forEachIndexed { index, message -> + append(" 第${index + 1}条：${message.content}") + } + } + } + familyBridgeService.recordToolCall( + toolName = "fetch_family_messages", + arguments = arguments, + outputText = outputText, + status = FamilyBridgeService.STATUS_SUCCESS, + callId = callId, + sessionId = currentSessionId, + ) + ToolExecutionResult(outputText, FamilyBridgeService.STATUS_SUCCESS) + }.getOrElse { error -> + val outputText = "暂时还没取到家人的新留言，请稍后再试一次。" + runCatching { + familyBridgeService.recordToolCall( + toolName = "fetch_family_messages", + arguments = arguments, + outputText = outputText, + status = FamilyBridgeService.STATUS_FAILURE, + callId = callId, + sessionId = currentSessionId, + ) + } + applyMessage("读取家人留言失败: ${error.message}") + ToolExecutionResult(outputText, FamilyBridgeService.STATUS_FAILURE) + } + } + + private data class ToolExecutionResult( + val outputText: String, + val status: String, + ) + private fun showRecordDialog(){ val audioRecordDialog = AudioRecordDialog(mContext, object : AudioRecordDialog.Listener{ override fun onFinish(path: String) { diff --git a/test/src/main/java/ai/guiji/duix/test/ui/activity/DeviceSettingsActivity.kt b/test/src/main/java/ai/guiji/duix/test/ui/activity/DeviceSettingsActivity.kt new file mode 100644 index 0000000..15b5d26 --- /dev/null +++ b/test/src/main/java/ai/guiji/duix/test/ui/activity/DeviceSettingsActivity.kt @@ -0,0 +1,80 @@ +package ai.guiji.duix.test.ui.activity + +import ai.guiji.duix.test.R +import ai.guiji.duix.test.databinding.ActivityDeviceSettingsBinding +import ai.guiji.duix.test.service.FamilyBridgeService +import ai.guiji.duix.test.util.QrCodeBitmapFactory +import android.os.Bundle +import android.util.TypedValue +import android.widget.Toast +import org.json.JSONObject + +class DeviceSettingsActivity : BaseActivity() { + + private lateinit var binding: ActivityDeviceSettingsBinding + private val familyBridgeService by lazy { FamilyBridgeService(applicationContext) } + + override fun onCreate(savedInstanceState: Bundle?) { + super.onCreate(savedInstanceState) + binding = ActivityDeviceSettingsBinding.inflate(layoutInflater) + setContentView(binding.root) + + binding.btnBack.setOnClickListener { finish() } + binding.btnRefresh.setOnClickListener { syncDeviceRegistration(true) } + + renderIdentityCard(familyBridgeService.getBindUrl()) + syncDeviceRegistration(false) + Thread { + runCatching { + familyBridgeService.recordUsageEvent( + FamilyBridgeService.EVENT_SETTINGS_OPENED, + JSONObject().put("screen", "device_settings"), + ) + } + }.start() + } + + private fun renderIdentityCard(bindUrl: String) { + val deviceUuid = familyBridgeService.getDeviceUuid() + binding.tvDeviceUuidValue.text = deviceUuid + binding.tvBindLinkValue.text = bindUrl + val qrSize = TypedValue.applyDimension( + TypedValue.COMPLEX_UNIT_DIP, + 220f, + resources.displayMetrics, + ).toInt() + binding.ivQrCode.setImageBitmap(QrCodeBitmapFactory.create(bindUrl, qrSize)) + } + + private fun syncDeviceRegistration(showToast: Boolean) { + binding.btnRefresh.isEnabled = false + binding.tvSyncStatus.text = getString(R.string.device_settings_syncing) + + Thread { + val result = runCatching { familyBridgeService.registerCurrentDevice() } + runOnUiThread { + binding.btnRefresh.isEnabled = true + result.onSuccess { registration -> + renderIdentityCard(registration.bindUrl) + binding.tvSyncStatus.text = getString(R.string.device_settings_sync_success) + if (showToast) { + Toast.makeText(mContext, R.string.device_settings_sync_success, Toast.LENGTH_SHORT).show() + } + }.onFailure { error -> + renderIdentityCard(familyBridgeService.getBindUrl()) + binding.tvSyncStatus.text = getString( + R.string.device_settings_sync_failed, + error.message ?: getString(R.string.ai_call_error_retry), + ) + if (showToast) { + Toast.makeText( + mContext, + error.message ?: getString(R.string.ai_call_error_retry), + Toast.LENGTH_SHORT, + ).show() + } + } + } + }.start() + } +} \ No newline at end of file diff --git a/test/src/main/java/ai/guiji/duix/test/ui/activity/MainActivity.kt b/test/src/main/java/ai/guiji/duix/test/ui/activity/MainActivity.kt index 1cc56f2..d994249 100644 --- a/test/src/main/java/ai/guiji/duix/test/ui/activity/MainActivity.kt +++ b/test/src/main/java/ai/guiji/duix/test/ui/activity/MainActivity.kt @@ -5,18 +5,21 @@ import ai.guiji.duix.test.R import ai.guiji.duix.test.databinding.ActivityMainBinding import ai.guiji.duix.test.model.AvatarCatalog import ai.guiji.duix.test.model.AvatarModel +import ai.guiji.duix.test.service.FamilyBridgeService import ai.guiji.duix.test.ui.adapter.AvatarAdapter import ai.guiji.duix.test.ui.dialog.LoadingDialog import android.content.Intent import android.os.Bundle import android.widget.Toast import androidx.recyclerview.widget.GridLayoutManager +import org.json.JSONObject import java.io.File class MainActivity : BaseActivity() { private lateinit var binding: ActivityMainBinding + private val familyBridgeService by lazy { FamilyBridgeService(applicationContext) } private var mLoadingDialog: LoadingDialog? = null private var mLastProgress = 0 @@ -32,6 +35,9 @@ class MainActivity : BaseActivity() { binding.btnVoiceClone.setOnClickListener { startActivity(Intent(mContext, VoiceCloneSetupActivity::class.java)) } + binding.btnSettings.setOnClickListener { + startActivity(Intent(mContext, SettingsActivity::class.java)) + } binding.rvAvatars.layoutManager = GridLayoutManager(this, 2) binding.rvAvatars.adapter = AvatarAdapter(AvatarCatalog.models, object : AvatarAdapter.Callback { @@ -40,6 +46,20 @@ class MainActivity : BaseActivity() { play(model) } }) + + bootstrapFamilyBridge() + } + + private fun bootstrapFamilyBridge() { + Thread { + runCatching { familyBridgeService.registerCurrentDevice() } + runCatching { + familyBridgeService.recordUsageEvent( + FamilyBridgeService.EVENT_APP_OPEN, + JSONObject().put("screen", "main"), + ) + } + }.start() } private fun play(model: AvatarModel) { diff --git a/test/src/main/java/ai/guiji/duix/test/ui/activity/SettingsActivity.kt b/test/src/main/java/ai/guiji/duix/test/ui/activity/SettingsActivity.kt new file mode 100644 index 0000000..ae9ec51 --- /dev/null +++ b/test/src/main/java/ai/guiji/duix/test/ui/activity/SettingsActivity.kt @@ -0,0 +1,41 @@ +package ai.guiji.duix.test.ui.activity + +import ai.guiji.duix.test.BuildConfig +import ai.guiji.duix.test.databinding.ActivitySettingsBinding +import ai.guiji.duix.test.service.DashScopeConfig +import ai.guiji.duix.test.service.FamilyBridgeService +import android.content.Intent +import android.net.Uri +import android.os.Bundle +import org.json.JSONObject + +class SettingsActivity : BaseActivity() { + + private lateinit var binding: ActivitySettingsBinding + private val familyBridgeService by lazy { FamilyBridgeService(applicationContext) } + + override fun onCreate(savedInstanceState: Bundle?) { + super.onCreate(savedInstanceState) + binding = ActivitySettingsBinding.inflate(layoutInflater) + setContentView(binding.root) + + binding.btnBack.setOnClickListener { finish() } + binding.rowDeviceCode.setOnClickListener { + startActivity(Intent(mContext, DeviceSettingsActivity::class.java)) + } + + binding.tvVersionValue.text = "${BuildConfig.VERSION_NAME} (${BuildConfig.VERSION_CODE})" + binding.tvServerValue.text = Uri.parse(DashScopeConfig.MONITOR_BASE_URL).host + ?: DashScopeConfig.MONITOR_BASE_URL + binding.tvDeviceValue.text = familyBridgeService.getDeviceUuid() + + Thread { + runCatching { + familyBridgeService.recordUsageEvent( + FamilyBridgeService.EVENT_SETTINGS_OPENED, + JSONObject().put("screen", "settings_home"), + ) + } + }.start() + } +} \ No newline at end of file diff --git a/test/src/main/java/ai/guiji/duix/test/util/QrCodeBitmapFactory.kt b/test/src/main/java/ai/guiji/duix/test/util/QrCodeBitmapFactory.kt new file mode 100644 index 0000000..8ee24d9 --- /dev/null +++ b/test/src/main/java/ai/guiji/duix/test/util/QrCodeBitmapFactory.kt @@ -0,0 +1,29 @@ +package ai.guiji.duix.test.util + +import android.graphics.Bitmap +import android.graphics.Color +import com.google.zxing.BarcodeFormat +import com.google.zxing.EncodeHintType +import com.google.zxing.qrcode.QRCodeWriter + +object QrCodeBitmapFactory { + + fun create(content: String, size: Int): Bitmap { + val hints = hashMapOf( + EncodeHintType.MARGIN to 1, + EncodeHintType.CHARACTER_SET to "UTF-8", + ) + val matrix = QRCodeWriter().encode(content, BarcodeFormat.QR_CODE, size, size, hints) + val pixels = IntArray(size * size) + + for (y in 0 until size) { + for (x in 0 until size) { + pixels[y * size + x] = if (matrix[x, y]) Color.BLACK else Color.WHITE + } + } + + return Bitmap.createBitmap(size, size, Bitmap.Config.ARGB_8888).apply { + setPixels(pixels, 0, size, 0, 0, size, size) + } + } +} \ No newline at end of file diff --git a/test/src/main/res/drawable/ic_arrow_back.xml b/test/src/main/res/drawable/ic_arrow_back.xml new file mode 100644 index 0000000..82e25b4 --- /dev/null +++ b/test/src/main/res/drawable/ic_arrow_back.xml @@ -0,0 +1,9 @@ + + + \ No newline at end of file diff --git a/test/src/main/res/drawable/ic_chevron_right.xml b/test/src/main/res/drawable/ic_chevron_right.xml new file mode 100644 index 0000000..40296a6 --- /dev/null +++ b/test/src/main/res/drawable/ic_chevron_right.xml @@ -0,0 +1,9 @@ + + + \ No newline at end of file diff --git a/test/src/main/res/drawable/ic_settings.xml b/test/src/main/res/drawable/ic_settings.xml new file mode 100644 index 0000000..241d102 --- /dev/null +++ b/test/src/main/res/drawable/ic_settings.xml @@ -0,0 +1,9 @@ + + + \ No newline at end of file diff --git a/test/src/main/res/drawable/shape_qr_surface.xml b/test/src/main/res/drawable/shape_qr_surface.xml new file mode 100644 index 0000000..6c81920 --- /dev/null +++ b/test/src/main/res/drawable/shape_qr_surface.xml @@ -0,0 +1,7 @@ + + + + + \ No newline at end of file diff --git a/test/src/main/res/drawable/shape_warm_icon_button.xml b/test/src/main/res/drawable/shape_warm_icon_button.xml new file mode 100644 index 0000000..aa83892 --- /dev/null +++ b/test/src/main/res/drawable/shape_warm_icon_button.xml @@ -0,0 +1,7 @@ + + + + + \ No newline at end of file diff --git a/test/src/main/res/layout/activity_device_settings.xml b/test/src/main/res/layout/activity_device_settings.xml new file mode 100644 index 0000000..f486d66 --- /dev/null +++ b/test/src/main/res/layout/activity_device_settings.xml @@ -0,0 +1,140 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/test/src/main/res/layout/activity_main.xml b/test/src/main/res/layout/activity_main.xml index ef3c29f..afe83ce 100644 --- a/test/src/main/res/layout/activity_main.xml +++ b/test/src/main/res/layout/activity_main.xml @@ -12,29 +12,49 @@ android:paddingTop="40dp" android:paddingBottom="20dp"> - + android:paddingStart="18dp" + android:paddingEnd="18dp"> - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/test/src/main/res/values-zh-rCN/strings.xml b/test/src/main/res/values-zh-rCN/strings.xml index df06302..649e652 100644 --- a/test/src/main/res/values-zh-rCN/strings.xml +++ b/test/src/main/res/values-zh-rCN/strings.xml @@ -60,6 +60,29 @@ 熟悉声音已经保存好了 熟悉声音保存失败，请稍后再试 请先在熟悉声音陪伴里保存一段声音 + 家人连接 + 打开设备码页面后，家人扫一扫就能把这台陪伴设备绑定到自己的账号，之后可以给您留言。 + 打开设备码 + 返回 + 家人连接 + 把这个二维码给家里人扫一扫，绑定后他们就能给您留言，也能看看最近的陪伴记录。 + 这台设备的二维码 + 家人用网页面板扫一扫，就能把这台设备连接到自己的账号。 + 设备 UUID + 绑定链接 + 重新同步二维码 + 正在为您同步设备连接信息… + 设备二维码已经准备好了 + 同步失败：%1$s + 打开设置 + 设置 + 把设备信息、家人连接入口和当前版本都放在这里，找起来更方便。 + 家人连接设备码 + 打开设备二维码页面后，家里人扫一扫就能把这台设备绑定到自己的账号。 + 关于这台设备 + 应用版本 + 家属面板域名 + 当前设备 UUID Github地址可能下载失败，您可以考虑使用代理或者将文件缓存到自己的存储服务 调试信息 diff --git a/test/src/main/res/values/strings.xml b/test/src/main/res/values/strings.xml index 1f94686..2058255 100644 --- a/test/src/main/res/values/strings.xml +++ b/test/src/main/res/values/strings.xml @@ -63,6 +63,29 @@ The familiar voice has been saved Failed to save the familiar voice, please try again later Please save a familiar voice first + Family Connection + Open the device code page so your family can scan and connect this companion device to their own account. + Open Device Code + Back + Family Connection + Show this QR code to your family. After scanning, they can leave you messages and check recent usage records. + This device QR code + Family members can scan this code with the web panel to bind this device. + Device UUID + Binding link + Refresh QR Code + Syncing device status… + Device QR code is ready + Sync failed: %1$s + Open settings + Settings + Keep device information, family connection entry and current version together in one place. + Family device code + Open the QR code page for your family to scan and bind this elder device. + About this device + App version + Family panel domain + Current device UUID The GitHub url may fail to download, you can consider using a proxy or caching the file to your own storage service Debug message \ No newline at end of file