添加联网搜索
This commit is contained in:
@@ -49,6 +49,12 @@ public class CameraFrameCapture {
|
||||
private Handler backgroundHandler;
|
||||
private final AtomicBoolean isCapturing = new AtomicBoolean(false);
|
||||
private final AtomicBoolean isRunning = new AtomicBoolean(false);
|
||||
private final Runnable captureRunnable = new Runnable() {
|
||||
@Override
|
||||
public void run() {
|
||||
captureStillFrame();
|
||||
}
|
||||
};
|
||||
|
||||
public CameraFrameCapture(Context context, FrameCallback callback) {
|
||||
this(context, callback, null);
|
||||
@@ -194,7 +200,7 @@ public class CameraFrameCapture {
|
||||
captureSession = session;
|
||||
isCapturing.set(true);
|
||||
startPreview();
|
||||
scheduleNextCapture();
|
||||
scheduleNextCapture(0L);
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -222,8 +228,14 @@ public class CameraFrameCapture {
|
||||
}
|
||||
}
|
||||
|
||||
private void scheduleNextCapture() {
|
||||
if (!isCapturing.get() || !isRunning.get() || captureSession == null || cameraDevice == null) return;
|
||||
private void scheduleNextCapture(long delayMs) {
|
||||
if (backgroundHandler == null) return;
|
||||
backgroundHandler.removeCallbacks(captureRunnable);
|
||||
backgroundHandler.postDelayed(captureRunnable, Math.max(delayMs, 0L));
|
||||
}
|
||||
|
||||
private void captureStillFrame() {
|
||||
if (!isCapturing.get() || !isRunning.get() || captureSession == null || cameraDevice == null || imageReader == null) return;
|
||||
try {
|
||||
CaptureRequest.Builder builder = cameraDevice.createCaptureRequest(CameraDevice.TEMPLATE_PREVIEW);
|
||||
builder.addTarget(imageReader.getSurface());
|
||||
@@ -235,7 +247,7 @@ public class CameraFrameCapture {
|
||||
@NonNull android.hardware.camera2.CaptureRequest request,
|
||||
@NonNull android.hardware.camera2.TotalCaptureResult result) {
|
||||
if (isRunning.get()) {
|
||||
backgroundHandler.postDelayed(CameraFrameCapture.this::scheduleNextCapture, CAPTURE_INTERVAL_MS);
|
||||
scheduleNextCapture(CAPTURE_INTERVAL_MS);
|
||||
}
|
||||
}
|
||||
}, backgroundHandler);
|
||||
@@ -244,6 +256,13 @@ public class CameraFrameCapture {
|
||||
}
|
||||
}
|
||||
|
||||
public void requestImmediateCapture() {
|
||||
if (!isRunning.get() || !isCapturing.get()) {
|
||||
return;
|
||||
}
|
||||
scheduleNextCapture(0L);
|
||||
}
|
||||
|
||||
private byte[] imageToJpeg(Image image) {
|
||||
Image.Plane[] planes = image.getPlanes();
|
||||
if (planes.length == 0) return null;
|
||||
@@ -257,6 +276,9 @@ public class CameraFrameCapture {
|
||||
isRunning.set(false);
|
||||
isCapturing.set(false);
|
||||
try {
|
||||
if (backgroundHandler != null) {
|
||||
backgroundHandler.removeCallbacks(captureRunnable);
|
||||
}
|
||||
if (captureSession != null) {
|
||||
captureSession.close();
|
||||
captureSession = null;
|
||||
|
||||
@@ -3,11 +3,13 @@ package ai.guiji.duix.test.realtime;
|
||||
import android.util.Base64;
|
||||
import android.util.Log;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONException;
|
||||
import org.json.JSONObject;
|
||||
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
|
||||
import okhttp3.OkHttpClient;
|
||||
import okhttp3.Request;
|
||||
@@ -17,25 +19,56 @@ import okhttp3.WebSocketListener;
|
||||
|
||||
/**
|
||||
* Qwen-Omni-Realtime WebSocket 客户端
|
||||
* 接入 qwen3-omni-flash-realtime-2025-12-01 模型
|
||||
* 默认接入 qwen3.5-omni-plus-realtime 模型
|
||||
*/
|
||||
public class OmniRealtimeClient {
|
||||
|
||||
private static final String TAG = "OmniRealtimeClient";
|
||||
|
||||
// Demo 配置(硬编码)
|
||||
private static final String API_KEY = "sk-7c32083ace99472e97209e611b9ecbb1";
|
||||
private static final String WS_URL = "wss://dashscope.aliyuncs.com/api-ws/v1/realtime?model=qwen3-omni-flash-realtime-2025-12-01";
|
||||
private static final String DEFAULT_API_KEY = "sk-7c32083ace99472e97209e611b9ecbb1";
|
||||
public static final String DEFAULT_MODEL = "qwen3.5-omni-plus-realtime";
|
||||
private static final String DEFAULT_BASE_URL = "wss://dashscope.aliyuncs.com/api-ws/v1/realtime";
|
||||
private static final String INPUT_AUDIO_TRANSCRIPTION_MODEL = "gummy-realtime-v1";
|
||||
private static final int MAX_IMAGE_BYTES = 500 * 1024;
|
||||
private static final long MIN_IMAGE_APPEND_INTERVAL_MS = 800L;
|
||||
private static final String DEFAULT_INSTRUCTIONS = "你是一名面向中国老年群体的心理陪伴 AI。你的职责是为老人提供温和、耐心、尊重、稳定的情感陪伴、情绪支持、回忆唤起、生活关怀和风险识别服务。你不是医生,也不是心理治疗师,不能替代专业诊疗,但应在必要时建议联系家属、社区、医生或急救资源。\n"
|
||||
+ "\n"
|
||||
+ "请始终遵循以下原则:\n"
|
||||
+ "\n"
|
||||
+ "先共情,再回应问题;\n"
|
||||
+ "\n"
|
||||
+ "先接住情绪,再给建议;\n"
|
||||
+ "\n"
|
||||
+ "使用简短、朴素、生活化的中文,始终用“您”称呼;\n"
|
||||
+ "\n"
|
||||
+ "不说教、不训诫、不否定、不敷衍;\n"
|
||||
+ "\n"
|
||||
+ "不把老人当小孩,不制造虚假亲密;\n"
|
||||
+ "\n"
|
||||
+ "一次不提供超过两个建议;\n"
|
||||
+ "\n"
|
||||
+ "鼓励老人表达感受,可适度引导回忆人生经历、兴趣、家庭和过往成就;\n"
|
||||
+ "\n"
|
||||
+ "对孤独、丧偶、被忽视、无价值感、失眠、焦虑、怕拖累家人等议题保持高度敏感;\n"
|
||||
+ "\n"
|
||||
+ "若出现自杀、自伤、重度绝望、幻觉妄想、意识混乱、虐待、严重医疗异常等风险,必须立即建议联系真人帮助和专业机构;\n"
|
||||
+ "\n"
|
||||
+ "不编造医疗知识、政策信息、机构电话,不确定时明确说明。\n"
|
||||
+ "\n"
|
||||
+ "你的回复风格应温和、平稳、尊重、有耐心。每次回答尽量包含:一句共情、一句理解或复述、一句温和追问或一个微小建议。核心目标是让老人感到:我被听见了,我被尊重了,有人愿意陪我。";
|
||||
|
||||
private final OkHttpClient okHttpClient;
|
||||
private WebSocket webSocket;
|
||||
private final Callback callback;
|
||||
private final String voice;
|
||||
private volatile SessionOptions sessionOptions;
|
||||
private final AtomicBoolean isConnected = new AtomicBoolean(false);
|
||||
private final AtomicBoolean userSpeaking = new AtomicBoolean(false);
|
||||
private final AtomicBoolean assistantResponding = new AtomicBoolean(false);
|
||||
private final AtomicBoolean imageWindowOpen = new AtomicBoolean(false);
|
||||
private final AtomicBoolean hasSentAnyAudio = new AtomicBoolean(false);
|
||||
private final AtomicLong lastImageAppendAtMs = new AtomicLong(0L);
|
||||
|
||||
public OmniRealtimeClient(String voice, Callback callback) {
|
||||
this.voice = voice;
|
||||
public OmniRealtimeClient(SessionOptions sessionOptions, Callback callback) {
|
||||
this.sessionOptions = sessionOptions;
|
||||
this.callback = callback;
|
||||
this.okHttpClient = new OkHttpClient.Builder()
|
||||
.connectTimeout(30, TimeUnit.SECONDS)
|
||||
@@ -45,14 +78,31 @@ public class OmniRealtimeClient {
|
||||
.build();
|
||||
}
|
||||
|
||||
public void updateSessionOptions(SessionOptions sessionOptions) {
|
||||
if (sessionOptions == null) {
|
||||
return;
|
||||
}
|
||||
this.sessionOptions = sessionOptions;
|
||||
if (isConnected.get()) {
|
||||
sendSessionUpdate();
|
||||
}
|
||||
}
|
||||
|
||||
public void connect() {
|
||||
if (isConnected.get()) {
|
||||
Log.w(TAG, "Already connected");
|
||||
return;
|
||||
}
|
||||
SessionOptions options = sessionOptions;
|
||||
if (options == null || options.getApiKey().trim().isEmpty()) {
|
||||
if (callback != null) {
|
||||
callback.onError("未配置 DashScope API Key");
|
||||
}
|
||||
return;
|
||||
}
|
||||
Request request = new Request.Builder()
|
||||
.url(WS_URL)
|
||||
.addHeader("Authorization", "Bearer " + API_KEY)
|
||||
.url(options.getWebSocketUrl())
|
||||
.addHeader("Authorization", "Bearer " + options.getApiKey())
|
||||
.build();
|
||||
|
||||
webSocket = okHttpClient.newWebSocket(request, new WebSocketListener() {
|
||||
@@ -75,6 +125,7 @@ public class OmniRealtimeClient {
|
||||
public void onFailure(WebSocket webSocket, Throwable t, Response response) {
|
||||
Log.e(TAG, "WebSocket error: " + t.getMessage());
|
||||
isConnected.set(false);
|
||||
resetStreamingState();
|
||||
if (callback != null) {
|
||||
callback.onError("连接失败: " + t.getMessage());
|
||||
}
|
||||
@@ -89,6 +140,7 @@ public class OmniRealtimeClient {
|
||||
public void onClosed(WebSocket webSocket, int code, String reason) {
|
||||
Log.i(TAG, "WebSocket closed: " + code + " " + reason);
|
||||
isConnected.set(false);
|
||||
resetStreamingState();
|
||||
if (callback != null) {
|
||||
callback.onDisconnected();
|
||||
}
|
||||
@@ -97,47 +149,37 @@ public class OmniRealtimeClient {
|
||||
}
|
||||
|
||||
private void sendSessionUpdate() {
|
||||
SessionOptions options = sessionOptions;
|
||||
if (options == null) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
JSONObject session = new JSONObject();
|
||||
session.put("modalities", new org.json.JSONArray().put("text").put("audio"));
|
||||
session.put("voice", voice);
|
||||
session.put("modalities", new JSONArray().put("text").put("audio"));
|
||||
session.put("voice", options.getVoice());
|
||||
session.put("input_audio_format", "pcm");
|
||||
session.put("output_audio_format", "pcm");
|
||||
session.put("instructions", "你是一名面向中国老年群体的心理陪伴 AI。你的职责是为老人提供温和、耐心、尊重、稳定的情感陪伴、情绪支持、回忆唤起、生活关怀和风险识别服务。你不是医生,也不是心理治疗师,不能替代专业诊疗,但应在必要时建议联系家属、社区、医生或急救资源。\n" +
|
||||
"\n" +
|
||||
"请始终遵循以下原则:\n" +
|
||||
"\n" +
|
||||
"先共情,再回应问题;\n" +
|
||||
"\n" +
|
||||
"先接住情绪,再给建议;\n" +
|
||||
"\n" +
|
||||
"使用简短、朴素、生活化的中文,始终用“您”称呼;\n" +
|
||||
"\n" +
|
||||
"不说教、不训诫、不否定、不敷衍;\n" +
|
||||
"\n" +
|
||||
"不把老人当小孩,不制造虚假亲密;\n" +
|
||||
"\n" +
|
||||
"一次不提供超过两个建议;\n" +
|
||||
"\n" +
|
||||
"鼓励老人表达感受,可适度引导回忆人生经历、兴趣、家庭和过往成就;\n" +
|
||||
"\n" +
|
||||
"对孤独、丧偶、被忽视、无价值感、失眠、焦虑、怕拖累家人等议题保持高度敏感;\n" +
|
||||
"\n" +
|
||||
"若出现自杀、自伤、重度绝望、幻觉妄想、意识混乱、虐待、严重医疗异常等风险,必须立即建议联系真人帮助和专业机构;\n" +
|
||||
"\n" +
|
||||
"不编造医疗知识、政策信息、机构电话,不确定时明确说明。\n" +
|
||||
"\n" +
|
||||
"你的回复风格应温和、平稳、尊重、有耐心。每次回答尽量包含:一句共情、一句理解或复述、一句温和追问或一个微小建议。核心目标是让老人感到:我被听见了,我被尊重了,有人愿意陪我。");
|
||||
session.put("instructions", options.getInstructions());
|
||||
session.put("enable_search", options.isEnableSearch());
|
||||
|
||||
// 服务端 VAD 模式
|
||||
JSONObject turnDetection = new JSONObject();
|
||||
turnDetection.put("type", "server_vad");
|
||||
turnDetection.put("threshold", 0.5);
|
||||
turnDetection.put("silence_duration_ms", 800);
|
||||
session.put("turn_detection", turnDetection);
|
||||
if (options.isEnableInputAudioTranscription()) {
|
||||
JSONObject inputAudioTranscription = new JSONObject();
|
||||
inputAudioTranscription.put("model", INPUT_AUDIO_TRANSCRIPTION_MODEL);
|
||||
session.put("input_audio_transcription", inputAudioTranscription);
|
||||
}
|
||||
|
||||
if (options.isEnableTurnDetection()) {
|
||||
JSONObject turnDetection = new JSONObject();
|
||||
turnDetection.put("type", "server_vad");
|
||||
turnDetection.put("threshold", 0.5);
|
||||
turnDetection.put("silence_duration_ms", 800);
|
||||
session.put("turn_detection", turnDetection);
|
||||
} else {
|
||||
session.put("turn_detection", JSONObject.NULL);
|
||||
}
|
||||
|
||||
JSONObject event = new JSONObject();
|
||||
event.put("event_id", "event_" + System.currentTimeMillis());
|
||||
event.put("event_id", nextEventId());
|
||||
event.put("type", "session.update");
|
||||
event.put("session", session);
|
||||
|
||||
@@ -161,6 +203,7 @@ public class OmniRealtimeClient {
|
||||
case "input_audio_buffer.speech_started":
|
||||
Log.d(TAG, "User started speaking");
|
||||
userSpeaking.set(true);
|
||||
imageWindowOpen.set(true);
|
||||
if (callback != null) {
|
||||
callback.onUserSpeechStarted();
|
||||
}
|
||||
@@ -169,11 +212,22 @@ public class OmniRealtimeClient {
|
||||
case "input_audio_buffer.speech_stopped":
|
||||
Log.d(TAG, "User stopped speaking");
|
||||
userSpeaking.set(false);
|
||||
imageWindowOpen.set(false);
|
||||
if (callback != null) {
|
||||
callback.onUserSpeechStopped();
|
||||
}
|
||||
break;
|
||||
|
||||
case "input_audio_buffer.committed":
|
||||
Log.d(TAG, "Audio buffer committed");
|
||||
imageWindowOpen.set(false);
|
||||
break;
|
||||
|
||||
case "response.created":
|
||||
assistantResponding.set(true);
|
||||
imageWindowOpen.set(false);
|
||||
break;
|
||||
|
||||
case "conversation.item.input_audio_transcription.completed":
|
||||
String userTranscript = event.optString("transcript", "");
|
||||
Log.d(TAG, "User said: " + userTranscript);
|
||||
@@ -214,19 +268,15 @@ public class OmniRealtimeClient {
|
||||
|
||||
case "response.done":
|
||||
Log.d(TAG, "Response done");
|
||||
assistantResponding.set(false);
|
||||
imageWindowOpen.set(false);
|
||||
if (callback != null) {
|
||||
callback.onResponseDone();
|
||||
}
|
||||
break;
|
||||
|
||||
case "error":
|
||||
String errMsg = event.optJSONObject("error") != null
|
||||
? event.optJSONObject("error").optString("message", "Unknown error")
|
||||
: "Unknown error";
|
||||
Log.e(TAG, "Server error: " + errMsg);
|
||||
if (callback != null) {
|
||||
callback.onError(errMsg);
|
||||
}
|
||||
handleServerError(event);
|
||||
break;
|
||||
|
||||
default:
|
||||
@@ -238,14 +288,18 @@ public class OmniRealtimeClient {
|
||||
}
|
||||
|
||||
public void appendAudio(byte[] pcmData) {
|
||||
if (!isConnected.get() || webSocket == null) return;
|
||||
if (!isConnected.get() || webSocket == null || pcmData == null || pcmData.length == 0) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
String audioB64 = Base64.encodeToString(pcmData, Base64.NO_WRAP);
|
||||
JSONObject event = new JSONObject();
|
||||
event.put("event_id", "event_" + System.currentTimeMillis());
|
||||
event.put("event_id", nextEventId());
|
||||
event.put("type", "input_audio_buffer.append");
|
||||
event.put("audio", audioB64);
|
||||
sendEvent(event);
|
||||
if (sendEvent(event)) {
|
||||
hasSentAnyAudio.set(true);
|
||||
}
|
||||
} catch (JSONException e) {
|
||||
Log.e(TAG, "Failed to append audio: " + e.getMessage());
|
||||
}
|
||||
@@ -255,24 +309,56 @@ public class OmniRealtimeClient {
|
||||
* 向图像缓冲区追加图像数据(用于视频通话)
|
||||
* 图片格式: JPG/JPEG,推荐 480P 或 720P,单张不超过 500KB
|
||||
*/
|
||||
public void appendImage(byte[] jpegData) {
|
||||
if (!isConnected.get() || webSocket == null) return;
|
||||
public boolean appendImage(byte[] jpegData) {
|
||||
if (!canAppendImage(jpegData)) {
|
||||
return false;
|
||||
}
|
||||
try {
|
||||
String imageB64 = Base64.encodeToString(jpegData, Base64.NO_WRAP);
|
||||
JSONObject event = new JSONObject();
|
||||
event.put("event_id", "event_" + System.currentTimeMillis());
|
||||
event.put("event_id", nextEventId());
|
||||
event.put("type", "input_image_buffer.append");
|
||||
event.put("image", imageB64);
|
||||
sendEvent(event);
|
||||
if (sendEvent(event)) {
|
||||
lastImageAppendAtMs.set(System.currentTimeMillis());
|
||||
return true;
|
||||
}
|
||||
} catch (JSONException e) {
|
||||
Log.e(TAG, "Failed to append image: " + e.getMessage());
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private void sendEvent(JSONObject event) {
|
||||
if (webSocket != null && isConnected.get()) {
|
||||
webSocket.send(event.toString());
|
||||
private boolean canAppendImage(byte[] jpegData) {
|
||||
if (!isConnected.get() || webSocket == null || jpegData == null || jpegData.length == 0) {
|
||||
return false;
|
||||
}
|
||||
if (jpegData.length >= MAX_IMAGE_BYTES) {
|
||||
Log.w(TAG, "Skip image larger than 500KB: " + jpegData.length);
|
||||
return false;
|
||||
}
|
||||
if (!hasSentAnyAudio.get()) {
|
||||
Log.d(TAG, "Skip image before the first audio frame is sent");
|
||||
return false;
|
||||
}
|
||||
if (!imageWindowOpen.get()) {
|
||||
Log.v(TAG, "Skip image outside active speech window");
|
||||
return false;
|
||||
}
|
||||
if (assistantResponding.get()) {
|
||||
Log.v(TAG, "Skip image while assistant is responding");
|
||||
return false;
|
||||
}
|
||||
long now = System.currentTimeMillis();
|
||||
long lastAppendAt = lastImageAppendAtMs.get();
|
||||
return now - lastAppendAt >= MIN_IMAGE_APPEND_INTERVAL_MS;
|
||||
}
|
||||
|
||||
private boolean sendEvent(JSONObject event) {
|
||||
if (webSocket != null && isConnected.get()) {
|
||||
return webSocket.send(event.toString());
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
public void disconnect() {
|
||||
@@ -281,12 +367,104 @@ public class OmniRealtimeClient {
|
||||
webSocket = null;
|
||||
}
|
||||
isConnected.set(false);
|
||||
resetStreamingState();
|
||||
}
|
||||
|
||||
public boolean isConnected() {
|
||||
return isConnected.get();
|
||||
}
|
||||
|
||||
private void handleServerError(JSONObject event) {
|
||||
String errMsg = event.optJSONObject("error") != null
|
||||
? event.optJSONObject("error").optString("message", "Unknown error")
|
||||
: "Unknown error";
|
||||
Log.e(TAG, "Server error: " + errMsg);
|
||||
if (isImageAppendError(errMsg)) {
|
||||
imageWindowOpen.set(false);
|
||||
lastImageAppendAtMs.set(System.currentTimeMillis());
|
||||
}
|
||||
assistantResponding.set(false);
|
||||
if (callback != null) {
|
||||
callback.onError(errMsg);
|
||||
}
|
||||
}
|
||||
|
||||
private boolean isImageAppendError(String message) {
|
||||
String lowerMessage = message == null ? "" : message.toLowerCase();
|
||||
return lowerMessage.contains("image")
|
||||
&& (lowerMessage.contains("append") || lowerMessage.contains("buffer"));
|
||||
}
|
||||
|
||||
private void resetStreamingState() {
|
||||
userSpeaking.set(false);
|
||||
assistantResponding.set(false);
|
||||
imageWindowOpen.set(false);
|
||||
hasSentAnyAudio.set(false);
|
||||
lastImageAppendAtMs.set(0L);
|
||||
}
|
||||
|
||||
private String nextEventId() {
|
||||
return "event_" + System.currentTimeMillis();
|
||||
}
|
||||
|
||||
public static class SessionOptions {
|
||||
private final String apiKey;
|
||||
private final String baseUrl;
|
||||
private final String model;
|
||||
private final String voice;
|
||||
private final String instructions;
|
||||
private final boolean enableSearch;
|
||||
private final boolean enableTurnDetection;
|
||||
private final boolean enableInputAudioTranscription;
|
||||
|
||||
public SessionOptions(String voice, boolean enableSearch) {
|
||||
this(DEFAULT_API_KEY, DEFAULT_BASE_URL, DEFAULT_MODEL, voice, DEFAULT_INSTRUCTIONS,
|
||||
enableSearch, true, true);
|
||||
}
|
||||
|
||||
public SessionOptions(String apiKey, String baseUrl, String model, String voice,
|
||||
String instructions, boolean enableSearch, boolean enableTurnDetection,
|
||||
boolean enableInputAudioTranscription) {
|
||||
this.apiKey = apiKey == null ? "" : apiKey;
|
||||
this.baseUrl = baseUrl == null || baseUrl.trim().isEmpty() ? DEFAULT_BASE_URL : baseUrl;
|
||||
this.model = model == null || model.trim().isEmpty() ? DEFAULT_MODEL : model;
|
||||
this.voice = voice == null || voice.trim().isEmpty() ? "Ethan" : voice;
|
||||
this.instructions = instructions == null || instructions.trim().isEmpty()
|
||||
? DEFAULT_INSTRUCTIONS : instructions;
|
||||
this.enableSearch = enableSearch;
|
||||
this.enableTurnDetection = enableTurnDetection;
|
||||
this.enableInputAudioTranscription = enableInputAudioTranscription;
|
||||
}
|
||||
|
||||
public String getApiKey() {
|
||||
return apiKey;
|
||||
}
|
||||
|
||||
public String getWebSocketUrl() {
|
||||
return baseUrl + "?model=" + model;
|
||||
}
|
||||
|
||||
public String getVoice() {
|
||||
return voice;
|
||||
}
|
||||
|
||||
public String getInstructions() {
|
||||
return instructions;
|
||||
}
|
||||
|
||||
public boolean isEnableSearch() {
|
||||
return enableSearch;
|
||||
}
|
||||
|
||||
public boolean isEnableTurnDetection() {
|
||||
return enableTurnDetection;
|
||||
}
|
||||
|
||||
public boolean isEnableInputAudioTranscription() {
|
||||
return enableInputAudioTranscription;
|
||||
}
|
||||
}
|
||||
|
||||
public interface Callback {
|
||||
void onConnected();
|
||||
|
||||
|
||||
@@ -72,6 +72,7 @@ class CallActivity : BaseActivity() {
|
||||
private val mainHandler = Handler(Looper.getMainLooper())
|
||||
private var assistantTextBuffer = StringBuilder()
|
||||
private var hasStartedPushForResponse = false
|
||||
private val enableWebSearch = true
|
||||
|
||||
override fun onCreate(savedInstanceState: Bundle?) {
|
||||
super.onCreate(savedInstanceState)
|
||||
@@ -247,6 +248,23 @@ class CallActivity : BaseActivity() {
|
||||
}
|
||||
}
|
||||
|
||||
private fun buildRealtimeSessionOptions(): OmniRealtimeClient.SessionOptions {
|
||||
return OmniRealtimeClient.SessionOptions(voice, enableWebSearch)
|
||||
}
|
||||
|
||||
private fun isImageAppendError(message: String): Boolean {
|
||||
val lowerMessage = message.lowercase()
|
||||
return lowerMessage.contains("image") && (lowerMessage.contains("append") || lowerMessage.contains("buffer"))
|
||||
}
|
||||
|
||||
private fun toFriendlyErrorMessage(message: String): String {
|
||||
return if (isImageAppendError(message)) {
|
||||
getString(R.string.ai_call_camera_paused)
|
||||
} else {
|
||||
getString(R.string.ai_call_error_retry)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
override fun onDestroy() {
|
||||
super.onDestroy()
|
||||
@@ -318,21 +336,23 @@ class CallActivity : BaseActivity() {
|
||||
}
|
||||
applyMessage("AI对话: 连接中...")
|
||||
binding.tvSubtitle.visibility = View.VISIBLE
|
||||
binding.tvSubtitle.text = "正在连接..."
|
||||
binding.tvSubtitle.text = getString(R.string.ai_call_connecting)
|
||||
binding.audioSpectrum.visibility = View.VISIBLE
|
||||
binding.audioSpectrum.clear()
|
||||
assistantTextBuffer.clear()
|
||||
hasStartedPushForResponse = false
|
||||
binding.btnAIConversation.setImageResource(R.drawable.ic_mic_off)
|
||||
binding.btnAIConversation.setBackgroundResource(R.drawable.shape_fab_ai_active)
|
||||
|
||||
omniClient = OmniRealtimeClient(voice, object : OmniRealtimeClient.Callback {
|
||||
omniClient = OmniRealtimeClient(buildRealtimeSessionOptions(), object : OmniRealtimeClient.Callback {
|
||||
override fun onConnected() {
|
||||
mainHandler.post {
|
||||
binding.tvSubtitle.text = "已连接,请对着麦克风说话"
|
||||
binding.tvSubtitle.text = getString(R.string.ai_call_connected)
|
||||
binding.btnCameraToggle.visibility = View.VISIBLE
|
||||
binding.btnCameraToggle.setBackgroundResource(R.drawable.shape_fab_camera_off)
|
||||
binding.btnCameraToggle.contentDescription = getString(R.string.camera_off)
|
||||
isCameraEnabled = false
|
||||
applyMessage("AI对话: 已连接")
|
||||
applyMessage("AI对话: 已连接, model=${OmniRealtimeClient.DEFAULT_MODEL}, search=on")
|
||||
streamingRecorder = StreamingAudioRecorder(mContext, object : StreamingAudioRecorder.StreamingCallback {
|
||||
override fun onAudioData(pcmData: ByteArray) {
|
||||
omniClient?.appendAudio(pcmData)
|
||||
@@ -354,26 +374,32 @@ class CallActivity : BaseActivity() {
|
||||
|
||||
override fun onError(message: String) {
|
||||
mainHandler.post {
|
||||
Toast.makeText(mContext, "AI错误: $message", Toast.LENGTH_SHORT).show()
|
||||
binding.tvSubtitle.text = "错误: $message"
|
||||
if (isImageAppendError(message)) {
|
||||
stopCamera()
|
||||
}
|
||||
val friendlyMessage = toFriendlyErrorMessage(message)
|
||||
Toast.makeText(mContext, friendlyMessage, Toast.LENGTH_SHORT).show()
|
||||
binding.tvSubtitle.text = friendlyMessage
|
||||
applyMessage("AI错误: $message")
|
||||
}
|
||||
}
|
||||
|
||||
override fun onUserSpeechStarted() {
|
||||
cameraFrameCapture?.requestImmediateCapture()
|
||||
mainHandler.post {
|
||||
binding.tvSubtitle.text = "正在聆听..."
|
||||
binding.tvSubtitle.text = getString(R.string.ai_call_listening)
|
||||
}
|
||||
}
|
||||
|
||||
override fun onUserSpeechStopped() {
|
||||
mainHandler.post {
|
||||
binding.tvSubtitle.text = "思考中..."
|
||||
binding.tvSubtitle.text = getString(R.string.ai_call_thinking)
|
||||
}
|
||||
}
|
||||
|
||||
override fun onUserTranscript(transcript: String) {
|
||||
mainHandler.post {
|
||||
binding.tvSubtitle.text = "您: $transcript"
|
||||
binding.tvSubtitle.text = getString(R.string.ai_call_user_said, transcript)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -434,6 +460,7 @@ class CallActivity : BaseActivity() {
|
||||
omniClient = null
|
||||
duix?.stopAudio()
|
||||
assistantTextBuffer.clear()
|
||||
hasStartedPushForResponse = false
|
||||
|
||||
mainHandler.post {
|
||||
if (!isFinishing) {
|
||||
|
||||
@@ -23,10 +23,10 @@ class MainActivity : BaseActivity() {
|
||||
|
||||
private val avatarModels = listOf(
|
||||
AvatarModel("晨煦", "Ethan", "https://dl.xn--876a.net/models/bendi3_20240518.zip", R.mipmap.ethan),
|
||||
AvatarModel("詹妮弗", "Jennifer", "https://dl.xn--876a.net/models/Emma.zip", R.mipmap.jennifer),
|
||||
AvatarModel("詹妮弗", "Maia", "https://dl.xn--876a.net/models/Emma.zip", R.mipmap.jennifer),
|
||||
AvatarModel("月白", "Moon", "https://dl.xn--876a.net/models/Kai.zip", R.mipmap.moon),
|
||||
AvatarModel("艾登", "Aiden", "https://dl.xn--876a.net/models/Leo.zip", R.mipmap.leo),
|
||||
AvatarModel("四月", "Maia", "https://dl.xn--876a.net/models/Lily.zip", R.mipmap.lily),
|
||||
AvatarModel("四月", "Serena", "https://dl.xn--876a.net/models/Lily.zip", R.mipmap.lily),
|
||||
AvatarModel("奥利弗", "Andre", "https://dl.xn--876a.net/models/Oliver.zip", R.mipmap.oliver),
|
||||
AvatarModel("索菲亚", "Sonrisa", "https://dl.xn--876a.net/models/Sofia.zip", R.mipmap.sofia),
|
||||
)
|
||||
|
||||
@@ -73,14 +73,15 @@
|
||||
<TextView
|
||||
android:id="@+id/tvSubtitle"
|
||||
android:text=""
|
||||
android:textSize="16sp"
|
||||
android:textSize="18sp"
|
||||
android:textColor="@color/white"
|
||||
android:shadowColor="#1a1e1f"
|
||||
android:shadowRadius="4"
|
||||
android:lineSpacingExtra="4dp"
|
||||
android:gravity="center"
|
||||
android:visibility="gone"
|
||||
android:background="#99000000"
|
||||
android:padding="12dp"
|
||||
android:padding="14dp"
|
||||
android:layout_width="0dp"
|
||||
android:layout_height="wrap_content"
|
||||
app:layout_constraintBottom_toTopOf="@+id/audioSpectrum"
|
||||
|
||||
@@ -23,6 +23,13 @@
|
||||
<string name="ai_fab_stop">停</string>
|
||||
<string name="camera_on">镜头开</string>
|
||||
<string name="camera_off">镜头关</string>
|
||||
<string name="ai_call_connecting">正在为您接通,请稍候</string>
|
||||
<string name="ai_call_connected">已经接通,您可以慢慢说</string>
|
||||
<string name="ai_call_listening">我在认真听您说</string>
|
||||
<string name="ai_call_thinking">我想一想,马上回答您</string>
|
||||
<string name="ai_call_user_said">您刚才说:%1$s</string>
|
||||
<string name="ai_call_camera_paused">镜头先休息一下,您继续说话就可以</string>
|
||||
<string name="ai_call_error_retry">刚刚有点忙,我们稍后再试一次</string>
|
||||
<string name="main_download_tips">Github地址可能下载失败,您可以考虑使用代理或者将文件缓存到自己的存储服务</string>
|
||||
<string name="debug_message">调试信息</string>
|
||||
|
||||
|
||||
@@ -26,6 +26,13 @@
|
||||
<string name="ai_fab_stop">停</string>
|
||||
<string name="camera_on">镜头开</string>
|
||||
<string name="camera_off">镜头关</string>
|
||||
<string name="ai_call_connecting">Connecting, please wait a moment</string>
|
||||
<string name="ai_call_connected">We are connected, you can speak slowly</string>
|
||||
<string name="ai_call_listening">I am listening to you carefully</string>
|
||||
<string name="ai_call_thinking">Let me think for a moment</string>
|
||||
<string name="ai_call_user_said">You just said: %1$s</string>
|
||||
<string name="ai_call_camera_paused">The camera is paused for now, you can keep talking</string>
|
||||
<string name="ai_call_error_retry">Something was a little busy just now, please try again shortly</string>
|
||||
<string name="main_download_tips">The GitHub url may fail to download, you can consider using a proxy or caching the file to your own storage service</string>
|
||||
<string name="debug_message">Debug message</string>
|
||||
</resources>
|
||||
Reference in New Issue
Block a user