From 1ad70525c1e3564bd33fa5f85c532a33f3da2e8c Mon Sep 17 00:00:00 2001 From: feie9456 Date: Fri, 17 Apr 2026 15:08:40 +0800 Subject: [PATCH] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E5=AE=B6=E5=B1=9E=E7=95=99?= =?UTF-8?q?=E8=A8=80=E5=A4=84=E7=90=86=E5=8A=9F=E8=83=BD=EF=BC=8C=E6=94=AF?= =?UTF-8?q?=E6=8C=81=E4=B8=BB=E5=8A=A8=E6=92=AD=E6=8A=A5=E5=92=8C=E6=B6=88?= =?UTF-8?q?=E6=81=AF=E7=A1=AE=E8=AE=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../test/realtime/OmniRealtimeClient.java | 12 ++ .../duix/test/service/FamilyBridgeService.kt | 56 +++++++ .../duix/test/ui/activity/CallActivity.kt | 137 +++++++++++++++--- 3 files changed, 186 insertions(+), 19 deletions(-) diff --git a/test/src/main/java/ai/guiji/duix/test/realtime/OmniRealtimeClient.java b/test/src/main/java/ai/guiji/duix/test/realtime/OmniRealtimeClient.java index e75c3e0..41d567c 100644 --- a/test/src/main/java/ai/guiji/duix/test/realtime/OmniRealtimeClient.java +++ b/test/src/main/java/ai/guiji/duix/test/realtime/OmniRealtimeClient.java @@ -446,6 +446,18 @@ public class OmniRealtimeClient { } } + public boolean requestResponse() { + try { + JSONObject responseCreate = new JSONObject(); + responseCreate.put("event_id", nextEventId()); + responseCreate.put("type", "response.create"); + return sendEvent(responseCreate); + } catch (JSONException e) { + Log.e(TAG, "Failed to send response.create: " + e.getMessage()); + return false; + } + } + private void handleServerError(JSONObject event) { String errMsg = event.optJSONObject("error") != null ? event.optJSONObject("error").optString("message", "Unknown error") diff --git a/test/src/main/java/ai/guiji/duix/test/service/FamilyBridgeService.kt b/test/src/main/java/ai/guiji/duix/test/service/FamilyBridgeService.kt index 62c7b70..72d1bdb 100644 --- a/test/src/main/java/ai/guiji/duix/test/service/FamilyBridgeService.kt +++ b/test/src/main/java/ai/guiji/duix/test/service/FamilyBridgeService.kt @@ -59,6 +59,9 @@ class FamilyBridgeService(context: Context) { content = item.optString("content"), importance = item.optString("importance"), createdAt = item.optString("createdAt"), + status = item.optString("status"), + deliveredAt = item.optString("deliveredAt").ifBlank { null }, + readAt = item.optString("readAt").ifBlank { null }, ) ) } @@ -72,6 +75,9 @@ class FamilyBridgeService(context: Context) { } fun markFamilyMessagesRead(messageIds: List) { + if (messageIds.isEmpty()) { + return + } val payload = JSONObject().apply { put("deviceUuid", getDeviceUuid()) put("messageIds", JSONArray(messageIds)) @@ -79,6 +85,26 @@ class FamilyBridgeService(context: Context) { postJson("/api/device/messages/read", payload) } + fun prepareConversationPrompt(take: Int = 3): ConversationPromptInjection? { + val result = fetchFamilyMessages(take) + val unreadMessages = result.messages + .filter { it.readAt.isNullOrBlank() } + .take(take.coerceIn(1, 5)) + + if (unreadMessages.isEmpty()) { + return null + } + + return ConversationPromptInjection( + messageIds = unreadMessages.map { it.id }, + promptText = buildConversationPrompt(unreadMessages), + ) + } + + fun acknowledgeFamilyMessages(messageIds: List) { + markFamilyMessagesRead(messageIds) + } + fun recordUsageEvent(eventType: String, detailJson: JSONObject? = null) { val payload = JSONObject().apply { put("deviceUuid", getDeviceUuid()) @@ -176,6 +202,28 @@ class FamilyBridgeService(context: Context) { .ifBlank { "长辈陪伴终端" } } + private fun buildConversationPrompt(messages: List): String { + return buildString { + append("当前开始一轮新对话，家里人刚刚给老人留了新留言。") + append("请在连接建立后的第一轮回复里，主动、自然、简短地先把下面这些留言转告给老人，不要提系统提示、后台、注入、工具或接口。") + append("转告完后，再继续正常陪伴和后续对话。") + messages.forEachIndexed { index, message -> + append("\n") + append(index + 1) + append(". ") + val importanceLabel = when (message.importance.trim().uppercase()) { + "HIGH" -> "希望尽快看到：" + "URGENT" -> "需要尽快联系：" + "LOW" -> "温和提醒：" + else -> "日常问候：" + } + append(importanceLabel) + append(message.content) + } + append("\n如果老人回应这些留言，就顺着留言继续温和交流；如果老人转到别的话题，就继续正常陪伴。") + } + } + data class DeviceRegistrationResult( val deviceUuid: String, val bindUrl: String, @@ -186,6 +234,9 @@ class FamilyBridgeService(context: Context) { val content: String, val importance: String, val createdAt: String, + val status: String, + val deliveredAt: String?, + val readAt: String?, ) data class FamilyMessageFetchResult( @@ -194,6 +245,11 @@ class FamilyBridgeService(context: Context) { val messages: List, ) + data class ConversationPromptInjection( + val messageIds: List, + val promptText: String, + ) + data class ConversationTurnPayload( val role: String, val content: String, diff --git a/test/src/main/java/ai/guiji/duix/test/ui/activity/CallActivity.kt b/test/src/main/java/ai/guiji/duix/test/ui/activity/CallActivity.kt index 962ed40..840e8d5 100644 --- a/test/src/main/java/ai/guiji/duix/test/ui/activity/CallActivity.kt +++ b/test/src/main/java/ai/guiji/duix/test/ui/activity/CallActivity.kt @@ -89,6 +89,8 @@ class CallActivity : BaseActivity() { private val familyBridgeService by lazy { FamilyBridgeService(applicationContext) } private val bridgeExecutor: ExecutorService = Executors.newSingleThreadExecutor() private var currentSessionId: String? = null + private var pendingConversationPrompt: FamilyBridgeService.ConversationPromptInjection? = null + private var waitingForInitialFamilyAnnouncement = false override fun onCreate(savedInstanceState: Bundle?) { super.onCreate(savedInstanceState) @@ -271,7 +273,9 @@ class CallActivity : BaseActivity() { } } - private fun buildRealtimeSessionOptions(): OmniRealtimeClient.SessionOptions { + private fun buildRealtimeSessionOptions( + conversationPrompt: FamilyBridgeService.ConversationPromptInjection? = null, + ): OmniRealtimeClient.SessionOptions { val outputMode = if (isCloneMode) { OmniRealtimeClient.SessionOptions.OutputMode.TEXT_ONLY } else { @@ -281,14 +285,22 @@ class CallActivity : BaseActivity() { voice, enableWebSearch, outputMode, - buildRealtimeInstructions(), + buildRealtimeInstructions(conversationPrompt), buildRealtimeTools(), ) } - private fun buildRealtimeInstructions(): String { + private fun buildRealtimeInstructions( + conversationPrompt: FamilyBridgeService.ConversationPromptInjection? = null, + ): String { if (!enableToolCalling) { - return OmniRealtimeClient.getDefaultInstructions() + return buildString { + append(OmniRealtimeClient.getDefaultInstructions()) + if (conversationPrompt != null) { + append("\n\n") + append(conversationPrompt.promptText) + } + } } return buildString { append(OmniRealtimeClient.getDefaultInstructions()) @@ -296,6 +308,10 @@ class CallActivity : BaseActivity() { append("当用户明确要给儿子、女儿、老伴、家里人或亲友带话、留言、报平安时，必须调用 leave_message_for_family 工具，不要假装已经转达。\n") append("当用户问家里人有没有留言、想听家人说了什么、想看看新消息时，必须调用 fetch_family_messages 工具，不要自己编造留言。\n") append("工具执行结果返回后，再用温和、简短、生活化的中文向老人说明情况。") + if (conversationPrompt != null) { + append("\n\n") + append(conversationPrompt.promptText) + } } } @@ -443,30 +459,53 @@ class CallActivity : BaseActivity() { assistantTextBuffer.clear() hasStartedPushForResponse = false currentSessionId = UUID.randomUUID().toString() + pendingConversationPrompt = null + waitingForInitialFamilyAnnouncement = false binding.btnAIConversation.setImageResource(R.drawable.ic_mic_off) binding.btnAIConversation.setBackgroundResource(R.drawable.shape_fab_ai_active) - omniClient = OmniRealtimeClient(buildRealtimeSessionOptions(), object : OmniRealtimeClient.Callback { + isAIConversationActive = true + + enqueueBridgeTask("家属留言预取失败") { + val initialConversationPrompt = runCatching { + familyBridgeService.prepareConversationPrompt(3) + }.onFailure { error -> + applyMessage("家属留言预取失败: ${error.message}") + }.getOrNull() + + mainHandler.post { + if (!isAIConversationActive || isFinishing) { + return@post + } + + pendingConversationPrompt = initialConversationPrompt + waitingForInitialFamilyAnnouncement = initialConversationPrompt != null + + omniClient = OmniRealtimeClient( + buildRealtimeSessionOptions(initialConversationPrompt), + object : OmniRealtimeClient.Callback { override fun onConnected() { mainHandler.post { - binding.tvSubtitle.text = getString(R.string.ai_call_connected) + binding.tvSubtitle.text = if (initialConversationPrompt != null) { + "我先跟您说说家里人的留言。" + } else { + getString(R.string.ai_call_connected) + } binding.btnCameraToggle.visibility = if (isCloneMode) View.GONE else View.VISIBLE binding.btnCameraToggle.setBackgroundResource(R.drawable.shape_fab_camera_off) binding.btnCameraToggle.contentDescription = getString(R.string.camera_off) isCameraEnabled = false applyMessage("AI对话: 已连接, model=${OmniRealtimeClient.DEFAULT_MODEL}, search=${if (enableWebSearch) "on" else "off"}, tools=${if (enableToolCalling) "on" else "off"}, cloneMode=$isCloneMode") - streamingRecorder = StreamingAudioRecorder(mContext, object : StreamingAudioRecorder.StreamingCallback { - override fun onAudioData(pcmData: ByteArray) { - omniClient?.appendAudio(pcmData) - mainHandler.post { binding.audioSpectrum.updateAudio(pcmData) } + if (initialConversationPrompt != null) { + applyMessage("AI对话: 检测到家属留言，准备主动播报") + val requested = omniClient?.requestResponse() == true + if (!requested) { + applyMessage("AI对话: 主动播报触发失败，直接进入普通对话") + cancelInitialFamilyAnnouncementIfNeeded() } - override fun onError(message: String) { - mainHandler.post { - Toast.makeText(mContext, message, Toast.LENGTH_SHORT).show() - } - } - }) - streamingRecorder?.start() + } else { + startStreamingRecorderIfNeeded() + } } enqueueBridgeTask("AI会话启动记录失败") { familyBridgeService.registerCurrentDevice() @@ -488,6 +527,9 @@ class CallActivity : BaseActivity() { if (isImageAppendError(message)) { stopCamera() } + if (waitingForInitialFamilyAnnouncement) { + cancelInitialFamilyAnnouncementIfNeeded() + } val friendlyMessage = toFriendlyErrorMessage(message) Toast.makeText(mContext, friendlyMessage, Toast.LENGTH_SHORT).show() binding.tvSubtitle.text = friendlyMessage @@ -582,12 +624,16 @@ class CallActivity : BaseActivity() { hasStartedPushForResponse = false } assistantTextBuffer.clear() + if (!isCloneMode) { + completeInitialFamilyAnnouncementIfNeeded() + } } } }) - isAIConversationActive = true - omniClient?.connect() + omniClient?.connect() + } + } } private fun stopAIConversation() { @@ -603,6 +649,8 @@ class CallActivity : BaseActivity() { duix?.stopAudio() assistantTextBuffer.clear() hasStartedPushForResponse = false + pendingConversationPrompt = null + waitingForInitialFamilyAnnouncement = false mainHandler.post { if (!isFinishing) { @@ -675,6 +723,9 @@ class CallActivity : BaseActivity() { hasStartedPushForResponse = false } ttsRealtimeClient = null + mainHandler.post { + completeInitialFamilyAnnouncementIfNeeded() + } } override fun onError(message: String) { @@ -761,6 +812,54 @@ class CallActivity : BaseActivity() { } } + private fun startStreamingRecorderIfNeeded() { + if (!isAIConversationActive || streamingRecorder != null) { + return + } + + streamingRecorder = StreamingAudioRecorder(mContext, object : StreamingAudioRecorder.StreamingCallback { + override fun onAudioData(pcmData: ByteArray) { + omniClient?.appendAudio(pcmData) + mainHandler.post { binding.audioSpectrum.updateAudio(pcmData) } + } + + override fun onError(message: String) { + mainHandler.post { + Toast.makeText(mContext, message, Toast.LENGTH_SHORT).show() + } + } + }) + streamingRecorder?.start() + } + + private fun completeInitialFamilyAnnouncementIfNeeded() { + if (!waitingForInitialFamilyAnnouncement) { + return + } + + val prompt = pendingConversationPrompt + pendingConversationPrompt = null + waitingForInitialFamilyAnnouncement = false + + if (prompt != null && prompt.messageIds.isNotEmpty()) { + enqueueBridgeTask("家属留言已读回写失败") { + familyBridgeService.acknowledgeFamilyMessages(prompt.messageIds) + } + } + + startStreamingRecorderIfNeeded() + } + + private fun cancelInitialFamilyAnnouncementIfNeeded() { + if (!waitingForInitialFamilyAnnouncement) { + return + } + + pendingConversationPrompt = null + waitingForInitialFamilyAnnouncement = false + startStreamingRecorderIfNeeded() + } + private fun uploadConversationTurnAsync(role: String, content: String, metaJson: JSONObject? = null) { enqueueBridgeTask("对话记录上传失败") { uploadConversationTurnInternal(role, content, metaJson)