feie9454/digital-human
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

添加家属留言处理功能，支持主动播报和消息确认

Browse Source
This commit is contained in:
feie9454
2026-04-17 15:08:40 +08:00
parent b177a7ae77
commit 1ad70525c1
3 changed files with 186 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/src/main/java/ai/guiji/duix/test/realtime/OmniRealtimeClient.java
+12
View File
@@ -446,6 +446,18 @@ public class OmniRealtimeClient {
}
}
public boolean requestResponse() {
try {
JSONObject responseCreate = new JSONObject();
responseCreate.put("event_id", nextEventId());
responseCreate.put("type", "response.create");
return sendEvent(responseCreate);
} catch (JSONException e) {
Log.e(TAG, "Failed to send response.create: " + e.getMessage());
return false;
}
}
private void handleServerError(JSONObject event) {
String errMsg = event.optJSONObject("error") != null
? event.optJSONObject("error").optString("message", "Unknown error")
test/src/main/java/ai/guiji/duix/test/service/FamilyBridgeService.kt
+56
View File
@@ -59,6 +59,9 @@ class FamilyBridgeService(context: Context) {
content = item.optString("content"),
importance = item.optString("importance"),
createdAt = item.optString("createdAt"),
status = item.optString("status"),
deliveredAt = item.optString("deliveredAt").ifBlank { null },
readAt = item.optString("readAt").ifBlank { null },
)
)
}
@@ -72,6 +75,9 @@ class FamilyBridgeService(context: Context) {
}
fun markFamilyMessagesRead(messageIds: List<String>) {
if (messageIds.isEmpty()) {
return
}
val payload = JSONObject().apply {
put("deviceUuid", getDeviceUuid())
put("messageIds", JSONArray(messageIds))
@@ -79,6 +85,26 @@ class FamilyBridgeService(context: Context) {
postJson("/api/device/messages/read", payload)
}
fun prepareConversationPrompt(take: Int = 3): ConversationPromptInjection? {
val result = fetchFamilyMessages(take)
val unreadMessages = result.messages
.filter { it.readAt.isNullOrBlank() }
.take(take.coerceIn(1, 5))
if (unreadMessages.isEmpty()) {
return null
}
return ConversationPromptInjection(
messageIds = unreadMessages.map { it.id },
promptText = buildConversationPrompt(unreadMessages),
)
}
fun acknowledgeFamilyMessages(messageIds: List<String>) {
markFamilyMessagesRead(messageIds)
}
fun recordUsageEvent(eventType: String, detailJson: JSONObject? = null) {
val payload = JSONObject().apply {
put("deviceUuid", getDeviceUuid())
@@ -176,6 +202,28 @@ class FamilyBridgeService(context: Context) {
.ifBlank { "长辈陪伴终端" }
}
private fun buildConversationPrompt(messages: List<FamilyMessageItem>): String {
return buildString {
append("当前开始一轮新对话，家里人刚刚给老人留了新留言。")
append("请在连接建立后的第一轮回复里，主动、自然、简短地先把下面这些留言转告给老人，不要提系统提示、后台、注入、工具或接口。")
append("转告完后，再继续正常陪伴和后续对话。")
messages.forEachIndexed { index, message ->
append("\n")
append(index + 1)
append(". ")
val importanceLabel = when (message.importance.trim().uppercase()) {
"HIGH" -> "希望尽快看到："
"URGENT" -> "需要尽快联系："
"LOW" -> "温和提醒："
else -> "日常问候："
}
append(importanceLabel)
append(message.content)
}
append("\n如果老人回应这些留言，就顺着留言继续温和交流；如果老人转到别的话题，就继续正常陪伴。")
}
}
data class DeviceRegistrationResult(
val deviceUuid: String,
val bindUrl: String,
@@ -186,6 +234,9 @@ class FamilyBridgeService(context: Context) {
val content: String,
val importance: String,
val createdAt: String,
val status: String,
val deliveredAt: String?,
val readAt: String?,
)
data class FamilyMessageFetchResult(
@@ -194,6 +245,11 @@ class FamilyBridgeService(context: Context) {
val messages: List<FamilyMessageItem>,
)
data class ConversationPromptInjection(
val messageIds: List<String>,
val promptText: String,
)
data class ConversationTurnPayload(
val role: String,
val content: String,
test/src/main/java/ai/guiji/duix/test/ui/activity/CallActivity.kt
+118 -19
View File
@@ -89,6 +89,8 @@ class CallActivity : BaseActivity() {
private val familyBridgeService by lazy { FamilyBridgeService(applicationContext) }
private val bridgeExecutor: ExecutorService = Executors.newSingleThreadExecutor()
private var currentSessionId: String? = null
private var pendingConversationPrompt: FamilyBridgeService.ConversationPromptInjection? = null
private var waitingForInitialFamilyAnnouncement = false
override fun onCreate(savedInstanceState: Bundle?) {
super.onCreate(savedInstanceState)
@@ -271,7 +273,9 @@ class CallActivity : BaseActivity() {
}
}
private fun buildRealtimeSessionOptions(): OmniRealtimeClient.SessionOptions {
private fun buildRealtimeSessionOptions(
conversationPrompt: FamilyBridgeService.ConversationPromptInjection? = null,
): OmniRealtimeClient.SessionOptions {
val outputMode = if (isCloneMode) {
OmniRealtimeClient.SessionOptions.OutputMode.TEXT_ONLY
} else {
@@ -281,14 +285,22 @@ class CallActivity : BaseActivity() {
voice,
enableWebSearch,
outputMode,
buildRealtimeInstructions(),
buildRealtimeInstructions(conversationPrompt),
buildRealtimeTools(),
)
}
private fun buildRealtimeInstructions(): String {
private fun buildRealtimeInstructions(
conversationPrompt: FamilyBridgeService.ConversationPromptInjection? = null,
): String {
if (!enableToolCalling) {
return OmniRealtimeClient.getDefaultInstructions()
return buildString {
append(OmniRealtimeClient.getDefaultInstructions())
if (conversationPrompt != null) {
append("\n\n")
append(conversationPrompt.promptText)
}
}
}
return buildString {
append(OmniRealtimeClient.getDefaultInstructions())
@@ -296,6 +308,10 @@ class CallActivity : BaseActivity() {
append("当用户明确要给儿子、女儿、老伴、家里人或亲友带话、留言、报平安时，必须调用 leave_message_for_family 工具，不要假装已经转达。\n")
append("当用户问家里人有没有留言、想听家人说了什么、想看看新消息时，必须调用 fetch_family_messages 工具，不要自己编造留言。\n")
append("工具执行结果返回后，再用温和、简短、生活化的中文向老人说明情况。")
if (conversationPrompt != null) {
append("\n\n")
append(conversationPrompt.promptText)
}
}
}
@@ -443,30 +459,53 @@ class CallActivity : BaseActivity() {
assistantTextBuffer.clear()
hasStartedPushForResponse = false
currentSessionId = UUID.randomUUID().toString()
pendingConversationPrompt = null
waitingForInitialFamilyAnnouncement = false
binding.btnAIConversation.setImageResource(R.drawable.ic_mic_off)
binding.btnAIConversation.setBackgroundResource(R.drawable.shape_fab_ai_active)
omniClient = OmniRealtimeClient(buildRealtimeSessionOptions(), object : OmniRealtimeClient.Callback {
isAIConversationActive = true
enqueueBridgeTask("家属留言预取失败") {
val initialConversationPrompt = runCatching {
familyBridgeService.prepareConversationPrompt(3)
}.onFailure { error ->
applyMessage("家属留言预取失败: ${error.message}")
}.getOrNull()
mainHandler.post {
if (!isAIConversationActive || isFinishing) {
return@post
}
pendingConversationPrompt = initialConversationPrompt
waitingForInitialFamilyAnnouncement = initialConversationPrompt != null
omniClient = OmniRealtimeClient(
buildRealtimeSessionOptions(initialConversationPrompt),
object : OmniRealtimeClient.Callback {
override fun onConnected() {
mainHandler.post {
binding.tvSubtitle.text = getString(R.string.ai_call_connected)
binding.tvSubtitle.text = if (initialConversationPrompt != null) {
"我先跟您说说家里人的留言。"
} else {
getString(R.string.ai_call_connected)
}
binding.btnCameraToggle.visibility = if (isCloneMode) View.GONE else View.VISIBLE
binding.btnCameraToggle.setBackgroundResource(R.drawable.shape_fab_camera_off)
binding.btnCameraToggle.contentDescription = getString(R.string.camera_off)
isCameraEnabled = false
applyMessage("AI对话: 已连接, model=${OmniRealtimeClient.DEFAULT_MODEL}, search=${if (enableWebSearch) "on" else "off"}, tools=${if (enableToolCalling) "on" else "off"}, cloneMode=$isCloneMode")
streamingRecorder = StreamingAudioRecorder(mContext, object : StreamingAudioRecorder.StreamingCallback {
override fun onAudioData(pcmData: ByteArray) {
omniClient?.appendAudio(pcmData)
mainHandler.post { binding.audioSpectrum.updateAudio(pcmData) }
if (initialConversationPrompt != null) {
applyMessage("AI对话: 检测到家属留言，准备主动播报")
val requested = omniClient?.requestResponse() == true
if (!requested) {
applyMessage("AI对话: 主动播报触发失败，直接进入普通对话")
cancelInitialFamilyAnnouncementIfNeeded()
}
override fun onError(message: String) {
mainHandler.post {
Toast.makeText(mContext, message, Toast.LENGTH_SHORT).show()
}
}
})
streamingRecorder?.start()
} else {
startStreamingRecorderIfNeeded()
}
}
enqueueBridgeTask("AI会话启动记录失败") {
familyBridgeService.registerCurrentDevice()
@@ -488,6 +527,9 @@ class CallActivity : BaseActivity() {
if (isImageAppendError(message)) {
stopCamera()
}
if (waitingForInitialFamilyAnnouncement) {
cancelInitialFamilyAnnouncementIfNeeded()
}
val friendlyMessage = toFriendlyErrorMessage(message)
Toast.makeText(mContext, friendlyMessage, Toast.LENGTH_SHORT).show()
binding.tvSubtitle.text = friendlyMessage
@@ -582,12 +624,16 @@ class CallActivity : BaseActivity() {
hasStartedPushForResponse = false
}
assistantTextBuffer.clear()
if (!isCloneMode) {
completeInitialFamilyAnnouncementIfNeeded()
}
}
}
})
isAIConversationActive = true
omniClient?.connect()
omniClient?.connect()
}
}
}
private fun stopAIConversation() {
@@ -603,6 +649,8 @@ class CallActivity : BaseActivity() {
duix?.stopAudio()
assistantTextBuffer.clear()
hasStartedPushForResponse = false
pendingConversationPrompt = null
waitingForInitialFamilyAnnouncement = false
mainHandler.post {
if (!isFinishing) {
@@ -675,6 +723,9 @@ class CallActivity : BaseActivity() {
hasStartedPushForResponse = false
}
ttsRealtimeClient = null
mainHandler.post {
completeInitialFamilyAnnouncementIfNeeded()
}
}
override fun onError(message: String) {
@@ -761,6 +812,54 @@ class CallActivity : BaseActivity() {
}
}
private fun startStreamingRecorderIfNeeded() {
if (!isAIConversationActive || streamingRecorder != null) {
return
}
streamingRecorder = StreamingAudioRecorder(mContext, object : StreamingAudioRecorder.StreamingCallback {
override fun onAudioData(pcmData: ByteArray) {
omniClient?.appendAudio(pcmData)
mainHandler.post { binding.audioSpectrum.updateAudio(pcmData) }
}
override fun onError(message: String) {
mainHandler.post {
Toast.makeText(mContext, message, Toast.LENGTH_SHORT).show()
}
}
})
streamingRecorder?.start()
}
private fun completeInitialFamilyAnnouncementIfNeeded() {
if (!waitingForInitialFamilyAnnouncement) {
return
}
val prompt = pendingConversationPrompt
pendingConversationPrompt = null
waitingForInitialFamilyAnnouncement = false
if (prompt != null && prompt.messageIds.isNotEmpty()) {
enqueueBridgeTask("家属留言已读回写失败") {
familyBridgeService.acknowledgeFamilyMessages(prompt.messageIds)
}
}
startStreamingRecorderIfNeeded()
}
private fun cancelInitialFamilyAnnouncementIfNeeded() {
if (!waitingForInitialFamilyAnnouncement) {
return
}
pendingConversationPrompt = null
waitingForInitialFamilyAnnouncement = false
startStreamingRecorderIfNeeded()
}
private fun uploadConversationTurnAsync(role: String, content: String, metaJson: JSONObject? = null) {
enqueueBridgeTask("对话记录上传失败") {
uploadConversationTurnInternal(role, content, metaJson)