diff --git a/test/build.gradle b/test/build.gradle index d54fb16..7c3a209 100644 --- a/test/build.gradle +++ b/test/build.gradle @@ -83,6 +83,7 @@ dependencies { implementation 'androidx.constraintlayout:constraintlayout:2.1.1' implementation "androidx.activity:activity:1.3.0" implementation "androidx.fragment:fragment:1.3.0" + implementation 'androidx.documentfile:documentfile:1.0.1' implementation 'com.github.bumptech.glide:glide:4.12.0' diff --git a/test/src/main/AndroidManifest.xml b/test/src/main/AndroidManifest.xml index 44501cf..4218c97 100644 --- a/test/src/main/AndroidManifest.xml +++ b/test/src/main/AndroidManifest.xml @@ -4,6 +4,9 @@ + + + @@ -38,6 +41,18 @@ android:theme="@style/Theme.DUIX.Call"> + + + + + + + extractor.setDataSource(descriptor.fileDescriptor, descriptor.startOffset, descriptor.length) + } ?: throw IllegalArgumentException("无法打开所选音频") + } + + selectedTrack = findFirstAudioTrack(extractor) + if (selectedTrack < 0) { + throw IllegalArgumentException("没有找到可读取的音频轨道") + } + + extractor.selectTrack(selectedTrack) + val inputFormat = extractor.getTrackFormat(selectedTrack) + val mime = inputFormat.getString(MediaFormat.KEY_MIME) + ?: throw IllegalArgumentException("无法识别音频格式") + sampleRate = inputFormat.optInt(MediaFormat.KEY_SAMPLE_RATE, TARGET_SAMPLE_RATE) + channelCount = inputFormat.optInt(MediaFormat.KEY_CHANNEL_COUNT, 1) + + decoder = MediaCodec.createDecoderByType(mime) + decoder.configure(inputFormat, null, null, 0) + decoder.start() + + val bufferInfo = MediaCodec.BufferInfo() + var inputDone = false + var outputDone = false + + while (!outputDone) { + if (!inputDone) { + val inputIndex = decoder.dequeueInputBuffer(TIMEOUT_US) + if (inputIndex >= 0) { + val inputBuffer = decoder.getInputBuffer(inputIndex) + ?: throw IllegalStateException("解码器输入缓冲区不可用") + val sampleSize = extractor.readSampleData(inputBuffer, 0) + if (sampleSize < 0) { + decoder.queueInputBuffer( + inputIndex, + 0, + 0, + 0L, + MediaCodec.BUFFER_FLAG_END_OF_STREAM, + ) + inputDone = true + } else { + decoder.queueInputBuffer( + inputIndex, + 0, + sampleSize, + extractor.sampleTime.coerceAtLeast(0L), + 0, + ) + extractor.advance() + } + } + } + + when (val outputIndex = decoder.dequeueOutputBuffer(bufferInfo, TIMEOUT_US)) { + MediaCodec.INFO_TRY_AGAIN_LATER -> Unit + MediaCodec.INFO_OUTPUT_FORMAT_CHANGED -> { + val outputFormat = decoder.outputFormat + sampleRate = outputFormat.optInt(MediaFormat.KEY_SAMPLE_RATE, sampleRate) + channelCount = outputFormat.optInt(MediaFormat.KEY_CHANNEL_COUNT, channelCount) + val pcmEncoding = outputFormat.optInt( + MediaFormat.KEY_PCM_ENCODING, + AudioFormat.ENCODING_PCM_16BIT, + ) + if (pcmEncoding != AudioFormat.ENCODING_PCM_16BIT) { + throw IllegalArgumentException("暂不支持该音频 PCM 编码") + } + } + else -> { + if (outputIndex >= 0) { + val outputBuffer = decoder.getOutputBuffer(outputIndex) + ?: throw IllegalStateException("解码器输出缓冲区不可用") + if (bufferInfo.size > 0) { + val chunk = ByteArray(bufferInfo.size) + outputBuffer.position(bufferInfo.offset) + outputBuffer.limit(bufferInfo.offset + bufferInfo.size) + outputBuffer.get(chunk) + rawPcmOutput.write(chunk) + } + val endOfStream = (bufferInfo.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0 + decoder.releaseOutputBuffer(outputIndex, false) + outputDone = endOfStream + } + } + } + } + + val converted = convertPcm16ToMono16k(rawPcmOutput.toByteArray(), sampleRate, channelCount) + if (converted.isEmpty()) { + throw IllegalArgumentException("音频解码后没有可用数据") + } + val durationMs = ((converted.size / 2L) * 1000L / TARGET_SAMPLE_RATE).coerceAtLeast(0L) + return DecodedAudio( + pcm16Mono16k = converted, + durationMs = durationMs, + ) + } finally { + runCatching { decoder?.stop() } + runCatching { decoder?.release() } + runCatching { if (selectedTrack >= 0) extractor.unselectTrack(selectedTrack) } + runCatching { extractor.release() } + runCatching { rawPcmOutput.close() } + } + } + + private fun findFirstAudioTrack(extractor: MediaExtractor): Int { + for (index in 0 until extractor.trackCount) { + val mime = extractor.getTrackFormat(index).getString(MediaFormat.KEY_MIME) ?: continue + if (mime.startsWith("audio/")) { + return index + } + } + return -1 + } + + private fun convertPcm16ToMono16k(input: ByteArray, inputSampleRate: Int, inputChannels: Int): ByteArray { + if (input.isEmpty()) { + return ByteArray(0) + } + if (inputChannels <= 0) { + throw IllegalArgumentException("无效的声道数") + } + + val frameCount = input.size / (2 * inputChannels) + if (frameCount <= 0) { + return ByteArray(0) + } + + val monoSamples = FloatArray(frameCount) + var byteOffset = 0 + for (frameIndex in 0 until frameCount) { + var sum = 0 + for (channelIndex in 0 until inputChannels) { + sum += readShortLE(input, byteOffset).toInt() + byteOffset += 2 + } + monoSamples[frameIndex] = sum.toFloat() / inputChannels.toFloat() + } + + if (inputSampleRate == TARGET_SAMPLE_RATE) { + val output = ByteArray(frameCount * 2) + for (index in monoSamples.indices) { + writeShortLE(output, index * 2, monoSamples[index].roundToInt().toShort()) + } + return output + } + + val outputFrameCount = ceil(frameCount.toDouble() * TARGET_SAMPLE_RATE / inputSampleRate).toInt() + val output = ByteArray(outputFrameCount * 2) + val positionScale = inputSampleRate.toDouble() / TARGET_SAMPLE_RATE.toDouble() + + for (outputIndex in 0 until outputFrameCount) { + val srcPosition = outputIndex * positionScale + val srcIndex0 = floor(srcPosition).toInt().coerceIn(0, monoSamples.lastIndex) + val srcIndex1 = (srcIndex0 + 1).coerceAtMost(monoSamples.lastIndex) + val fraction = srcPosition - srcIndex0 + val interpolated = monoSamples[srcIndex0] + + ((monoSamples[srcIndex1] - monoSamples[srcIndex0]) * fraction.toFloat()) + val clamped = interpolated.roundToInt().coerceIn(Short.MIN_VALUE.toInt(), Short.MAX_VALUE.toInt()) + writeShortLE(output, outputIndex * 2, clamped.toShort()) + } + return output + } + + private fun readShortLE(data: ByteArray, offset: Int): Short { + val low = data[offset].toInt() and 0xFF + val high = data[offset + 1].toInt() shl 8 + return (low or high).toShort() + } + + private fun writeShortLE(data: ByteArray, offset: Int, value: Short) { + data[offset] = (value.toInt() and 0xFF).toByte() + data[offset + 1] = ((value.toInt() ushr 8) and 0xFF).toByte() + } + + private fun MediaFormat.optInt(key: String, fallback: Int): Int { + return runCatching { + if (containsKey(key)) getInteger(key) else fallback + }.getOrDefault(fallback) + } + + private const val TIMEOUT_US = 10_000L +} \ No newline at end of file diff --git a/test/src/main/java/ai/guiji/duix/test/model/BatchAudioJob.kt b/test/src/main/java/ai/guiji/duix/test/model/BatchAudioJob.kt new file mode 100644 index 0000000..64d1872 --- /dev/null +++ b/test/src/main/java/ai/guiji/duix/test/model/BatchAudioJob.kt @@ -0,0 +1,23 @@ +package ai.guiji.duix.test.model + +import android.net.Uri + +data class BatchAudioJob( + val sourceUri: Uri, + val sourceRelativePath: String, + val displayName: String, + val outputRelativePath: String, + val attemptCount: Int = 0, + val status: Status = Status.PENDING, + val statusText: String = "", + val detailText: String = "", + val outputUri: Uri? = null, +) { + enum class Status { + PENDING, + RUNNING, + COMPLETED, + FAILED, + SKIPPED, + } +} \ No newline at end of file diff --git a/test/src/main/java/ai/guiji/duix/test/realtime/OmniRealtimeClient.java b/test/src/main/java/ai/guiji/duix/test/realtime/OmniRealtimeClient.java index 2939c02..7dea69c 100644 --- a/test/src/main/java/ai/guiji/duix/test/realtime/OmniRealtimeClient.java +++ b/test/src/main/java/ai/guiji/duix/test/realtime/OmniRealtimeClient.java @@ -446,6 +446,22 @@ public class OmniRealtimeClient { } } + public boolean commitInputAudio() { + try { + JSONObject event = new JSONObject(); + event.put("event_id", nextEventId()); + event.put("type", "input_audio_buffer.commit"); + boolean sent = sendEvent(event); + if (sent) { + imageWindowOpen.set(false); + } + return sent; + } catch (JSONException e) { + Log.e(TAG, "Failed to send input_audio_buffer.commit: " + e.getMessage()); + return false; + } + } + private void handleServerError(JSONObject event) { String errMsg = event.optJSONObject("error") != null ? event.optJSONObject("error").optString("message", "Unknown error") diff --git a/test/src/main/java/ai/guiji/duix/test/service/DocumentTreeAudioStore.kt b/test/src/main/java/ai/guiji/duix/test/service/DocumentTreeAudioStore.kt new file mode 100644 index 0000000..0b2e4df --- /dev/null +++ b/test/src/main/java/ai/guiji/duix/test/service/DocumentTreeAudioStore.kt @@ -0,0 +1,138 @@ +package ai.guiji.duix.test.service + +import android.content.Context +import android.net.Uri +import androidx.documentfile.provider.DocumentFile +import java.io.File +import java.util.Locale + +object DocumentTreeAudioStore { + + data class AudioEntry( + val uri: Uri, + val displayName: String, + val relativePath: String, + ) + + data class ScanResult( + val rootName: String, + val files: List, + ) + + fun scanAudioFiles(context: Context, treeUri: Uri): ScanResult { + val root = DocumentFile.fromTreeUri(context, treeUri) + ?: throw IllegalArgumentException("无法读取所选文件夹") + if (!root.isDirectory) { + throw IllegalArgumentException("所选内容不是文件夹") + } + + val files = mutableListOf() + traverse(root, "", files) + return ScanResult( + rootName = root.name ?: "selected-folder", + files = files.sortedBy { it.relativePath.lowercase(Locale.US) }, + ) + } + + fun writeVideoResult( + context: Context, + treeUri: Uri, + outputRootName: String, + sourceRelativePath: String, + localFile: File, + ): Uri { + val root = DocumentFile.fromTreeUri(context, treeUri) + ?: throw IllegalArgumentException("无法写入所选文件夹") + if (!root.isDirectory) { + throw IllegalArgumentException("所选内容不是文件夹") + } + + var currentDir = findOrCreateDirectory(root, outputRootName) + val normalizedPath = sourceRelativePath.replace('\\', '/') + val pathSegments = normalizedPath.split('/').filter { it.isNotBlank() } + if (pathSegments.isEmpty()) { + throw IllegalArgumentException("无效的源文件路径") + } + + for (segment in pathSegments.dropLast(1)) { + currentDir = findOrCreateDirectory(currentDir, segment) + } + + val outputFileName = replaceExtension(pathSegments.last(), "mp4") + currentDir.findFile(outputFileName)?.delete() + val outputDocument = currentDir.createFile(VIDEO_MIME_TYPE, outputFileName) + ?: throw IllegalStateException("无法创建输出文件") + + localFile.inputStream().use { input -> + context.contentResolver.openOutputStream(outputDocument.uri, "w")?.use { output -> + input.copyTo(output) + } ?: throw IllegalStateException("无法打开输出文件") + } + return outputDocument.uri + } + + fun buildOutputRelativePath(outputRootName: String, sourceRelativePath: String): String { + val normalizedPath = sourceRelativePath.replace('\\', '/') + return outputRootName + "/" + replaceExtension(normalizedPath, "mp4") + } + + private fun traverse(directory: DocumentFile, parentPath: String, output: MutableList) { + val children = directory.listFiles().sortedBy { it.name?.lowercase(Locale.US) ?: "" } + for (child in children) { + val childName = child.name ?: continue + val childRelativePath = if (parentPath.isEmpty()) childName else "$parentPath/$childName" + when { + child.isDirectory -> traverse(child, childRelativePath, output) + child.isFile && isAudioFile(child) -> { + output += AudioEntry( + uri = child.uri, + displayName = childName, + relativePath = childRelativePath, + ) + } + } + } + } + + private fun isAudioFile(document: DocumentFile): Boolean { + val mimeType = document.type?.lowercase(Locale.US).orEmpty() + if (mimeType.startsWith("audio/")) { + return true + } + val name = document.name?.lowercase(Locale.US).orEmpty() + return AUDIO_EXTENSIONS.any { extension -> name.endsWith(".$extension") } + } + + private fun findOrCreateDirectory(parent: DocumentFile, name: String): DocumentFile { + parent.findFile(name)?.let { existing -> + if (existing.isDirectory) { + return existing + } + existing.delete() + } + return parent.createDirectory(name) + ?: throw IllegalStateException("无法创建目录: $name") + } + + private fun replaceExtension(name: String, extension: String): String { + val dotIndex = name.lastIndexOf('.') + val baseName = if (dotIndex > 0) name.substring(0, dotIndex) else name + return "$baseName.$extension" + } + + private val AUDIO_EXTENSIONS = setOf( + "aac", + "amr", + "flac", + "m4a", + "mp3", + "ogg", + "opus", + "wav", + "webm", + "wma", + "3gp", + ) + + private const val VIDEO_MIME_TYPE = "video/mp4" +} \ No newline at end of file diff --git a/test/src/main/java/ai/guiji/duix/test/service/LocalFileAudioStore.kt b/test/src/main/java/ai/guiji/duix/test/service/LocalFileAudioStore.kt new file mode 100644 index 0000000..a77f496 --- /dev/null +++ b/test/src/main/java/ai/guiji/duix/test/service/LocalFileAudioStore.kt @@ -0,0 +1,130 @@ +package ai.guiji.duix.test.service + +import android.net.Uri +import java.io.File +import java.util.Locale + +object LocalFileAudioStore { + + data class AudioEntry( + val uri: Uri, + val displayName: String, + val relativePath: String, + ) + + data class ScanResult( + val rootDir: File, + val rootName: String, + val files: List, + ) + + data class WriteResult( + val file: File, + val skipped: Boolean, + ) + + fun scanAudioFiles(rootDir: File): ScanResult { + val normalizedRoot = rootDir.canonicalFile + if (!normalizedRoot.exists() || !normalizedRoot.isDirectory) { + throw IllegalArgumentException("无法读取所选文件夹") + } + + val files = mutableListOf() + traverse(normalizedRoot, "", files) + return ScanResult( + rootDir = normalizedRoot, + rootName = normalizedRoot.name.ifBlank { normalizedRoot.absolutePath }, + files = files.sortedBy { it.relativePath.lowercase(Locale.US) }, + ) + } + + fun writeVideoResult( + rootDir: File, + outputRootName: String, + sourceRelativePath: String, + localFile: File, + ): WriteResult { + val normalizedRoot = rootDir.canonicalFile + if (!normalizedRoot.exists() || !normalizedRoot.isDirectory) { + throw IllegalArgumentException("无法写入所选文件夹") + } + + val outputFile = resolveOutputFile(normalizedRoot, outputRootName, sourceRelativePath) + if (outputFile.exists()) { + return WriteResult(outputFile, skipped = true) + } + outputFile.parentFile?.let { parent -> + if (!parent.exists() && !parent.mkdirs()) { + throw IllegalStateException("无法创建输出目录") + } + } + return runCatching { + localFile.copyTo(outputFile, overwrite = false) + WriteResult(outputFile, skipped = false) + }.getOrElse { error -> + if (outputFile.exists()) { + WriteResult(outputFile, skipped = true) + } else { + throw error + } + } + } + + fun resolveOutputFile( + rootDir: File, + outputRootName: String, + sourceRelativePath: String, + ): File { + val normalizedRoot = rootDir.canonicalFile + return File(normalizedRoot, buildOutputRelativePath(outputRootName, sourceRelativePath)) + } + + fun buildOutputRelativePath(outputRootName: String, sourceRelativePath: String): String { + val normalizedPath = sourceRelativePath.replace('\\', '/') + return outputRootName + "/" + replaceExtension(normalizedPath, "mp4") + } + + private fun traverse(directory: File, parentPath: String, output: MutableList) { + val children = directory.listFiles() + ?.sortedBy { it.name.lowercase(Locale.US) } + .orEmpty() + for (child in children) { + val childRelativePath = if (parentPath.isEmpty()) child.name else "$parentPath/${child.name}" + when { + child.isDirectory -> traverse(child, childRelativePath, output) + child.isFile && isAudioFile(child) -> { + output += AudioEntry( + uri = Uri.fromFile(child), + displayName = child.name, + relativePath = childRelativePath, + ) + } + } + } + } + + private fun isAudioFile(file: File): Boolean { + val extension = file.extension.lowercase(Locale.US) + return extension in AUDIO_EXTENSIONS + } + + private fun replaceExtension(name: String, extension: String): String { + val dotIndex = name.lastIndexOf('.') + val baseName = if (dotIndex > 0) name.substring(0, dotIndex) else name + return "$baseName.$extension" + } + + private val AUDIO_EXTENSIONS = setOf( + "aac", + "amr", + "flac", + "m4a", + "mp3", + "ogg", + "opus", + "wav", + "webm", + "wma", + "3gp", + ) +} \ No newline at end of file diff --git a/test/src/main/java/ai/guiji/duix/test/ui/activity/DigitalHumanTestActivity.kt b/test/src/main/java/ai/guiji/duix/test/ui/activity/DigitalHumanTestActivity.kt new file mode 100644 index 0000000..11a29a3 --- /dev/null +++ b/test/src/main/java/ai/guiji/duix/test/ui/activity/DigitalHumanTestActivity.kt @@ -0,0 +1,1287 @@ +package ai.guiji.duix.test.ui.activity + +import ai.guiji.duix.sdk.client.Constant +import ai.guiji.duix.sdk.client.DUIX +import ai.guiji.duix.sdk.client.VirtualModelUtil +import ai.guiji.duix.sdk.client.bean.ImageFrame +import ai.guiji.duix.sdk.client.render.DUIXRenderer +import ai.guiji.duix.sdk.client.render.RenderSink +import ai.guiji.duix.test.R +import ai.guiji.duix.test.audio.AudioFileDecoder +import ai.guiji.duix.test.audio.AudioResampler +import ai.guiji.duix.test.databinding.ActivityDigitalHumanTestBinding +import ai.guiji.duix.test.model.AvatarCatalog +import ai.guiji.duix.test.model.AvatarModel +import ai.guiji.duix.test.model.BatchAudioJob +import ai.guiji.duix.test.realtime.OmniRealtimeClient +import ai.guiji.duix.test.service.DashScopeConfig +import ai.guiji.duix.test.service.LocalFileAudioStore +import ai.guiji.duix.test.service.VoiceCloneProfileStore +import ai.guiji.duix.test.ui.adapter.AvatarAdapter +import ai.guiji.duix.test.ui.adapter.DigitalHumanBatchJobAdapter +import ai.guiji.duix.test.ui.dialog.LoadingDialog +import ai.guiji.duix.test.video.DigitalHumanMp4Exporter +import android.app.Activity +import android.content.pm.PackageManager +import android.content.Intent +import android.net.Uri +import android.opengl.GLSurfaceView +import android.os.Build +import android.os.Bundle +import android.os.Environment +import android.os.Handler +import android.os.Looper +import android.provider.Settings +import android.view.View +import android.widget.Toast +import androidx.activity.result.contract.ActivityResultContracts +import androidx.core.content.ContextCompat +import androidx.recyclerview.widget.GridLayoutManager +import androidx.recyclerview.widget.LinearLayoutManager +import com.bumptech.glide.Glide +import java.io.File +import java.util.Locale +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +class DigitalHumanTestActivity : BaseActivity() { + + private lateinit var binding: ActivityDigitalHumanTestBinding + private lateinit var avatarAdapter: AvatarAdapter + private lateinit var batchJobAdapter: DigitalHumanBatchJobAdapter + + private val mainHandler = Handler(Looper.getMainLooper()) + private val workerExecutor: ExecutorService = Executors.newSingleThreadExecutor() + private val profileStore by lazy { VoiceCloneProfileStore(this) } + + private var selectedAvatar: AvatarModel = AvatarCatalog.models.first() + private var selectedFolderDir: File? = null + private var selectedFolderName: String? = null + private var selectedFolderPath: String? = null + private var batchJobs = mutableListOf() + private var outputRootName = DEFAULT_OUTPUT_ROOT + + private var duix: DUIX? = null + private var duixRenderer: DUIXRenderer? = null + private var omniClient: OmniRealtimeClient? = null + private var exporter: DigitalHumanMp4Exporter? = null + private var currentTempOutputFile: File? = null + private var loadingDialog: LoadingDialog? = null + private var lastProgress = -1 + + private var currentJobIndex = -1 + private var isBatchRunning = false + private var isScanningFolder = false + private var isFinalizingExport = false + private var hasStartedPushForResponse = false + private var assistantAudioDone = false + private var responseDone = false + private var avatarPlaybackEnded = false + private var receivedAssistantAudio = false + private var currentJobAttemptId = 0L + private var nextJobAttemptId = 0L + private var activeConnectionId = 0L + private var nextConnectionId = 0L + + private val assistantTextBuffer = StringBuilder() + + private val folderPickerLauncher = registerForActivityResult( + ActivityResultContracts.StartActivityForResult(), + ) { result -> + if (result.resultCode != Activity.RESULT_OK) { + return@registerForActivityResult + } + val selectedPath = result.data?.getStringExtra(FolderPickerActivity.EXTRA_SELECTED_PATH) + ?: return@registerForActivityResult + handleFolderSelected(File(selectedPath)) + } + + private val manageStorageAccessLauncher = registerForActivityResult( + ActivityResultContracts.StartActivityForResult(), + ) { + if (hasLocalFolderAccess()) { + launchLocalFolderPicker() + } else { + updateStatus(getString(R.string.digital_human_test_manage_storage_denied)) + Toast.makeText(this, R.string.digital_human_test_manage_storage_denied, Toast.LENGTH_SHORT).show() + } + } + + private val storagePermissionLauncher = registerForActivityResult( + ActivityResultContracts.RequestMultiplePermissions(), + ) { result -> + val granted = result.values.all { it } + if (granted) { + launchLocalFolderPicker() + } else { + updateStatus(getString(R.string.digital_human_test_legacy_storage_denied)) + Toast.makeText(this, R.string.digital_human_test_legacy_storage_denied, Toast.LENGTH_SHORT).show() + } + } + + private val renderSink = object : RenderSink { + override fun onVideoFrame(imageFrame: ImageFrame) { + duixRenderer?.onVideoFrame(imageFrame) + exporter?.recordFrame(imageFrame) + } + } + + override fun onCreate(savedInstanceState: Bundle?) { + super.onCreate(savedInstanceState) + keepScreenOn() + + binding = ActivityDigitalHumanTestBinding.inflate(layoutInflater) + setContentView(binding.root) + + selectedAvatar = profileStore.load()?.toAvatarModel() ?: AvatarCatalog.models.first() + outputRootName = buildOutputRootName() + + setupPreview() + setupAvatarList() + setupBatchList() + setupActions() + + Glide.with(this).load("file:///android_asset/bg/bg1.png").into(binding.ivBg) + + clearCurrentText() + updateFolderSummary() + updateQueueSummary() + updateStatus(getString(R.string.digital_human_test_status_idle, selectedAvatar.name)) + updatePreviewHint() + updateUiState() + } + + override fun onDestroy() { + super.onDestroy() + dismissLoadingDialog() + isBatchRunning = false + clearCurrentJobRuntime(cancelExporter = true) + disconnectOmni() + workerExecutor.shutdownNow() + releaseDuix() + duixRenderer?.release() + } + + private fun setupPreview() { + binding.glTextureView.setEGLContextClientVersion(GL_CONTEXT_VERSION) + binding.glTextureView.setEGLConfigChooser(8, 8, 8, 8, 16, 0) + binding.glTextureView.isOpaque = false + duixRenderer = DUIXRenderer(this, binding.glTextureView) + binding.glTextureView.setRenderer(duixRenderer) + binding.glTextureView.renderMode = GLSurfaceView.RENDERMODE_WHEN_DIRTY + } + + private fun setupAvatarList() { + avatarAdapter = AvatarAdapter( + AvatarCatalog.models, + object : AvatarAdapter.Callback { + override fun onSelect(model: AvatarModel) { + if (isBatchRunning) { + return + } + selectedAvatar = model + outputRootName = buildOutputRootName() + avatarAdapter.setSelectedModel(model) + rebuildBatchJobsForSelectedOutput() + updateFolderSummary() + updateQueueSummary() + updateStatus(buildReadyStatusText()) + updatePreviewHint() + } + }, + enableSelection = true, + ) + + binding.rvAvatars.layoutManager = GridLayoutManager(this, 2) + binding.rvAvatars.adapter = avatarAdapter + avatarAdapter.setSelectedModel(selectedAvatar) + } + + private fun setupBatchList() { + batchJobAdapter = DigitalHumanBatchJobAdapter() + binding.rvBatchJobs.layoutManager = LinearLayoutManager(this) + binding.rvBatchJobs.adapter = batchJobAdapter + } + + private fun setupActions() { + binding.btnBack.setOnClickListener { finish() } + binding.btnSelectFolder.setOnClickListener { + if (!isBatchRunning && !isScanningFolder) { + openFolderPicker() + } + } + binding.btnPrimaryAction.setOnClickListener { + if (!isBatchRunning && !isScanningFolder) { + startBatch() + } + } + } + + private fun openFolderPicker() { + when { + Build.VERSION.SDK_INT >= Build.VERSION_CODES.R && hasLocalFolderAccess() -> { + launchLocalFolderPicker() + } + Build.VERSION.SDK_INT >= Build.VERSION_CODES.R -> { + requestManageStorageAccess() + } + Build.VERSION.SDK_INT < Build.VERSION_CODES.M -> { + launchLocalFolderPicker() + } + hasLegacyStoragePermissions() -> { + launchLocalFolderPicker() + } + else -> { + storagePermissionLauncher.launch( + arrayOf( + android.Manifest.permission.READ_EXTERNAL_STORAGE, + android.Manifest.permission.WRITE_EXTERNAL_STORAGE, + ), + ) + } + } + } + + private fun launchLocalFolderPicker() { + folderPickerLauncher.launch( + FolderPickerActivity.createIntent(this, selectedFolderDir?.absolutePath), + ) + } + + private fun requestManageStorageAccess() { + updateStatus(getString(R.string.digital_human_test_need_storage_access)) + Toast.makeText(this, R.string.digital_human_test_need_storage_access, Toast.LENGTH_SHORT).show() + val appSpecificIntent = Intent(Settings.ACTION_MANAGE_APP_ALL_FILES_ACCESS_PERMISSION).apply { + data = Uri.parse("package:$packageName") + } + val fallbackIntent = Intent(Settings.ACTION_MANAGE_ALL_FILES_ACCESS_PERMISSION) + val intent = if (appSpecificIntent.resolveActivity(packageManager) != null) { + appSpecificIntent + } else { + fallbackIntent + } + manageStorageAccessLauncher.launch(intent) + } + + private fun handleFolderSelected(directory: File) { + val normalizedDirectory = runCatching { directory.canonicalFile }.getOrElse { directory } + selectedFolderDir = normalizedDirectory + selectedFolderName = null + selectedFolderPath = normalizedDirectory.absolutePath + outputRootName = buildOutputRootName() + batchJobs.clear() + batchJobAdapter.submitList(emptyList()) + currentJobIndex = -1 + clearCurrentText() + updateQueueSummary() + updateFolderSummary() + updateStatus(getString(R.string.digital_human_test_status_scanning_folder)) + updateUiState() + isScanningFolder = true + showLoadingDialog(getString(R.string.digital_human_test_status_scanning_folder)) + + workerExecutor.execute { + runCatching { + LocalFileAudioStore.scanAudioFiles(normalizedDirectory) + }.onSuccess { scanResult -> + mainHandler.post { + if (isFinishing || isDestroyed) { + return@post + } + isScanningFolder = false + dismissLoadingDialog() + selectedFolderName = scanResult.rootName + selectedFolderPath = scanResult.rootDir.absolutePath + outputRootName = buildOutputRootName() + batchJobs = scanResult.files.map { buildPendingJob(it) }.toMutableList() + rebuildBatchJobsForSelectedOutput() + updateFolderSummary() + updateQueueSummary() + updateStatus( + if (batchJobs.isEmpty()) { + getString(R.string.digital_human_test_no_audio_found) + } else { + buildReadyStatusText() + }, + ) + updateUiState() + if (batchJobs.isEmpty()) { + Toast.makeText(this, R.string.digital_human_test_no_audio_found, Toast.LENGTH_SHORT).show() + } + } + }.onFailure { error -> + mainHandler.post { + isScanningFolder = false + dismissLoadingDialog() + batchJobs.clear() + batchJobAdapter.submitList(emptyList()) + updateFolderSummary() + updateQueueSummary() + updateStatus(safeErrorMessage(error)) + updateUiState() + Toast.makeText(this, safeErrorMessage(error), Toast.LENGTH_SHORT).show() + } + } + } + } + + private fun buildPendingJob(entry: LocalFileAudioStore.AudioEntry): BatchAudioJob { + val outputRelativePath = LocalFileAudioStore.buildOutputRelativePath( + outputRootName, + entry.relativePath, + ) + return BatchAudioJob( + sourceUri = entry.uri, + sourceRelativePath = entry.relativePath, + displayName = entry.displayName, + outputRelativePath = outputRelativePath, + status = BatchAudioJob.Status.PENDING, + statusText = getString(R.string.digital_human_test_job_pending), + detailText = getString(R.string.digital_human_test_job_output, outputRelativePath), + ) + } + + private fun rebuildBatchJobsForSelectedOutput() { + outputRootName = buildOutputRootName() + val folderDir = selectedFolderDir + batchJobs = batchJobs.map { job -> + val newOutputRelativePath = LocalFileAudioStore.buildOutputRelativePath( + outputRootName, + job.sourceRelativePath, + ) + val existingOutput = folderDir?.let { + LocalFileAudioStore.resolveOutputFile(it, outputRootName, job.sourceRelativePath) + } + val shouldSkip = existingOutput?.isFile == true + job.copy( + outputRelativePath = newOutputRelativePath, + attemptCount = 0, + status = if (shouldSkip) BatchAudioJob.Status.SKIPPED else BatchAudioJob.Status.PENDING, + statusText = if (shouldSkip) { + getString(R.string.digital_human_test_job_skipped) + } else { + getString(R.string.digital_human_test_job_pending) + }, + detailText = getString(R.string.digital_human_test_job_output, newOutputRelativePath), + outputUri = if (shouldSkip && existingOutput != null) Uri.fromFile(existingOutput) else null, + ) + }.toMutableList() + batchJobAdapter.submitList(batchJobs) + } + + private fun startBatch() { + if (selectedFolderDir == null) { + Toast.makeText(this, R.string.digital_human_test_need_folder, Toast.LENGTH_SHORT).show() + return + } + if (batchJobs.isEmpty()) { + Toast.makeText(this, R.string.digital_human_test_no_audio_found, Toast.LENGTH_SHORT).show() + return + } + + rebuildBatchJobsForSelectedOutput() + if (pendingPreviewJobCount() == 0) { + updateQueueSummary() + updateStatus(getString(R.string.digital_human_test_all_outputs_exist)) + updateUiState() + Toast.makeText(this, R.string.digital_human_test_all_outputs_exist, Toast.LENGTH_SHORT).show() + return + } + + isBatchRunning = true + currentJobIndex = -1 + clearCurrentText() + updateQueueSummary() + updateStatus(getString(R.string.digital_human_test_status_preparing_resources, selectedAvatar.name)) + updatePreviewHint() + updateUiState() + + ensureAvatarResources { + initDuixAndStartBatch() + } + } + + private fun ensureAvatarResources(onReady: () -> Unit) { + if (VirtualModelUtil.checkBaseConfig(this)) { + ensureAvatarModel(onReady) + } else { + downloadBaseConfig { + ensureAvatarModel(onReady) + } + } + } + + private fun ensureAvatarModel(onReady: () -> Unit) { + if (VirtualModelUtil.checkModel(this, selectedAvatar.modelUrl)) { + onReady() + } else { + downloadAvatarModel(onReady) + } + } + + private fun downloadBaseConfig(onReady: () -> Unit) { + showLoadingDialog(getString(R.string.digital_human_test_status_preparing_resources, selectedAvatar.name)) + VirtualModelUtil.baseConfigDownload(this, BASE_CONFIG_URL, object : VirtualModelUtil.ModelDownloadCallback { + override fun onDownloadProgress(url: String?, current: Long, total: Long) { + updateLoadingProgress(current, total) { progress -> + "基础资源下载中(${progress}%)" + } + } + + override fun onUnzipProgress(url: String?, current: Long, total: Long) { + updateLoadingProgress(current, total) { progress -> + "基础资源解压中(${progress}%)" + } + } + + override fun onDownloadComplete(url: String?, dir: File?) { + runOnUiThread { + dismissLoadingDialog() + if (isBatchRunning) { + onReady() + } + } + } + + override fun onDownloadFail(url: String?, code: Int, msg: String?) { + mainHandler.post { + stopBatchWithError(msg ?: getString(R.string.ai_call_error_retry)) + } + } + }) + } + + private fun downloadAvatarModel(onReady: () -> Unit) { + showLoadingDialog(getString(R.string.digital_human_test_status_preparing_resources, selectedAvatar.name)) + VirtualModelUtil.modelDownload(this, selectedAvatar.modelUrl, object : VirtualModelUtil.ModelDownloadCallback { + override fun onDownloadProgress(url: String?, current: Long, total: Long) { + updateLoadingProgress(current, total) { progress -> + "${selectedAvatar.name} 下载中(${progress}%)" + } + } + + override fun onUnzipProgress(url: String?, current: Long, total: Long) { + updateLoadingProgress(current, total) { progress -> + "${selectedAvatar.name} 解压中(${progress}%)" + } + } + + override fun onDownloadComplete(url: String?, dir: File?) { + runOnUiThread { + dismissLoadingDialog() + if (isBatchRunning) { + onReady() + } + } + } + + override fun onDownloadFail(url: String?, code: Int, msg: String?) { + mainHandler.post { + stopBatchWithError(msg ?: getString(R.string.ai_call_error_retry)) + } + } + }) + } + + private fun initDuixAndStartBatch() { + updateStatus(getString(R.string.digital_human_test_status_loading_avatar, selectedAvatar.name)) + updatePreviewHint() + releaseDuix() + + duix = DUIX(this, selectedAvatar.modelUrl, renderSink) { event, msg, _ -> + when (event) { + Constant.CALLBACK_EVENT_INIT_READY -> { + mainHandler.post { + if (isBatchRunning && !isFinishing) { + startNextJob() + } + } + } + + Constant.CALLBACK_EVENT_INIT_ERROR, + Constant.CALLBACK_EVENT_AUDIO_PLAY_ERROR -> { + mainHandler.post { + stopBatchWithError(msg.ifBlank { getString(R.string.ai_call_error_retry) }) + } + } + + Constant.CALLBACK_EVENT_AUDIO_PLAY_END -> { + mainHandler.post { + avatarPlaybackEnded = true + if (isBatchRunning && assistantAudioDone && responseDone) { + finalizeCurrentJob() + } + } + } + } + } + duix?.init() + } + + private fun startNextJob() { + if (!isBatchRunning) { + return + } + var nextIndex = currentJobIndex + 1 + while (nextIndex < batchJobs.size && batchJobs[nextIndex].status != BatchAudioJob.Status.PENDING) { + nextIndex += 1 + } + if (nextIndex >= batchJobs.size) { + finishBatch() + return + } + + currentJobIndex = nextIndex + prepareCurrentJob() + } + + private fun prepareCurrentJob() { + val job = currentJobOrNull() ?: return + val folderDir = selectedFolderDir + if (folderDir != null) { + val existingOutput = LocalFileAudioStore.resolveOutputFile(folderDir, outputRootName, job.sourceRelativePath) + if (existingOutput.isFile) { + completeCurrentJobAsSkipped(existingOutput) + return + } + } + + currentJobAttemptId = ++nextJobAttemptId + assistantAudioDone = false + responseDone = false + avatarPlaybackEnded = false + receivedAssistantAudio = false + isFinalizingExport = false + hasStartedPushForResponse = false + clearCurrentText() + duix?.stopAudio() + + currentTempOutputFile = createTempOutputFile(job) + exporter = DigitalHumanMp4Exporter(currentTempOutputFile!!) + + updateCurrentJob( + job.copy( + status = BatchAudioJob.Status.RUNNING, + statusText = getString(R.string.digital_human_test_job_decoding), + detailText = getString(R.string.digital_human_test_job_output, job.outputRelativePath), + ), + ) + updateQueueSummary() + updateStatus( + getString( + R.string.digital_human_test_status_batch_current, + currentJobIndex + 1, + batchJobs.size, + job.sourceRelativePath, + ), + ) + updatePreviewHint() + binding.rvBatchJobs.post { + binding.rvBatchJobs.smoothScrollToPosition(currentJobIndex) + } + decodeCurrentAudioAndConnect(job, currentJobAttemptId) + } + + private fun decodeCurrentAudioAndConnect(job: BatchAudioJob, attemptId: Long) { + workerExecutor.execute { + runCatching { + AudioFileDecoder.decodeToPcm16Mono16k(applicationContext, job.sourceUri) + }.onSuccess { decodedAudio -> + mainHandler.post { + if (!isCurrentAttempt(attemptId) || currentJobOrNull()?.sourceUri != job.sourceUri || isFinishing) { + return@post + } + connectOmniAndSend(decodedAudio, attemptId) + } + }.onFailure { error -> + mainHandler.post { + if (!isCurrentAttempt(attemptId)) { + return@post + } + handleCurrentJobFailure(safeErrorMessage(error), allowRetry = false) + } + } + } + } + + private fun connectOmniAndSend(decodedAudio: AudioFileDecoder.DecodedAudio, attemptId: Long) { + val currentJob = currentJobOrNull() ?: return + updateCurrentJob( + currentJob.copy( + status = BatchAudioJob.Status.RUNNING, + statusText = getString(R.string.digital_human_test_job_connecting), + ), + ) + assistantTextBuffer.clear() + val connectionId = ++nextConnectionId + lateinit var client: OmniRealtimeClient + client = OmniRealtimeClient( + OmniRealtimeClient.SessionOptions( + DashScopeConfig.DEFAULT_API_KEY, + DashScopeConfig.REALTIME_BASE_URL, + OmniRealtimeClient.DEFAULT_MODEL, + selectedAvatar.voice, + OmniRealtimeClient.getDefaultInstructions(), + false, + false, + true, + OmniRealtimeClient.SessionOptions.OutputMode.AUDIO_AND_TEXT, + ), + object : OmniRealtimeClient.Callback { + override fun onConnected() { + if (!isActiveConnection(attemptId, connectionId)) { + return + } + mainHandler.post { + if (!isActiveConnection(attemptId, connectionId)) { + return@post + } + val job = currentJobOrNull() ?: return@post + updateCurrentJob( + job.copy( + status = BatchAudioJob.Status.RUNNING, + statusText = getString(R.string.digital_human_test_job_uploading), + ), + ) + } + workerExecutor.execute { + runCatching { + sendDecodedAudio(client, decodedAudio, attemptId, connectionId) + }.onFailure { error -> + mainHandler.post { + if (!isActiveConnection(attemptId, connectionId)) { + return@post + } + handleCurrentJobFailure(safeErrorMessage(error), allowRetry = true) + } + } + } + } + + override fun onDisconnected() { + mainHandler.post { + if (!isActiveConnection(attemptId, connectionId)) { + return@post + } + if (isFinalizingExport) { + return@post + } + if (isBatchRunning && !isFinishing) { + handleCurrentJobFailure( + getString(R.string.digital_human_test_connection_lost), + allowRetry = true, + ) + } + } + } + + override fun onError(message: String) { + mainHandler.post { + if (!isActiveConnection(attemptId, connectionId)) { + return@post + } + if (isFinalizingExport) { + return@post + } + handleCurrentJobFailure( + message.ifBlank { getString(R.string.ai_call_error_retry) }, + allowRetry = true, + ) + } + } + + override fun onUserSpeechStarted() = Unit + + override fun onUserSpeechStopped() = Unit + + override fun onUserTranscript(transcript: String) { + if (!isActiveConnection(attemptId, connectionId)) { + return + } + mainHandler.post { + if (!isActiveConnection(attemptId, connectionId)) { + return@post + } + updateInputTranscript(transcript) + } + } + + override fun onAssistantTextDelta(delta: String) { + if (!isActiveConnection(attemptId, connectionId)) { + return + } + mainHandler.post { + if (!isActiveConnection(attemptId, connectionId)) { + return@post + } + assistantTextBuffer.append(delta) + binding.tvSubtitle.visibility = View.VISIBLE + binding.tvSubtitle.text = assistantTextBuffer.toString() + } + } + + override fun onAssistantTranscriptDone(transcript: String) { + if (!isActiveConnection(attemptId, connectionId)) { + return + } + mainHandler.post { + if (!isActiveConnection(attemptId, connectionId)) { + return@post + } + assistantTextBuffer.clear() + assistantTextBuffer.append(transcript) + binding.tvSubtitle.visibility = View.VISIBLE + binding.tvSubtitle.text = transcript + } + } + + override fun onToolCallRequested(toolName: String, callId: String, argumentsJson: String) = Unit + + override fun onAssistantAudioDelta(pcmData: ByteArray) { + if (!isActiveConnection(attemptId, connectionId)) { + return + } + receivedAssistantAudio = true + exporter?.recordAudioChunk(pcmData) + if (!hasStartedPushForResponse) { + hasStartedPushForResponse = true + duix?.startPush() + } + val resampled = AudioResampler.resample24kTo16k(pcmData) + if (resampled.isNotEmpty()) { + duix?.pushPcm(resampled) + } + mainHandler.post { + val job = currentJobOrNull() ?: return@post + updateCurrentJob( + job.copy( + status = BatchAudioJob.Status.RUNNING, + statusText = getString(R.string.digital_human_test_job_generating), + ), + ) + updatePreviewHint() + } + } + + override fun onAssistantAudioDone() { + if (!isActiveConnection(attemptId, connectionId)) { + return + } + assistantAudioDone = true + if (hasStartedPushForResponse) { + duix?.stopPush() + hasStartedPushForResponse = false + } + mainHandler.post { + if (!isActiveConnection(attemptId, connectionId)) { + return@post + } + if (responseDone && avatarPlaybackEnded) { + finalizeCurrentJob() + } + } + } + + override fun onResponseDone() { + mainHandler.post { + if (!isActiveConnection(attemptId, connectionId)) { + return@post + } + responseDone = true + if (!receivedAssistantAudio) { + handleCurrentJobFailure( + getString(R.string.digital_human_test_no_audio_response), + allowRetry = true, + ) + return@post + } + val job = currentJobOrNull() ?: return@post + updateCurrentJob( + job.copy( + status = BatchAudioJob.Status.RUNNING, + statusText = getString(R.string.digital_human_test_job_muxing), + ), + ) + if (assistantAudioDone && avatarPlaybackEnded) { + finalizeCurrentJob() + } + } + } + }, + ) + activeConnectionId = connectionId + omniClient = client + client.connect() + } + + private fun sendDecodedAudio( + client: OmniRealtimeClient, + decodedAudio: AudioFileDecoder.DecodedAudio, + attemptId: Long, + connectionId: Long, + ) { + if (!isActiveConnection(attemptId, connectionId)) { + return + } + var offset = 0 + while (offset < decodedAudio.pcm16Mono16k.size) { + if (!isActiveConnection(attemptId, connectionId)) { + return + } + val end = (offset + OMNI_INPUT_CHUNK_BYTES).coerceAtMost(decodedAudio.pcm16Mono16k.size) + client.appendAudio(decodedAudio.pcm16Mono16k.copyOfRange(offset, end)) + offset = end + } + if (!isActiveConnection(attemptId, connectionId)) { + return + } + val committed = client.commitInputAudio() + val requested = committed && client.requestResponse() + if (!requested) { + throw IllegalStateException(getString(R.string.ai_call_error_retry)) + } + } + + private fun finalizeCurrentJob() { + if (!isBatchRunning || isFinalizingExport) { + return + } + val folderDir = selectedFolderDir ?: run { + stopBatchWithError(getString(R.string.digital_human_test_need_folder)) + return + } + val activeExporter = exporter ?: run { + handleCurrentJobFailure(getString(R.string.digital_human_test_no_audio_response), allowRetry = true) + return + } + val currentJob = currentJobOrNull() ?: return + val attemptId = currentJobAttemptId + isFinalizingExport = true + + updateCurrentJob( + currentJob.copy( + status = BatchAudioJob.Status.RUNNING, + statusText = getString(R.string.digital_human_test_job_muxing), + ), + ) + + workerExecutor.execute { + var localOutputFile: File? = null + runCatching { + val exportResult = activeExporter.finish() + localOutputFile = exportResult.file + if (!isCurrentAttempt(attemptId)) { + return@runCatching LocalFileAudioStore.WriteResult(exportResult.file, skipped = true) + } + mainHandler.post { + if (!isCurrentAttempt(attemptId)) { + return@post + } + val job = currentJobOrNull() ?: return@post + updateCurrentJob( + job.copy( + status = BatchAudioJob.Status.RUNNING, + statusText = getString(R.string.digital_human_test_job_copying), + ), + ) + updateStatus(getString(R.string.digital_human_test_status_copying)) + } + LocalFileAudioStore.writeVideoResult( + folderDir, + outputRootName, + currentJob.sourceRelativePath, + exportResult.file, + ) + }.onSuccess { writeResult -> + localOutputFile?.delete() + mainHandler.post { + if (!isCurrentAttempt(attemptId)) { + return@post + } + if (writeResult.skipped) { + completeCurrentJobAsSkipped(writeResult.file) + } else { + completeCurrentJob(Uri.fromFile(writeResult.file)) + } + } + }.onFailure { error -> + localOutputFile?.delete() + mainHandler.post { + if (!isCurrentAttempt(attemptId)) { + return@post + } + handleCurrentJobFailure(safeErrorMessage(error), allowRetry = false) + } + } + } + } + + private fun completeCurrentJob(outputUri: Uri) { + val job = currentJobOrNull() ?: return + clearCurrentJobRuntime(cancelExporter = false) + updateCurrentJob( + job.copy( + status = BatchAudioJob.Status.COMPLETED, + statusText = getString(R.string.digital_human_test_job_done), + detailText = getString(R.string.digital_human_test_job_output, job.outputRelativePath), + outputUri = outputUri, + ), + ) + updateQueueSummary() + startNextJob() + } + + private fun completeCurrentJobAsSkipped(outputFile: File) { + val job = currentJobOrNull() ?: return + clearCurrentJobRuntime(cancelExporter = false) + updateCurrentJob( + job.copy( + status = BatchAudioJob.Status.SKIPPED, + statusText = getString(R.string.digital_human_test_job_skipped), + detailText = getString(R.string.digital_human_test_job_output, job.outputRelativePath), + outputUri = Uri.fromFile(outputFile), + ), + ) + updateQueueSummary() + startNextJob() + } + + private fun handleCurrentJobFailure(message: String, allowRetry: Boolean) { + if (!isBatchRunning) { + return + } + val job = currentJobOrNull() + if (job != null && allowRetry && job.attemptCount < MAX_RETRY_COUNT) { + val nextRetryAttempt = job.attemptCount + 1 + clearCurrentJobRuntime(cancelExporter = true) + updateCurrentJob( + job.copy( + attemptCount = nextRetryAttempt, + status = BatchAudioJob.Status.RUNNING, + statusText = getString( + R.string.digital_human_test_job_retrying, + nextRetryAttempt, + MAX_RETRY_COUNT, + ), + detailText = message, + ), + ) + updateQueueSummary() + updateStatus( + getString( + R.string.digital_human_test_status_retrying, + nextRetryAttempt, + MAX_RETRY_COUNT, + job.sourceRelativePath, + ), + ) + mainHandler.postDelayed({ + if (!isBatchRunning) { + return@postDelayed + } + val pendingRetryJob = currentJobOrNull() ?: return@postDelayed + if (pendingRetryJob.attemptCount != nextRetryAttempt || pendingRetryJob.status != BatchAudioJob.Status.RUNNING) { + return@postDelayed + } + prepareCurrentJob() + }, RETRY_DELAY_MS) + return + } + clearCurrentJobRuntime(cancelExporter = true) + if (job != null) { + updateCurrentJob( + job.copy( + status = BatchAudioJob.Status.FAILED, + statusText = getString(R.string.digital_human_test_job_failed), + detailText = message, + ), + ) + updateQueueSummary() + } + startNextJob() + } + + private fun finishBatch() { + isBatchRunning = false + currentJobIndex = -1 + updateQueueSummary() + updateStatus( + getString( + R.string.digital_human_test_status_batch_done, + completedJobCount(), + skippedJobCount(), + failedJobCount(), + ), + ) + updatePreviewHint() + updateUiState() + } + + private fun stopBatchWithError(message: String) { + dismissLoadingDialog() + isBatchRunning = false + clearCurrentJobRuntime(cancelExporter = true) + updateStatus(message) + updatePreviewHint() + updateUiState() + Toast.makeText(this, message, Toast.LENGTH_SHORT).show() + } + + private fun clearCurrentJobRuntime(cancelExporter: Boolean) { + currentJobAttemptId = 0L + disconnectOmni() + if (hasStartedPushForResponse) { + duix?.stopPush() + hasStartedPushForResponse = false + } + duix?.stopAudio() + if (cancelExporter) { + runCatching { exporter?.cancel() } + } + exporter = null + currentTempOutputFile?.let { file -> + if (file.exists()) { + file.delete() + } + } + currentTempOutputFile = null + isFinalizingExport = false + assistantAudioDone = false + responseDone = false + avatarPlaybackEnded = false + receivedAssistantAudio = false + clearCurrentText() + } + + private fun disconnectOmni() { + activeConnectionId = 0L + omniClient?.disconnect() + omniClient = null + } + + private fun releaseDuix() { + duix?.stopAudio() + duix?.release() + duix = null + } + + private fun updateStatus(text: String) { + binding.tvStatus.text = text + } + + private fun updateFolderSummary() { + if (selectedFolderDir == null) { + binding.tvFolderName.text = getString(R.string.digital_human_test_folder_none) + binding.tvFolderMeta.text = getString(R.string.digital_human_test_folder_hint) + return + } + + binding.tvFolderName.text = selectedFolderName ?: selectedFolderPath ?: getString(R.string.digital_human_test_folder_none) + binding.tvFolderMeta.text = if (batchJobs.isEmpty()) { + if (isScanningFolder) { + getString(R.string.digital_human_test_status_scanning_folder) + } else { + getString(R.string.digital_human_test_no_audio_found) + } + } else { + getString( + R.string.digital_human_test_folder_summary, + batchJobs.size, + selectedFolderPath ?: "", + outputRootName, + ) + } + } + + private fun updateQueueSummary() { + binding.tvProgressSummary.text = getString( + R.string.digital_human_test_progress_summary, + batchJobs.size, + completedJobCount(), + skippedJobCount(), + failedJobCount(), + ) + binding.tvProgressEmpty.visibility = if (batchJobs.isEmpty()) View.VISIBLE else View.GONE + binding.rvBatchJobs.visibility = if (batchJobs.isEmpty()) View.GONE else View.VISIBLE + } + + private fun updateInputTranscript(transcript: String?) { + val visible = !transcript.isNullOrBlank() + binding.tvInputTranscriptLabel.visibility = if (visible) View.VISIBLE else View.GONE + binding.tvInputTranscript.visibility = if (visible) View.VISIBLE else View.GONE + binding.tvInputTranscript.text = transcript.orEmpty() + } + + private fun clearCurrentText() { + assistantTextBuffer.clear() + binding.tvSubtitle.visibility = View.GONE + binding.tvSubtitle.text = "" + binding.tvInputTranscriptLabel.visibility = View.GONE + binding.tvInputTranscript.visibility = View.GONE + binding.tvInputTranscript.text = "" + } + + private fun updatePreviewHint() { + val showHint = !isBatchRunning || duix?.isReady() != true + binding.tvPreviewHint.visibility = if (showHint) View.VISIBLE else View.GONE + binding.tvPreviewHint.text = if (isBatchRunning) { + getString(R.string.digital_human_test_preview_loading, selectedAvatar.name) + } else { + getString(R.string.digital_human_test_preview_idle, selectedAvatar.name) + } + } + + private fun updateUiState() { + binding.btnPrimaryAction.text = if (isBatchRunning) { + getString(R.string.digital_human_test_running) + } else { + getString(R.string.digital_human_test_generate) + } + binding.btnPrimaryAction.isEnabled = !isBatchRunning && !isScanningFolder && canStartBatch() + binding.btnSelectFolder.isEnabled = !isBatchRunning && !isScanningFolder + binding.layoutAvatarSection.visibility = if (isBatchRunning) View.GONE else View.VISIBLE + binding.layoutProgressSection.visibility = if (shouldShowProgressSection()) View.VISIBLE else View.GONE + } + + private fun shouldShowProgressSection(): Boolean { + return isBatchRunning || batchJobs.any { it.status != BatchAudioJob.Status.PENDING } + } + + private fun showLoadingDialog(message: String) { + if (loadingDialog == null) { + loadingDialog = LoadingDialog(this, message) + } + lastProgress = -1 + loadingDialog?.show() + loadingDialog?.setContent(message) + } + + private fun dismissLoadingDialog() { + loadingDialog?.dismiss() + loadingDialog = null + lastProgress = -1 + } + + private fun updateLoadingProgress(current: Long, total: Long, messageBuilder: (Int) -> String) { + if (total <= 0L) { + return + } + val progress = ((current * 100L) / total).toInt() + if (progress == lastProgress) { + return + } + lastProgress = progress + runOnUiThread { + loadingDialog?.setContent(messageBuilder(progress)) + } + } + + private fun updateCurrentJob(updatedJob: BatchAudioJob) { + if (currentJobIndex !in batchJobs.indices) { + return + } + batchJobs[currentJobIndex] = updatedJob + batchJobAdapter.updateItem(currentJobIndex, updatedJob) + } + + private fun currentJobOrNull(): BatchAudioJob? { + return batchJobs.getOrNull(currentJobIndex) + } + + private fun isCurrentAttempt(attemptId: Long): Boolean { + return attemptId != 0L && currentJobAttemptId == attemptId && isBatchRunning + } + + private fun isActiveConnection(attemptId: Long, connectionId: Long): Boolean { + return isCurrentAttempt(attemptId) && activeConnectionId == connectionId + } + + private fun hasLocalFolderAccess(): Boolean { + return Build.VERSION.SDK_INT < Build.VERSION_CODES.R || Environment.isExternalStorageManager() + } + + private fun hasLegacyStoragePermissions(): Boolean { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) { + return true + } + val permissions = arrayOf( + android.Manifest.permission.READ_EXTERNAL_STORAGE, + android.Manifest.permission.WRITE_EXTERNAL_STORAGE, + ) + return permissions.all { permission -> + ContextCompat.checkSelfPermission(this, permission) == PackageManager.PERMISSION_GRANTED + } + } + + private fun buildOutputRootName(): String { + val normalizedVoice = selectedAvatar.voice + .lowercase(Locale.US) + .replace(Regex("[^a-z0-9]+"), "_") + .trim('_') + return if (normalizedVoice.isBlank()) { + DEFAULT_OUTPUT_ROOT + } else { + "${DEFAULT_OUTPUT_ROOT}_$normalizedVoice" + } + } + + private fun createTempOutputFile(job: BatchAudioJob): File { + val tempDir = File(cacheDir, TEMP_OUTPUT_DIR_NAME).apply { mkdirs() } + val safeName = sanitizeFileName(job.displayName) + return File(tempDir, "${System.currentTimeMillis()}_${currentJobIndex + 1}_$safeName.mp4") + } + + private fun sanitizeFileName(name: String): String { + val baseName = name.substringBeforeLast('.', name) + val sanitized = baseName.replace(Regex("[^a-zA-Z0-9._-]+"), "_").trim('_') + return if (sanitized.isBlank()) "digital_human" else sanitized + } + + private fun safeErrorMessage(error: Throwable): String { + return error.message?.takeIf { it.isNotBlank() } ?: getString(R.string.ai_call_error_retry) + } + + private fun buildReadyStatusText(): String { + return when { + batchJobs.isEmpty() -> getString(R.string.digital_human_test_status_idle, selectedAvatar.name) + pendingPreviewJobCount() == 0 -> getString(R.string.digital_human_test_all_outputs_exist) + else -> getString(R.string.digital_human_test_status_batch_ready, pendingPreviewJobCount()) + } + } + + private fun pendingPreviewJobCount(): Int { + return batchJobs.count { it.status == BatchAudioJob.Status.PENDING } + } + + private fun canStartBatch(): Boolean { + val folderDir = selectedFolderDir ?: return false + if (batchJobs.isEmpty()) { + return false + } + return batchJobs.any { job -> + !LocalFileAudioStore.resolveOutputFile(folderDir, outputRootName, job.sourceRelativePath).exists() + } + } + + private fun completedJobCount(): Int { + return batchJobs.count { it.status == BatchAudioJob.Status.COMPLETED } + } + + private fun skippedJobCount(): Int { + return batchJobs.count { it.status == BatchAudioJob.Status.SKIPPED } + } + + private fun failedJobCount(): Int { + return batchJobs.count { it.status == BatchAudioJob.Status.FAILED } + } + + companion object { + private const val GL_CONTEXT_VERSION = 2 + private const val BASE_CONFIG_URL = "https://dl.xn--876a.net/models/gj_dh_res.zip" + private const val OMNI_INPUT_CHUNK_BYTES = 3200 + private const val DEFAULT_OUTPUT_ROOT = "digital_human_output" + private const val TEMP_OUTPUT_DIR_NAME = "digital_human_batch_temp" + private const val MAX_RETRY_COUNT = 2 + private const val RETRY_DELAY_MS = 1500L + } +} \ No newline at end of file diff --git a/test/src/main/java/ai/guiji/duix/test/ui/activity/FolderPickerActivity.kt b/test/src/main/java/ai/guiji/duix/test/ui/activity/FolderPickerActivity.kt new file mode 100644 index 0000000..1e68f27 --- /dev/null +++ b/test/src/main/java/ai/guiji/duix/test/ui/activity/FolderPickerActivity.kt @@ -0,0 +1,133 @@ +package ai.guiji.duix.test.ui.activity + +import ai.guiji.duix.test.R +import ai.guiji.duix.test.databinding.ActivityFolderPickerBinding +import ai.guiji.duix.test.ui.adapter.FolderBrowserAdapter +import android.app.Activity +import android.content.Context +import android.content.Intent +import android.os.Bundle +import android.os.Environment +import android.view.View +import androidx.activity.OnBackPressedCallback +import androidx.recyclerview.widget.LinearLayoutManager +import java.io.File + +class FolderPickerActivity : BaseActivity() { + + private lateinit var binding: ActivityFolderPickerBinding + private lateinit var folderAdapter: FolderBrowserAdapter + private lateinit var browseRoot: File + private lateinit var currentDirectory: File + + override fun onCreate(savedInstanceState: Bundle?) { + super.onCreate(savedInstanceState) + binding = ActivityFolderPickerBinding.inflate(layoutInflater) + setContentView(binding.root) + + browseRoot = resolveBrowseRoot() + currentDirectory = resolveInitialDirectory(intent.getStringExtra(EXTRA_INITIAL_PATH)) + + onBackPressedDispatcher.addCallback(this, object : OnBackPressedCallback(true) { + override fun handleOnBackPressed() { + if (canNavigateUp()) { + navigateUp() + } else { + isEnabled = false + onBackPressedDispatcher.onBackPressed() + } + } + }) + + setupList() + setupActions() + renderCurrentDirectory() + } + + private fun setupList() { + folderAdapter = FolderBrowserAdapter { folder -> + currentDirectory = folder + renderCurrentDirectory() + } + binding.rvFolders.layoutManager = LinearLayoutManager(this) + binding.rvFolders.adapter = folderAdapter + } + + private fun setupActions() { + binding.btnBack.setOnClickListener { finish() } + binding.btnNavigateUp.setOnClickListener { navigateUp() } + binding.btnSelectCurrent.setOnClickListener { + setResult( + Activity.RESULT_OK, + Intent().putExtra(EXTRA_SELECTED_PATH, currentDirectory.absolutePath), + ) + finish() + } + } + + private fun renderCurrentDirectory() { + val subFolders = currentDirectory.listFiles() + ?.filter { it.isDirectory && it.canRead() } + ?.sortedBy { it.name.lowercase() } + .orEmpty() + + binding.tvCurrentFolderName.text = currentDirectory.name.ifBlank { currentDirectory.absolutePath } + binding.tvCurrentPath.text = currentDirectory.absolutePath + binding.btnNavigateUp.isEnabled = canNavigateUp() + binding.tvEmpty.visibility = if (subFolders.isEmpty()) View.VISIBLE else View.GONE + binding.rvFolders.visibility = if (subFolders.isEmpty()) View.GONE else View.VISIBLE + folderAdapter.submitList(subFolders) + } + + private fun navigateUp() { + if (!canNavigateUp()) { + return + } + currentDirectory.parentFile?.let { parent -> + currentDirectory = parent + renderCurrentDirectory() + } + } + + private fun canNavigateUp(): Boolean { + val rootPath = browseRoot.canonicalPath + val currentPath = currentDirectory.canonicalPath + return currentPath != rootPath && currentDirectory.parentFile != null && currentPath.startsWith(rootPath) + } + + private fun resolveBrowseRoot(): File { + return Environment.getExternalStorageDirectory().canonicalFile + } + + private fun resolveInitialDirectory(initialPath: String?): File { + val candidates = listOfNotNull( + initialPath?.takeIf { it.isNotBlank() }?.let(::File), + Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_DOCUMENTS), + Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_MUSIC), + Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_MOVIES), + Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_DOWNLOADS), + browseRoot, + ) + for (candidate in candidates) { + val file = runCatching { candidate.canonicalFile }.getOrNull() ?: continue + if (!file.exists() || !file.isDirectory || !file.canRead()) { + continue + } + if (file.canonicalPath.startsWith(browseRoot.canonicalPath)) { + return file + } + } + return browseRoot + } + + companion object { + const val EXTRA_INITIAL_PATH = "extra_initial_path" + const val EXTRA_SELECTED_PATH = "extra_selected_path" + + fun createIntent(context: Context, initialPath: String?): Intent { + return Intent(context, FolderPickerActivity::class.java).apply { + putExtra(EXTRA_INITIAL_PATH, initialPath) + } + } + } +} \ No newline at end of file diff --git a/test/src/main/java/ai/guiji/duix/test/ui/activity/SettingsActivity.kt b/test/src/main/java/ai/guiji/duix/test/ui/activity/SettingsActivity.kt index 16611d5..1be43b0 100644 --- a/test/src/main/java/ai/guiji/duix/test/ui/activity/SettingsActivity.kt +++ b/test/src/main/java/ai/guiji/duix/test/ui/activity/SettingsActivity.kt @@ -28,6 +28,9 @@ class SettingsActivity : BaseActivity() { binding.rowDebugMode.setOnClickListener { binding.switchDebugMode.performClick() } + binding.rowDigitalHumanTest.setOnClickListener { + startActivity(Intent(mContext, DigitalHumanTestActivity::class.java)) + } binding.tvVersionValue.text = "${BuildConfig.VERSION_NAME} (${BuildConfig.VERSION_CODE})" binding.tvServerValue.text = Uri.parse(DashScopeConfig.MONITOR_BASE_URL).host diff --git a/test/src/main/java/ai/guiji/duix/test/ui/adapter/DigitalHumanBatchJobAdapter.kt b/test/src/main/java/ai/guiji/duix/test/ui/adapter/DigitalHumanBatchJobAdapter.kt new file mode 100644 index 0000000..c9bd0e4 --- /dev/null +++ b/test/src/main/java/ai/guiji/duix/test/ui/adapter/DigitalHumanBatchJobAdapter.kt @@ -0,0 +1,62 @@ +package ai.guiji.duix.test.ui.adapter + +import ai.guiji.duix.test.R +import ai.guiji.duix.test.databinding.ItemDigitalHumanBatchJobBinding +import ai.guiji.duix.test.model.BatchAudioJob +import android.view.LayoutInflater +import android.view.ViewGroup +import androidx.core.content.ContextCompat +import androidx.recyclerview.widget.RecyclerView +import java.util.Locale + +class DigitalHumanBatchJobAdapter : RecyclerView.Adapter() { + + private val items = mutableListOf() + + class ViewHolder(val binding: ItemDigitalHumanBatchJobBinding) : + RecyclerView.ViewHolder(binding.root) + + override fun onCreateViewHolder(parent: ViewGroup, viewType: Int): ViewHolder { + val binding = ItemDigitalHumanBatchJobBinding.inflate( + LayoutInflater.from(parent.context), + parent, + false, + ) + return ViewHolder(binding) + } + + override fun onBindViewHolder(holder: ViewHolder, position: Int) { + val item = items[position] + val context = holder.binding.root.context + holder.binding.tvIndex.text = String.format(Locale.getDefault(), "%02d", position + 1) + holder.binding.tvPath.text = item.sourceRelativePath + holder.binding.tvStatus.text = item.statusText + holder.binding.tvDetail.text = item.detailText + holder.binding.tvDetail.alpha = if (item.detailText.isBlank()) 0.55f else 1f + + val statusColor = when (item.status) { + BatchAudioJob.Status.PENDING -> R.color.warm_status_pending + BatchAudioJob.Status.RUNNING -> R.color.warm_status_running + BatchAudioJob.Status.COMPLETED -> R.color.warm_status_success + BatchAudioJob.Status.FAILED -> R.color.warm_status_error + BatchAudioJob.Status.SKIPPED -> R.color.warm_status_skipped + } + holder.binding.tvStatus.setTextColor(ContextCompat.getColor(context, statusColor)) + } + + override fun getItemCount(): Int = items.size + + fun submitList(newItems: List) { + items.clear() + items.addAll(newItems) + notifyDataSetChanged() + } + + fun updateItem(index: Int, item: BatchAudioJob) { + if (index !in items.indices) { + return + } + items[index] = item + notifyItemChanged(index) + } +} \ No newline at end of file diff --git a/test/src/main/java/ai/guiji/duix/test/ui/adapter/FolderBrowserAdapter.kt b/test/src/main/java/ai/guiji/duix/test/ui/adapter/FolderBrowserAdapter.kt new file mode 100644 index 0000000..8bea085 --- /dev/null +++ b/test/src/main/java/ai/guiji/duix/test/ui/adapter/FolderBrowserAdapter.kt @@ -0,0 +1,42 @@ +package ai.guiji.duix.test.ui.adapter + +import ai.guiji.duix.test.databinding.ItemFolderEntryBinding +import android.view.LayoutInflater +import android.view.ViewGroup +import androidx.recyclerview.widget.RecyclerView +import java.io.File + +class FolderBrowserAdapter( + private val onFolderSelected: (File) -> Unit, +) : RecyclerView.Adapter() { + + private val items = mutableListOf() + + class ViewHolder(val binding: ItemFolderEntryBinding) : RecyclerView.ViewHolder(binding.root) + + override fun onCreateViewHolder(parent: ViewGroup, viewType: Int): ViewHolder { + val binding = ItemFolderEntryBinding.inflate( + LayoutInflater.from(parent.context), + parent, + false, + ) + return ViewHolder(binding) + } + + override fun onBindViewHolder(holder: ViewHolder, position: Int) { + val item = items[position] + holder.binding.tvFolderName.text = item.name.ifBlank { item.absolutePath } + holder.binding.tvFolderPath.text = item.absolutePath + holder.binding.root.setOnClickListener { + onFolderSelected(item) + } + } + + override fun getItemCount(): Int = items.size + + fun submitList(newItems: List) { + items.clear() + items.addAll(newItems) + notifyDataSetChanged() + } +} \ No newline at end of file diff --git a/test/src/main/java/ai/guiji/duix/test/video/DigitalHumanMp4Exporter.kt b/test/src/main/java/ai/guiji/duix/test/video/DigitalHumanMp4Exporter.kt new file mode 100644 index 0000000..4ef2e89 --- /dev/null +++ b/test/src/main/java/ai/guiji/duix/test/video/DigitalHumanMp4Exporter.kt @@ -0,0 +1,560 @@ +package ai.guiji.duix.test.video + +import ai.guiji.duix.sdk.client.bean.ImageFrame +import android.media.MediaCodec +import android.media.MediaCodecInfo +import android.media.MediaFormat +import android.media.MediaMuxer +import java.io.File +import java.nio.ByteBuffer +import java.util.concurrent.CountDownLatch +import java.util.concurrent.Executors +import java.util.concurrent.TimeUnit +import kotlin.math.max +import kotlin.math.min + +class DigitalHumanMp4Exporter( + private val outputFile: File, +) { + + data class ExportResult( + val file: File, + val durationMs: Long, + ) + + private data class QueuedAudio(val data: ByteArray) + + private data class FramePacket( + val raw: ByteArray, + val mask: ByteArray, + val width: Int, + val height: Int, + val timestampNs: Long, + ) + + private data class EncodedSample( + val data: ByteArray, + val info: MediaCodec.BufferInfo, + ) + + private val executor = Executors.newSingleThreadExecutor() + + private var startNs = -1L + private var initialized = false + private var closed = false + + private var videoWidth = 0 + private var videoHeight = 0 + private var videoColorFormat = MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Flexible + + private var muxer: MediaMuxer? = null + private var videoEncoder: MediaCodec? = null + private var audioEncoder: MediaCodec? = null + + private var videoTrackIndex = -1 + private var audioTrackIndex = -1 + private var muxerStarted = false + + private val pendingAudioChunks = ArrayList() + private val pendingVideoSamples = ArrayList() + private val pendingAudioSamples = ArrayList() + + private var lastFramePacket: FramePacket? = null + private var lastVideoPtsUs = -1L + private var audioBytesQueued = 0L + + fun recordAudioChunk(pcm24k: ByteArray, timestampNs: Long = System.nanoTime()) { + if (pcm24k.isEmpty() || closed) { + return + } + val copy = pcm24k.clone() + executor.execute { + if (closed) { + return@execute + } + ensureStarted(timestampNs) + if (!initialized) { + pendingAudioChunks.add(QueuedAudio(copy)) + } else { + queueAudioInternal(copy) + } + } + } + + fun recordFrame(imageFrame: ImageFrame, timestampNs: Long = System.nanoTime()) { + if (closed) { + return + } + val rawSize = imageFrame.width * imageFrame.height * RGB_CHANNELS + val rawCopy = copyBufferBytes(imageFrame.rawBuffer, rawSize) + val maskCopy = copyBufferBytes(imageFrame.maskBuffer, rawSize) + val packet = FramePacket(rawCopy, maskCopy, imageFrame.width, imageFrame.height, timestampNs) + executor.execute { + if (closed) { + return@execute + } + ensureStarted(timestampNs) + if (!initialized) { + initializeEncoders(packet.width, packet.height) + } + queueVideoInternal(packet) + lastFramePacket = packet + if (pendingAudioChunks.isNotEmpty()) { + flushPendingAudioChunks() + } + } + } + + fun finish(): ExportResult { + val latch = CountDownLatch(1) + var result: ExportResult? = null + var failure: Throwable? = null + + executor.execute { + try { + closed = true + if (!initialized) { + throw IllegalStateException("未收到数字人渲染帧，无法导出视频") + } + if (pendingAudioChunks.isNotEmpty()) { + flushPendingAudioChunks() + } + if (audioBytesQueued <= 0L) { + throw IllegalStateException("未收到模型音频响应") + } + + val audioDurationUs = calculateAudioDurationUs() + val lastFrame = lastFramePacket + if (lastFrame != null && audioDurationUs > lastVideoPtsUs + FRAME_PADDING_US) { + queueVideoInternal(lastFrame.copy(timestampNs = startNs + (audioDurationUs * 1000L))) + } + + signalAudioEndOfStream(audioDurationUs) + drainAudioEncoder(endOfStream = true) + + val finalVideoPtsUs = max(lastVideoPtsUs, audioDurationUs) + signalVideoEndOfStream(finalVideoPtsUs) + drainVideoEncoder(endOfStream = true) + + releaseInternal(deleteOutput = false) + result = ExportResult( + file = outputFile, + durationMs = (audioDurationUs / 1000L).coerceAtLeast(0L), + ) + } catch (throwable: Throwable) { + failure = throwable + releaseInternal(deleteOutput = true) + } finally { + latch.countDown() + } + } + + if (!latch.await(FINISH_TIMEOUT_SECONDS, TimeUnit.SECONDS)) { + executor.shutdownNow() + throw IllegalStateException("视频导出超时") + } + executor.shutdown() + failure?.let { throw it } + return result ?: throw IllegalStateException("视频导出失败") + } + + fun cancel() { + val latch = CountDownLatch(1) + executor.execute { + try { + closed = true + releaseInternal(deleteOutput = true) + } finally { + latch.countDown() + } + } + latch.await(CANCEL_TIMEOUT_SECONDS, TimeUnit.SECONDS) + executor.shutdownNow() + } + + private fun ensureStarted(timestampNs: Long) { + if (startNs < 0L) { + startNs = timestampNs + } + } + + private fun initializeEncoders(width: Int, height: Int) { + if (initialized) { + return + } + if (width <= 0 || height <= 0 || width % 2 != 0 || height % 2 != 0) { + throw IllegalArgumentException("渲染帧尺寸无效: ${width}x${height}") + } + + outputFile.parentFile?.mkdirs() + if (outputFile.exists()) { + outputFile.delete() + } + + videoWidth = width + videoHeight = height + muxer = MediaMuxer(outputFile.absolutePath, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4) + + videoEncoder = MediaCodec.createEncoderByType(VIDEO_MIME_TYPE).apply { + videoColorFormat = selectVideoColorFormat(codecInfo) + val format = MediaFormat.createVideoFormat(VIDEO_MIME_TYPE, width, height).apply { + setInteger(MediaFormat.KEY_COLOR_FORMAT, videoColorFormat) + setInteger(MediaFormat.KEY_BIT_RATE, max(width * height * VIDEO_BITRATE_FACTOR, MIN_VIDEO_BITRATE)) + setInteger(MediaFormat.KEY_FRAME_RATE, VIDEO_FRAME_RATE) + setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, VIDEO_I_FRAME_INTERVAL) + } + configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE) + start() + } + + audioEncoder = MediaCodec.createEncoderByType(AUDIO_MIME_TYPE).apply { + val format = MediaFormat.createAudioFormat(AUDIO_MIME_TYPE, AUDIO_SAMPLE_RATE, AUDIO_CHANNEL_COUNT).apply { + setInteger(MediaFormat.KEY_AAC_PROFILE, MediaCodecInfo.CodecProfileLevel.AACObjectLC) + setInteger(MediaFormat.KEY_BIT_RATE, AUDIO_BITRATE) + setInteger(MediaFormat.KEY_MAX_INPUT_SIZE, AUDIO_MAX_INPUT_SIZE) + } + configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE) + start() + } + + initialized = true + } + + private fun flushPendingAudioChunks() { + if (pendingAudioChunks.isEmpty()) { + return + } + val iterator = pendingAudioChunks.iterator() + while (iterator.hasNext()) { + queueAudioInternal(iterator.next().data) + iterator.remove() + } + } + + private fun queueAudioInternal(pcm24k: ByteArray) { + var offset = 0 + val encoder = audioEncoder ?: throw IllegalStateException("音频编码器尚未初始化") + while (offset < pcm24k.size) { + val inputIndex = dequeueInputBuffer(encoder) + val inputBuffer = encoder.getInputBuffer(inputIndex) + ?: throw IllegalStateException("音频输入缓冲区不可用") + inputBuffer.clear() + val chunkSize = min(inputBuffer.remaining(), pcm24k.size - offset) + inputBuffer.put(pcm24k, offset, chunkSize) + val ptsUs = audioBytesQueued * MICROS_PER_SECOND / AUDIO_BYTES_PER_SECOND + encoder.queueInputBuffer(inputIndex, 0, chunkSize, ptsUs, 0) + audioBytesQueued += chunkSize.toLong() + offset += chunkSize + drainAudioEncoder(endOfStream = false) + } + } + + private fun queueVideoInternal(packet: FramePacket) { + if (packet.width != videoWidth || packet.height != videoHeight) { + throw IllegalStateException("渲染帧尺寸发生变化，无法继续编码") + } + val encoder = videoEncoder ?: throw IllegalStateException("视频编码器尚未初始化") + val inputIndex = dequeueInputBuffer(encoder) + val inputBuffer = encoder.getInputBuffer(inputIndex) + ?: throw IllegalStateException("视频输入缓冲区不可用") + inputBuffer.clear() + + val yuv = convertFrameToYuv(packet.raw, packet.mask, packet.width, packet.height, videoColorFormat) + if (inputBuffer.remaining() < yuv.size) { + throw IllegalStateException("视频输入缓冲区容量不足") + } + + inputBuffer.put(yuv) + val ptsUs = computeVideoPresentationTimeUs(packet.timestampNs) + encoder.queueInputBuffer(inputIndex, 0, yuv.size, ptsUs, 0) + lastVideoPtsUs = ptsUs + drainVideoEncoder(endOfStream = false) + } + + private fun signalAudioEndOfStream(finalPtsUs: Long) { + val encoder = audioEncoder ?: return + val inputIndex = dequeueInputBuffer(encoder) + encoder.queueInputBuffer(inputIndex, 0, 0, finalPtsUs, MediaCodec.BUFFER_FLAG_END_OF_STREAM) + } + + private fun signalVideoEndOfStream(finalPtsUs: Long) { + val encoder = videoEncoder ?: return + val inputIndex = dequeueInputBuffer(encoder) + encoder.queueInputBuffer(inputIndex, 0, 0, finalPtsUs, MediaCodec.BUFFER_FLAG_END_OF_STREAM) + } + + private fun drainAudioEncoder(endOfStream: Boolean) { + drainEncoder( + encoder = audioEncoder, + pendingSamples = pendingAudioSamples, + getTrackIndex = { audioTrackIndex }, + setTrackIndex = { audioTrackIndex = it }, + endOfStream = endOfStream, + ) + } + + private fun drainVideoEncoder(endOfStream: Boolean) { + drainEncoder( + encoder = videoEncoder, + pendingSamples = pendingVideoSamples, + getTrackIndex = { videoTrackIndex }, + setTrackIndex = { videoTrackIndex = it }, + endOfStream = endOfStream, + ) + } + + private fun drainEncoder( + encoder: MediaCodec?, + pendingSamples: MutableList, + getTrackIndex: () -> Int, + setTrackIndex: (Int) -> Unit, + endOfStream: Boolean, + ) { + if (encoder == null) { + return + } + val bufferInfo = MediaCodec.BufferInfo() + while (true) { + val outputIndex = encoder.dequeueOutputBuffer(bufferInfo, TIMEOUT_US) + when { + outputIndex == MediaCodec.INFO_TRY_AGAIN_LATER -> { + if (!endOfStream) { + return + } + } + outputIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED -> { + val muxerInstance = muxer ?: throw IllegalStateException("复用器未初始化") + val trackIndex = muxerInstance.addTrack(encoder.outputFormat) + setTrackIndex(trackIndex) + startMuxerIfReady() + } + outputIndex >= 0 -> { + val outputBuffer = encoder.getOutputBuffer(outputIndex) + ?: throw IllegalStateException("编码器输出缓冲区不可用") + + if ((bufferInfo.flags and MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0) { + bufferInfo.size = 0 + } + + if (bufferInfo.size > 0) { + val trackIndex = getTrackIndex() + if (trackIndex < 0) { + throw IllegalStateException("编码轨道尚未创建") + } + val sample = copyEncodedSample(outputBuffer, bufferInfo) + if (muxerStarted) { + writeEncodedSample(trackIndex, sample) + } else { + pendingSamples.add(sample) + } + } + + val endReached = (bufferInfo.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0 + encoder.releaseOutputBuffer(outputIndex, false) + if (endReached) { + return + } + } + } + } + } + + private fun startMuxerIfReady() { + if (muxerStarted || videoTrackIndex < 0 || audioTrackIndex < 0) { + return + } + val muxerInstance = muxer ?: throw IllegalStateException("复用器未初始化") + muxerInstance.start() + muxerStarted = true + + flushPendingSamples(videoTrackIndex, pendingVideoSamples) + flushPendingSamples(audioTrackIndex, pendingAudioSamples) + } + + private fun flushPendingSamples(trackIndex: Int, pendingSamples: MutableList) { + if (pendingSamples.isEmpty()) { + return + } + pendingSamples.sortBy { it.info.presentationTimeUs } + val iterator = pendingSamples.iterator() + while (iterator.hasNext()) { + writeEncodedSample(trackIndex, iterator.next()) + iterator.remove() + } + } + + private fun writeEncodedSample(trackIndex: Int, sample: EncodedSample) { + val muxerInstance = muxer ?: throw IllegalStateException("复用器未初始化") + muxerInstance.writeSampleData(trackIndex, ByteBuffer.wrap(sample.data), sample.info) + } + + private fun copyEncodedSample(buffer: ByteBuffer, info: MediaCodec.BufferInfo): EncodedSample { + val data = ByteArray(info.size) + buffer.position(info.offset) + buffer.limit(info.offset + info.size) + buffer.get(data) + val copiedInfo = MediaCodec.BufferInfo().apply { + offset = 0 + size = data.size + presentationTimeUs = info.presentationTimeUs + flags = info.flags + } + return EncodedSample(data, copiedInfo) + } + + private fun computeVideoPresentationTimeUs(timestampNs: Long): Long { + val relativeUs = ((timestampNs - startNs) / 1_000L).coerceAtLeast(0L) + return if (lastVideoPtsUs < 0L) { + relativeUs + } else { + max(relativeUs, lastVideoPtsUs + 1L) + } + } + + private fun calculateAudioDurationUs(): Long { + return audioBytesQueued * MICROS_PER_SECOND / AUDIO_BYTES_PER_SECOND + } + + private fun dequeueInputBuffer(encoder: MediaCodec): Int { + while (true) { + val inputIndex = encoder.dequeueInputBuffer(TIMEOUT_US) + if (inputIndex >= 0) { + return inputIndex + } + if (inputIndex != MediaCodec.INFO_TRY_AGAIN_LATER) { + throw IllegalStateException("获取编码器输入缓冲区失败") + } + if (closed) { + throw IllegalStateException("编码任务已结束") + } + } + } + + private fun releaseInternal(deleteOutput: Boolean) { + runCatching { + if (muxerStarted) { + muxer?.stop() + } + } + runCatching { muxer?.release() } + runCatching { videoEncoder?.stop() } + runCatching { videoEncoder?.release() } + runCatching { audioEncoder?.stop() } + runCatching { audioEncoder?.release() } + + muxer = null + videoEncoder = null + audioEncoder = null + muxerStarted = false + initialized = false + pendingAudioChunks.clear() + pendingAudioSamples.clear() + pendingVideoSamples.clear() + + if (deleteOutput && outputFile.exists()) { + outputFile.delete() + } + } + + private fun selectVideoColorFormat(codecInfo: MediaCodecInfo): Int { + val capabilities = codecInfo.getCapabilitiesForType(VIDEO_MIME_TYPE) + val candidates = intArrayOf( + MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420SemiPlanar, + MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Planar, + MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Flexible, + ) + for (candidate in candidates) { + if (capabilities.colorFormats.contains(candidate)) { + return candidate + } + } + throw IllegalStateException("当前设备不支持可用的 YUV420 视频编码格式") + } + + private fun convertFrameToYuv( + raw: ByteArray, + mask: ByteArray, + width: Int, + height: Int, + colorFormat: Int, + ): ByteArray { + val frameSize = width * height + val output = ByteArray(frameSize + frameSize / 2) + val isSemiPlanar = colorFormat == MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420SemiPlanar + var yIndex = 0 + var uIndex = frameSize + var vIndex = if (isSemiPlanar) frameSize + 1 else frameSize + frameSize / 4 + + for (row in 0 until height) { + for (col in 0 until width) { + val pixelIndex = row * width + col + val rgbOffset = pixelIndex * RGB_CHANNELS + + val alpha = ((mask[rgbOffset].toInt() and 0xFF) + + (mask[rgbOffset + 1].toInt() and 0xFF) + + (mask[rgbOffset + 2].toInt() and 0xFF)) / RGB_CHANNELS + + val blue = ((raw[rgbOffset].toInt() and 0xFF) * alpha) / 255 + val green = ((raw[rgbOffset + 1].toInt() and 0xFF) * alpha) / 255 + val red = ((raw[rgbOffset + 2].toInt() and 0xFF) * alpha) / 255 + + val y = clampToByte(((66 * red + 129 * green + 25 * blue + 128) shr 8) + 16) + val u = clampToByte(((-38 * red - 74 * green + 112 * blue + 128) shr 8) + 128) + val v = clampToByte(((112 * red - 94 * green - 18 * blue + 128) shr 8) + 128) + + output[yIndex++] = y.toByte() + + if (row % 2 == 0 && col % 2 == 0) { + if (isSemiPlanar) { + output[uIndex] = u.toByte() + output[vIndex] = v.toByte() + uIndex += 2 + vIndex += 2 + } else { + output[uIndex++] = u.toByte() + output[vIndex++] = v.toByte() + } + } + } + } + return output + } + + private fun clampToByte(value: Int): Int { + return value.coerceIn(0, 255) + } + + private fun copyBufferBytes(buffer: ByteBuffer, size: Int): ByteArray { + if (buffer.hasArray()) { + return buffer.array().copyOf(size) + } + val duplicate = buffer.duplicate() + duplicate.position(0) + val output = ByteArray(size) + duplicate.get(output, 0, min(size, duplicate.remaining())) + return output + } + + companion object { + private const val VIDEO_MIME_TYPE = "video/avc" + private const val AUDIO_MIME_TYPE = "audio/mp4a-latm" + + private const val AUDIO_SAMPLE_RATE = 24_000 + private const val AUDIO_CHANNEL_COUNT = 1 + private const val AUDIO_BYTES_PER_SECOND = AUDIO_SAMPLE_RATE * AUDIO_CHANNEL_COUNT * 2L + private const val AUDIO_BITRATE = 64_000 + private const val AUDIO_MAX_INPUT_SIZE = 48_000 + + private const val VIDEO_FRAME_RATE = 24 + private const val VIDEO_I_FRAME_INTERVAL = 1 + private const val VIDEO_BITRATE_FACTOR = 6 + private const val MIN_VIDEO_BITRATE = 1_200_000 + + private const val RGB_CHANNELS = 3 + private const val FRAME_PADDING_US = 80_000L + private const val MICROS_PER_SECOND = 1_000_000L + private const val TIMEOUT_US = 10_000L + private const val FINISH_TIMEOUT_SECONDS = 60L + private const val CANCEL_TIMEOUT_SECONDS = 5L + } +} \ No newline at end of file diff --git a/test/src/main/res/layout/activity_digital_human_test.xml b/test/src/main/res/layout/activity_digital_human_test.xml new file mode 100644 index 0000000..e72c18e --- /dev/null +++ b/test/src/main/res/layout/activity_digital_human_test.xml @@ -0,0 +1,258 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/test/src/main/res/layout/activity_folder_picker.xml b/test/src/main/res/layout/activity_folder_picker.xml new file mode 100644 index 0000000..5b6ff0d --- /dev/null +++ b/test/src/main/res/layout/activity_folder_picker.xml @@ -0,0 +1,132 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/test/src/main/res/layout/activity_settings.xml b/test/src/main/res/layout/activity_settings.xml index 5ab5d40..ea1e36b 100644 --- a/test/src/main/res/layout/activity_settings.xml +++ b/test/src/main/res/layout/activity_settings.xml @@ -134,6 +134,51 @@ android:text="" /> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/test/src/main/res/layout/item_folder_entry.xml b/test/src/main/res/layout/item_folder_entry.xml new file mode 100644 index 0000000..0dc5fe4 --- /dev/null +++ b/test/src/main/res/layout/item_folder_entry.xml @@ -0,0 +1,41 @@ + + + + + + + + + + + + \ No newline at end of file diff --git a/test/src/main/res/values-zh-rCN/strings.xml b/test/src/main/res/values-zh-rCN/strings.xml index ab223c1..899c66c 100644 --- a/test/src/main/res/values-zh-rCN/strings.xml +++ b/test/src/main/res/values-zh-rCN/strings.xml @@ -82,10 +82,60 @@ 打开设备二维码页面后，家里人扫一扫就能把这台设备绑定到自己的账号。 显示调试详情 开启后，聊天页面会显示完整的交互日志，包括工具调用的输入和输出。 + 数字人测试 + 选择一个文件夹，批量把里面的音频送给 Omni，关闭 search 和 tooluse，并按原目录结构导出数字人 mp4。 关于这台设备 应用版本 家属面板域名 当前设备 UUID + 数字人测试 + 选择一个文件夹后，应用会递归处理其中的音频，关闭 search 和 tools，并按原目录结构批量导出数字人回应 mp4。 + 为 %1$s 选好一个文件夹后，我会按顺序处理里面的音频，并批量导出完整 mp4。 + 正在扫描所选文件夹里的音频… + 正在为 %1$s 准备测试资源… + 正在让 %1$s 准备好批量处理… + 正在解析您选好的音频… + 正在连接 Omni… + 正在把音频送给 Omni… + Omni 正在回应，数字人也在跟着这段声音做同步反应。 + 回应已经完成，正在整理并封装最终 mp4… + 当前这条完整 mp4 已经保存好了。 + 已准备好 %1$d 条待处理任务，点击开始后会按顺序处理。 + 正在处理第 %1$d/%2$d 条：%3$s + 批处理完成，成功 %1$d 条，跳过 %2$d 条，失败 %3$d 条。 + 正在按原目录结构保存结果… + 当前任务失败，正在重试 %3$s（%1$d/%2$d）… + %1$s 会在这里依次回应当前队列里的音频。 + %1$s 正在准备当前这条任务… + 选择输入文件夹 + 还没有选文件夹 + 直接选择一个文件夹，应用会递归扫描其中的常见音频文件，并按相同目录层级写回 mp4。 + 已扫描 %1$d 段音频，结果会保存到 %2$s/%3$s 下，并保留原目录结构。 + 选择文件夹 + 开始批量生成 + 批处理中… + 请先选择一个文件夹 + 目标 mp4 都已存在，当前没有需要处理的任务。 + 当前输入识别 + 暂时还没有识别内容 + 处理进度 + 总计 %1$d 条，已完成 %2$d 条，已跳过 %3$d 条，失败 %4$d 条。 + 点击开始后，这里会按顺序显示每条音频的处理进度。 + 等待处理 + 解析音频 + 连接 Omni + 发送音频 + 生成响应 + 封装 mp4 + 保存结果 + 重试中（%1$d/%2$d） + 已完成 + 失败 + 已跳过 + 输出：%1$s + 所选文件夹里没有找到可处理的音频文件 + 这次 Omni 没有返回可用的响应音频 + 测试连接中途断开了 Github地址可能下载失败，您可以考虑使用代理或者将文件缓存到自己的存储服务 调试信息 diff --git a/test/src/main/res/values/colors.xml b/test/src/main/res/values/colors.xml index 47b74d5..75d152b 100644 --- a/test/src/main/res/values/colors.xml +++ b/test/src/main/res/values/colors.xml @@ -25,4 +25,9 @@ #FFF0E0D2 #FFF8F1EB #FFD7B39B + #FF9C7C6E + #FFC16845 + #FF5C8C5A + #FFB85C4A + #FF7B8A94 \ No newline at end of file diff --git a/test/src/main/res/values/strings.xml b/test/src/main/res/values/strings.xml index fef24e2..954d432 100644 --- a/test/src/main/res/values/strings.xml +++ b/test/src/main/res/values/strings.xml @@ -84,10 +84,69 @@ Open the QR code page for your family to scan and bind this elder device. Show debug details When enabled, the chat page will show full interaction logs, including tool input and output. + Digital Human Test + Choose a local folder, batch-send the audio files inside to Omni, and export avatar MP4s while keeping the original directory structure. About this device App version Family panel domain Current device UUID + Digital Human Test + Choose a local folder, recursively process its audio files without search or tools, and export avatar MP4s while preserving the original directory structure. + Choose a local folder for %1$s, then I will process the audio files one by one and export complete MP4 responses in batch. + Scanning the selected folder for audio files… + Preparing %1$s for the test… + Loading %1$s for the batch… + Decoding the selected audio… + Connecting to Omni… + Sending the selected audio to Omni… + Omni is answering. The avatar is following the response audio now. + The answer is ready. Packing the final MP4… + The current MP4 has been saved. + %1$d jobs are ready to run. Tap start to process them one by one. + Processing %1$d/%2$d: %3$s + Batch completed. %1$d succeeded, %2$d were skipped, and %3$d failed. + Saving the result with the original folder structure… + Retrying %3$s (%1$d/%2$d)… + %1$s will respond to the queued audio files here. + Getting %1$s ready for the current job… + Choose an input folder + No folder selected yet + Choose a local folder directly. On Android 11 and above, the app will first ask for file management access so it can avoid the system folder picker restrictions. + Found %1$d audio files. Results will be saved under %2$s/%3$s while keeping the original structure. + Choose Folder + Start Batch Export + Processing… + Choose a folder first + All target MP4 files already exist. There is nothing to process right now. + Allow file management access first, then choose the input folder. + File management access is still disabled, so the app cannot browse local folders yet. + Storage permission is required to browse local folders on this Android version. + Current recognized input + No recognized transcript yet + Progress + %1$d total, %2$d completed, %3$d skipped, %4$d failed. + After the batch starts, each audio job will appear here in order. + Queued + Decoding audio + Connecting to Omni + Uploading audio + Generating response + Packing MP4 + Saving result + Retrying (%1$d/%2$d) + Completed + Failed + Skipped + Output: %1$s + No supported audio files were found in the selected folder + Omni did not return response audio this time + The test connection was interrupted + Choose Folder + Browse the device storage directly, enter the target folder, and then use the current folder as the batch input root. + Use Current Folder + Up One Level + Subfolders + There are no readable subfolders here. You can still use the current folder. The GitHub url may fail to download, you can consider using a proxy or caching the file to your own storage service Debug message \ No newline at end of file