From 76f1d47da5bdd55213f2b9b1c147e2bc5e4da79c Mon Sep 17 00:00:00 2001 From: Henoch Minich <301610408+minichhenoch-sir@users.noreply.github.com> Date: Tue, 20 Oct 2026 14:12:59 -0500 Subject: [PATCH] feat(chat): rework playback of voice messages and audio files Audio messages were downloaded completely before playback, their state was mirrored into the chat messages and reset by list updates, taps on another message were ignored while one was playing, the speed button had no effect, and playback ignored audio focus and unplugged headphones. VoiceMessageMediaService now streams audio messages through the app's HTTP client into a size-limited disk cache and is the single source of truth for chat audio. It handles audio focus, becoming noisy, wake locks and retries after network errors, and exposes playback to the notification, the lock screen and headsets. Voice messages continue with the newer voice messages; audio files are played among the audio files of the conversation and can be shuffled and repeated. The service only accepts media items from Talk itself, and account credentials are only sent to the WebDAV files of that account. The chat and the conversation list observe the playback through ChatAudioPlayer, a MediaController that is connected while the screen is started. Tapping another message switches playback, seeking happens once a drag ends, and a player bar shows what is playing, also for another conversation. Tapping the bar opens a full player with seeking, previous and next, speed, shuffle, repeat and the list of queued items, scrolled to the current one. Speeds are kept per account and separately for voice messages and audio files, and are shared between screens. Opening the playing message, from the notification, the player bar or another chat, switches to its account and opens the thread it was played in. ChatAudioStore keeps durations, waveforms and the position of long recordings. Waveforms are computed from the cached file once the player has loaded it, so the file is not downloaded twice. Playback starts from the persisted position once the metadata is loaded. When an account is removed, by the user or by a remote wipe, playback of its audio is stopped and unloaded, and its metadata is deleted from memory and from the metadata cache. Assisted-by: Claude Code:claude-opus-5-5 Co-Authored-By: Claude Opus 5.5 Co-authored-by: Marcel Hibbe Co-authored-by: rapterjet2004 Signed-off-by: Henoch Minich <301610408+minichhenoch-sir@users.noreply.github.com> Signed-off-by: Marcel Hibbe Signed-off-by: rapterjet2004 --- app/build.gradle.kts | 2 + .../com/nextcloud/talk/chat/ChatActivity.kt | 428 +++++------------- .../talk/chat/audio/ChatAudioCredentials.kt | 26 ++ .../talk/chat/audio/ChatAudioDataSources.kt | 78 ++++ .../talk/chat/audio/ChatAudioIntents.kt | 32 ++ .../nextcloud/talk/chat/audio/ChatAudioKey.kt | 51 +++ .../talk/chat/audio/ChatAudioMetadata.kt | 74 +++ .../talk/chat/audio/ChatAudioPlaybackState.kt | 104 +++++ .../talk/chat/audio/ChatAudioPlayer.kt | 337 ++++++++++++++ .../talk/chat/audio/ChatAudioQueue.kt | 39 ++ .../talk/chat/audio/ChatAudioSpeeds.kt | 53 +++ .../talk/chat/audio/ChatAudioStore.kt | 195 ++++++++ .../talk/chat/audio/ChatAudioTrack.kt | 142 ++++++ .../chat/audio/VoiceMessageWaveformLoader.kt | 95 ++++ .../nextcloud/talk/chat/audio/Waveforms.kt | 61 +++ .../talk/chat/data/io/MediaPlayerManager.kt | 208 +-------- .../chat/data/io/VoiceMessageMediaService.kt | 325 ++++++++++++- .../talk/chat/data/model/ChatMessage.kt | 18 - .../talk/chat/ui/model/ChatMessageUi.kt | 46 +- .../talk/chat/viewmodels/ChatViewModel.kt | 109 ----- .../chat/viewmodels/MessageInputViewModel.kt | 2 +- .../ConversationsListActivity.kt | 45 +- .../ui/ConversationsListScreen.kt | 4 +- .../talk/dagger/modules/ManagerModule.kt | 6 +- .../talk/jobs/AccountRemovalWorker.java | 4 + .../talk/ui/BackgroundVoiceMessageCard.kt | 255 ----------- .../talk/ui/ComposeWaveformSeekbar.kt | 7 +- .../com/nextcloud/talk/ui/WaveformSeekBar.kt | 4 +- .../talk/ui/chat/AudioFileMessage.kt | 86 ++-- .../com/nextcloud/talk/ui/chat/ChatAudio.kt | 149 ++++++ .../talk/ui/chat/ChatAudioPlayerBar.kt | 150 ++++++ .../talk/ui/chat/ChatAudioPlayerHost.kt | 95 ++++ .../talk/ui/chat/ChatAudioPlayerSheet.kt | 376 +++++++++++++++ .../nextcloud/talk/ui/chat/ChatMessageView.kt | 49 +- .../com/nextcloud/talk/ui/chat/ChatView.kt | 2 +- .../nextcloud/talk/ui/chat/VoiceMessage.kt | 111 ++--- .../com/nextcloud/talk/utils/AudioUtils.kt | 301 +++++------- .../utils/preferences/AppPreferences.java | 16 +- .../utils/preferences/AppPreferencesImpl.kt | 63 ++- app/src/main/res/layout/activity_chat.xml | 5 + app/src/main/res/values/strings.xml | 17 + .../chat/audio/ChatAudioCredentialsTest.kt | 82 ++++ .../talk/chat/audio/ChatAudioKeyTest.kt | 40 ++ .../talk/chat/audio/ChatAudioMetadataTest.kt | 53 +++ .../chat/audio/ChatAudioPlaybackStateTest.kt | 79 ++++ .../talk/chat/audio/ChatAudioQueueTest.kt | 72 +++ .../chat/audio/ChatAudioRepeatModeTest.kt | 28 ++ .../talk/chat/audio/ChatAudioStoreTest.kt | 145 ++++++ .../talk/chat/audio/ChatAudioTrackTest.kt | 137 ++++++ .../talk/chat/audio/WaveformsTest.kt | 59 +++ 50 files changed, 3560 insertions(+), 1305 deletions(-) create mode 100644 app/src/main/java/com/nextcloud/talk/chat/audio/ChatAudioCredentials.kt create mode 100644 app/src/main/java/com/nextcloud/talk/chat/audio/ChatAudioDataSources.kt create mode 100644 app/src/main/java/com/nextcloud/talk/chat/audio/ChatAudioIntents.kt create mode 100644 app/src/main/java/com/nextcloud/talk/chat/audio/ChatAudioKey.kt create mode 100644 app/src/main/java/com/nextcloud/talk/chat/audio/ChatAudioMetadata.kt create mode 100644 app/src/main/java/com/nextcloud/talk/chat/audio/ChatAudioPlaybackState.kt create mode 100644 app/src/main/java/com/nextcloud/talk/chat/audio/ChatAudioPlayer.kt create mode 100644 app/src/main/java/com/nextcloud/talk/chat/audio/ChatAudioQueue.kt create mode 100644 app/src/main/java/com/nextcloud/talk/chat/audio/ChatAudioSpeeds.kt create mode 100644 app/src/main/java/com/nextcloud/talk/chat/audio/ChatAudioStore.kt create mode 100644 app/src/main/java/com/nextcloud/talk/chat/audio/ChatAudioTrack.kt create mode 100644 app/src/main/java/com/nextcloud/talk/chat/audio/VoiceMessageWaveformLoader.kt create mode 100644 app/src/main/java/com/nextcloud/talk/chat/audio/Waveforms.kt delete mode 100644 app/src/main/java/com/nextcloud/talk/ui/BackgroundVoiceMessageCard.kt create mode 100644 app/src/main/java/com/nextcloud/talk/ui/chat/ChatAudio.kt create mode 100644 app/src/main/java/com/nextcloud/talk/ui/chat/ChatAudioPlayerBar.kt create mode 100644 app/src/main/java/com/nextcloud/talk/ui/chat/ChatAudioPlayerHost.kt create mode 100644 app/src/main/java/com/nextcloud/talk/ui/chat/ChatAudioPlayerSheet.kt create mode 100644 app/src/test/java/com/nextcloud/talk/chat/audio/ChatAudioCredentialsTest.kt create mode 100644 app/src/test/java/com/nextcloud/talk/chat/audio/ChatAudioKeyTest.kt create mode 100644 app/src/test/java/com/nextcloud/talk/chat/audio/ChatAudioMetadataTest.kt create mode 100644 app/src/test/java/com/nextcloud/talk/chat/audio/ChatAudioPlaybackStateTest.kt create mode 100644 app/src/test/java/com/nextcloud/talk/chat/audio/ChatAudioQueueTest.kt create mode 100644 app/src/test/java/com/nextcloud/talk/chat/audio/ChatAudioRepeatModeTest.kt create mode 100644 app/src/test/java/com/nextcloud/talk/chat/audio/ChatAudioStoreTest.kt create mode 100644 app/src/test/java/com/nextcloud/talk/chat/audio/ChatAudioTrackTest.kt create mode 100644 app/src/test/java/com/nextcloud/talk/chat/audio/WaveformsTest.kt diff --git a/app/build.gradle.kts b/app/build.gradle.kts index 561b0fa0404..c096999b238 100644 --- a/app/build.gradle.kts +++ b/app/build.gradle.kts @@ -298,6 +298,8 @@ dependencies { implementation("com.google.code.gson:gson:2.14.0") implementation("androidx.media3:media3-exoplayer:$media3Version") + implementation("androidx.media3:media3-datasource-okhttp:$media3Version") + implementation("androidx.media3:media3-database:$media3Version") implementation("androidx.media3:media3-ui:$media3Version") implementation("androidx.media3:media3-transformer:$media3Version") implementation("androidx.media3:media3-effect:$media3Version") diff --git a/app/src/main/java/com/nextcloud/talk/chat/ChatActivity.kt b/app/src/main/java/com/nextcloud/talk/chat/ChatActivity.kt index 7c6b0a0f0f6..3f915a1f830 100644 --- a/app/src/main/java/com/nextcloud/talk/chat/ChatActivity.kt +++ b/app/src/main/java/com/nextcloud/talk/chat/ChatActivity.kt @@ -18,13 +18,11 @@ import android.Manifest import android.annotation.SuppressLint import android.content.ClipData import android.content.ClipboardManager -import android.content.ComponentName import android.content.Context import android.content.Intent import android.content.pm.PackageManager import android.content.res.AssetFileDescriptor import android.database.Cursor -import android.media.MediaMetadataRetriever import android.net.Uri import android.os.Build import android.os.Bundle @@ -68,7 +66,6 @@ import androidx.compose.runtime.remember import androidx.compose.runtime.rememberCoroutineScope import androidx.compose.runtime.setValue import androidx.compose.ui.platform.LocalDensity -import androidx.core.content.ContextCompat import androidx.core.content.FileProvider import androidx.core.content.PermissionChecker import androidx.core.content.PermissionChecker.PERMISSION_GRANTED @@ -82,11 +79,6 @@ import androidx.fragment.app.commit import androidx.lifecycle.ViewModelProvider import androidx.lifecycle.compose.collectAsStateWithLifecycle import androidx.lifecycle.lifecycleScope -import androidx.media3.common.MediaItem -import androidx.media3.common.MediaMetadata -import androidx.media3.common.Player -import androidx.media3.session.MediaController -import androidx.media3.session.SessionToken import androidx.recyclerview.widget.LinearLayoutManager import androidx.work.Data import androidx.work.OneTimeWorkRequest @@ -97,7 +89,6 @@ import androidx.work.WorkManager import autodagger.AutoInjector import com.google.android.material.dialog.MaterialAlertDialogBuilder import com.google.android.material.snackbar.Snackbar -import com.google.common.util.concurrent.ListenableFuture import com.nextcloud.android.common.ui.color.ColorUtil import com.nextcloud.talk.BuildConfig import com.nextcloud.talk.R @@ -108,7 +99,15 @@ import com.nextcloud.talk.api.NcApi import com.nextcloud.talk.api.NcApiCoroutines import com.nextcloud.talk.application.NextcloudTalkApplication import com.nextcloud.talk.attachmentpreview.FileAttachmentPreviewFragment -import com.nextcloud.talk.chat.data.io.VoiceMessageMediaService +import com.nextcloud.talk.chat.audio.ChatAudioError +import com.nextcloud.talk.chat.audio.ChatAudioKey +import com.nextcloud.talk.chat.audio.ChatAudioKind +import com.nextcloud.talk.chat.audio.ChatAudioPlayer +import com.nextcloud.talk.chat.audio.ChatAudioQueue +import com.nextcloud.talk.chat.audio.ChatAudioSpeeds +import com.nextcloud.talk.chat.audio.ChatAudioStore +import com.nextcloud.talk.chat.audio.openMessageIntent +import com.nextcloud.talk.chat.audio.toChatAudioTrack import com.nextcloud.talk.chat.data.model.ChatMessage import com.nextcloud.talk.chat.data.model.FileParameters import com.nextcloud.talk.chat.ui.ChatEmptyState @@ -123,7 +122,6 @@ import com.nextcloud.talk.chat.ui.ShowReactionsModalBottomSheet import com.nextcloud.talk.chat.ui.TempMessageActionsBottomSheet import com.nextcloud.talk.chat.ui.TypingIndicatorBanner import com.nextcloud.talk.chat.ui.buildMessageActionsState -import com.nextcloud.talk.chat.ui.model.MessageTypeContent import com.nextcloud.talk.chat.viewmodels.ChatViewModel import com.nextcloud.talk.chat.viewmodels.MessageInputViewModel import com.nextcloud.talk.conversationinfo.ConversationInfoActivity @@ -165,12 +163,16 @@ import com.nextcloud.talk.ui.OutOfOfficeViewData import com.nextcloud.talk.ui.PinnedMessageView import com.nextcloud.talk.ui.PlaybackSpeed import com.nextcloud.talk.ui.UpcomingEventView +import com.nextcloud.talk.ui.chat.ChatAudioPlayerHost +import com.nextcloud.talk.ui.chat.ChatAudioUi import com.nextcloud.talk.ui.chat.ChatMessageCallbacks import com.nextcloud.talk.ui.chat.ChatView import com.nextcloud.talk.ui.chat.ChatViewCallbacks import com.nextcloud.talk.ui.chat.ChatViewState +import com.nextcloud.talk.ui.chat.LocalChatAudioUi import com.nextcloud.talk.ui.chat.LocalUploadProgressProvider import com.nextcloud.talk.ui.chat.LocalUploadedLocalPreviewProvider +import com.nextcloud.talk.ui.chat.chatAudioPlayerCallbacks import com.nextcloud.talk.ui.dialog.DateTimeCompose import com.nextcloud.talk.ui.dialog.GetPinnedOptionsDialog import com.nextcloud.talk.ui.dialog.SaveToStorageDialogFragment @@ -178,7 +180,6 @@ import com.nextcloud.talk.ui.theme.LocalMessageUtils import com.nextcloud.talk.ui.theme.LocalOpenGraphFetcher import com.nextcloud.talk.ui.theme.LocalViewThemeUtils import com.nextcloud.talk.utils.ApiUtils -import com.nextcloud.talk.utils.AudioUtils import com.nextcloud.talk.utils.CapabilitiesUtil import com.nextcloud.talk.utils.CapabilitiesUtil.hasSpreedFeatureCapability import com.nextcloud.talk.utils.CapabilitiesUtil.retentionOfClassifiedRoom @@ -225,7 +226,6 @@ import io.reactivex.android.schedulers.AndroidSchedulers import io.reactivex.disposables.Disposable import io.reactivex.schedulers.Schedulers import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.FlowPreview import kotlinx.coroutines.Job import kotlinx.coroutines.delay @@ -236,11 +236,10 @@ import kotlinx.coroutines.flow.collect import kotlinx.coroutines.flow.collectLatest import kotlinx.coroutines.flow.distinctUntilChanged import kotlinx.coroutines.flow.first +import kotlinx.coroutines.flow.launchIn import kotlinx.coroutines.flow.map import kotlinx.coroutines.flow.onEach -import kotlinx.coroutines.isActive import kotlinx.coroutines.launch -import kotlinx.coroutines.withContext import org.greenrobot.eventbus.Subscribe import org.greenrobot.eventbus.ThreadMode import java.io.File @@ -253,7 +252,6 @@ import java.time.ZonedDateTime import java.util.Date import java.util.Locale import java.util.UUID -import java.util.concurrent.CancellationException import java.util.concurrent.ExecutionException import javax.inject.Inject import kotlin.math.abs @@ -289,6 +287,12 @@ class ChatActivity : @Inject lateinit var networkMonitor: NetworkMonitor + @Inject + lateinit var chatAudioStore: ChatAudioStore + + @Inject + lateinit var chatAudioSpeeds: ChatAudioSpeeds + @Inject lateinit var chatViewModelFactory: ChatViewModel.ChatViewModelFactory @@ -318,8 +322,17 @@ class ChatActivity : mutableStateOf(ChatViewModel.UpcomingEventUIState.None) private val overflowContainerHeightPx = mutableIntStateOf(0) - private var mediaControllerFuture: ListenableFuture? = null - private var mediaController: MediaController? = null + private lateinit var chatAudioPlayer: ChatAudioPlayer + private val voiceMessageSpeed = MutableStateFlow(PlaybackSpeed.NORMAL) + private val chatAudioUi by lazy { + ChatAudioUi( + playbackState = chatAudioPlayer.state, + metadata = chatAudioStore.metadata, + voiceSpeed = voiceMessageSpeed, + internalUserId = { if (::initialUser.isInitialized) conversationUser.id else null }, + loadMetadata = { chatAudioStore.load(it) } + ) + } private val startSelectContactForResult = registerForActivityResult( ActivityResultContracts @@ -431,76 +444,6 @@ class ChatActivity : private lateinit var pickMultipleMedia: ActivityResultLauncher - private var progressJob: Job? = null - - private fun updateSeekbarUi() { - mediaController?.let { controller -> - val currentPosition = controller.currentPosition - val duration = controller.duration.takeIf { it > 0 } ?: 1 - - val progress = (currentPosition.toFloat() / duration) * FLOAT_100 - val secondsPlayed = (currentPosition / MILLIS_1000) - - val msg = chatViewModel.currentVoiceMessage?.apply { - voiceMessageSeekbarProgress = kotlin.math.ceil(progress).toInt() - voiceMessagePlayedSeconds = secondsPlayed.toInt() - if (controller.duration > 0) { - voiceMessageDuration = (controller.duration / MILLIS_1000).toInt() - } - } - - val currentMediaId = controller.currentMediaItem?.mediaId - if (currentMediaId != null && msg != null) { - chatViewModel.syncVoiceMessageUiState(msg) - } - } - } - - private fun startProgressPolling() { - progressJob?.cancel() - progressJob = lifecycleScope.launch { - while (isActive) { - updateSeekbarUi() - // Poll every 150ms for smooth seekbar updates - delay(MILLIS_150) - } - } - } - - private fun stopProgressPolling() { - progressJob?.cancel() - progressJob = null - } - - private val playerListener = object : Player.Listener { - override fun onIsPlayingChanged(isPlaying: Boolean) { - if (isPlaying) { - startProgressPolling() - } else { - stopProgressPolling() - updateSeekbarUi() - } - - chatViewModel.currentVoiceMessage?.apply { - isPlayingVoiceMessage = isPlaying - }?.let { chatViewModel.syncVoiceMessageUiState(it) } - } - - // Catches instant changes (e.g., manual seeks, skipping to the next track) - override fun onPositionDiscontinuity( - oldPosition: Player.PositionInfo, - newPosition: Player.PositionInfo, - reason: Int - ) { - updateSeekbarUi() - } - - override fun onMediaItemTransition(mediaItem: MediaItem?, reason: Int) { - // Ensures the UI resets immediately when transitioning to the next voice message - updateSeekbarUi() - } - } - private val onBackPressedCallback = object : OnBackPressedCallback(true) { override fun handleOnBackPressed() { if (chatViewModel.chatMode.value == ChatViewModel.ChatMode.SEARCH_MODE) { @@ -604,6 +547,9 @@ class ChatActivity : // and registerForActivityResult() must be called before the activity is started. initialUser = setUpBoundUserOrFinish() ?: return + chatAudioPlayer = ChatAudioPlayer(this, chatAudioStore, lifecycleScope) + lifecycle.addObserver(chatAudioPlayer) + binding = ActivityChatBinding.inflate(layoutInflater) setContentView(binding.root) keepContentOutOfDisplayCutout() @@ -642,12 +588,20 @@ class ChatActivity : setPinnedMessageContent() + setAudioPlayerBarContent() + setUpcomingEventContent() binding.chatOverflowContainer.viewTreeObserver.addOnGlobalLayoutListener { overflowContainerHeightPx.intValue = binding.chatOverflowContainer.height } + initialUser.userId?.let { + voiceMessageSpeed.value = chatAudioSpeeds.speedFor(it, ChatAudioKind.VOICE_MESSAGE) + chatAudioSpeeds.speedFlow(it, ChatAudioKind.VOICE_MESSAGE) + .onEach { speed -> voiceMessageSpeed.value = speed } + .launchIn(lifecycleScope) + } handleIntent(intent) chatViewModel.initData() if (sharedText.isNotEmpty()) { @@ -905,7 +859,8 @@ class ChatActivity : LocalMessageUtils provides messageUtils, LocalOpenGraphFetcher provides { url -> chatViewModel.fetchOpenGraph(url) }, LocalUploadProgressProvider provides { refId -> uploadProgressMap[refId] }, - LocalUploadedLocalPreviewProvider provides { refId -> uploadedLocalPreviewMap[refId] } + LocalUploadedLocalPreviewProvider provides { refId -> uploadedLocalPreviewMap[refId] }, + LocalChatAudioUi provides chatAudioUi ) { val isOneToOneConversation = uiState.isOneToOneConversation @@ -948,16 +903,9 @@ class ChatActivity : onSwipeReply = { handleSwipeToReply(it) }, onFileClick = { downloadAndOpenFile(it, openWhenDownloadState, downloadingFileState) }, onPollClick = { pollId, pollName -> openPollDialog(pollId, pollName) }, - onVoicePlayPauseClick = { onVoiceClick(it) }, - onVoiceSeek = { id, progress -> - mediaController?.let { controller -> - if (id.toString() == controller.currentMediaItem?.mediaId) { - val pos = controller.duration * progress / 100f - controller.seekTo(pos.toLong()) - } - } - }, - onVoiceSpeedClick = { onVoiceSpeedClickCompose(it) }, + onVoicePlayPauseClick = { onAudioMessageClick(it) }, + onVoiceSeek = { messageId, progress -> onAudioMessageSeek(messageId, progress) }, + onVoiceSpeedClick = { onVoiceMessageSpeedClick() }, onReactionClick = { messageId, emoji -> handleReactionClick(messageId, emoji) }, onReactionLongClick = { messageId -> openReactionsDialog(messageId) }, onOpenThreadClick = { messageId -> openThread(messageId.toLong()) }, @@ -1084,138 +1032,90 @@ class ChatActivity : } } - @Suppress("ReturnCount", "CyclomaticComplexMethod") - private fun onVoiceClick(messageId: Int) { - fun getAudioDuration(audioFilePath: String): Long { - val retriever = MediaMetadataRetriever() - - return try { - retriever.setDataSource(audioFilePath) - val durationString = retriever.extractMetadata(MediaMetadataRetriever.METADATA_KEY_DURATION) - val durationLong = durationString?.toLong() ?: 0L - - durationLong / ONE_SECOND_IN_MILLIS - } catch (e: IllegalArgumentException) { - logger.e(TAG, "Failed to read audio duration for $audioFilePath", e) - 0L - } finally { - retriever.release() - } + private fun onAudioMessageClick(messageId: Int) { + if (!::initialUser.isInitialized) { + return } - - fun setupAndPlay(controller: MediaController, message: ChatMessage, file: String) { - val avatarUrl = chatViewModel.getAvatarUrl(message) - - val artist = if (message.isVoiceMessage) { - "Voice Message" - } else { - message.fileParameters.name - } - val metadata = MediaMetadata.Builder() - .setTitle(message.actorDisplayName) - .setArtist(artist) - .setArtworkUri(avatarUrl.toUri()) - .build() - - val mediaItem = MediaItem.Builder() - .setMediaId(message.jsonMessageId.toString()) - .setMediaMetadata(metadata) - .setUri(file) - .build() - - controller.setMediaItem(mediaItem) - - controller.prepare() - controller.play() + val voiceMessageLabel = getString(R.string.nc_voice_message) + val tracks = chatViewModel.uiState.value.items + .asReversed() + .mapNotNull { (it as? ChatViewModel.ChatItem.MessageItem)?.uiMessage } + .mapNotNull { it.toChatAudioTrack(conversationUser, voiceMessageLabel, openThreadId()) } + val queue = ChatAudioQueue.build(tracks, messageId, { it.key.messageId }, { it.kind }) + if (queue == null) { + showAudioError(ChatAudioError.UNAVAILABLE) + } else { + val speed = speedFor(queue.items[queue.startIndex].kind) + chatAudioPlayer.playOrPause(queue.items, queue.startIndex, speed.value) } + } - fun setUpWaveform(message: ChatMessage, file: File) { - if (message.voiceMessageFloatArray != null) return - - val filename = message.fileParameters.name - message.isDownloadingVoiceMessage = true - - chatViewModel.syncVoiceMessageUiState(message) - - CoroutineScope(Dispatchers.Default).launch { - val waveform = AudioUtils.audioFileToFloatArray(file) - appPreferences.saveWaveFormForFile(filename, waveform.toTypedArray()) - message.voiceMessageFloatArray = waveform - - withContext(Dispatchers.Main) { - message.isDownloadingVoiceMessage = false - chatViewModel.syncVoiceMessageUiState(message) - } - } + private fun speedFor(kind: ChatAudioKind): PlaybackSpeed = + if (kind == ChatAudioKind.VOICE_MESSAGE) { + voiceMessageSpeed.value + } else { + conversationUser.userId?.let { chatAudioSpeeds.speedFor(it, kind) } ?: PlaybackSpeed.NORMAL } - fun prepareVoiceMessage(controller: MediaController, message: ChatMessage): Boolean { - val currentMessageId = message.jsonMessageId.toString() - val filename = message.fileParameters.name - if (filename.isEmpty()) { - return true - } - - val file = FileUtils.resolveSharedAttachmentFile(context.cacheDir, filename) ?: return true - val fileURI = file.toUri() - val filePath = fileURI.toString() - - chatViewModel.syncVoiceMessageUiState( - message.apply { - voiceMessageDuration = getAudioDuration(file.absolutePath).toInt() - } - ) - - fun finishPreparing() { - setupAndPlay(controller, message, filePath) - if (message.isVoiceMessage) { - setUpWaveform(message, file) - } else { - message.isDownloadingVoiceMessage = false - chatViewModel.syncVoiceMessageUiState(message) - } - } - - val alreadyLoaded = controller.currentMediaItem?.mediaId == currentMessageId - if (!alreadyLoaded || !file.exists()) { - if (!file.exists()) { - downloadFileToCache(message, true) { - chatViewModel.syncVoiceMessageUiState( - message.apply { - voiceMessageDuration = getAudioDuration(file.absolutePath).toInt() - } - ) - finishPreparing() - } - } else { - finishPreparing() - } + private fun onAudioMessageSeek(messageId: Int, progress: Float) { + val internalUserId = if (::initialUser.isInitialized) conversationUser.id else null + internalUserId?.let { chatAudioPlayer.seekTo(ChatAudioKey(it, roomToken, messageId), progress) } + } - return true - } - return false + private fun onVoiceMessageSpeedClick() { + val nextSpeed = voiceMessageSpeed.value.next() + voiceMessageSpeed.value = nextSpeed + if (::initialUser.isInitialized) { + conversationUser.userId?.let { chatAudioSpeeds.save(it, ChatAudioKind.VOICE_MESSAGE, nextSpeed) } + conversationUser.id?.let { chatAudioPlayer.setSpeed(it, ChatAudioKind.VOICE_MESSAGE, nextSpeed.value) } } + } - lifecycleScope.launch { - val message = chatViewModel.getMessageById(messageId.toLong()).first() - - mediaController?.let { controller -> - // If a message is still playing, it must be paused first before another can be loaded - val currentMessageId = message.jsonMessageId.toString() - if (controller.isPlaying && controller.currentMediaItem?.mediaId != currentMessageId) return@launch + private fun openAudioMessage(key: ChatAudioKey, threadId: Long?) { + val isThisView = key.roomToken == roomToken && + threadId == openThreadId() && + ::initialUser.isInitialized && + key.internalUserId == conversationUser.id + if (isThisView) { + chatViewModel.jumpToQuotedMessage(key.messageId.toLong()) + } else { + startActivity(key.openMessageIntent(this, threadId)) + } + } - chatViewModel.currentVoiceMessage = message + /** The thread this chat shows, or null if it shows the conversation itself. */ + private fun openThreadId(): Long? = conversationThreadId?.takeIf { it > 0L } - // If the controller is playing a new voice message, initialize - if (prepareVoiceMessage(controller, message)) return@launch + private fun showAudioError(error: ChatAudioError) { + val message = when (error) { + ChatAudioError.NETWORK -> R.string.nc_audio_error_network + ChatAudioError.UNAVAILABLE -> R.string.nc_audio_error_unavailable + ChatAudioError.UNSUPPORTED -> R.string.nc_audio_error_unsupported + ChatAudioError.UNKNOWN -> R.string.nc_audio_error_generic + } + Snackbar.make(binding.root, message, Snackbar.LENGTH_LONG).show() + } - // If the controller is playing the same voice message, pause/resume - if (controller.isPlaying) { - controller.pause() - } else { - controller.play() + private fun setAudioPlayerBarContent() { + val callbacks = chatAudioPlayerCallbacks( + player = chatAudioPlayer, + speeds = chatAudioSpeeds, + scope = lifecycleScope, + onOpenMessage = { key, threadId -> openAudioMessage(key, threadId) }, + onSpeedChanged = { key, kind, speed -> + val isOwnVoiceMessage = kind == ChatAudioKind.VOICE_MESSAGE && + ::initialUser.isInitialized && + key.internalUserId == conversationUser.id + if (isOwnVoiceMessage) { + voiceMessageSpeed.value = speed } } + ) + binding.audioPlayerBarComposeView.setContent { + MaterialTheme(colorScheme = viewThemeUtils.getColorScheme(this@ChatActivity)) { + val state by chatAudioPlayer.state.collectAsStateWithLifecycle() + ChatAudioPlayerHost(state = state, callbacks = callbacks) + } } } @@ -1278,19 +1178,6 @@ class ChatActivity : } } - private fun onVoiceSpeedClickCompose(messageId: Int) { - val currentSpeed = chatViewModel.uiState.value.items - .mapNotNull { (it as? ChatViewModel.ChatItem.MessageItem)?.uiMessage } - .firstOrNull { it.id == messageId } - ?.content - ?.let { it as? MessageTypeContent.Voice } - ?.playbackSpeed ?: PlaybackSpeed.NORMAL - val nextSpeed = currentSpeed.next() - chatViewModel.setPlayBack(nextSpeed) - appPreferences.savePreferredPlayback(conversationUser!!.userId, nextSpeed) - chatViewModel.setVoiceMessageSpeed(messageId, nextSpeed) - } - fun downloadAndOpenFile( messageId: Int, openWhenDownloadState: MutableState, @@ -1419,38 +1306,7 @@ class ChatActivity : override fun onStart() { super.onStart() active = true - this.lifecycle.addObserver(AudioUtils) this.lifecycle.addObserver(chatViewModel) - - val sessionToken = SessionToken(this, ComponentName(this, VoiceMessageMediaService::class.java)) - val future = MediaController.Builder(this, sessionToken) - .setListener(object : MediaController.Listener { - override fun onDisconnected(controller: MediaController) { - mediaController?.removeListener(playerListener) - stopProgressPolling() - mediaController = null - } - }) - .buildAsync() - mediaControllerFuture = future - - future.addListener({ - if (future !== mediaControllerFuture) { - return@addListener - } - - mediaController = try { - future.get() - } catch (_: CancellationException) { - null - } - - mediaController?.addListener(playerListener) - - if (mediaController?.isPlaying == true) { - startProgressPolling() - } - }, ContextCompat.getMainExecutor(this)) } override fun onSaveInstanceState(outState: Bundle) { @@ -1463,14 +1319,7 @@ class ChatActivity : override fun onStop() { super.onStop() active = false - this.lifecycle.removeObserver(AudioUtils) this.lifecycle.removeObserver(chatViewModel) - - mediaController?.removeListener(playerListener) - stopProgressPolling() - - mediaControllerFuture?.let { MediaController.releaseFuture(it) } - mediaController = null } @OptIn(FlowPreview::class) @@ -1815,10 +1664,11 @@ class ChatActivity : } } - this.lifecycleScope.launch { - chatViewModel.mediaPlayerSeekbarObserver.onEach { msg -> - }.collect() - } + chatAudioPlayer.state + .map { it.currentKey to it.error } + .distinctUntilChanged() + .onEach { (_, error) -> error?.let { showAudioError(it) } } + .launchIn(lifecycleScope) messageInputViewModel.editMessageViewState.observe(this) { state -> when (state) { @@ -1882,6 +1732,7 @@ class ChatActivity : } lastRecordingInProgress = voiceRecordingInProgress if (voiceRecordingInProgress) { + chatAudioPlayer.pause() window.addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON) } else { window.clearFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON) @@ -2406,31 +2257,12 @@ class ChatActivity : } } - // override fun updateMediaPlayerProgressBySlider(message: ChatMessage, progress: Int) { - // chatViewModel.seekToMediaPlayer(progress) - // } - // - // override fun registerMessageToObservePlaybackSpeedPreferences( - // userId: String, - // listener: (speed: PlaybackSpeed) -> Unit - // ) { - // CoroutineScope(Dispatchers.Default).launch { - // chatViewModel.voiceMessagePlayBackUIFlow.onEach { speed -> - // withContext(Dispatchers.Main) { - // listener(speed) - // } - // }.collect() - // } - // } - @SuppressLint("LongLogTag") private fun downloadFileToCache( message: ChatMessage, openWhenDownloaded: Boolean, funToCallWhenDownloadSuccessful: (() -> Unit) ) { - message.isDownloadingVoiceMessage = true - chatViewModel.syncVoiceMessageUiState(message) message.openWhenDownloaded = openWhenDownloaded val attachmentFolder = CapabilitiesUtil.getAttachmentFolder( @@ -4296,9 +4128,6 @@ class ChatActivity : private const val GET_ROOM_INFO_DELAY_NORMAL: Long = 30000 private const val GET_ROOM_INFO_DELAY_LOBBY: Long = 5000 private const val MILLIS_250 = 250L - private const val MILLIS_150 = 150L - private const val MILLIS_1000 = 1000L - private const val FLOAT_100 = 100f private const val AGE_THRESHOLD_FOR_DELETE_MESSAGE: Int = 21600000 // (6 hours in millis = 6 * 3600 * 1000) private const val REQUEST_SHARE_FILE_PERMISSION: Int = 221 private const val REQUEST_RECORD_AUDIO_PERMISSION = 222 @@ -4307,7 +4136,6 @@ class ChatActivity : private const val FILE_DATE_PATTERN = "yyyy-MM-dd HH-mm-ss" private const val VIDEO_SUFFIX = ".mp4" private const val PICTURE_SUFFIX = ".jpg" - private const val VOICE_MESSAGE_SEEKBAR_BASE = 1000 private const val HTTP_BAD_REQUEST = 400 private const val HTTP_FORBIDDEN = 403 private const val HTTP_NOT_FOUND = 404 @@ -4320,20 +4148,12 @@ class ChatActivity : private const val NOTIFICATION_LEVEL_NEVER = 3 private const val ONE_SECOND_IN_MILLIS = 1000 private const val MILLISEC_15: Long = 15 - private const val CURRENT_AUDIO_MESSAGE_KEY = "CURRENT_AUDIO_MESSAGE" - private const val CURRENT_AUDIO_POSITION_KEY = "CURRENT_AUDIO_POSITION" - private const val CURRENT_AUDIO_WAS_PLAYING_KEY = "CURRENT_AUDIO_PLAYING" - private const val RESUME_AUDIO_TAG = "RESUME_AUDIO_TAG" private const val FIVE_MINUTES_IN_SECONDS: Long = 300 private const val ROOM_TYPE_ONE_TO_ONE = "1" private const val ACTOR_TYPE = "users" const val CONVERSATION_INTERNAL_ID = "CONVERSATION_INTERNAL_ID" const val NO_OFFLINE_MESSAGES_FOUND = "NO_OFFLINE_MESSAGES_FOUND" private const val NO_MORE_RESULTS_TOAST_THROTTLE_MS: Long = 2000 - const val VOICE_MESSAGE_CONTINUOUS_BEFORE = -5 - const val VOICE_MESSAGE_CONTINUOUS_AFTER = 5 - const val VOICE_MESSAGE_PLAY_ADD_THRESHOLD = 0.1 - const val VOICE_MESSAGE_MARK_PLAYED_FACTOR = 20 const val OUT_OF_OFFICE_ALPHA = 76 const val ZERO_INDEX = 0 const val ONE_INDEX = 1 diff --git a/app/src/main/java/com/nextcloud/talk/chat/audio/ChatAudioCredentials.kt b/app/src/main/java/com/nextcloud/talk/chat/audio/ChatAudioCredentials.kt new file mode 100644 index 00000000000..cffc39445a8 --- /dev/null +++ b/app/src/main/java/com/nextcloud/talk/chat/audio/ChatAudioCredentials.kt @@ -0,0 +1,26 @@ +/* + * Nextcloud Talk - Android Client + * + * SPDX-FileCopyrightText: 2026 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: GPL-3.0-or-later + */ + +package com.nextcloud.talk.chat.audio + +/** + * Decides whether the credentials of an account may be attached to an audio request. Only requests for the WebDAV + * files of that account on its own server qualify, so credentials are never sent to any other host, whatever URL + * ends up in the player. + */ +object ChatAudioCredentials { + private const val USER_FILES_PATH = "/remote.php/dav/files/" + + /** + * The prefix of every file URL of the account, built the same way as + * [com.nextcloud.talk.utils.ApiUtils.getUrlForFileDownload]. + */ + fun userFilesPrefix(baseUrl: String, userId: String): String = "$baseUrl$USER_FILES_PATH$userId/" + + fun belongsToUser(url: String, baseUrl: String?, userId: String?): Boolean = + !baseUrl.isNullOrEmpty() && !userId.isNullOrEmpty() && url.startsWith(userFilesPrefix(baseUrl, userId)) +} diff --git a/app/src/main/java/com/nextcloud/talk/chat/audio/ChatAudioDataSources.kt b/app/src/main/java/com/nextcloud/talk/chat/audio/ChatAudioDataSources.kt new file mode 100644 index 00000000000..1a77b517bde --- /dev/null +++ b/app/src/main/java/com/nextcloud/talk/chat/audio/ChatAudioDataSources.kt @@ -0,0 +1,78 @@ +/* + * Nextcloud Talk - Android Client + * + * SPDX-FileCopyrightText: 2026 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: GPL-3.0-or-later + */ + +package com.nextcloud.talk.chat.audio + +import android.content.Context +import androidx.annotation.OptIn +import androidx.media3.common.util.UnstableApi +import androidx.media3.database.StandaloneDatabaseProvider +import androidx.media3.datasource.DataSpec +import androidx.media3.datasource.DefaultDataSource +import androidx.media3.datasource.ResolvingDataSource +import androidx.media3.datasource.cache.CacheDataSource +import androidx.media3.datasource.cache.LeastRecentlyUsedCacheEvictor +import androidx.media3.datasource.cache.SimpleCache +import androidx.media3.datasource.okhttp.OkHttpDataSource +import com.nextcloud.talk.users.UserManager +import com.nextcloud.talk.utils.ApiUtils +import kotlinx.coroutines.runBlocking +import okhttp3.OkHttpClient +import java.io.File +import javax.inject.Inject +import javax.inject.Singleton + +/** + * Data sources for chat audio. Files are streamed through the app's HTTP client, so proxy settings and user + * accepted certificates apply, and are kept in a size limited disk cache that is shared by the player and the + * waveform extraction. There must be only one instance per process, as a cache directory can only be used once. + */ +@OptIn(UnstableApi::class) +@Singleton +class ChatAudioDataSources @Inject constructor( + context: Context, + okHttpClient: OkHttpClient, + private val userManager: UserManager +) { + private val cache = SimpleCache( + File(context.cacheDir, CACHE_DIRECTORY), + LeastRecentlyUsedCacheEvictor(MAX_CACHE_SIZE_BYTES), + StandaloneDatabaseProvider(context) + ) + + val cacheDataSourceFactory: CacheDataSource.Factory = CacheDataSource.Factory() + .setCache(cache) + .setUpstreamDataSourceFactory( + DefaultDataSource.Factory( + context, + ResolvingDataSource.Factory(OkHttpDataSource.Factory(okHttpClient)) { dataSpec -> + authenticate(dataSpec) + } + ) + ) + .setFlags(CacheDataSource.FLAG_IGNORE_CACHE_ON_ERROR) + + private fun authenticate(dataSpec: DataSpec): DataSpec { + val credentials = credentialsFor(dataSpec.uri.toString()) + return if (credentials == null) { + dataSpec + } else { + dataSpec.withAdditionalHeaders(mapOf(AUTHORIZATION_HEADER to credentials)) + } + } + + private fun credentialsFor(url: String): String? = + runBlocking { userManager.getUsers() } + .firstOrNull { user -> ChatAudioCredentials.belongsToUser(url, user.baseUrl, user.userId) } + ?.let { user -> ApiUtils.getCredentials(user.username, user.token) } + + companion object { + private const val CACHE_DIRECTORY = "chat_audio" + private const val MAX_CACHE_SIZE_BYTES = 200L * 1024 * 1024 + private const val AUTHORIZATION_HEADER = "Authorization" + } +} diff --git a/app/src/main/java/com/nextcloud/talk/chat/audio/ChatAudioIntents.kt b/app/src/main/java/com/nextcloud/talk/chat/audio/ChatAudioIntents.kt new file mode 100644 index 00000000000..5c73eb42739 --- /dev/null +++ b/app/src/main/java/com/nextcloud/talk/chat/audio/ChatAudioIntents.kt @@ -0,0 +1,32 @@ +/* + * Nextcloud Talk - Android Client + * + * SPDX-FileCopyrightText: 2026 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: GPL-3.0-or-later + */ + +package com.nextcloud.talk.chat.audio + +import android.content.Context +import android.content.Intent +import android.os.Bundle +import com.nextcloud.talk.activities.MainActivity +import com.nextcloud.talk.utils.bundle.BundleKeys + +/** + * Opens the conversation of the audio message at the message, switching to its account if needed. A message of + * the thread [threadId] is opened in that thread. + */ +fun ChatAudioKey.openMessageIntent(context: Context, threadId: Long?): Intent = + Intent(context, MainActivity::class.java).apply { + putExtra(BundleKeys.KEY_INTERNAL_USER_ID, internalUserId) + putExtra(BundleKeys.KEY_ROOM_TOKEN, roomToken) + putExtras(openMessageExtras(threadId)) + } + +/** The extras that make the chat jump to the audio message, in the thread [threadId] if it belongs to one. */ +fun ChatAudioKey.openMessageExtras(threadId: Long?): Bundle = + Bundle().apply { + putString(BundleKeys.KEY_MESSAGE_ID, messageId.toString()) + threadId?.let { putLong(BundleKeys.KEY_THREAD_ID, it) } + } diff --git a/app/src/main/java/com/nextcloud/talk/chat/audio/ChatAudioKey.kt b/app/src/main/java/com/nextcloud/talk/chat/audio/ChatAudioKey.kt new file mode 100644 index 00000000000..67027be951a --- /dev/null +++ b/app/src/main/java/com/nextcloud/talk/chat/audio/ChatAudioKey.kt @@ -0,0 +1,51 @@ +/* + * Nextcloud Talk - Android Client + * + * SPDX-FileCopyrightText: 2026 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: GPL-3.0-or-later + */ + +package com.nextcloud.talk.chat.audio + +/** + * Identifies an audio message in a conversation of one account. [mediaId] is the media id of the corresponding + * item in the playback service, so the key of whatever the service currently plays can always be restored. + */ +data class ChatAudioKey(val internalUserId: Long, val roomToken: String, val messageId: Int) { + + val mediaId: String + get() = "${mediaIdPrefixOf(internalUserId)}$roomToken$SEPARATOR$messageId" + + companion object { + private const val MEDIA_ID_PREFIX = "talk-audio:" + private const val SEPARATOR = "/" + private const val PART_COUNT = 3 + + /** The start of the media ids of all audio messages of the account [internalUserId]. */ + fun mediaIdPrefixOf(internalUserId: Long): String = "$MEDIA_ID_PREFIX$internalUserId$SEPARATOR" + + fun fromMediaId(mediaId: String?): ChatAudioKey? { + val parts = mediaId + ?.takeIf { it.startsWith(MEDIA_ID_PREFIX) } + ?.removePrefix(MEDIA_ID_PREFIX) + ?.split(SEPARATOR) + ?.takeIf { it.size == PART_COUNT && it[1].isNotEmpty() } + val internalUserId = parts?.get(0)?.toLongOrNull() + val messageId = parts?.get(2)?.toIntOrNull() + return if (parts != null && internalUserId != null && messageId != null) { + ChatAudioKey(internalUserId, parts[1], messageId) + } else { + null + } + } + } +} + +/** + * The kinds of chat audio. A queue only contains messages of one kind, and each kind is played with its own + * audio attributes and playback speed. + */ +enum class ChatAudioKind { + VOICE_MESSAGE, + AUDIO_FILE +} diff --git a/app/src/main/java/com/nextcloud/talk/chat/audio/ChatAudioMetadata.kt b/app/src/main/java/com/nextcloud/talk/chat/audio/ChatAudioMetadata.kt new file mode 100644 index 00000000000..9a85645ac96 --- /dev/null +++ b/app/src/main/java/com/nextcloud/talk/chat/audio/ChatAudioMetadata.kt @@ -0,0 +1,74 @@ +/* + * Nextcloud Talk - Android Client + * + * SPDX-FileCopyrightText: 2026 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: GPL-3.0-or-later + */ + +package com.nextcloud.talk.chat.audio + +import java.io.ByteArrayInputStream +import java.io.ByteArrayOutputStream +import java.io.DataInputStream +import java.io.DataOutputStream +import java.io.IOException + +/** + * What is known about an audio message beyond the chat message itself: its duration, the position to resume + * from and, for voice messages, the waveform. + */ +data class ChatAudioMetadata(val durationMs: Long = 0L, val positionMs: Long = 0L, val waveform: List? = null) { + + /** Only long recordings keep their position across app restarts, short ones start from the beginning again. */ + val persistablePosition: Long + get() = if (durationMs >= LONG_AUDIO_DURATION_MS) positionMs else 0L + + /** The position playback should start from, or 0 if the message was (almost) played to the end. */ + val resumePosition: Long + get() = if (positionMs > 0L && (durationMs <= 0L || positionMs < durationMs - END_TOLERANCE_MS)) { + positionMs + } else { + 0L + } + + companion object { + const val LONG_AUDIO_DURATION_MS = 5 * 60 * 1000L + const val END_TOLERANCE_MS = 1000L + } +} + +/** Binary representation of [ChatAudioMetadata] used by the small on-disk metadata cache. */ +object ChatAudioMetadataCodec { + private const val VERSION = 1 + private const val MAX_WAVEFORM_SIZE = 1024 + + fun encode(metadata: ChatAudioMetadata): ByteArray { + val bytes = ByteArrayOutputStream() + DataOutputStream(bytes).use { output -> + val waveform = metadata.waveform.orEmpty().take(MAX_WAVEFORM_SIZE) + output.writeInt(VERSION) + output.writeLong(metadata.durationMs) + output.writeLong(metadata.persistablePosition) + output.writeInt(waveform.size) + waveform.forEach { output.writeFloat(it) } + } + return bytes.toByteArray() + } + + fun decode(bytes: ByteArray): ChatAudioMetadata? = + try { + DataInputStream(ByteArrayInputStream(bytes)).use { input -> + require(input.readInt() == VERSION) + val durationMs = input.readLong() + val positionMs = input.readLong() + val waveformSize = input.readInt() + require(waveformSize in 0..MAX_WAVEFORM_SIZE) + val waveform = List(waveformSize) { input.readFloat() } + ChatAudioMetadata(durationMs, positionMs, waveform.takeIf { it.isNotEmpty() }) + } + } catch (e: IOException) { + null + } catch (e: IllegalArgumentException) { + null + } +} diff --git a/app/src/main/java/com/nextcloud/talk/chat/audio/ChatAudioPlaybackState.kt b/app/src/main/java/com/nextcloud/talk/chat/audio/ChatAudioPlaybackState.kt new file mode 100644 index 00000000000..5c9d3c21ccc --- /dev/null +++ b/app/src/main/java/com/nextcloud/talk/chat/audio/ChatAudioPlaybackState.kt @@ -0,0 +1,104 @@ +/* + * Nextcloud Talk - Android Client + * + * SPDX-FileCopyrightText: 2026 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: GPL-3.0-or-later + */ + +package com.nextcloud.talk.chat.audio + +/** Why the current audio message could not be played. */ +enum class ChatAudioError { + NETWORK, + UNAVAILABLE, + UNSUPPORTED, + UNKNOWN +} + +/** Repeat modes of audio files, in the order a repeat button switches through them. */ +enum class ChatAudioRepeatMode { + OFF, + ALL, + ONE; + + fun next(): ChatAudioRepeatMode = entries[(ordinal + 1) % entries.size] + + companion object { + fun fromName(name: String?): ChatAudioRepeatMode = entries.firstOrNull { it.name == name } ?: OFF + } +} + +/** + * State of the chat audio playback as seen by the UI. It only describes the item that is loaded in the playback + * service; other messages are described by their [ChatAudioMetadata]. + * + * @param currentThreadId the thread [currentKey] was played in, or null if it was played in the conversation + * @param isPlaying true while playback is requested, including while it waits for data + * @param isBuffering true while playback is requested but waits for data + * @param hasNext whether the queue has an item after the current one + */ +data class ChatAudioPlaybackState( + val currentKey: ChatAudioKey? = null, + val currentThreadId: Long? = null, + val kind: ChatAudioKind? = null, + val title: String = "", + val subtitle: String = "", + val isPlaying: Boolean = false, + val isBuffering: Boolean = false, + val positionMs: Long = 0L, + val durationMs: Long = 0L, + val speed: Float = 1f, + val error: ChatAudioError? = null, + val shuffleEnabled: Boolean = false, + val repeatMode: ChatAudioRepeatMode = ChatAudioRepeatMode.OFF, + val hasNext: Boolean = false, + val queue: List = emptyList() +) { + val isActive: Boolean + get() = currentKey != null + + /** The state of the message with the given [key] for its bubble in the chat. */ + fun messageState(key: ChatAudioKey?, metadata: ChatAudioMetadata?): ChatAudioMessageState = + if (key != null && key == currentKey) { + ChatAudioMessageState( + isCurrent = true, + isPlaying = isPlaying, + isBuffering = isBuffering, + positionMs = positionMs, + durationMs = durationMs.takeIf { it > 0L } ?: metadata?.durationMs ?: 0L, + error = error + ) + } else { + ChatAudioMessageState( + positionMs = metadata?.resumePosition ?: 0L, + durationMs = metadata?.durationMs ?: 0L + ) + } +} + +/** An item of the loaded queue; [index] is its position in the queue, not in the shuffled playback order. */ +data class ChatAudioQueueItem(val index: Int, val key: ChatAudioKey, val title: String, val subtitle: String) + +/** What the bubble of a single audio message shows. */ +data class ChatAudioMessageState( + val isCurrent: Boolean = false, + val isPlaying: Boolean = false, + val isBuffering: Boolean = false, + val positionMs: Long = 0L, + val durationMs: Long = 0L, + val error: ChatAudioError? = null +) { + val progress: Float + get() = if (durationMs > 0L) (positionMs.toFloat() / durationMs).coerceIn(0f, 1f) else 0f + + val canSeek: Boolean + get() = durationMs > 0L + + /** The elapsed time while the message is played or paused in between, otherwise its duration. */ + val displayedTimeMs: Long? + get() = when { + isCurrent || positionMs > 0L -> positionMs + durationMs > 0L -> durationMs + else -> null + } +} diff --git a/app/src/main/java/com/nextcloud/talk/chat/audio/ChatAudioPlayer.kt b/app/src/main/java/com/nextcloud/talk/chat/audio/ChatAudioPlayer.kt new file mode 100644 index 00000000000..076c50c010b --- /dev/null +++ b/app/src/main/java/com/nextcloud/talk/chat/audio/ChatAudioPlayer.kt @@ -0,0 +1,337 @@ +/* + * Nextcloud Talk - Android Client + * + * SPDX-FileCopyrightText: 2026 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: GPL-3.0-or-later + */ + +package com.nextcloud.talk.chat.audio + +import android.content.ComponentName +import android.content.Context +import android.os.Handler +import android.os.Looper +import android.util.Log +import androidx.core.content.ContextCompat +import androidx.lifecycle.DefaultLifecycleObserver +import androidx.lifecycle.LifecycleOwner +import androidx.media3.common.MediaItem +import androidx.media3.common.Player +import androidx.media3.session.MediaController +import androidx.media3.session.SessionToken +import com.google.common.util.concurrent.ListenableFuture +import com.nextcloud.talk.chat.data.io.VoiceMessageMediaService +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.launch +import java.util.concurrent.CancellationException +import java.util.concurrent.ExecutionException + +/** + * Connects a screen to [VoiceMessageMediaService] while the screen is started, exposes the playback as [state] + * and controls it. Playback continues in the service when the screen is gone. [scope] is the scope of the screen. + */ +@Suppress("TooManyFunctions") +class ChatAudioPlayer(context: Context, private val store: ChatAudioStore, private val scope: CoroutineScope) : + DefaultLifecycleObserver { + + private val context = context.applicationContext + private val handler = Handler(Looper.getMainLooper()) + private var controllerFuture: ListenableFuture? = null + private var controller: MediaController? = null + private var pendingAction: ((MediaController) -> Unit)? = null + private var isStarted = false + + private val _state = MutableStateFlow(ChatAudioPlaybackState()) + val state: StateFlow = _state.asStateFlow() + + private val progressUpdater = Runnable { updateState() } + + private val playerListener = object : Player.Listener { + override fun onEvents(player: Player, events: Player.Events) { + updateState() + } + } + + override fun onStart(owner: LifecycleOwner) { + isStarted = true + connect() + } + + override fun onStop(owner: LifecycleOwner) { + isStarted = false + disconnect() + } + + /** + * Pauses or resumes the item at [startIndex] of [queue] if it is loaded; otherwise starts playing [queue] from + * that item, at the position the message was left at. + */ + fun playOrPause(queue: List, startIndex: Int, speed: Float) { + val track = queue.getOrNull(startIndex) ?: return + runWithController { controller -> + if (controller.chatAudioKey() == track.key) { + controller.pauseOrResume() + } else { + withResumePosition(controller, track.key) { startPositionMs -> + controller.startQueue(store, queue, startIndex, startPositionMs, speed) + } + } + } + } + + /** Pauses or resumes whatever is loaded. */ + fun togglePlayback() { + runWithController { it.pauseOrResume() } + } + + /** Seeks the message to [fraction] of its duration, or remembers it as start position if it is not loaded. */ + fun seekTo(key: ChatAudioKey, fraction: Float) { + val controller = controller + if (controller != null && controller.chatAudioKey() == key && controller.duration > 0L) { + controller.seekTo((controller.duration * fraction).toLong()) + } else { + val durationMs = store.metadataFor(key)?.durationMs ?: 0L + if (durationMs > 0L) { + store.rememberPosition(key, (durationMs * fraction).toLong(), durationMs) + } + } + updateState() + } + + /** Applies [speed] if a message of [kind] of the account [internalUserId] is loaded. */ + fun setSpeed(internalUserId: Long, kind: ChatAudioKind, speed: Float) { + val state = _state.value + controller + ?.takeIf { state.kind == kind && state.currentKey?.internalUserId == internalUserId } + ?.setPlaybackSpeed(speed) + } + + /** Skips to the next item, in shuffled order if shuffling. */ + fun skipToNext() { + runWithController { it.seekToNext() } + } + + /** Restarts the current item, or skips to the previous one if the current one only just started. */ + fun skipToPrevious() { + runWithController { it.seekToPrevious() } + } + + /** Sets shuffling and repeating; the playback service only applies them to audio files. */ + fun setPlaybackModes(shuffleEnabled: Boolean, repeatMode: ChatAudioRepeatMode) { + runWithController { + it.shuffleModeEnabled = shuffleEnabled + it.repeatMode = repeatMode.toPlayerRepeatMode() + } + } + + /** Plays the item at [index] of the queue, a long recording from where it was left; pauses it if it is current. */ + fun playQueueItem(index: Int) { + runWithController { controller -> + if (index == controller.currentMediaItemIndex) { + controller.pauseOrResume() + } else if (index in 0 until controller.mediaItemCount) { + val key = ChatAudioKey.fromMediaId(controller.getMediaItemAt(index).mediaId) + withResumePosition(controller, key) { startPositionMs -> + controller.seekTo(index, startPositionMs) + if (controller.playbackState == Player.STATE_IDLE) { + controller.prepare() + } + controller.play() + } + } + } + } + + fun pause() { + controller?.pause() + } + + /** Stops playback and unloads the queue, which also removes the media notification. */ + fun stop() { + controller?.let { controller -> + controller.rememberPositionIn(store) + controller.stop() + controller.clearMediaItems() + } + } + + private fun connect() { + val sessionToken = SessionToken(context, ComponentName(context, VoiceMessageMediaService::class.java)) + val future = MediaController.Builder(context, sessionToken) + .setListener(object : MediaController.Listener { + override fun onDisconnected(controller: MediaController) { + if (controller === this@ChatAudioPlayer.controller) { + disconnect() + _state.value = ChatAudioPlaybackState() + } + } + }) + .buildAsync() + controllerFuture = future + future.addListener( + { + if (future === controllerFuture) { + val connected = future.controllerOrNull() + if (connected == null) { + // Allows the next action to connect again. + controllerFuture = null + } else { + controller = connected + connected.addListener(playerListener) + pendingAction?.invoke(connected) + updateState() + } + pendingAction = null + } + }, + ContextCompat.getMainExecutor(context) + ) + } + + private fun disconnect() { + handler.removeCallbacks(progressUpdater) + controller?.removeListener(playerListener) + controller = null + pendingAction = null + controllerFuture?.let { MediaController.releaseFuture(it) } + controllerFuture = null + } + + /** + * Runs [action] with the position [key] should resume from, once what is persisted about it is loaded, and only + * if the screen is still connected through [controller]. + */ + private fun withResumePosition(controller: MediaController, key: ChatAudioKey?, action: (Long) -> Unit) { + scope.launch { + val startPositionMs = key?.let { store.loadNow(it)?.resumePosition } ?: 0L + if (controller === this@ChatAudioPlayer.controller) { + action(startPositionMs) + updateState() + } + } + } + + private fun runWithController(action: (MediaController) -> Unit) { + val connected = controller + if (connected == null) { + pendingAction = action + if (controllerFuture == null && isStarted) { + connect() + } + } else { + action(connected) + updateState() + } + } + + private fun updateState() { + handler.removeCallbacks(progressUpdater) + val controller = controller ?: return + _state.value = controller.toPlaybackState() + if (controller.isPlaying) { + handler.postDelayed(progressUpdater, PROGRESS_UPDATE_INTERVAL_MS) + } + } + + companion object { + private const val PROGRESS_UPDATE_INTERVAL_MS = 100L + } +} + +private val TAG = ChatAudioPlayer::class.java.simpleName + +private fun ListenableFuture.controllerOrNull(): MediaController? = + try { + get() + } catch (e: ExecutionException) { + Log.w(TAG, "Failed to connect to the chat audio playback", e) + null + } catch (e: CancellationException) { + Log.d(TAG, "Connection to the chat audio playback was cancelled", e) + null + } + +private fun Player.chatAudioKey(): ChatAudioKey? = ChatAudioKey.fromMediaId(currentMediaItem?.mediaId) + +private fun Player.startQueue( + store: ChatAudioStore, + queue: List, + startIndex: Int, + startPositionMs: Long, + speed: Float +) { + rememberPositionIn(store) + setMediaItems(queue.map { it.toMediaItem() }, startIndex, startPositionMs) + setPlaybackSpeed(speed) + prepare() + play() +} + +private fun Player.rememberPositionIn(store: ChatAudioStore) { + val key = chatAudioKey() + if (key != null && playbackState != Player.STATE_ENDED) { + store.rememberPosition(key, currentPosition, duration) + } +} + +/** True while playback is requested and not held back by an error, the end of the queue or another app. */ +private fun Player.wantsToPlay(): Boolean = + playWhenReady && + playbackSuppressionReason == Player.PLAYBACK_SUPPRESSION_REASON_NONE && + playerError == null && + playbackState != Player.STATE_ENDED + +private fun Player.pauseOrResume() { + if (wantsToPlay()) { + pause() + } else { + when (playbackState) { + Player.STATE_IDLE -> prepare() + Player.STATE_ENDED -> seekToDefaultPosition() + Player.STATE_BUFFERING, Player.STATE_READY -> Unit + } + play() + } +} + +private fun Player.toPlaybackState(): ChatAudioPlaybackState { + val mediaItem: MediaItem? = currentMediaItem + val key = chatAudioKey() + return if (mediaItem == null || key == null) { + ChatAudioPlaybackState() + } else { + ChatAudioPlaybackState( + currentKey = key, + currentThreadId = mediaItem.chatAudioThreadId(), + kind = mediaItem.chatAudioKind(), + title = mediaItem.mediaMetadata.title?.toString().orEmpty(), + subtitle = mediaItem.mediaMetadata.artist?.toString().orEmpty(), + isPlaying = wantsToPlay(), + isBuffering = wantsToPlay() && playbackState == Player.STATE_BUFFERING, + positionMs = currentPosition.coerceAtLeast(0L), + durationMs = duration.coerceAtLeast(0L), + speed = playbackParameters.speed, + error = playerError?.toChatAudioError(), + shuffleEnabled = shuffleModeEnabled, + repeatMode = chatAudioRepeatModeOf(repeatMode), + hasNext = hasNextMediaItem(), + queue = queueItems() + ) + } +} + +private fun Player.queueItems(): List = + (0 until mediaItemCount).mapNotNull { index -> + val mediaItem = getMediaItemAt(index) + ChatAudioKey.fromMediaId(mediaItem.mediaId)?.let { key -> + ChatAudioQueueItem( + index = index, + key = key, + title = mediaItem.mediaMetadata.title?.toString().orEmpty(), + subtitle = mediaItem.mediaMetadata.artist?.toString().orEmpty() + ) + } + } diff --git a/app/src/main/java/com/nextcloud/talk/chat/audio/ChatAudioQueue.kt b/app/src/main/java/com/nextcloud/talk/chat/audio/ChatAudioQueue.kt new file mode 100644 index 00000000000..c509c3b5369 --- /dev/null +++ b/app/src/main/java/com/nextcloud/talk/chat/audio/ChatAudioQueue.kt @@ -0,0 +1,39 @@ +/* + * Nextcloud Talk - Android Client + * + * SPDX-FileCopyrightText: 2026 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: GPL-3.0-or-later + */ + +package com.nextcloud.talk.chat.audio + +/** + * Builds the queue that is played when an audio message is tapped. A voice message is followed by the newer voice + * messages, so a conversation is listened to in order. An audio file is played within the audio files around it, + * like in a playlist that can be skipped through, shuffled and repeated. + */ +object ChatAudioQueue { + const val MAX_SIZE = 50 + + /** The items of a queue and the index of the tapped one among them. */ + data class Selection(val items: List, val startIndex: Int) + + /** + * @param messages the audio messages of the conversation, oldest first + * @param startId the id of the tapped message + * @param idOf returns the message id of an element of [messages] + * @param kindOf returns the audio kind of an element of [messages], or null if it cannot be played + * @return the queue, or null if the tapped message cannot be played + */ + fun build(messages: List, startId: Int, idOf: (T) -> Int, kindOf: (T) -> ChatAudioKind?): Selection? { + val kind = messages.firstOrNull { idOf(it) == startId }?.let(kindOf) ?: return null + val sameKind = messages.filter { kindOf(it) == kind } + val index = sameKind.indexOfFirst { idOf(it) == startId } + return if (kind == ChatAudioKind.VOICE_MESSAGE) { + Selection(sameKind.subList(index, sameKind.size).take(MAX_SIZE), 0) + } else { + val first = (index - MAX_SIZE / 2).coerceIn(0, (sameKind.size - MAX_SIZE).coerceAtLeast(0)) + Selection(sameKind.subList(first, minOf(sameKind.size, first + MAX_SIZE)), index - first) + } + } +} diff --git a/app/src/main/java/com/nextcloud/talk/chat/audio/ChatAudioSpeeds.kt b/app/src/main/java/com/nextcloud/talk/chat/audio/ChatAudioSpeeds.kt new file mode 100644 index 00000000000..3e96f05b186 --- /dev/null +++ b/app/src/main/java/com/nextcloud/talk/chat/audio/ChatAudioSpeeds.kt @@ -0,0 +1,53 @@ +/* + * Nextcloud Talk - Android Client + * + * SPDX-FileCopyrightText: 2026 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: GPL-3.0-or-later + */ + +package com.nextcloud.talk.chat.audio + +import com.nextcloud.talk.ui.PlaybackSpeed +import com.nextcloud.talk.users.UserManager +import com.nextcloud.talk.utils.preferences.AppPreferences +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.distinctUntilChanged +import kotlinx.coroutines.flow.map +import kotlinx.coroutines.flow.update +import javax.inject.Inject +import javax.inject.Singleton + +/** Playback speeds of chat audio, kept per account and separately for voice messages and audio files. */ +@Singleton +class ChatAudioSpeeds @Inject constructor( + private val appPreferences: AppPreferences, + private val userManager: UserManager +) { + private val savedSpeeds = MutableStateFlow, PlaybackSpeed>>(emptyMap()) + + fun speedFor(userId: String, kind: ChatAudioKind): PlaybackSpeed = + when (kind) { + ChatAudioKind.VOICE_MESSAGE -> appPreferences.getPreferredPlayback(userId) + ChatAudioKind.AUDIO_FILE -> appPreferences.getAudioFilePlaybackSpeed(userId) + } + + /** The speed of [kind] for the account [userId], also when it is changed on another screen. */ + fun speedFlow(userId: String, kind: ChatAudioKind): Flow = + savedSpeeds + .map { it[userId to kind] ?: speedFor(userId, kind) } + .distinctUntilChanged() + + fun save(userId: String, kind: ChatAudioKind, speed: PlaybackSpeed) { + when (kind) { + ChatAudioKind.VOICE_MESSAGE -> appPreferences.savePreferredPlayback(userId, speed) + ChatAudioKind.AUDIO_FILE -> appPreferences.saveAudioFilePlaybackSpeed(userId, speed) + } + savedSpeeds.update { it + ((userId to kind) to speed) } + } + + /** Saves [speed] for the account of [key], which is not necessarily the account of the current screen. */ + suspend fun save(key: ChatAudioKey, kind: ChatAudioKind, speed: PlaybackSpeed) { + userManager.getUserWithId(key.internalUserId)?.userId?.let { save(it, kind, speed) } + } +} diff --git a/app/src/main/java/com/nextcloud/talk/chat/audio/ChatAudioStore.kt b/app/src/main/java/com/nextcloud/talk/chat/audio/ChatAudioStore.kt new file mode 100644 index 00000000000..baed40dc2a7 --- /dev/null +++ b/app/src/main/java/com/nextcloud/talk/chat/audio/ChatAudioStore.kt @@ -0,0 +1,195 @@ +/* + * Nextcloud Talk - Android Client + * + * SPDX-FileCopyrightText: 2026 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: GPL-3.0-or-later + */ + +package com.nextcloud.talk.chat.audio + +import android.content.Context +import android.util.Log +import kotlinx.coroutines.CoroutineDispatcher +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Deferred +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.async +import kotlinx.coroutines.flow.MutableSharedFlow +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.SharedFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asSharedFlow +import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.flow.update +import kotlinx.coroutines.launch +import java.io.File +import java.io.IOException +import java.util.concurrent.ConcurrentHashMap +import javax.inject.Inject +import javax.inject.Singleton + +/** + * Remembers [ChatAudioMetadata] of audio messages for the whole process, so progress, duration and waveform + * survive leaving a conversation. It is also written to a small file cache, which keeps durations, waveforms and + * the position of long recordings across app restarts. + */ +@Singleton +class ChatAudioStore internal constructor(private val directory: File, ioDispatcher: CoroutineDispatcher) { + + @Inject + constructor(context: Context) : this( + File(context.cacheDir, METADATA_DIRECTORY), + Dispatchers.IO.limitedParallelism(1) + ) + + private val ioScope = CoroutineScope(SupervisorJob() + ioDispatcher) + private val loads = ConcurrentHashMap>() + private val positionChangedKeys: MutableSet = ConcurrentHashMap.newKeySet() + private val removedAccounts: MutableSet = ConcurrentHashMap.newKeySet() + + private val _metadata = MutableStateFlow>(emptyMap()) + val metadata: StateFlow> = _metadata.asStateFlow() + + private val _accountRemovals = MutableSharedFlow(extraBufferCapacity = ACCOUNT_REMOVALS_BUFFER) + + /** Emits the internal user id of every account removed by [forgetAccount]. */ + val accountRemovals: SharedFlow = _accountRemovals.asSharedFlow() + + fun metadataFor(key: ChatAudioKey): ChatAudioMetadata? = _metadata.value[key] + + /** Reads what is persisted about [key], at most once per process. */ + fun load(key: ChatAudioKey) { + startLoad(key) + } + + /** Like [load], but waits until what is persisted about [key] is known. */ + suspend fun loadNow(key: ChatAudioKey): ChatAudioMetadata? { + startLoad(key)?.await() + return metadataFor(key) + } + + private fun startLoad(key: ChatAudioKey): Deferred? = + if (isAccountRemoved(key.internalUserId)) { + null + } else { + loads.computeIfAbsent(key) { + ioScope.async { + val persisted = fileFor(key) + .takeIf { it.isFile } + ?.let { readQuietly(it) } + ?.let { ChatAudioMetadataCodec.decode(it) } + if (persisted != null && !isAccountRemoved(key.internalUserId)) { + _metadata.update { current -> current + (key to merge(key, current[key], persisted)) } + // Values changed before the load, like a position that was reset, also replace the file. + if (metadataFor(key) != persisted) { + persist(key) + } + } + } + } + } + + fun rememberDuration(key: ChatAudioKey, durationMs: Long) { + if (durationMs > 0L) { + change(key) { it.copy(durationMs = durationMs) } + } + } + + fun rememberPosition(key: ChatAudioKey, positionMs: Long, durationMs: Long) { + positionChangedKeys.add(key) + change(key) { + it.copy( + positionMs = positionMs.coerceAtLeast(0L), + durationMs = durationMs.takeIf { duration -> duration > 0L } ?: it.durationMs + ) + } + } + + fun forgetPosition(key: ChatAudioKey) { + positionChangedKeys.add(key) + if ((metadataFor(key)?.positionMs ?: 0L) != 0L) { + change(key) { it.copy(positionMs = 0L) } + } + } + + fun rememberWaveform(key: ChatAudioKey, waveform: FloatArray) { + change(key) { it.copy(waveform = waveform.toList()) } + } + + fun isAccountRemoved(internalUserId: Long): Boolean = internalUserId in removedAccounts + + /** + * Deletes everything remembered about the audio messages of a removed account, in memory and on disk, and + * ignores them from now on. Internal user ids are never reused, so this lasts for the rest of the process. + */ + fun forgetAccount(internalUserId: Long) { + removedAccounts.add(internalUserId) + val belongsToAccount = { key: ChatAudioKey -> key.internalUserId == internalUserId } + _metadata.update { current -> current.filterKeys { !belongsToAccount(it) } } + loads.keys.removeIf(belongsToAccount) + positionChangedKeys.removeIf(belongsToAccount) + _accountRemovals.tryEmit(internalUserId) + ioScope.launch { + val prefix = fileNameOf(ChatAudioKey.mediaIdPrefixOf(internalUserId)) + directory.listFiles { file -> file.name.startsWith(prefix) } + ?.forEach { file -> + if (!file.delete()) { + Log.w(TAG, "Failed to delete audio metadata") + } + } + } + } + + private fun change(key: ChatAudioKey, transform: (ChatAudioMetadata) -> ChatAudioMetadata) { + if (isAccountRemoved(key.internalUserId)) { + return + } + val before = _metadata.value[key] + _metadata.update { current -> current + (key to transform(current[key] ?: ChatAudioMetadata())) } + if (_metadata.value[key] != before) { + persist(key) + } + } + + private fun merge(key: ChatAudioKey, current: ChatAudioMetadata?, persisted: ChatAudioMetadata) = + ChatAudioMetadata( + durationMs = current?.durationMs?.takeIf { it > 0L } ?: persisted.durationMs, + positionMs = if (key in positionChangedKeys) current?.positionMs ?: 0L else persisted.positionMs, + waveform = current?.waveform ?: persisted.waveform + ) + + private fun persist(key: ChatAudioKey) { + ioScope.launch { + val metadata = _metadata.value[key] + if (metadata != null) { + try { + directory.mkdirs() + fileFor(key).writeBytes(ChatAudioMetadataCodec.encode(metadata)) + } catch (e: IOException) { + Log.w(TAG, "Failed to store audio metadata", e) + } + } + } + } + + private fun fileFor(key: ChatAudioKey): File = File(directory, fileNameOf(key.mediaId)) + + companion object { + private const val METADATA_DIRECTORY = "chat_audio_metadata" + private const val ACCOUNT_REMOVALS_BUFFER = 8 + } +} + +private val TAG = ChatAudioStore::class.java.simpleName +private val unsafeFileNameCharacters = Regex("[^A-Za-z0-9_-]") + +private fun fileNameOf(mediaId: String): String = mediaId.replace(unsafeFileNameCharacters, "_") + +private fun readQuietly(file: File): ByteArray? = + try { + file.readBytes() + } catch (e: IOException) { + Log.w(TAG, "Failed to read audio metadata", e) + null + } diff --git a/app/src/main/java/com/nextcloud/talk/chat/audio/ChatAudioTrack.kt b/app/src/main/java/com/nextcloud/talk/chat/audio/ChatAudioTrack.kt new file mode 100644 index 00000000000..6aa85f6973f --- /dev/null +++ b/app/src/main/java/com/nextcloud/talk/chat/audio/ChatAudioTrack.kt @@ -0,0 +1,142 @@ +/* + * Nextcloud Talk - Android Client + * + * SPDX-FileCopyrightText: 2026 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: GPL-3.0-or-later + */ + +package com.nextcloud.talk.chat.audio + +import android.os.Bundle +import androidx.annotation.OptIn +import androidx.core.net.toUri +import androidx.media3.common.MediaItem +import androidx.media3.common.MediaMetadata +import androidx.media3.common.PlaybackException +import androidx.media3.common.Player +import androidx.media3.common.util.UnstableApi +import com.nextcloud.talk.chat.ui.model.ChatMessageUi +import com.nextcloud.talk.chat.ui.model.MessageTypeContent +import com.nextcloud.talk.data.user.model.User +import com.nextcloud.talk.utils.ApiUtils +import com.nextcloud.talk.utils.CharacterAvatarUtils + +/** An audio message as it is handed to the playback service; [threadId] is the thread it was played in, if any. */ +data class ChatAudioTrack( + val key: ChatAudioKey, + val kind: ChatAudioKind, + val uri: String, + val cacheKey: String, + val title: String, + val artist: String, + val artworkUri: String?, + val threadId: Long? = null +) + +private const val FILE_PARAMETER = "file" +private const val FILE_PATH = "path" +private const val FILE_ID = "id" +private const val FILE_ETAG = "etag" +private const val EXTRA_KIND = "com.nextcloud.talk.chat.audio.KIND" +private const val EXTRA_THREAD_ID = "com.nextcloud.talk.chat.audio.THREAD_ID" + +fun ChatMessageUi.chatAudioKind(): ChatAudioKind? = + when (content) { + is MessageTypeContent.Voice -> ChatAudioKind.VOICE_MESSAGE + is MessageTypeContent.AudioFile -> ChatAudioKind.AUDIO_FILE + else -> null + } + +/** + * Creates the track that plays this message for [user], or null if the message is no playable audio message. + * The file is streamed from the user's WebDAV files, where Talk shares attachments to. [threadId] is the thread + * the message is shown in, if any. + */ +fun ChatMessageUi.toChatAudioTrack(user: User, voiceMessageLabel: String, threadId: Long? = null): ChatAudioTrack? { + val kind = chatAudioKind() ?: return null + val file = messageParameters[FILE_PARAMETER].orEmpty() + val path = file[FILE_PATH].orEmpty() + val baseUrl = user.baseUrl.orEmpty() + val userId = user.userId.orEmpty() + val key = user.id?.let { internalUserId -> roomToken?.let { ChatAudioKey(internalUserId, it, id) } } + return if (key == null || path.isEmpty() || baseUrl.isEmpty() || userId.isEmpty()) { + null + } else { + val uri = ApiUtils.getUrlForFileDownload(baseUrl, userId, path) + val fileId = file[FILE_ID].orEmpty() + val fileName = (content as? MessageTypeContent.AudioFile)?.fileName.orEmpty() + val hasServerAvatar = CharacterAvatarUtils.avatarFor(actorType, actorId, actorDisplayName, null) == null + ChatAudioTrack( + key = key, + kind = kind, + uri = uri, + cacheKey = if (fileId.isEmpty()) uri else "$baseUrl/$fileId/${file[FILE_ETAG].orEmpty()}", + title = if (kind == ChatAudioKind.AUDIO_FILE && fileName.isNotEmpty()) fileName else actorDisplayName, + artist = if (kind == ChatAudioKind.VOICE_MESSAGE) voiceMessageLabel else actorDisplayName, + artworkUri = actorId?.takeIf { hasServerAvatar }?.let { ApiUtils.getUrlForAvatar(baseUrl, it, true) }, + threadId = threadId + ) + } +} + +@OptIn(UnstableApi::class) +fun ChatAudioTrack.toMediaItem(): MediaItem = + MediaItem.Builder() + .setMediaId(key.mediaId) + .setUri(uri) + .setCustomCacheKey(cacheKey) + .setMediaMetadata( + MediaMetadata.Builder() + .setTitle(title) + .setArtist(artist) + .setArtworkUri(artworkUri?.toUri()) + .setExtras( + Bundle().apply { + putString(EXTRA_KIND, kind.name) + threadId?.let { putLong(EXTRA_THREAD_ID, it) } + } + ) + .build() + ) + .build() + +fun ChatAudioRepeatMode.toPlayerRepeatMode(): Int = + when (this) { + ChatAudioRepeatMode.OFF -> Player.REPEAT_MODE_OFF + ChatAudioRepeatMode.ALL -> Player.REPEAT_MODE_ALL + ChatAudioRepeatMode.ONE -> Player.REPEAT_MODE_ONE + } + +fun chatAudioRepeatModeOf(playerRepeatMode: Int): ChatAudioRepeatMode = + when (playerRepeatMode) { + Player.REPEAT_MODE_ALL -> ChatAudioRepeatMode.ALL + Player.REPEAT_MODE_ONE -> ChatAudioRepeatMode.ONE + else -> ChatAudioRepeatMode.OFF + } + +fun MediaItem.chatAudioKind(): ChatAudioKind { + val name = mediaMetadata.extras?.getString(EXTRA_KIND) + return ChatAudioKind.entries.firstOrNull { it.name == name } ?: ChatAudioKind.VOICE_MESSAGE +} + +/** The thread the item was played in, or null if it was played in the conversation itself. */ +fun MediaItem.chatAudioThreadId(): Long? = mediaMetadata.extras?.getLong(EXTRA_THREAD_ID)?.takeIf { it > 0L } + +fun PlaybackException.toChatAudioError(): ChatAudioError = + when (errorCode) { + PlaybackException.ERROR_CODE_IO_NETWORK_CONNECTION_FAILED, + PlaybackException.ERROR_CODE_IO_NETWORK_CONNECTION_TIMEOUT -> ChatAudioError.NETWORK + + PlaybackException.ERROR_CODE_IO_BAD_HTTP_STATUS, + PlaybackException.ERROR_CODE_IO_FILE_NOT_FOUND, + PlaybackException.ERROR_CODE_IO_NO_PERMISSION -> ChatAudioError.UNAVAILABLE + + PlaybackException.ERROR_CODE_PARSING_CONTAINER_MALFORMED, + PlaybackException.ERROR_CODE_PARSING_CONTAINER_UNSUPPORTED, + PlaybackException.ERROR_CODE_DECODER_INIT_FAILED, + PlaybackException.ERROR_CODE_DECODING_FAILED, + PlaybackException.ERROR_CODE_DECODING_FORMAT_UNSUPPORTED, + PlaybackException.ERROR_CODE_DECODING_FORMAT_EXCEEDS_CAPABILITIES -> ChatAudioError.UNSUPPORTED + + else -> ChatAudioError.UNKNOWN + } diff --git a/app/src/main/java/com/nextcloud/talk/chat/audio/VoiceMessageWaveformLoader.kt b/app/src/main/java/com/nextcloud/talk/chat/audio/VoiceMessageWaveformLoader.kt new file mode 100644 index 00000000000..f8bf1b50dbe --- /dev/null +++ b/app/src/main/java/com/nextcloud/talk/chat/audio/VoiceMessageWaveformLoader.kt @@ -0,0 +1,95 @@ +/* + * Nextcloud Talk - Android Client + * + * SPDX-FileCopyrightText: 2026 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: GPL-3.0-or-later + */ + +package com.nextcloud.talk.chat.audio + +import android.content.Context +import android.net.Uri +import android.util.Log +import androidx.annotation.OptIn +import androidx.annotation.WorkerThread +import androidx.media3.common.C +import androidx.media3.common.util.UnstableApi +import androidx.media3.datasource.DataSpec +import com.nextcloud.talk.utils.AudioUtils +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.launch +import java.io.File +import java.io.FileOutputStream +import java.io.IOException +import java.util.concurrent.ConcurrentHashMap +import javax.inject.Inject +import javax.inject.Singleton + +/** + * Computes the waveform of voice messages. The file is read through the audio cache, so it is not downloaded a + * second time once the player loaded it, and the result is kept in [ChatAudioStore]. + */ +@OptIn(UnstableApi::class) +@Singleton +class VoiceMessageWaveformLoader @Inject constructor( + private val context: Context, + private val dataSources: ChatAudioDataSources, + private val store: ChatAudioStore +) { + private val scope = CoroutineScope(SupervisorJob() + Dispatchers.IO) + private val loadingKeys: MutableSet = ConcurrentHashMap.newKeySet() + + fun load(key: ChatAudioKey, uri: Uri, cacheKey: String?) { + if (store.metadataFor(key)?.waveform == null && loadingKeys.add(key)) { + scope.launch { + try { + extractWaveform(uri, cacheKey)?.let { store.rememberWaveform(key, it) } + } finally { + loadingKeys.remove(key) + } + } + } + } + + @WorkerThread + private fun extractWaveform(uri: Uri, cacheKey: String?): FloatArray? { + var file: File? = null + return try { + file = File.createTempFile(TEMP_FILE_PREFIX, null, context.cacheDir) + copyFromCache(DataSpec.Builder().setUri(uri).setKey(cacheKey).build(), file) + // A file that cannot be decoded gets an empty waveform, so it is not decoded again and again. + AudioUtils.extractWaveformLevels(file)?.let { Waveforms.reduce(it, WAVEFORM_BARS) } ?: FloatArray(0) + } catch (e: IOException) { + Log.w(TAG, "Failed to load a voice message for its waveform", e) + null + } finally { + file?.delete() + } + } + + private fun copyFromCache(dataSpec: DataSpec, target: File) { + val dataSource = dataSources.cacheDataSourceFactory.createDataSource() + try { + dataSource.open(dataSpec) + FileOutputStream(target).use { output -> + val buffer = ByteArray(BUFFER_SIZE) + var read = dataSource.read(buffer, 0, buffer.size) + while (read != C.RESULT_END_OF_INPUT) { + output.write(buffer, 0, read) + read = dataSource.read(buffer, 0, buffer.size) + } + } + } finally { + dataSource.close() + } + } + + companion object { + private val TAG = VoiceMessageWaveformLoader::class.java.simpleName + private const val TEMP_FILE_PREFIX = "voice_message_waveform" + private const val BUFFER_SIZE = 16 * 1024 + const val WAVEFORM_BARS = 100 + } +} diff --git a/app/src/main/java/com/nextcloud/talk/chat/audio/Waveforms.kt b/app/src/main/java/com/nextcloud/talk/chat/audio/Waveforms.kt new file mode 100644 index 00000000000..b485857d844 --- /dev/null +++ b/app/src/main/java/com/nextcloud/talk/chat/audio/Waveforms.kt @@ -0,0 +1,61 @@ +/* + * Nextcloud Talk - Android Client + * + * SPDX-FileCopyrightText: 2026 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: GPL-3.0-or-later + */ + +package com.nextcloud.talk.chat.audio + +import kotlin.math.sqrt + +/** Turns decoded audio into the bars of a voice message waveform. */ +object Waveforms { + + /** + * Reduces [levels] to [size] bars, each the maximum of an equally sized range of [levels] (repeating levels if + * there are fewer than [size]), scaled so that the highest bar is 1. + */ + fun reduce(levels: FloatArray, size: Int): FloatArray { + if (levels.isEmpty() || size <= 0) { + return FloatArray(size.coerceAtLeast(0)) + } + val bars = FloatArray(size) { index -> + val start = (index.toLong() * levels.size / size).toInt() + val end = ((index + 1).toLong() * levels.size / size).toInt().coerceIn(start + 1, levels.size) + var max = 0f + for (position in start until end) { + max = maxOf(max, levels[position]) + } + max + } + val highest = bars.max() + return if (highest > 0f) FloatArray(size) { bars[it] / highest } else bars + } +} + +/** Collects the loudness (root mean square) of consecutive chunks of decoded audio. */ +class WaveformLevels { + private var levels = FloatArray(INITIAL_CAPACITY) + + var size: Int = 0 + private set + + /** Adds the level of a chunk from the sum of its squared samples, each sample being in the range -1..1. */ + fun addChunk(sumOfSquares: Double, sampleCount: Int) { + if (sampleCount <= 0) { + return + } + if (size == levels.size) { + levels = levels.copyOf(size * 2) + } + levels[size] = sqrt(sumOfSquares / sampleCount).toFloat() + size++ + } + + fun toArray(): FloatArray = levels.copyOf(size) + + companion object { + private const val INITIAL_CAPACITY = 256 + } +} diff --git a/app/src/main/java/com/nextcloud/talk/chat/data/io/MediaPlayerManager.kt b/app/src/main/java/com/nextcloud/talk/chat/data/io/MediaPlayerManager.kt index 1916c85c2f3..42ed107b25a 100644 --- a/app/src/main/java/com/nextcloud/talk/chat/data/io/MediaPlayerManager.kt +++ b/app/src/main/java/com/nextcloud/talk/chat/data/io/MediaPlayerManager.kt @@ -8,15 +8,10 @@ package com.nextcloud.talk.chat.data.io import android.util.Log import androidx.media3.common.MediaItem -import androidx.media3.common.PlaybackParameters import androidx.media3.common.Player import androidx.media3.exoplayer.ExoPlayer import com.nextcloud.talk.application.NextcloudTalkApplication import com.nextcloud.talk.chat.ChatActivity -import com.nextcloud.talk.chat.data.model.ChatMessage -import com.nextcloud.talk.ui.PlaybackSpeed -import com.nextcloud.talk.utils.preferences.AppPreferences -import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.MainScope import kotlinx.coroutines.cancel @@ -24,39 +19,24 @@ import kotlinx.coroutines.delay import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.MutableSharedFlow import kotlinx.coroutines.flow.MutableStateFlow -import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.isActive import kotlinx.coroutines.launch import kotlinx.coroutines.withContext -import java.io.File -import java.io.FileNotFoundException import kotlin.math.ceil import kotlin.time.Duration.Companion.milliseconds /** - * Abstraction over the [MediaPlayer](https://developer.android.com/reference/android/media/MediaPlayer) class used - * to manage the MediaPlayer instance. + * Abstraction over an [ExoPlayer] instance, used to play back a voice recording before it is sent. Audio messages + * of the chat are played by [VoiceMessageMediaService] instead. */ -@Suppress("TooManyFunctions", "TooGenericExceptionCaught") +@Suppress("TooGenericExceptionCaught") class MediaPlayerManager : LifecycleAwareManager { companion object { val TAG: String = MediaPlayerManager::class.java.simpleName private const val SEEKBAR_UPDATE_DELAY = 150L - private const val ONE_SEC = 1000 private const val DIVIDER = 100f - private const val IS_PLAYED_CUTOFF = 5 - - @JvmStatic - private val manager: MediaPlayerManager = MediaPlayerManager() - - fun sharedInstance(preferences: AppPreferences): MediaPlayerManager = - manager.apply { - appPreferences = preferences - } } - lateinit var appPreferences: AppPreferences - enum class MediaPlayerManagerState { DEFAULT, SETUP, @@ -67,19 +47,7 @@ class MediaPlayerManager : LifecycleAwareManager { ERROR } - val backgroundPlayUIFlow: StateFlow - get() = _backgroundPlayUIFlow - private val _backgroundPlayUIFlow = MutableStateFlow(null) - - val managerState: Flow - get() = _managerState - private val _managerState = MutableStateFlow(MediaPlayerManagerState.DEFAULT) - - private val playQueue = mutableListOf>() - - val mediaPlayerSeekBarPositionMsg: Flow - get() = _mediaPlayerSeekBarPositionMsg - private val _mediaPlayerSeekBarPositionMsg: MutableSharedFlow = MutableSharedFlow() + private val managerState = MutableStateFlow(MediaPlayerManagerState.DEFAULT) val mediaPlayerSeekBarPosition: Flow get() = _mediaPlayerSeekBarPosition @@ -89,14 +57,8 @@ class MediaPlayerManager : LifecycleAwareManager { private var loop = false private var scope = MainScope() - private val _currentCycledMessage = MutableStateFlow(null) - val currentCycledMessage: StateFlow - get() = _currentCycledMessage - - private var currentDataSource: String = "" var mediaPlayerDuration: Int = 0 var mediaPlayerPosition: Int = 0 - private var requestedPlaybackSpeed: PlaybackSpeed? = null /** * Starts playing audio from the given path, initializes or resumes if the player is already created. @@ -109,28 +71,7 @@ class MediaPlayerManager : LifecycleAwareManager { if (mediaPlayer == null || !scope.isActive) { init(path) } else { - _managerState.value = MediaPlayerManagerState.RESUMED - mediaPlayer!!.play() - loop = true - scope.launch { seekbarUpdateObserver() } - } - } - - /** - * Starting cycling through the playQueue, playing messages automatically unless stop() is called. - * - */ - fun startCycling() { - if (mediaPlayer != null && mediaPlayer!!.isPlaying) { - stop() - } - - val shouldReset = playQueue.isNotEmpty() && playQueue.first().first != currentDataSource - - if (mediaPlayer == null || !scope.isActive || shouldReset) { - initCycling() - } else { - _managerState.value = MediaPlayerManagerState.RESUMED + managerState.value = MediaPlayerManagerState.RESUMED mediaPlayer!!.play() loop = true scope.launch { seekbarUpdateObserver() } @@ -148,24 +89,19 @@ class MediaPlayerManager : LifecycleAwareManager { mediaPlayer!!.stop() mediaPlayer!!.release() mediaPlayer = null - _currentCycledMessage.value = null - _backgroundPlayUIFlow.tryEmit(null) - _managerState.value = MediaPlayerManagerState.STOPPED + managerState.value = MediaPlayerManagerState.STOPPED } } /** * Pauses the player. */ - fun pause(notifyUI: Boolean) { + fun pause() { if (mediaPlayer != null) { Log.d(TAG, "media player paused") - _managerState.value = MediaPlayerManagerState.PAUSED + managerState.value = MediaPlayerManagerState.PAUSED mediaPlayer!!.pause() loop = false - if (notifyUI) { - _backgroundPlayUIFlow.tryEmit(null) - } } } @@ -181,8 +117,6 @@ class MediaPlayerManager : LifecycleAwareManager { } private suspend fun seekbarUpdateObserver() { - _currentCycledMessage.value?.voiceMessageDuration = mediaPlayerDuration / ONE_SEC - _currentCycledMessage.value?.resetVoiceMessage = false withContext(Dispatchers.IO) { while (true) { if (!loop) { @@ -203,16 +137,7 @@ class MediaPlayerManager : LifecycleAwareManager { val pos = p.currentPosition mediaPlayerPosition = pos.toInt() val progress = (pos.toFloat() / mediaPlayerDuration) * DIVIDER - val progressI = ceil(progress).toInt() - val seconds = (pos / ONE_SEC).toInt() - _mediaPlayerSeekBarPosition.emit(progressI) - _currentCycledMessage.value?.let { msg -> - msg.isPlayingVoiceMessage = true - msg.voiceMessageSeekbarProgress = progressI - msg.voiceMessagePlayedSeconds = seconds - if (progressI >= IS_PLAYED_CUTOFF) msg.wasPlayedVoiceMessage = true - _mediaPlayerSeekBarPositionMsg.emit(msg) - } + _mediaPlayerSeekBarPosition.emit(ceil(progress).toInt()) } } @@ -221,135 +146,33 @@ class MediaPlayerManager : LifecycleAwareManager { } } - /** - * Adds a audio file to the play queue. for cycling through - * - * @throws FileNotFoundException if the file is not downloaded to cache first - */ - fun addToPlayList(path: String, chatMessage: ChatMessage) { - val file = File(path) - if (!file.exists()) { - throw FileNotFoundException("Cannot add to playlist without downloading to cache first for path\n$path") - } - - for (pair in playQueue) { - if (pair.first == path) return - } - - playQueue.add(Pair(path, chatMessage)) - } - - fun clearPlayList() { - playQueue.clear() - } - - /** - * Sets the player speed. - */ - fun setPlayBackSpeed(speed: PlaybackSpeed) { - requestedPlaybackSpeed = speed - if (mediaPlayer != null && mediaPlayer!!.isPlaying) { - mediaPlayer!!.playbackParameters = PlaybackParameters(speed.value) - } - } - private fun init(path: String) { try { val context = NextcloudTalkApplication.sharedApplication!!.applicationContext mediaPlayer = ExoPlayer.Builder(context).build().apply { - _managerState.value = MediaPlayerManagerState.SETUP + managerState.value = MediaPlayerManagerState.SETUP setMediaItem(MediaItem.fromUri(path)) - currentDataSource = path - prepare() - addListener(object : Player.Listener { - override fun onPlaybackStateChanged(playbackState: Int) { - if (playbackState == Player.STATE_READY && - _managerState.value == MediaPlayerManagerState.SETUP - ) { - onPrepare() - } - } - }) - } - } catch (e: Exception) { - Log.e(ChatActivity.TAG, "failed to initialize mediaPlayer", e) - _managerState.value = MediaPlayerManagerState.ERROR - } - } - - private fun initCycling() { - try { - val context = NextcloudTalkApplication.sharedApplication!!.applicationContext - mediaPlayer = ExoPlayer.Builder(context).build().apply { - _managerState.value = MediaPlayerManagerState.SETUP - val pair = playQueue.iterator().next() - setMediaItem(MediaItem.fromUri(pair.first)) - currentDataSource = pair.first - _currentCycledMessage.value = pair.second - playQueue.removeAt(0) prepare() addListener(object : Player.Listener { override fun onPlaybackStateChanged(playbackState: Int) { if (playbackState == Player.STATE_READY && - _managerState.value == MediaPlayerManagerState.SETUP + managerState.value == MediaPlayerManagerState.SETUP ) { onPrepare() - } else if (playbackState == Player.STATE_ENDED) { - if (playQueue.iterator().hasNext() && playQueue.first().first != currentDataSource) { - _managerState.value = MediaPlayerManagerState.SETUP - val nextPair = playQueue.iterator().next() - playQueue.removeAt(0) - mediaPlayer?.setMediaItem(MediaItem.fromUri(nextPair.first)) - currentDataSource = nextPair.first - _currentCycledMessage.value = nextPair.second - prepare() - } else { - mediaPlayer?.release() - mediaPlayer = null - _backgroundPlayUIFlow.tryEmit(null) - _currentCycledMessage.value?.let { - it.resetVoiceMessage = true - it.isPlayingVoiceMessage = false - it.voiceMessageSeekbarProgress = 0 - it.voiceMessagePlayedSeconds = 0 - } - val completedMessage = _currentCycledMessage.value - _currentCycledMessage.value = null - if (completedMessage != null) { - scope.launch { - _mediaPlayerSeekBarPositionMsg.emit(completedMessage) - } - } - loop = false - _managerState.value = MediaPlayerManagerState.STOPPED - } } } }) } } catch (e: Exception) { Log.e(ChatActivity.TAG, "failed to initialize mediaPlayer", e) - _managerState.value = MediaPlayerManagerState.ERROR + managerState.value = MediaPlayerManagerState.ERROR } } private fun ExoPlayer.onPrepare() { mediaPlayerDuration = this.duration.toInt() - - val playBackSpeed = requestedPlaybackSpeed?.value - ?: if (_currentCycledMessage.value?.actorId == null) { - PlaybackSpeed.NORMAL.value - } else { - appPreferences.getPreferredPlayback(_currentCycledMessage.value?.actorId).value - } - playbackParameters = PlaybackParameters(playBackSpeed) - play() - _managerState.value = MediaPlayerManagerState.STARTED - _currentCycledMessage.value?.let { - it.isPlayingVoiceMessage = true - _backgroundPlayUIFlow.tryEmit(it) - } + managerState.value = MediaPlayerManagerState.STARTED loop = true scope = MainScope() scope.launch { seekbarUpdateObserver() } @@ -367,10 +190,5 @@ class MediaPlayerManager : LifecycleAwareManager { override fun handleOnStop() { loop = false - if (mediaPlayer != null && _currentCycledMessage.value != null && mediaPlayer!!.isPlaying) { - CoroutineScope(Dispatchers.Default).launch { - _backgroundPlayUIFlow.tryEmit(_currentCycledMessage.value) - } - } } } diff --git a/app/src/main/java/com/nextcloud/talk/chat/data/io/VoiceMessageMediaService.kt b/app/src/main/java/com/nextcloud/talk/chat/data/io/VoiceMessageMediaService.kt index 3deed37744b..e31c781acb0 100644 --- a/app/src/main/java/com/nextcloud/talk/chat/data/io/VoiceMessageMediaService.kt +++ b/app/src/main/java/com/nextcloud/talk/chat/data/io/VoiceMessageMediaService.kt @@ -7,28 +7,343 @@ package com.nextcloud.talk.chat.data.io +import android.app.PendingIntent +import android.content.Intent import androidx.annotation.OptIn +import androidx.media3.common.AudioAttributes +import androidx.media3.common.C +import androidx.media3.common.MediaItem +import androidx.media3.common.PlaybackException +import androidx.media3.common.Player +import androidx.media3.common.Timeline import androidx.media3.common.util.UnstableApi import androidx.media3.exoplayer.ExoPlayer +import androidx.media3.exoplayer.analytics.AnalyticsListener +import androidx.media3.exoplayer.source.DefaultMediaSourceFactory +import androidx.media3.exoplayer.source.LoadEventInfo +import androidx.media3.exoplayer.source.MediaLoadData +import androidx.media3.exoplayer.source.ShuffleOrder +import androidx.media3.session.DefaultMediaNotificationProvider import androidx.media3.session.MediaSession import androidx.media3.session.MediaSessionService +import autodagger.AutoInjector +import com.google.common.util.concurrent.Futures +import com.google.common.util.concurrent.ListenableFuture +import com.nextcloud.talk.R +import com.nextcloud.talk.application.NextcloudTalkApplication +import com.nextcloud.talk.chat.audio.ChatAudioDataSources +import com.nextcloud.talk.chat.audio.ChatAudioError +import com.nextcloud.talk.chat.audio.ChatAudioKey +import com.nextcloud.talk.chat.audio.ChatAudioKind +import com.nextcloud.talk.chat.audio.ChatAudioRepeatMode +import com.nextcloud.talk.chat.audio.ChatAudioStore +import com.nextcloud.talk.chat.audio.VoiceMessageWaveformLoader +import com.nextcloud.talk.chat.audio.chatAudioKind +import com.nextcloud.talk.chat.audio.chatAudioRepeatModeOf +import com.nextcloud.talk.chat.audio.chatAudioThreadId +import com.nextcloud.talk.chat.audio.openMessageIntent +import com.nextcloud.talk.chat.audio.toChatAudioError +import com.nextcloud.talk.chat.audio.toPlayerRepeatMode +import com.nextcloud.talk.data.network.NetworkMonitor +import com.nextcloud.talk.utils.preferences.AppPreferences +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job +import kotlinx.coroutines.MainScope +import kotlinx.coroutines.cancel +import kotlinx.coroutines.delay +import kotlinx.coroutines.flow.first +import kotlinx.coroutines.launch +import javax.inject.Inject +import kotlin.random.Random +/** + * Plays voice messages and audio files of chats, also in the background, and exposes them to the notification, + * the lock screen, headsets and Bluetooth devices. It is the single source of truth for chat audio playback; the + * UI only controls it through a [androidx.media3.session.MediaController]. + */ +@OptIn(UnstableApi::class) +@AutoInjector(NextcloudTalkApplication::class) class VoiceMessageMediaService : MediaSessionService() { + + @Inject + lateinit var dataSources: ChatAudioDataSources + + @Inject + lateinit var store: ChatAudioStore + + @Inject + lateinit var waveformLoader: VoiceMessageWaveformLoader + + @Inject + lateinit var networkMonitor: NetworkMonitor + + @Inject + lateinit var appPreferences: AppPreferences + private var mediaSession: MediaSession? = null + private val serviceScope = MainScope() + private var retryJob: Job? = null + private var automaticRetries = 0 + private var currentKey: ChatAudioKey? = null + private var audioFileShuffle = false + private var audioFileRepeatMode = ChatAudioRepeatMode.OFF - @OptIn(UnstableApi::class) override fun onCreate() { super.onCreate() - val player = ExoPlayer.Builder(this).build() - mediaSession = MediaSession.Builder(this, player).build() + NextcloudTalkApplication.sharedApplication!!.componentApplication.inject(this) + audioFileShuffle = appPreferences.getChatAudioShuffle() + audioFileRepeatMode = ChatAudioRepeatMode.fromName(appPreferences.getChatAudioRepeatMode()) + + val player = ExoPlayer.Builder(this) + .setMediaSourceFactory(DefaultMediaSourceFactory(dataSources.cacheDataSourceFactory)) + .setAudioAttributes(audioAttributesFor(ChatAudioKind.VOICE_MESSAGE), true) + .setHandleAudioBecomingNoisy(true) + .setWakeMode(C.WAKE_MODE_NETWORK) + .build() + player.addListener(PlayerListener(player)) + player.addAnalyticsListener(LoadListener()) + serviceScope.launch { + store.accountRemovals.collect { internalUserId -> stopIfPlayingAccount(player, internalUserId) } + } + + mediaSession = MediaSession.Builder(this, player) + .setCallback(SessionCallback()) + .build() + + setMediaNotificationProvider( + DefaultMediaNotificationProvider.Builder(this).build().apply { + setSmallIcon(R.drawable.ic_notification) + } + ) } override fun onGetSession(controllerInfo: MediaSession.ControllerInfo): MediaSession? = mediaSession override fun onDestroy() { - mediaSession?.player?.release() - mediaSession?.release() + serviceScope.cancel() + mediaSession?.let { session -> + rememberPosition(session.player) + session.player.release() + session.release() + } mediaSession = null super.onDestroy() } + + private fun rememberPosition(player: Player) { + val key = currentKey + if (key != null && player.playbackState != Player.STATE_ENDED) { + store.rememberPosition(key, player.currentPosition, player.duration) + } + } + + /** + * Stops playback and unloads the queue if it plays audio of a removed account, so neither its audio nor its + * senders stay in the notification or on the lock screen. + */ + private fun stopIfPlayingAccount(player: ExoPlayer, internalUserId: Long) { + if (currentKey?.internalUserId == internalUserId) { + currentKey = null + player.stop() + player.clearMediaItems() + } + } + + private fun onItemStarted(player: ExoPlayer, mediaItem: MediaItem?, isNewQueue: Boolean) { + val key = ChatAudioKey.fromMediaId(mediaItem?.mediaId) + if (key != null && store.isAccountRemoved(key.internalUserId)) { + currentKey = key + stopIfPlayingAccount(player, key.internalUserId) + return + } + currentKey = key + automaticRetries = 0 + retryJob?.cancel() + mediaSession?.setSessionActivity(key?.let { sessionActivityFor(it, mediaItem?.chatAudioThreadId()) }) + if (mediaItem == null) { + // Nothing is left to play, so the service only keeps running while a screen is bound to it. + stopSelf() + } + if (key != null && mediaItem != null) { + val kind = mediaItem.chatAudioKind() + player.setAudioAttributes(audioAttributesFor(kind), true) + applyPlaybackModes(player, kind, isNewQueue) + store.forgetPosition(key) + // A prebuffered item starts while the player stays ready, so its duration is only known here. + store.rememberDuration(key, player.duration) + } + } + + /** Voice messages are always played in order, audio files are shuffled and repeated as the user chose. */ + private fun applyPlaybackModes(player: ExoPlayer, kind: ChatAudioKind, isNewQueue: Boolean) { + val isAudioFile = kind == ChatAudioKind.AUDIO_FILE + if (isAudioFile && isNewQueue) { + player.setShuffleOrder(shuffleOrderStartingAt(player.currentMediaItemIndex, player.mediaItemCount)) + } + player.shuffleModeEnabled = isAudioFile && audioFileShuffle + player.repeatMode = if (isAudioFile) audioFileRepeatMode.toPlayerRepeatMode() else Player.REPEAT_MODE_OFF + } + + private fun loadWaveform(mediaItem: MediaItem) { + val key = ChatAudioKey.fromMediaId(mediaItem.mediaId) + val localConfiguration = mediaItem.localConfiguration + if (key != null && localConfiguration != null && mediaItem.chatAudioKind() == ChatAudioKind.VOICE_MESSAGE) { + waveformLoader.load(key, localConfiguration.uri, localConfiguration.customCacheKey) + } + } + + private fun sessionActivityFor(key: ChatAudioKey, threadId: Long?): PendingIntent { + val intent = key.openMessageIntent(this, threadId).apply { + flags = Intent.FLAG_ACTIVITY_CLEAR_TOP or Intent.FLAG_ACTIVITY_SINGLE_TOP + } + return PendingIntent.getActivity( + this, + SESSION_ACTIVITY_REQUEST_CODE, + intent, + PendingIntent.FLAG_IMMUTABLE or PendingIntent.FLAG_UPDATE_CURRENT + ) + } + + /** Retries after a network error once the device is online again, like a stalled download is resumed. */ + private fun scheduleRetry(player: ExoPlayer, error: PlaybackException) { + if (error.toChatAudioError() == ChatAudioError.NETWORK && automaticRetries < MAX_AUTOMATIC_RETRIES) { + retryJob?.cancel() + retryJob = serviceScope.launch { + networkMonitor.isOnline.first { it } + delay(RETRY_DELAY_MS) + if (player.playerError != null) { + automaticRetries++ + player.prepare() + } + } + } + } + + private inner class PlayerListener(private val player: ExoPlayer) : Player.Listener { + + override fun onMediaItemTransition(mediaItem: MediaItem?, reason: Int) { + val isNewQueue = reason == Player.MEDIA_ITEM_TRANSITION_REASON_PLAYLIST_CHANGED + onItemStarted(player, mediaItem, isNewQueue) + } + + override fun onShuffleModeEnabledChanged(shuffleModeEnabled: Boolean) { + if (player.currentMediaItem?.chatAudioKind() == ChatAudioKind.AUDIO_FILE) { + if (shuffleModeEnabled) { + player.setShuffleOrder(shuffleOrderStartingAt(player.currentMediaItemIndex, player.mediaItemCount)) + } + if (shuffleModeEnabled != audioFileShuffle) { + audioFileShuffle = shuffleModeEnabled + serviceScope.launch(Dispatchers.IO) { appPreferences.setChatAudioShuffle(shuffleModeEnabled) } + } + } + } + + override fun onRepeatModeChanged(repeatMode: Int) { + val mode = chatAudioRepeatModeOf(repeatMode) + if (player.currentMediaItem?.chatAudioKind() == ChatAudioKind.AUDIO_FILE && mode != audioFileRepeatMode) { + audioFileRepeatMode = mode + serviceScope.launch(Dispatchers.IO) { appPreferences.setChatAudioRepeatMode(mode.name) } + } + } + + override fun onPositionDiscontinuity( + oldPosition: Player.PositionInfo, + newPosition: Player.PositionInfo, + reason: Int + ) { + val oldKey = ChatAudioKey.fromMediaId(oldPosition.mediaItem?.mediaId) + if (oldKey != null && oldPosition.mediaItem?.mediaId != newPosition.mediaItem?.mediaId) { + if (reason == Player.DISCONTINUITY_REASON_AUTO_TRANSITION) { + store.forgetPosition(oldKey) + } else { + store.rememberPosition(oldKey, oldPosition.positionMs, C.TIME_UNSET) + } + } + } + + override fun onPlaybackStateChanged(playbackState: Int) { + val key = currentKey + if (key != null && playbackState == Player.STATE_READY) { + store.rememberDuration(key, player.duration) + } + if (playbackState == Player.STATE_ENDED) { + key?.let { store.forgetPosition(it) } + player.stop() + player.clearMediaItems() + } + } + + override fun onIsPlayingChanged(isPlaying: Boolean) { + if (isPlaying) { + automaticRetries = 0 + } else { + rememberPosition(player) + } + } + + override fun onPlayerError(error: PlaybackException) { + scheduleRetry(player, error) + } + } + + /** + * Starts computing the waveform of a voice message once the player loaded all of it, as the file is locked in + * the cache while it is loaded and would otherwise be downloaded a second time. + */ + private inner class LoadListener : AnalyticsListener { + override fun onLoadCompleted( + eventTime: AnalyticsListener.EventTime, + loadEventInfo: LoadEventInfo, + mediaLoadData: MediaLoadData + ) { + val timeline = eventTime.timeline + if (mediaLoadData.dataType == C.DATA_TYPE_MEDIA && eventTime.windowIndex < timeline.windowCount) { + loadWaveform(timeline.getWindow(eventTime.windowIndex, Timeline.Window()).mediaItem) + } + } + } + + /** + * Only accepts media items of this app. Other apps may control playback, e.g. from the lock screen or a + * headset, but can never make the service load their URLs with the credentials of an account. + */ + private inner class SessionCallback : MediaSession.Callback { + override fun onAddMediaItems( + mediaSession: MediaSession, + controller: MediaSession.ControllerInfo, + mediaItems: MutableList + ): ListenableFuture> = + if (controller.packageName == packageName && mediaItems.all { it.localConfiguration != null }) { + Futures.immediateFuture(mediaItems) + } else { + Futures.immediateFailedFuture>( + UnsupportedOperationException("Only Talk can add chat audio") + ) + } + } + + companion object { + private const val SESSION_ACTIVITY_REQUEST_CODE = 4711 + private const val MAX_AUTOMATIC_RETRIES = 3 + private const val RETRY_DELAY_MS = 1000L + + private fun audioAttributesFor(kind: ChatAudioKind): AudioAttributes = + AudioAttributes.Builder() + .setUsage(C.USAGE_MEDIA) + .setContentType( + if (kind == ChatAudioKind.VOICE_MESSAGE) { + C.AUDIO_CONTENT_TYPE_SPEECH + } else { + C.AUDIO_CONTENT_TYPE_MUSIC + } + ) + .build() + } +} + +/** A shuffle order that plays the item at [firstIndex] first and all other items afterwards in random order. */ +@OptIn(UnstableApi::class) +private fun shuffleOrderStartingAt(firstIndex: Int, length: Int): ShuffleOrder { + val others = (0 until length).filter { it != firstIndex }.shuffled() + return ShuffleOrder.DefaultShuffleOrder((listOf(firstIndex) + others).toIntArray(), Random.nextLong()) } diff --git a/app/src/main/java/com/nextcloud/talk/chat/data/model/ChatMessage.kt b/app/src/main/java/com/nextcloud/talk/chat/data/model/ChatMessage.kt index 836d5c2cb16..425a2da31ea 100644 --- a/app/src/main/java/com/nextcloud/talk/chat/data/model/ChatMessage.kt +++ b/app/src/main/java/com/nextcloud/talk/chat/data/model/ChatMessage.kt @@ -91,24 +91,6 @@ data class ChatMessage( var incoming: Boolean = false, - var isDownloadingVoiceMessage: Boolean = false, - - var resetVoiceMessage: Boolean = false, - - var isPlayingVoiceMessage: Boolean = false, - - var wasPlayedVoiceMessage: Boolean = false, - - var voiceMessageDuration: Int = 0, - - var voiceMessagePlayedSeconds: Int = 0, - - var voiceMessageDownloadProgress: Int = 0, - - var voiceMessageSeekbarProgress: Int = 0, - - var voiceMessageFloatArray: FloatArray? = null, - var expandableParent: Boolean = false, var isExpanded: Boolean = false, diff --git a/app/src/main/java/com/nextcloud/talk/chat/ui/model/ChatMessageUi.kt b/app/src/main/java/com/nextcloud/talk/chat/ui/model/ChatMessageUi.kt index 7a059e30b19..88b3cf7caf4 100644 --- a/app/src/main/java/com/nextcloud/talk/chat/ui/model/ChatMessageUi.kt +++ b/app/src/main/java/com/nextcloud/talk/chat/ui/model/ChatMessageUi.kt @@ -14,7 +14,6 @@ import com.nextcloud.talk.application.NextcloudTalkApplication.Companion.sharedA import com.nextcloud.talk.chat.data.model.ChatMessage import com.nextcloud.talk.data.database.model.SendStatus import com.nextcloud.talk.data.user.model.User -import com.nextcloud.talk.ui.PlaybackSpeed import com.nextcloud.talk.utils.ApiUtils import com.nextcloud.talk.utils.CapabilitiesUtil import com.nextcloud.talk.utils.DrawableUtils @@ -103,26 +102,10 @@ sealed interface MessageTypeContent { data class Deck(val cardName: String, val stackName: String, val boardName: String, val cardLink: String) : MessageTypeContent - data class Voice( - val actorId: String?, - val isPlaying: Boolean, - val wasPlayed: Boolean, - val isDownloading: Boolean, - val durationSeconds: Int, - val playedSeconds: Int, - val seekbarProgress: Int, - val waveform: List, - val playbackSpeed: PlaybackSpeed = PlaybackSpeed.NORMAL - ) : MessageTypeContent + // The playback state of audio messages is not part of the message, see com.nextcloud.talk.ui.chat.ChatAudioUi + data object Voice : MessageTypeContent - data class AudioFile( - val fileName: String, - val isPlaying: Boolean, - val isDownloading: Boolean, - val durationSeconds: Int, - val playedSeconds: Int, - val seekbarProgress: Int - ) : MessageTypeContent + data class AudioFile(val fileName: String) : MessageTypeContent } enum class MessageStatusIcon { @@ -282,7 +265,7 @@ fun getMessageTypeContent(user: User, message: ChatMessage, isClassified: Boolea if (message.isSystemMessage) { MessageTypeContent.SystemMessage } else if (message.isVoiceMessage) { - getVoiceContent(message) + MessageTypeContent.Voice } else if (message.hasFileAttachment && message.isTemporary) { getUploadingMediaContent(message) } else if (message.hasFileAttachment && message.fileParameters.mimetype.startsWith(Mimetype.AUDIO_PREFIX)) { @@ -406,24 +389,5 @@ fun getDeckContent(message: ChatMessage): MessageTypeContent.Deck = cardLink = message.deckCardParameters.link ) -fun getVoiceContent(message: ChatMessage): MessageTypeContent.Voice = - MessageTypeContent.Voice( - actorId = message.actorId, - isPlaying = message.isPlayingVoiceMessage, - wasPlayed = message.wasPlayedVoiceMessage, - isDownloading = message.isDownloadingVoiceMessage, - durationSeconds = message.voiceMessageDuration, - playedSeconds = message.voiceMessagePlayedSeconds, - seekbarProgress = message.voiceMessageSeekbarProgress, - waveform = message.voiceMessageFloatArray?.toList().orEmpty() - ) - fun getAudioFileContent(message: ChatMessage): MessageTypeContent.AudioFile = - MessageTypeContent.AudioFile( - fileName = message.fileParameters.name, - isPlaying = message.isPlayingVoiceMessage, - isDownloading = message.isDownloadingVoiceMessage, - durationSeconds = message.voiceMessageDuration, - playedSeconds = message.voiceMessagePlayedSeconds, - seekbarProgress = message.voiceMessageSeekbarProgress - ) + MessageTypeContent.AudioFile(fileName = message.fileParameters.name) diff --git a/app/src/main/java/com/nextcloud/talk/chat/viewmodels/ChatViewModel.kt b/app/src/main/java/com/nextcloud/talk/chat/viewmodels/ChatViewModel.kt index f005033c954..95399a9533a 100644 --- a/app/src/main/java/com/nextcloud/talk/chat/viewmodels/ChatViewModel.kt +++ b/app/src/main/java/com/nextcloud/talk/chat/viewmodels/ChatViewModel.kt @@ -27,7 +27,6 @@ import com.nextcloud.talk.application.NextcloudTalkApplication import com.nextcloud.talk.arbitrarystorage.ArbitraryStorageManager import com.nextcloud.talk.chat.data.ChatMessageRepository import com.nextcloud.talk.chat.data.io.AudioFocusRequestManager -import com.nextcloud.talk.chat.data.io.MediaPlayerManager import com.nextcloud.talk.chat.data.io.MediaRecorderManager import com.nextcloud.talk.chat.data.model.ChatMessage import com.nextcloud.talk.chat.data.model.FileParameters @@ -72,7 +71,6 @@ import com.nextcloud.talk.models.json.userAbsence.UserAbsenceDataDto import com.nextcloud.talk.repositories.reactions.ReactionsRepository import com.nextcloud.talk.repositories.unifiedsearch.UnifiedSearchRepository import com.nextcloud.talk.threadsoverview.data.ThreadsRepository -import com.nextcloud.talk.ui.PlaybackSpeed import com.nextcloud.talk.users.UserManager import com.nextcloud.talk.utils.ApiUtils import com.nextcloud.talk.utils.CapabilitiesUtil.hasSpreedFeatureCapability @@ -371,7 +369,6 @@ class ChatViewModel @AssistedInject constructor( private var showUnreadMessagesMarker: Boolean = true private var isLoadMoreInProgress = false - private val mediaPlayerManager: MediaPlayerManager = MediaPlayerManager.sharedInstance(appPreferences) lateinit var currentLifeCycleFlag: LifeCycleFlag val disposableSet = mutableSetOf() @@ -476,7 +473,6 @@ class ChatViewModel @AssistedInject constructor( currentLifeCycleFlag = LifeCycleFlag.RESUMED mediaRecorderManager.handleOnResume() chatRepository.handleOnResume() - mediaPlayerManager.handleOnResume() if (isReturningFromBackground) { viewModelScope.launch { chatRepository.fetchNewMessages() @@ -495,7 +491,6 @@ class ChatViewModel @AssistedInject constructor( messageSearchHelper?.cancelSearch() mediaRecorderManager.handleOnPause() chatRepository.handleOnPause() - mediaPlayerManager.handleOnPause() saveMessageDraft() } @@ -508,7 +503,6 @@ class ChatViewModel @AssistedInject constructor( mediaRecorderManager.handleOnStop() } chatRepository.handleOnStop() - mediaPlayerManager.handleOnStop() } fun onSignalingChatMessageReceived(chatMessages: List) { @@ -521,15 +515,6 @@ class ChatViewModel @AssistedInject constructor( showUnreadMessagesMarker = shouldShow } - val mediaPlayerSeekbarObserver: Flow - get() = mediaPlayerManager.mediaPlayerSeekBarPositionMsg - - val currentlyPlayedMessageId: Flow = mediaPlayerManager.currentCycledMessage.map { it?.jsonMessageId } - - fun setPlayBack(speed: PlaybackSpeed) { - mediaPlayerManager.setPlayBackSpeed(speed) - } - private val _recordTouchObserver: MutableLiveData = MutableLiveData() val recordTouchObserver: LiveData get() = _recordTouchObserver @@ -554,10 +539,6 @@ class ChatViewModel @AssistedInject constructor( val unbindRoomResult: LiveData get() = _unbindRoomResult - private val _voiceMessagePlaybackSpeedPreferences: MutableLiveData> = MutableLiveData() - val voiceMessagePlaybackSpeedPreferences: LiveData> - get() = _voiceMessagePlaybackSpeedPreferences - private val _threadRetrieveState = MutableStateFlow(ThreadRetrieveUiState.None) val threadRetrieveState: StateFlow = _threadRetrieveState @@ -830,7 +811,6 @@ class ChatViewModel @AssistedInject constructor( observeLobbyState() observeLobbyPolling() observeMessages() - observeMediaPlayerProgressForCompose() observePinnedMessage() observeRoomRefresh() observeIncomingMessages() @@ -1080,95 +1060,6 @@ class ChatViewModel @AssistedInject constructor( firstUnreadMessageId = null } - private fun observeMediaPlayerProgressForCompose() { - mediaPlayerSeekbarObserver - .onEach { message -> - syncVoiceMessageUiState(message) - } - .launchIn(viewModelScope) - } - - fun pauseVoiceMessageUiState(messageId: Int) { - _uiState.update { current -> - val updatedItems = current.items.map { item -> - if (item is ChatItem.MessageItem && item.uiMessage.id == messageId) { - val voiceContent = item.uiMessage.content as? MessageTypeContent.Voice - if (voiceContent != null) { - item.copy(uiMessage = item.uiMessage.copy(content = voiceContent.copy(isPlaying = false))) - } else { - item - } - } else { - item - } - } - current.copy(items = updatedItems) - } - } - - fun setVoiceMessageSpeed(messageId: Int, speed: PlaybackSpeed) { - _uiState.update { current -> - val updatedItems = current.items.map { item -> - if (item is ChatItem.MessageItem && item.uiMessage.id == messageId) { - val voiceContent = item.uiMessage.content as? MessageTypeContent.Voice - if (voiceContent != null) { - item.copy(uiMessage = item.uiMessage.copy(content = voiceContent.copy(playbackSpeed = speed))) - } else { - item - } - } else { - item - } - } - current.copy(items = updatedItems) - } - } - - var currentVoiceMessage: ChatMessage? = null - - fun syncVoiceMessageUiState(message: ChatMessage) { - currentVoiceMessage = message - _uiState.update { current -> - val updatedItems = current.items.map { item -> - if (item is ChatItem.MessageItem && item.uiMessage.id == message.jsonMessageId) { - when (val content = item.uiMessage.content) { - is MessageTypeContent.Voice -> { - val updatedVoiceContent = content.copy( - actorId = message.actorId, - isPlaying = message.isPlayingVoiceMessage, - wasPlayed = message.wasPlayedVoiceMessage, - isDownloading = message.isDownloadingVoiceMessage, - durationSeconds = message.voiceMessageDuration, - playedSeconds = message.voiceMessagePlayedSeconds, - seekbarProgress = message.voiceMessageSeekbarProgress, - waveform = message.voiceMessageFloatArray?.toList() ?: content.waveform - // playbackSpeed is preserved from existing content - ) - item.copy(uiMessage = item.uiMessage.copy(content = updatedVoiceContent)) - } - - is MessageTypeContent.AudioFile -> { - val updatedAudioFileContent = content.copy( - isPlaying = message.isPlayingVoiceMessage, - isDownloading = message.isDownloadingVoiceMessage, - durationSeconds = message.voiceMessageDuration, - playedSeconds = message.voiceMessagePlayedSeconds, - seekbarProgress = message.voiceMessageSeekbarProgress - ) - item.copy(uiMessage = item.uiMessage.copy(content = updatedAudioFileContent)) - } - - else -> item - } - } else { - item - } - } - - current.copy(items = updatedItems) - } - } - // ------------------------------ // Observe conversation // ------------------------------ diff --git a/app/src/main/java/com/nextcloud/talk/chat/viewmodels/MessageInputViewModel.kt b/app/src/main/java/com/nextcloud/talk/chat/viewmodels/MessageInputViewModel.kt index 55c9a2f8320..5cbd10adeba 100644 --- a/app/src/main/java/com/nextcloud/talk/chat/viewmodels/MessageInputViewModel.kt +++ b/app/src/main/java/com/nextcloud/talk/chat/viewmodels/MessageInputViewModel.kt @@ -361,7 +361,7 @@ class MessageInputViewModel : fun pauseMediaPlayer() { audioFocusRequestManager.audioFocusRequest(false) { - mediaPlayerManager.pause(false) + mediaPlayerManager.pause() _isVoicePreviewPlaying.postValue(false) } } diff --git a/app/src/main/java/com/nextcloud/talk/conversationlist/ConversationsListActivity.kt b/app/src/main/java/com/nextcloud/talk/conversationlist/ConversationsListActivity.kt index f03d2abe101..1770c167ea3 100644 --- a/app/src/main/java/com/nextcloud/talk/conversationlist/ConversationsListActivity.kt +++ b/app/src/main/java/com/nextcloud/talk/conversationlist/ConversationsListActivity.kt @@ -32,7 +32,9 @@ import androidx.core.content.pm.ShortcutInfoCompat import androidx.core.content.pm.ShortcutManagerCompat import androidx.core.graphics.drawable.IconCompat import androidx.core.net.toUri +import androidx.compose.runtime.getValue import androidx.lifecycle.Lifecycle +import androidx.lifecycle.compose.collectAsStateWithLifecycle import androidx.lifecycle.lifecycleScope import androidx.lifecycle.repeatOnLifecycle import androidx.work.Data @@ -52,6 +54,12 @@ import com.nextcloud.talk.activities.MainActivity import com.nextcloud.talk.api.NcApiCoroutines import com.nextcloud.talk.application.NextcloudTalkApplication import com.nextcloud.talk.chat.ChatActivity +import com.nextcloud.talk.chat.audio.ChatAudioKey +import com.nextcloud.talk.chat.audio.ChatAudioPlayer +import com.nextcloud.talk.chat.audio.ChatAudioSpeeds +import com.nextcloud.talk.chat.audio.ChatAudioStore +import com.nextcloud.talk.chat.audio.openMessageExtras +import com.nextcloud.talk.chat.audio.openMessageIntent import com.nextcloud.talk.contacts.ContactsActivity import com.nextcloud.talk.contacts.ContactsViewModel import com.nextcloud.talk.conversation.RenameConversationDialogFragment @@ -78,6 +86,8 @@ import com.nextcloud.talk.models.domain.SearchMessageEntry import com.nextcloud.talk.models.json.conversations.ConversationEnums import com.nextcloud.talk.settings.SettingsActivity import com.nextcloud.talk.threadsoverview.ThreadsOverviewActivity +import com.nextcloud.talk.ui.chat.ChatAudioPlayerHost +import com.nextcloud.talk.ui.chat.chatAudioPlayerCallbacks import com.nextcloud.talk.ui.chooseaccount.ChooseAccountShareToDialogFragment import com.nextcloud.talk.ui.dialog.FilterConversationFragment import com.nextcloud.talk.ui.dialog.FilterConversationFragment.Companion.ARCHIVE @@ -166,6 +176,14 @@ class ConversationsListActivity : BaseActivity() { val contactsViewModel: ContactsViewModel by assistedViewModels { contactsViewModelFactory.build(currentUser) } + @Inject + lateinit var chatAudioStore: ChatAudioStore + + @Inject + lateinit var chatAudioSpeeds: ChatAudioSpeeds + + private lateinit var chatAudioPlayer: ChatAudioPlayer + val conversationsListViewModel: ConversationsListViewModel by assistedViewModels { conversationsListViewModelFactory.build(currentUser) } @@ -230,6 +248,8 @@ class ConversationsListActivity : BaseActivity() { NextcloudTalkApplication.sharedApplication!!.componentApplication.inject(this) currentUser = setUpBoundUserOrFinish() ?: return ecosystemManager = EcosystemManager(this@ConversationsListActivity) + chatAudioPlayer = ChatAudioPlayer(this, chatAudioStore, lifecycleScope) + lifecycle.addObserver(chatAudioPlayer) if (!currentUser.current) { // The shown account becomes the last used one, which the account switcher and status views rely on. @@ -245,12 +265,22 @@ class ConversationsListActivity : BaseActivity() { } onBackPressedDispatcher.addCallback(this, onBackPressedCallback) + val audioPlayerCallbacks = chatAudioPlayerCallbacks( + player = chatAudioPlayer, + speeds = chatAudioSpeeds, + scope = lifecycleScope, + onOpenMessage = { key, threadId -> openAudioMessage(key, threadId) } + ) setContent { ConversationsListScreen( viewModel = conversationsListViewModel, tagsViewModel = conversationTagsViewModel, state = buildScreenState(), - callbacks = buildScreenCallbacks() + callbacks = buildScreenCallbacks(), + audioPlayer = { + val audioState by chatAudioPlayer.state.collectAsStateWithLifecycle() + ChatAudioPlayerHost(state = audioState, callbacks = audioPlayerCallbacks) + } ) } @@ -698,6 +728,19 @@ class ConversationsListActivity : BaseActivity() { startActivity(intent) } + /** + * Opens the chat of an audio message at the message, in its thread if it has one; messages of another account + * switch to that account. + */ + private fun openAudioMessage(key: ChatAudioKey, threadId: Long?) { + if (key.internalUserId == currentUser?.id) { + val extras = key.openMessageExtras(threadId) + startActivity(ChatActivity.createIntent(context, key.internalUserId, key.roomToken, extras)) + } else { + startActivity(key.openMessageIntent(this, threadId)) + } + } + fun filterConversation() { conversationsListViewModel.reloadFilterFromStorage(UserIdUtils.getIdForUser(currentUser)) } diff --git a/app/src/main/java/com/nextcloud/talk/conversationlist/ui/ConversationsListScreen.kt b/app/src/main/java/com/nextcloud/talk/conversationlist/ui/ConversationsListScreen.kt index 0cc1bcbf7ff..df100782446 100644 --- a/app/src/main/java/com/nextcloud/talk/conversationlist/ui/ConversationsListScreen.kt +++ b/app/src/main/java/com/nextcloud/talk/conversationlist/ui/ConversationsListScreen.kt @@ -131,7 +131,8 @@ fun ConversationsListScreen( viewModel: ConversationsListViewModel, tagsViewModel: ConversationTagsViewModel, state: ConversationsListScreenState, - callbacks: ConversationsListScreenCallbacks + callbacks: ConversationsListScreenCallbacks, + audioPlayer: @Composable () -> Unit = {} ) { val context = LocalContext.current val activity = context as Activity @@ -273,6 +274,7 @@ fun ConversationsListScreen( onAccountChooserClick = callbacks.onAccountChooserClick ) ) + audioPlayer() } } ) { paddingValues -> diff --git a/app/src/main/java/com/nextcloud/talk/dagger/modules/ManagerModule.kt b/app/src/main/java/com/nextcloud/talk/dagger/modules/ManagerModule.kt index cc7b14ff60b..bad3b1034ce 100644 --- a/app/src/main/java/com/nextcloud/talk/dagger/modules/ManagerModule.kt +++ b/app/src/main/java/com/nextcloud/talk/dagger/modules/ManagerModule.kt @@ -12,7 +12,6 @@ import com.nextcloud.talk.chat.data.io.AudioFocusRequestManager import com.nextcloud.talk.chat.data.io.AudioRecorderManager import com.nextcloud.talk.chat.data.io.MediaPlayerManager import com.nextcloud.talk.chat.data.io.MediaRecorderManager -import com.nextcloud.talk.utils.preferences.AppPreferences import dagger.Module import dagger.Provides @@ -26,10 +25,7 @@ class ManagerModule { fun provideAudioRecorderManager(): AudioRecorderManager = AudioRecorderManager() @Provides - fun provideMediaPlayerManager(preferences: AppPreferences): MediaPlayerManager = - MediaPlayerManager().apply { - appPreferences = preferences - } + fun provideMediaPlayerManager(): MediaPlayerManager = MediaPlayerManager() @Provides fun provideAudioFocusManager(context: Context): AudioFocusRequestManager = AudioFocusRequestManager(context) diff --git a/app/src/main/java/com/nextcloud/talk/jobs/AccountRemovalWorker.java b/app/src/main/java/com/nextcloud/talk/jobs/AccountRemovalWorker.java index 4062d2d27b4..5eeb8b528a8 100644 --- a/app/src/main/java/com/nextcloud/talk/jobs/AccountRemovalWorker.java +++ b/app/src/main/java/com/nextcloud/talk/jobs/AccountRemovalWorker.java @@ -15,6 +15,7 @@ import com.nextcloud.talk.api.NcApi; import com.nextcloud.talk.application.NextcloudTalkApplication; import com.nextcloud.talk.arbitrarystorage.ArbitraryStorageManager; +import com.nextcloud.talk.chat.audio.ChatAudioStore; import com.nextcloud.talk.data.database.dao.ChatBlocksDao; import com.nextcloud.talk.data.database.dao.ChatMessagesDao; import com.nextcloud.talk.data.database.dao.ConversationsDao; @@ -74,6 +75,8 @@ public class AccountRemovalWorker extends Worker { @Inject AccountCookieInterceptor accountCookieInterceptor; + @Inject ChatAudioStore chatAudioStore; + NcApi ncApi; public AccountRemovalWorker(@NonNull Context context, @NonNull WorkerParameters workerParams) { @@ -198,6 +201,7 @@ private void initiateUserDeletion(User user) { long id = user.getId(); DirectShareHelper.INSTANCE.removeShortcutsForUser(getApplicationContext(), id); WebSocketConnectionHelper.deleteExternalSignalingInstanceForUserEntity(id); + chatAudioStore.forgetAccount(id); try { arbitraryStorageManager.deleteAllEntriesForAccountIdentifier(id); diff --git a/app/src/main/java/com/nextcloud/talk/ui/BackgroundVoiceMessageCard.kt b/app/src/main/java/com/nextcloud/talk/ui/BackgroundVoiceMessageCard.kt deleted file mode 100644 index 604aecfac61..00000000000 --- a/app/src/main/java/com/nextcloud/talk/ui/BackgroundVoiceMessageCard.kt +++ /dev/null @@ -1,255 +0,0 @@ -/* - * Nextcloud Talk - Android Client - * - * SPDX-FileCopyrightText: 2024 Julius Linus - * SPDX-License-Identifier: GPL-3.0-or-later - */ - -package com.nextcloud.talk.ui - -import android.animation.ValueAnimator -import android.content.Context -import androidx.compose.foundation.background -import androidx.compose.foundation.layout.Box -import androidx.compose.foundation.layout.Row -import androidx.compose.foundation.layout.Spacer -import androidx.compose.foundation.layout.fillMaxWidth -import androidx.compose.foundation.layout.height -import androidx.compose.foundation.layout.offset -import androidx.compose.foundation.layout.padding -import androidx.compose.foundation.layout.size -import androidx.compose.foundation.shape.RoundedCornerShape -import androidx.compose.material3.ColorScheme -import androidx.compose.material3.Icon -import androidx.compose.material3.IconButton -import androidx.compose.material3.MaterialTheme -import androidx.compose.material3.Surface -import androidx.compose.material3.Text -import androidx.compose.runtime.Composable -import androidx.compose.runtime.getValue -import androidx.compose.runtime.mutableFloatStateOf -import androidx.compose.runtime.mutableStateOf -import androidx.compose.runtime.remember -import androidx.compose.runtime.setValue -import androidx.compose.ui.Alignment -import androidx.compose.ui.Modifier -import androidx.compose.ui.graphics.vector.ImageVector -import androidx.compose.ui.platform.LocalContext -import androidx.compose.ui.res.stringResource -import androidx.compose.ui.res.vectorResource -import androidx.compose.ui.tooling.preview.Preview -import androidx.compose.ui.unit.dp -import coil.compose.AsyncImage -import com.nextcloud.talk.R -import com.nextcloud.talk.contacts.loadImage -import com.nextcloud.talk.ui.theme.ViewThemeUtils -import com.nextcloud.talk.utils.preview.ComposePreviewUtils - -@Suppress("LongParameterList") -class BackgroundVoiceMessageCard( - val name: String, - val duration: Int, - private val offset: Float, - private val imageURI: String, - private val conversationImageURI: String, - private var viewThemeUtils: ViewThemeUtils, - private var context: Context -) { - - private val progressState = mutableFloatStateOf(0.0f) - private val animator = ValueAnimator.ofFloat(offset, 1.0f) - - init { - animator.duration = duration.toLong() - animator.addUpdateListener { animation -> - progressState.floatValue = animation.animatedValue as Float - } - - animator.start() - } - - @Suppress("FunctionNaming") - @Composable - fun GetView(onPlayPaused: (isPaused: Boolean) -> Unit, onClosed: () -> Unit) { - var isPausedIcon by remember { mutableStateOf(false) } - val errorPlaceholderImage: Int = R.drawable.account_circle_96dp - val loadedImage = loadImage(imageURI, context, errorPlaceholderImage) - val conversationImage = loadImage(conversationImageURI, context, errorPlaceholderImage) - BackgroundVoiceMessageCardContent( - name = name, - isPaused = isPausedIcon, - progress = progressState.floatValue, - userImageModel = loadedImage, - conversationImageModel = conversationImage, - colorScheme = viewThemeUtils.getColorScheme(context), - onPlayPauseClicked = { - isPausedIcon = !isPausedIcon - onPlayPaused(isPausedIcon) - if (isPausedIcon) animator.pause() else animator.resume() - }, - onClosed = onClosed - ) - } -} - -private const val TITLE_SPACE = 0.8f - -@Suppress("LongParameterList", "LongMethod") -@Composable -fun BackgroundVoiceMessageCardContent( - name: String, - isPaused: Boolean, - progress: Float, - userImageModel: Any?, - conversationImageModel: Any?, - colorScheme: ColorScheme, - onPlayPauseClicked: () -> Unit, - onClosed: () -> Unit -) { - MaterialTheme(colorScheme = colorScheme) { - Surface( - shape = RoundedCornerShape(8.dp), - color = MaterialTheme.colorScheme.onPrimary, - modifier = Modifier.padding(16.dp, 0.dp) - ) { - Box( - modifier = Modifier - .background(MaterialTheme.colorScheme.primary, shape = RoundedCornerShape(8.dp)) - .fillMaxWidth(progress) - .height(4.dp) - ) - Row( - modifier = Modifier - .padding(16.dp) - .fillMaxWidth() - ) { - Box( - contentAlignment = Alignment.Center, - modifier = Modifier.align(Alignment.CenterVertically) - ) { - IconButton(onClick = onPlayPauseClicked) { - Icon( - imageVector = if (isPaused) { - ImageVector.vectorResource(R.drawable.ic_baseline_play_arrow_voice_message_24) - } else { - ImageVector.vectorResource(R.drawable.ic_baseline_pause_voice_message_24) - }, - contentDescription = stringResource(R.string.play_pause_voice_message), - modifier = Modifier.size(24.dp), - tint = MaterialTheme.colorScheme.primary - ) - } - } - Spacer(modifier = Modifier.size(16.dp)) - - Box( - modifier = Modifier - .weight(TITLE_SPACE) - .align(Alignment.CenterVertically), - contentAlignment = Alignment.Center - ) { - Row { - Box { - AsyncImage( - model = conversationImageModel, - contentDescription = stringResource(R.string.user_avatar), - modifier = Modifier - .size(width = 45.dp, height = 45.dp) - .padding(8.dp) - .offset(10.dp, 10.dp) - ) - AsyncImage( - model = userImageModel, - contentDescription = stringResource(R.string.user_avatar), - modifier = Modifier - .size(width = 45.dp, height = 45.dp) - .padding(8.dp) - ) - } - Text( - name, - modifier = Modifier - .align(Alignment.CenterVertically) - .padding(8.dp), - color = MaterialTheme.colorScheme.onBackground - ) - } - } - - Box( - contentAlignment = Alignment.Center, - modifier = Modifier.align(Alignment.CenterVertically) - ) { - IconButton(onClick = onClosed) { - Icon( - imageVector = ImageVector.vectorResource(R.drawable.ic_baseline_close_24), - contentDescription = stringResource(R.string.close), - modifier = Modifier - .size(24.dp) - .padding(2.dp), - tint = MaterialTheme.colorScheme.primary - ) - } - } - } - } - } -} - -@Preview(name = "Playing - Light Mode") -@Composable -fun BackgroundVoiceMessageCardPlayingPreview() { - BackgroundVoiceMessageCardPreview(isPaused = false) -} - -@Preview( - name = "Playing - Dark Mode", - uiMode = android.content.res.Configuration.UI_MODE_NIGHT_YES -) -@Composable -fun BackgroundVoiceMessageCardPlayingDarkPreview() { - BackgroundVoiceMessageCardPreview(isPaused = false) -} - -@Preview(name = "Playing - RTL / Arabic", locale = "ar") -@Composable -fun BackgroundVoiceMessageCardPlayingRtlPreview() { - BackgroundVoiceMessageCardPreview(isPaused = false) -} - -@Preview(name = "Paused - Light Mode") -@Composable -fun BackgroundVoiceMessageCardPausedPreview() { - BackgroundVoiceMessageCardPreview(isPaused = true) -} - -@Preview( - name = "Paused - Dark Mode", - uiMode = android.content.res.Configuration.UI_MODE_NIGHT_YES -) -@Composable -fun BackgroundVoiceMessageCardPausedDarkPreview() { - BackgroundVoiceMessageCardPreview(isPaused = true) -} - -@Preview(name = "Paused - RTL / Arabic", locale = "ar") -@Composable -fun BackgroundVoiceMessageCardPausedRtlPreview() { - BackgroundVoiceMessageCardPreview(isPaused = true) -} - -@Composable -private fun BackgroundVoiceMessageCardPreview(isPaused: Boolean) { - val context = LocalContext.current - val colorScheme = ComposePreviewUtils.getInstance(context).viewThemeUtils.getColorScheme(context) - BackgroundVoiceMessageCardContent( - name = "Alice", - isPaused = isPaused, - progress = 0.4f, - userImageModel = null, - conversationImageModel = null, - colorScheme = colorScheme, - onPlayPauseClicked = {}, - onClosed = {} - ) -} diff --git a/app/src/main/java/com/nextcloud/talk/ui/ComposeWaveformSeekbar.kt b/app/src/main/java/com/nextcloud/talk/ui/ComposeWaveformSeekbar.kt index 2765aee7b68..ba4d1af7c97 100644 --- a/app/src/main/java/com/nextcloud/talk/ui/ComposeWaveformSeekbar.kt +++ b/app/src/main/java/com/nextcloud/talk/ui/ComposeWaveformSeekbar.kt @@ -33,6 +33,7 @@ const val MAX_HEIGHT = 100 const val OVERLAP = 0.025 const val PROGRESS = 0.97f +@Suppress("LongParameterList") @OptIn(ExperimentalMaterial3Api::class) @Composable fun ComposeWaveformSeekBar( @@ -40,7 +41,9 @@ fun ComposeWaveformSeekBar( onValueChange: (Float) -> Unit, modifier: Modifier, waveData: FloatArray, - enabled: Boolean + enabled: Boolean, + onValueChangeFinished: () -> Unit = {}, + sliderModifier: Modifier = Modifier ) { val barWidth = Stroke.DefaultMiter val thumbSize = WAVEFORM_THUMB_SIZE.dp @@ -51,6 +54,8 @@ fun ComposeWaveformSeekBar( value = value, enabled = enabled, onValueChange = onValueChange, + onValueChangeFinished = onValueChangeFinished, + modifier = sliderModifier, track = { Box( modifier = modifier diff --git a/app/src/main/java/com/nextcloud/talk/ui/WaveformSeekBar.kt b/app/src/main/java/com/nextcloud/talk/ui/WaveformSeekBar.kt index d6d4bfff7c9..c4a171cbd95 100644 --- a/app/src/main/java/com/nextcloud/talk/ui/WaveformSeekBar.kt +++ b/app/src/main/java/com/nextcloud/talk/ui/WaveformSeekBar.kt @@ -15,7 +15,7 @@ import android.util.AttributeSet import androidx.annotation.ColorInt import androidx.appcompat.widget.AppCompatSeekBar import androidx.core.graphics.toColorInt -import com.nextcloud.talk.utils.AudioUtils +import com.nextcloud.talk.chat.audio.Waveforms import kotlin.math.roundToInt import androidx.core.graphics.withSave @@ -74,7 +74,7 @@ class WaveformSeekBar : AppCompatSeekBar { if (usableWidth > MINIMUM_WIDTH && rawData.isNotEmpty() && usableWidth != savedMeasure) { savedMeasure = usableWidth val numBars = if (usableWidth > VALUE_100) (usableWidth / WIDTH_DIVISOR) else usableWidth / 2f - waveData = AudioUtils.shrinkFloatArray(rawData, numBars.roundToInt()) + waveData = Waveforms.reduce(rawData, numBars.roundToInt()) invalidate() } } diff --git a/app/src/main/java/com/nextcloud/talk/ui/chat/AudioFileMessage.kt b/app/src/main/java/com/nextcloud/talk/ui/chat/AudioFileMessage.kt index 2c85897e1bf..d0fff2802a2 100644 --- a/app/src/main/java/com/nextcloud/talk/ui/chat/AudioFileMessage.kt +++ b/app/src/main/java/com/nextcloud/talk/ui/chat/AudioFileMessage.kt @@ -7,42 +7,35 @@ package com.nextcloud.talk.ui.chat -import android.text.format.DateUtils import androidx.compose.foundation.layout.Column import androidx.compose.foundation.layout.Row import androidx.compose.foundation.layout.fillMaxWidth import androidx.compose.foundation.layout.padding import androidx.compose.foundation.layout.size -import androidx.compose.material.icons.Icons -import androidx.compose.material.icons.filled.Pause -import androidx.compose.material.icons.filled.PlayArrow -import androidx.compose.material3.CircularProgressIndicator -import androidx.compose.material3.ExperimentalMaterial3Api import androidx.compose.material3.Icon -import androidx.compose.material3.IconButton import androidx.compose.material3.MaterialTheme.colorScheme import androidx.compose.material3.Slider import androidx.compose.material3.SliderDefaults import androidx.compose.material3.Text import androidx.compose.runtime.Composable import androidx.compose.runtime.getValue -import androidx.compose.runtime.mutableFloatStateOf +import androidx.compose.runtime.mutableStateOf import androidx.compose.runtime.remember import androidx.compose.runtime.setValue import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier import androidx.compose.ui.res.painterResource import androidx.compose.ui.res.stringResource +import androidx.compose.ui.semantics.contentDescription +import androidx.compose.ui.semantics.semantics import androidx.compose.ui.text.style.TextOverflow import androidx.compose.ui.unit.dp import com.nextcloud.talk.R import com.nextcloud.talk.chat.ui.model.ChatMessageUi import com.nextcloud.talk.chat.ui.model.MessageTypeContent -private const val SEEKBAR_MAX = 100 private const val INACTIVE_TRACK_ALPHA = 0.4f -@OptIn(ExperimentalMaterial3Api::class) @Suppress("Detekt.LongMethod") @Composable fun AudioFileMessage( @@ -50,8 +43,7 @@ fun AudioFileMessage( message: ChatMessageUi, isOneToOneConversation: Boolean = false, conversationThreadId: Long? = null, - onPlayPauseClick: (Int) -> Unit = {}, - onSeek: (messageId: Int, progress: Int) -> Unit = { _, _ -> } + callbacks: ChatMessageCallbacks = ChatMessageCallbacks() ) { MessageScaffold( uiMessage = message, @@ -59,8 +51,9 @@ fun AudioFileMessage( conversationThreadId = conversationThreadId, forceTimeBelow = true, content = { - val remainingSeconds = (typeContent.durationSeconds - typeContent.playedSeconds) - val icon = if (typeContent.isPlaying) Icons.Filled.Pause else Icons.Filled.PlayArrow + val audioState = chatAudioMessageState(message) + var draggedProgress by remember { mutableStateOf(null) } + val seekDescription = stringResource(R.string.nc_audio_seek) Column { Row( @@ -88,43 +81,34 @@ fun AudioFileMessage( verticalAlignment = Alignment.CenterVertically, modifier = Modifier.fillMaxWidth() ) { - if (typeContent.isDownloading) { - CircularProgressIndicator(modifier = Modifier.size(48.dp), strokeWidth = 2.dp) - } else { - IconButton( - onClick = { onPlayPauseClick(message.id) }, - modifier = Modifier.size(48.dp) - ) { - Icon( - imageVector = icon, - contentDescription = stringResource(R.string.play_pause_audio_file), - modifier = Modifier.size(40.dp) - ) - } - } - - var sliderValue by remember { mutableFloatStateOf(0f) } - sliderValue = typeContent.seekbarProgress * 1f / SEEKBAR_MAX + ChatAudioPlayPauseButton( + state = audioState, + contentDescription = stringResource(R.string.play_pause_audio_file), + onClick = { callbacks.onVoicePlayPauseClick(message.id) } + ) Slider( - value = sliderValue, - onValueChange = { - val progressI = (it * SEEKBAR_MAX).toInt() - onSeek(message.id, progressI) + value = draggedProgress ?: audioState.progress, + onValueChange = { draggedProgress = it }, + onValueChangeFinished = { + draggedProgress?.let { callbacks.onVoiceSeek(message.id, it) } + draggedProgress = null }, + enabled = audioState.canSeek, colors = SliderDefaults.colors( thumbColor = colorScheme.primary, activeTrackColor = colorScheme.primary, inactiveTrackColor = colorScheme.onPrimaryContainer.copy(alpha = INACTIVE_TRACK_ALPHA) ), modifier = Modifier - .fillMaxWidth() + .weight(1f) .padding(8.dp) + .semantics { contentDescription = seekDescription } ) } Text( - text = DateUtils.formatElapsedTime(remainingSeconds.toLong()), + text = audioTimeText(audioState.displayedTimeMs, audioState.durationMs), color = colorScheme.onPrimaryContainer, modifier = Modifier.padding(start = 4.dp) ) @@ -133,29 +117,21 @@ fun AudioFileMessage( ) } +private fun audioTimeText(displayedTimeMs: Long?, durationMs: Long): String = + if (durationMs > 0L && displayedTimeMs != null && displayedTimeMs != durationMs) { + "${formatAudioTime(displayedTimeMs)} / ${formatAudioTime(durationMs)}" + } else { + formatAudioTime(displayedTimeMs) + } + @ChatMessagePreviews @Composable private fun AudioFileMessagePreview() { PreviewContainer { + val content = MessageTypeContent.AudioFile(fileName = "podcast-episode-42.mp3") AudioFileMessage( - typeContent = MessageTypeContent.AudioFile( - fileName = "podcast-episode-42.mp3", - isPlaying = false, - isDownloading = false, - durationSeconds = 245, - playedSeconds = 60, - seekbarProgress = 24 - ), - message = createBaseMessageWithoutCaption( - MessageTypeContent.AudioFile( - fileName = "podcast-episode-42.mp3", - isPlaying = false, - isDownloading = false, - durationSeconds = 245, - playedSeconds = 60, - seekbarProgress = 24 - ) - ) + typeContent = content, + message = createBaseMessageWithoutCaption(content) ) } } diff --git a/app/src/main/java/com/nextcloud/talk/ui/chat/ChatAudio.kt b/app/src/main/java/com/nextcloud/talk/ui/chat/ChatAudio.kt new file mode 100644 index 00000000000..0c209b7bfe0 --- /dev/null +++ b/app/src/main/java/com/nextcloud/talk/ui/chat/ChatAudio.kt @@ -0,0 +1,149 @@ +/* + * Nextcloud Talk - Android Client + * + * SPDX-FileCopyrightText: 2026 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: GPL-3.0-or-later + */ + +package com.nextcloud.talk.ui.chat + +import android.text.format.DateUtils +import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.layout.size +import androidx.compose.material.icons.Icons +import androidx.compose.material.icons.filled.Pause +import androidx.compose.material.icons.filled.PlayArrow +import androidx.compose.material.icons.filled.Refresh +import androidx.compose.material3.CircularProgressIndicator +import androidx.compose.material3.Icon +import androidx.compose.material3.IconButton +import androidx.compose.runtime.Composable +import androidx.compose.runtime.LaunchedEffect +import androidx.compose.runtime.Stable +import androidx.compose.runtime.collectAsState +import androidx.compose.runtime.derivedStateOf +import androidx.compose.runtime.getValue +import androidx.compose.runtime.remember +import androidx.compose.runtime.staticCompositionLocalOf +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.res.stringResource +import androidx.compose.ui.unit.dp +import com.nextcloud.talk.R +import com.nextcloud.talk.chat.audio.ChatAudioKey +import com.nextcloud.talk.chat.audio.ChatAudioMessageState +import com.nextcloud.talk.chat.audio.ChatAudioMetadata +import com.nextcloud.talk.chat.audio.ChatAudioPlaybackState +import com.nextcloud.talk.chat.ui.model.ChatMessageUi +import com.nextcloud.talk.ui.PlaybackSpeed +import kotlinx.coroutines.flow.StateFlow +import kotlin.math.abs + +private const val MILLIS_PER_SECOND = 1000L +private const val UNKNOWN_TIME = "--:--" + +/** + * The audio playback as needed by audio message bubbles. Screens that play audio provide it via + * [LocalChatAudioUi]; without it, audio messages are shown without playback state. + */ +@Stable +class ChatAudioUi( + val playbackState: StateFlow, + val metadata: StateFlow>, + val voiceSpeed: StateFlow, + private val internalUserId: () -> Long?, + private val loadMetadata: (ChatAudioKey) -> Unit +) { + fun keyFor(message: ChatMessageUi): ChatAudioKey? { + val userId = internalUserId() + val roomToken = message.roomToken + return if (userId == null || roomToken == null) null else ChatAudioKey(userId, roomToken, message.id) + } + + fun load(key: ChatAudioKey) = loadMetadata(key) +} + +val LocalChatAudioUi = staticCompositionLocalOf { null } + +/** The playback state of [message], only recomposing when it changes for this very message. */ +@Composable +fun chatAudioMessageState(message: ChatMessageUi): ChatAudioMessageState { + val audioUi = LocalChatAudioUi.current + val key = remember(audioUi, message.roomToken, message.id) { audioUi?.keyFor(message) } + if (audioUi == null || key == null) { + return ChatAudioMessageState() + } + LaunchedEffect(audioUi, key) { audioUi.load(key) } + val playbackState = audioUi.playbackState.collectAsState() + val metadata = audioUi.metadata.collectAsState() + val messageState by remember(key, playbackState, metadata) { + derivedStateOf { playbackState.value.messageState(key, metadata.value[key]) } + } + return messageState +} + +/** The waveform of the voice message [message], or null while it is not known yet. */ +@Composable +fun chatAudioWaveform(message: ChatMessageUi): List? { + val audioUi = LocalChatAudioUi.current + val key = remember(audioUi, message.roomToken, message.id) { audioUi?.keyFor(message) } + if (audioUi == null || key == null) { + return null + } + val metadata = audioUi.metadata.collectAsState() + val waveform by remember(key, metadata) { derivedStateOf { metadata.value[key]?.waveform } } + return waveform +} + +@Composable +fun chatAudioVoiceSpeed(): PlaybackSpeed { + val voiceSpeed = LocalChatAudioUi.current?.voiceSpeed ?: return PlaybackSpeed.NORMAL + val speed by voiceSpeed.collectAsState() + return speed +} + +/** The speed step that is closest to the playback speed [value]. */ +fun speedOf(value: Float): PlaybackSpeed = PlaybackSpeed.entries.minBy { abs(it.value - value) } + +/** Formats a playback time like the chat does, or a placeholder if the time is not known yet. */ +fun formatAudioTime(timeMs: Long?): String = + timeMs?.let { DateUtils.formatElapsedTime(it / MILLIS_PER_SECOND) } ?: UNKNOWN_TIME + +/** + * Play/pause button of an audio message. It shows a progress ring while the audio is loading and turns into a + * retry button after an error. + */ +@Composable +fun ChatAudioPlayPauseButton( + state: ChatAudioMessageState, + contentDescription: String, + onClick: () -> Unit, + modifier: Modifier = Modifier +) { + Box(contentAlignment = Alignment.Center, modifier = modifier.size(48.dp)) { + if (state.isBuffering) { + CircularProgressIndicator(modifier = Modifier.size(44.dp), strokeWidth = 2.dp) + } + IconButton(onClick = onClick, modifier = Modifier.size(48.dp)) { + when { + state.error != null -> Icon( + imageVector = Icons.Filled.Refresh, + contentDescription = stringResource(R.string.nc_audio_retry), + modifier = Modifier.size(32.dp) + ) + + state.isPlaying -> Icon( + imageVector = Icons.Filled.Pause, + contentDescription = contentDescription, + modifier = Modifier.size(40.dp) + ) + + else -> Icon( + imageVector = Icons.Filled.PlayArrow, + contentDescription = contentDescription, + modifier = Modifier.size(40.dp) + ) + } + } + } +} diff --git a/app/src/main/java/com/nextcloud/talk/ui/chat/ChatAudioPlayerBar.kt b/app/src/main/java/com/nextcloud/talk/ui/chat/ChatAudioPlayerBar.kt new file mode 100644 index 00000000000..13f3897fbaf --- /dev/null +++ b/app/src/main/java/com/nextcloud/talk/ui/chat/ChatAudioPlayerBar.kt @@ -0,0 +1,150 @@ +/* + * Nextcloud Talk - Android Client + * + * SPDX-FileCopyrightText: 2026 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: GPL-3.0-or-later + */ + +package com.nextcloud.talk.ui.chat + +import androidx.compose.foundation.clickable +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.height +import androidx.compose.foundation.layout.padding +import androidx.compose.material.icons.Icons +import androidx.compose.material.icons.filled.Close +import androidx.compose.material3.Icon +import androidx.compose.material3.IconButton +import androidx.compose.material3.LinearProgressIndicator +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.Surface +import androidx.compose.material3.Text +import androidx.compose.material3.TextButton +import androidx.compose.runtime.Composable +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.res.stringResource +import androidx.compose.ui.semantics.Role +import androidx.compose.ui.text.style.TextOverflow +import androidx.compose.ui.tooling.preview.Preview +import androidx.compose.ui.unit.dp +import com.nextcloud.talk.R +import com.nextcloud.talk.chat.audio.ChatAudioKey +import com.nextcloud.talk.chat.audio.ChatAudioKind +import com.nextcloud.talk.chat.audio.ChatAudioPlaybackState + +/** + * Compact player shown while an audio message is loaded, also if it belongs to another conversation. Tapping it + * opens the full player. + */ +@Composable +fun ChatAudioPlayerBar( + state: ChatAudioPlaybackState, + callbacks: ChatAudioPlayerCallbacks, + onOpenPlayer: () -> Unit, + modifier: Modifier = Modifier +) { + val messageState = state.messageState(state.currentKey, null) + Surface( + color = MaterialTheme.colorScheme.surfaceContainerHigh, + modifier = modifier.fillMaxWidth() + ) { + Column { + Row( + verticalAlignment = Alignment.CenterVertically, + modifier = Modifier + .fillMaxWidth() + .clickable( + onClickLabel = stringResource(R.string.nc_audio_player_open), + role = Role.Button, + onClick = onOpenPlayer + ) + .padding(horizontal = 4.dp) + ) { + ChatAudioPlayPauseButton( + state = messageState, + contentDescription = stringResource(R.string.nc_audio_player_play_pause), + onClick = callbacks.onPlayPause + ) + ChatAudioPlayerBarTitle( + title = state.title, + subtitle = "${state.subtitle} · ${formatAudioTime(messageState.displayedTimeMs)}", + modifier = Modifier + .weight(1f) + .padding(horizontal = 4.dp) + ) + if (state.kind == ChatAudioKind.VOICE_MESSAGE) { + TextButton(onClick = callbacks.onSpeedClick) { + Text(text = speedOf(state.speed).label) + } + } + IconButton(onClick = callbacks.onClose) { + Icon( + imageVector = Icons.Filled.Close, + contentDescription = stringResource(R.string.nc_audio_player_close) + ) + } + } + LinearProgressIndicator( + progress = { messageState.progress }, + modifier = Modifier + .fillMaxWidth() + .height(2.dp) + ) + } + } +} + +@Composable +private fun ChatAudioPlayerBarTitle(title: String, subtitle: String, modifier: Modifier = Modifier) { + Column(modifier = modifier) { + Text( + text = title, + style = MaterialTheme.typography.titleSmall, + maxLines = 1, + overflow = TextOverflow.Ellipsis + ) + Text( + text = subtitle, + style = MaterialTheme.typography.bodySmall, + color = MaterialTheme.colorScheme.onSurfaceVariant, + maxLines = 1, + overflow = TextOverflow.Ellipsis + ) + } +} + +/** What the controls of [ChatAudioPlayerBar] and [ChatAudioPlayerSheet] do. */ +data class ChatAudioPlayerCallbacks( + val onPlayPause: () -> Unit = {}, + val onSeek: (Float) -> Unit = {}, + val onPrevious: () -> Unit = {}, + val onNext: () -> Unit = {}, + val onSpeedClick: () -> Unit = {}, + val onShuffleClick: () -> Unit = {}, + val onRepeatClick: () -> Unit = {}, + val onQueueItemClick: (Int) -> Unit = {}, + val onOpenMessage: () -> Unit = {}, + val onClose: () -> Unit = {} +) + +@Preview +@Composable +private fun ChatAudioPlayerBarPreview() { + ChatAudioPlayerBar( + state = ChatAudioPlaybackState( + currentKey = ChatAudioKey(1L, "preview", 1), + kind = ChatAudioKind.VOICE_MESSAGE, + title = "John Doe", + subtitle = "Voice message", + isPlaying = true, + positionMs = 12_000L, + durationMs = 45_000L, + speed = 1.5f + ), + callbacks = ChatAudioPlayerCallbacks(), + onOpenPlayer = {} + ) +} diff --git a/app/src/main/java/com/nextcloud/talk/ui/chat/ChatAudioPlayerHost.kt b/app/src/main/java/com/nextcloud/talk/ui/chat/ChatAudioPlayerHost.kt new file mode 100644 index 00000000000..259c93c1135 --- /dev/null +++ b/app/src/main/java/com/nextcloud/talk/ui/chat/ChatAudioPlayerHost.kt @@ -0,0 +1,95 @@ +/* + * Nextcloud Talk - Android Client + * + * SPDX-FileCopyrightText: 2026 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: GPL-3.0-or-later + */ + +package com.nextcloud.talk.ui.chat + +import androidx.compose.runtime.Composable +import androidx.compose.runtime.LaunchedEffect +import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.saveable.rememberSaveable +import androidx.compose.runtime.setValue +import androidx.compose.ui.Modifier +import com.nextcloud.talk.chat.audio.ChatAudioKey +import com.nextcloud.talk.chat.audio.ChatAudioKind +import com.nextcloud.talk.chat.audio.ChatAudioPlaybackState +import com.nextcloud.talk.chat.audio.ChatAudioPlayer +import com.nextcloud.talk.chat.audio.ChatAudioSpeeds +import com.nextcloud.talk.ui.PlaybackSpeed +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.launch + +/** The player bar while an audio message is loaded, and the full player once the bar is tapped. */ +@Composable +fun ChatAudioPlayerHost( + state: ChatAudioPlaybackState, + callbacks: ChatAudioPlayerCallbacks, + modifier: Modifier = Modifier +) { + var isPlayerOpen by rememberSaveable { mutableStateOf(false) } + val isLoaded = state.currentKey != null + LaunchedEffect(isLoaded) { + if (!isLoaded) { + isPlayerOpen = false + } + } + if (isLoaded) { + ChatAudioPlayerBar( + state = state, + callbacks = callbacks, + onOpenPlayer = { isPlayerOpen = true }, + modifier = modifier + ) + } + if (isLoaded && isPlayerOpen) { + ChatAudioPlayerSheet(state = state, callbacks = callbacks, onDismiss = { isPlayerOpen = false }) + } +} + +/** + * Controls [player] from [ChatAudioPlayerHost]. A changed speed is saved for the account of the playing message + * and reported to [onSpeedChanged]. + */ +fun chatAudioPlayerCallbacks( + player: ChatAudioPlayer, + speeds: ChatAudioSpeeds, + scope: CoroutineScope, + onOpenMessage: (ChatAudioKey, Long?) -> Unit, + onSpeedChanged: (ChatAudioKey, ChatAudioKind, PlaybackSpeed) -> Unit = { _, _, _ -> } +): ChatAudioPlayerCallbacks = + ChatAudioPlayerCallbacks( + onPlayPause = { player.togglePlayback() }, + onSeek = { fraction -> player.state.value.currentKey?.let { player.seekTo(it, fraction) } }, + onPrevious = { player.skipToPrevious() }, + onNext = { player.skipToNext() }, + onSpeedClick = { + val state = player.state.value + val key = state.currentKey + val kind = state.kind + if (key != null && kind != null) { + val speed = speedOf(state.speed).next() + player.setSpeed(key.internalUserId, kind, speed.value) + onSpeedChanged(key, kind, speed) + scope.launch(Dispatchers.IO) { speeds.save(key, kind, speed) } + } + }, + onShuffleClick = { + val state = player.state.value + player.setPlaybackModes(!state.shuffleEnabled, state.repeatMode) + }, + onRepeatClick = { + val state = player.state.value + player.setPlaybackModes(state.shuffleEnabled, state.repeatMode.next()) + }, + onQueueItemClick = { player.playQueueItem(it) }, + onOpenMessage = { + val state = player.state.value + state.currentKey?.let { onOpenMessage(it, state.currentThreadId) } + }, + onClose = { player.stop() } + ) diff --git a/app/src/main/java/com/nextcloud/talk/ui/chat/ChatAudioPlayerSheet.kt b/app/src/main/java/com/nextcloud/talk/ui/chat/ChatAudioPlayerSheet.kt new file mode 100644 index 00000000000..524cdeb0b1f --- /dev/null +++ b/app/src/main/java/com/nextcloud/talk/ui/chat/ChatAudioPlayerSheet.kt @@ -0,0 +1,376 @@ +/* + * Nextcloud Talk - Android Client + * + * SPDX-FileCopyrightText: 2026 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: GPL-3.0-or-later + */ + +package com.nextcloud.talk.ui.chat + +import androidx.compose.foundation.clickable +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.BoxWithConstraints +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.PaddingValues +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.layout.size +import androidx.compose.foundation.lazy.LazyColumn +import androidx.compose.foundation.lazy.LazyListScope +import androidx.compose.foundation.lazy.items +import androidx.compose.foundation.lazy.rememberLazyListState +import androidx.compose.foundation.shape.CircleShape +import androidx.compose.material.icons.Icons +import androidx.compose.material.icons.filled.Mic +import androidx.compose.material.icons.filled.MusicNote +import androidx.compose.material.icons.filled.PlayArrow +import androidx.compose.material.icons.filled.Repeat +import androidx.compose.material.icons.filled.RepeatOne +import androidx.compose.material.icons.filled.Shuffle +import androidx.compose.material.icons.filled.SkipNext +import androidx.compose.material.icons.filled.SkipPrevious +import androidx.compose.material3.ExperimentalMaterial3Api +import androidx.compose.material3.HorizontalDivider +import androidx.compose.material3.Icon +import androidx.compose.material3.IconButton +import androidx.compose.material3.IconToggleButton +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.ModalBottomSheet +import androidx.compose.material3.Slider +import androidx.compose.material3.Surface +import androidx.compose.material3.Text +import androidx.compose.material3.TextButton +import androidx.compose.material3.rememberModalBottomSheetState +import androidx.compose.runtime.Composable +import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.remember +import androidx.compose.runtime.setValue +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.res.stringResource +import androidx.compose.ui.semantics.Role +import androidx.compose.ui.semantics.contentDescription +import androidx.compose.ui.semantics.selected +import androidx.compose.ui.semantics.semantics +import androidx.compose.ui.semantics.stateDescription +import androidx.compose.ui.text.style.TextAlign +import androidx.compose.ui.text.style.TextOverflow +import androidx.compose.ui.tooling.preview.Preview +import androidx.compose.ui.unit.dp +import com.nextcloud.talk.R +import com.nextcloud.talk.chat.audio.ChatAudioKey +import com.nextcloud.talk.chat.audio.ChatAudioKind +import com.nextcloud.talk.chat.audio.ChatAudioMessageState +import com.nextcloud.talk.chat.audio.ChatAudioPlaybackState +import com.nextcloud.talk.chat.audio.ChatAudioQueueItem +import com.nextcloud.talk.chat.audio.ChatAudioRepeatMode + +private val minHeightForPinnedPlayer = 560.dp + +/** The full player with all controls and the queue, shown on top of the current screen. */ +@OptIn(ExperimentalMaterial3Api::class) +@Composable +fun ChatAudioPlayerSheet(state: ChatAudioPlaybackState, callbacks: ChatAudioPlayerCallbacks, onDismiss: () -> Unit) { + ModalBottomSheet( + onDismissRequest = onDismiss, + sheetState = rememberModalBottomSheetState(skipPartiallyExpanded = true) + ) { + ChatAudioPlayerSheetContent( + state = state, + callbacks = callbacks.copy( + onOpenMessage = { + onDismiss() + callbacks.onOpenMessage() + } + ) + ) + } +} + +/** + * Keeps the player in place above the queue, which scrolls on its own. In a lower window, e.g. on a phone in + * landscape, the player scrolls together with the queue instead, so that every control stays reachable. + */ +@Composable +private fun ChatAudioPlayerSheetContent( + state: ChatAudioPlaybackState, + callbacks: ChatAudioPlayerCallbacks, + modifier: Modifier = Modifier +) { + val hasQueue = state.queue.size > 1 + BoxWithConstraints(modifier = modifier.fillMaxWidth()) { + if (hasQueue && maxHeight >= minHeightForPinnedPlayer) { + Column(modifier = Modifier.padding(horizontal = 24.dp)) { + ChatAudioSheetPlayer(state = state, callbacks = callbacks) + ChatAudioQueueHeader() + ChatAudioQueueList(state = state, callbacks = callbacks, modifier = Modifier.weight(1f, fill = false)) + } + } else { + LazyColumn(contentPadding = PaddingValues(start = 24.dp, end = 24.dp, bottom = 24.dp)) { + item { + ChatAudioSheetPlayer(state = state, callbacks = callbacks) + } + if (hasQueue) { + item { + ChatAudioQueueHeader() + } + queueItems(state = state, callbacks = callbacks) + } + } + } + } +} + +/** Opens at the current item, so that it is visible together with the items that follow it. */ +@Composable +private fun ChatAudioQueueList( + state: ChatAudioPlaybackState, + callbacks: ChatAudioPlayerCallbacks, + modifier: Modifier = Modifier +) { + val currentIndex = state.queue.indexOfFirst { it.key == state.currentKey } + LazyColumn( + state = rememberLazyListState(initialFirstVisibleItemIndex = currentIndex.coerceAtLeast(0)), + contentPadding = PaddingValues(bottom = 24.dp), + modifier = modifier + ) { + queueItems(state = state, callbacks = callbacks) + } +} + +private fun LazyListScope.queueItems(state: ChatAudioPlaybackState, callbacks: ChatAudioPlayerCallbacks) { + items(state.queue, key = { it.key.mediaId }) { item -> + ChatAudioQueueRow( + item = item, + kind = state.kind, + isCurrent = item.key == state.currentKey, + onClick = { callbacks.onQueueItemClick(item.index) } + ) + } +} + +@Composable +private fun ChatAudioQueueHeader() { + Column { + HorizontalDivider(modifier = Modifier.padding(vertical = 8.dp)) + Text( + text = stringResource(R.string.nc_audio_player_queue), + style = MaterialTheme.typography.titleSmall, + modifier = Modifier.padding(vertical = 8.dp) + ) + } +} + +@Composable +private fun ChatAudioSheetPlayer(state: ChatAudioPlaybackState, callbacks: ChatAudioPlayerCallbacks) { + val messageState = state.messageState(state.currentKey, null) + Column( + horizontalAlignment = Alignment.CenterHorizontally, + verticalArrangement = Arrangement.spacedBy(8.dp), + modifier = Modifier.fillMaxWidth() + ) { + Surface(shape = CircleShape, color = MaterialTheme.colorScheme.primaryContainer) { + Icon( + imageVector = kindIcon(state.kind), + contentDescription = null, + modifier = Modifier + .padding(16.dp) + .size(40.dp) + ) + } + Text( + text = state.title, + style = MaterialTheme.typography.titleLarge, + textAlign = TextAlign.Center, + maxLines = 2, + overflow = TextOverflow.Ellipsis + ) + Text( + text = state.subtitle, + style = MaterialTheme.typography.bodyMedium, + color = MaterialTheme.colorScheme.onSurfaceVariant + ) + ChatAudioSheetSeekBar(messageState = messageState, onSeek = callbacks.onSeek) + ChatAudioSheetTransport(state = state, messageState = messageState, callbacks = callbacks) + ChatAudioSheetOptions(state = state, callbacks = callbacks) + TextButton(onClick = callbacks.onOpenMessage) { + Text(text = stringResource(R.string.nc_audio_player_show_message)) + } + } +} + +@Composable +private fun ChatAudioQueueRow(item: ChatAudioQueueItem, kind: ChatAudioKind?, isCurrent: Boolean, onClick: () -> Unit) { + val color = if (isCurrent) MaterialTheme.colorScheme.primary else MaterialTheme.colorScheme.onSurface + Row( + verticalAlignment = Alignment.CenterVertically, + horizontalArrangement = Arrangement.spacedBy(16.dp), + modifier = Modifier + .fillMaxWidth() + .clickable( + onClickLabel = stringResource(R.string.nc_audio_player_play_pause), + role = Role.Button, + onClick = onClick + ) + .semantics { selected = isCurrent } + .padding(vertical = 8.dp) + ) { + Icon( + imageVector = if (isCurrent) Icons.Filled.PlayArrow else kindIcon(kind), + contentDescription = null, + tint = if (isCurrent) color else MaterialTheme.colorScheme.onSurfaceVariant + ) + Column(modifier = Modifier.weight(1f)) { + Text( + text = item.title, + style = MaterialTheme.typography.bodyLarge, + color = color, + maxLines = 1, + overflow = TextOverflow.Ellipsis + ) + Text( + text = item.subtitle, + style = MaterialTheme.typography.bodySmall, + color = MaterialTheme.colorScheme.onSurfaceVariant, + maxLines = 1, + overflow = TextOverflow.Ellipsis + ) + } + } +} + +@Composable +private fun ChatAudioSheetSeekBar(messageState: ChatAudioMessageState, onSeek: (Float) -> Unit) { + var draggedProgress by remember { mutableStateOf(null) } + val seekDescription = stringResource(R.string.nc_audio_seek) + val shownPositionMs = draggedProgress?.let { (it * messageState.durationMs).toLong() } ?: messageState.positionMs + Column(modifier = Modifier.fillMaxWidth()) { + Slider( + value = draggedProgress ?: messageState.progress, + onValueChange = { draggedProgress = it }, + onValueChangeFinished = { + draggedProgress?.let(onSeek) + draggedProgress = null + }, + enabled = messageState.canSeek, + modifier = Modifier.semantics { contentDescription = seekDescription } + ) + Row(modifier = Modifier.fillMaxWidth(), horizontalArrangement = Arrangement.SpaceBetween) { + Text(text = formatAudioTime(shownPositionMs), style = MaterialTheme.typography.bodySmall) + Text( + text = formatAudioTime(messageState.durationMs.takeIf { it > 0L }), + style = MaterialTheme.typography.bodySmall + ) + } + } +} + +@Composable +private fun ChatAudioSheetTransport( + state: ChatAudioPlaybackState, + messageState: ChatAudioMessageState, + callbacks: ChatAudioPlayerCallbacks +) { + Row(verticalAlignment = Alignment.CenterVertically, horizontalArrangement = Arrangement.spacedBy(16.dp)) { + IconButton(onClick = callbacks.onPrevious) { + Icon( + imageVector = Icons.Filled.SkipPrevious, + contentDescription = stringResource(R.string.nc_audio_player_previous) + ) + } + ChatAudioPlayPauseButton( + state = messageState, + contentDescription = stringResource(R.string.nc_audio_player_play_pause), + onClick = callbacks.onPlayPause + ) + IconButton(onClick = callbacks.onNext, enabled = state.hasNext) { + Icon( + imageVector = Icons.Filled.SkipNext, + contentDescription = stringResource(R.string.nc_audio_player_next) + ) + } + } +} + +/** Speed for every message; shuffle and repeat only for audio files, as voice messages are listened to in order. */ +@Composable +private fun ChatAudioSheetOptions(state: ChatAudioPlaybackState, callbacks: ChatAudioPlayerCallbacks) { + val isAudioFile = state.kind == ChatAudioKind.AUDIO_FILE + val speedDescription = stringResource(R.string.playback_speed_control) + val speedLabel = speedOf(state.speed).label + Row(verticalAlignment = Alignment.CenterVertically, horizontalArrangement = Arrangement.spacedBy(16.dp)) { + if (isAudioFile) { + IconToggleButton(checked = state.shuffleEnabled, onCheckedChange = { callbacks.onShuffleClick() }) { + Icon( + imageVector = Icons.Filled.Shuffle, + contentDescription = stringResource(R.string.nc_audio_player_shuffle), + tint = activeTint(state.shuffleEnabled) + ) + } + } + TextButton( + onClick = callbacks.onSpeedClick, + modifier = Modifier.semantics { + contentDescription = speedDescription + stateDescription = speedLabel + } + ) { + Text(text = speedLabel) + } + if (isAudioFile) { + IconButton(onClick = callbacks.onRepeatClick) { + Icon( + imageVector = if (state.repeatMode == ChatAudioRepeatMode.ONE) { + Icons.Filled.RepeatOne + } else { + Icons.Filled.Repeat + }, + contentDescription = stringResource(repeatModeDescription(state.repeatMode)), + tint = activeTint(state.repeatMode != ChatAudioRepeatMode.OFF) + ) + } + } + } +} + +@Composable +private fun activeTint(isActive: Boolean) = + if (isActive) MaterialTheme.colorScheme.primary else MaterialTheme.colorScheme.onSurfaceVariant + +private fun kindIcon(kind: ChatAudioKind?) = + if (kind == ChatAudioKind.AUDIO_FILE) Icons.Filled.MusicNote else Icons.Filled.Mic + +private fun repeatModeDescription(repeatMode: ChatAudioRepeatMode): Int = + when (repeatMode) { + ChatAudioRepeatMode.OFF -> R.string.nc_audio_player_repeat_off + ChatAudioRepeatMode.ALL -> R.string.nc_audio_player_repeat_all + ChatAudioRepeatMode.ONE -> R.string.nc_audio_player_repeat_one + } + +@Preview(heightDp = 720) +@Composable +private fun ChatAudioPlayerSheetContentPreview() { + val queue = listOf("intro.mp3", "podcast-episode-42.mp3", "outro.mp3").mapIndexed { index, title -> + ChatAudioQueueItem(index, ChatAudioKey(1L, "preview", index + 1), title, "John Doe") + } + Surface { + ChatAudioPlayerSheetContent( + state = ChatAudioPlaybackState( + currentKey = queue[1].key, + kind = ChatAudioKind.AUDIO_FILE, + title = queue[1].title, + subtitle = queue[1].subtitle, + isPlaying = true, + positionMs = 754_000L, + durationMs = 2_710_000L, + shuffleEnabled = true, + repeatMode = ChatAudioRepeatMode.ALL, + hasNext = true, + queue = queue + ), + callbacks = ChatAudioPlayerCallbacks() + ) + } +} diff --git a/app/src/main/java/com/nextcloud/talk/ui/chat/ChatMessageView.kt b/app/src/main/java/com/nextcloud/talk/ui/chat/ChatMessageView.kt index 95cb0549711..11b9965fb67 100644 --- a/app/src/main/java/com/nextcloud/talk/ui/chat/ChatMessageView.kt +++ b/app/src/main/java/com/nextcloud/talk/ui/chat/ChatMessageView.kt @@ -27,9 +27,14 @@ import androidx.compose.runtime.saveable.rememberSaveable import androidx.compose.ui.Modifier import androidx.compose.ui.unit.dp import com.nextcloud.talk.R +import com.nextcloud.talk.chat.audio.ChatAudioKey +import com.nextcloud.talk.chat.audio.ChatAudioMetadata +import com.nextcloud.talk.chat.audio.ChatAudioPlaybackState import com.nextcloud.talk.chat.ui.model.ChatMessageUi import com.nextcloud.talk.chat.ui.model.MessageTypeContent +import com.nextcloud.talk.ui.PlaybackSpeed import kotlinx.coroutines.delay +import kotlinx.coroutines.flow.MutableStateFlow private const val QUOTE_HIGHLIGHT_INITIAL_ALPHA = 0.15f private const val PREVIEW_WAVEFORM_POINT_ONE = 0.1f @@ -38,6 +43,10 @@ private const val PREVIEW_WAVEFORM_POINT_THREE = 0.4f private const val PREVIEW_WAVEFORM_POINT_FOUR = 0.15f private const val PREVIEW_WAVEFORM_POINT_FIVE = 0.3f private const val PREVIEW_WAVEFORM_POINT_SIX = 0.5f +private const val PREVIEW_USER_ID = 1L +private const val PREVIEW_ROOM_TOKEN = "preview" +private const val PREVIEW_VOICE_DURATION_MS = 16_000L +private const val PREVIEW_VOICE_POSITION_MS = 4_000L private val previewWaveform = listOf( PREVIEW_WAVEFORM_POINT_ONE, @@ -65,7 +74,7 @@ data class ChatMessageCallbacks( val onFileClick: (Int) -> Unit = {}, val onPollClick: (String, String) -> Unit = { _, _ -> }, val onVoicePlayPauseClick: (Int) -> Unit = {}, - val onVoiceSeek: (Int, Int) -> Unit = { _, _ -> }, + val onVoiceSeek: (Int, Float) -> Unit = { _, _ -> }, val onVoiceSpeedClick: (Int) -> Unit = {}, val onReactionClick: (Int, String) -> Unit = { _, _ -> }, val onReactionLongClick: (Int) -> Unit = {}, @@ -178,13 +187,10 @@ fun ChatMessageView( is MessageTypeContent.Voice -> { VoiceMessage( - typeContent = content, message = message, isOneToOneConversation = context.isOneToOneConversation, conversationThreadId = context.conversationThreadId, - onPlayPauseClick = callbacks.onVoicePlayPauseClick, - onSeek = callbacks.onVoiceSeek, - onSpeedClick = callbacks.onVoiceSpeedClick + callbacks = callbacks ) } @@ -194,8 +200,7 @@ fun ChatMessageView( message = message, isOneToOneConversation = context.isOneToOneConversation, conversationThreadId = context.conversationThreadId, - onPlayPauseClick = callbacks.onVoicePlayPauseClick, - onSeek = callbacks.onVoiceSeek + callbacks = callbacks ) } @@ -330,19 +335,25 @@ private fun ChatMessageViewGeolocationPreview() { @Composable private fun ChatMessageViewVoicePreview() { PreviewContainer { - val uiMessage = createBaseMessage( - MessageTypeContent.Voice( - actorId = "john", - isPlaying = false, - wasPlayed = false, - isDownloading = false, - durationSeconds = 16, - playedSeconds = 4, - seekbarProgress = 25, - waveform = previewWaveform - ) + val uiMessage = createBaseMessage(MessageTypeContent.Voice).copy(roomToken = PREVIEW_ROOM_TOKEN) + val audioUi = ChatAudioUi( + playbackState = MutableStateFlow(ChatAudioPlaybackState()), + metadata = MutableStateFlow( + mapOf( + ChatAudioKey(PREVIEW_USER_ID, PREVIEW_ROOM_TOKEN, uiMessage.id) to ChatAudioMetadata( + durationMs = PREVIEW_VOICE_DURATION_MS, + positionMs = PREVIEW_VOICE_POSITION_MS, + waveform = previewWaveform + ) + ) + ), + voiceSpeed = MutableStateFlow(PlaybackSpeed.NORMAL), + internalUserId = { PREVIEW_USER_ID }, + loadMetadata = {} ) - ChatMessageView(message = uiMessage) + CompositionLocalProvider(LocalChatAudioUi provides audioUi) { + ChatMessageView(message = uiMessage) + } } } diff --git a/app/src/main/java/com/nextcloud/talk/ui/chat/ChatView.kt b/app/src/main/java/com/nextcloud/talk/ui/chat/ChatView.kt index 0799becf3b9..f0c57bf57c9 100644 --- a/app/src/main/java/com/nextcloud/talk/ui/chat/ChatView.kt +++ b/app/src/main/java/com/nextcloud/talk/ui/chat/ChatView.kt @@ -385,7 +385,7 @@ fun ChatView( state = listState, reverseLayout = true, verticalArrangement = Arrangement.spacedBy(2.dp), - contentPadding = PaddingValues(bottom = 20.dp), + contentPadding = PaddingValues(top = state.stickyHeaderTopOffset, bottom = 20.dp), modifier = Modifier .fillMaxSize() .semantics { diff --git a/app/src/main/java/com/nextcloud/talk/ui/chat/VoiceMessage.kt b/app/src/main/java/com/nextcloud/talk/ui/chat/VoiceMessage.kt index 7f40a8a0301..3edcad7639e 100644 --- a/app/src/main/java/com/nextcloud/talk/ui/chat/VoiceMessage.kt +++ b/app/src/main/java/com/nextcloud/talk/ui/chat/VoiceMessage.kt @@ -7,56 +7,43 @@ package com.nextcloud.talk.ui.chat -import android.text.format.DateUtils import androidx.compose.animation.core.animateIntAsState +import androidx.compose.foundation.layout.Box import androidx.compose.foundation.layout.Column import androidx.compose.foundation.layout.Row import androidx.compose.foundation.layout.fillMaxWidth import androidx.compose.foundation.layout.height import androidx.compose.foundation.layout.padding -import androidx.compose.foundation.layout.size -import androidx.compose.material.icons.Icons -import androidx.compose.material.icons.filled.Pause -import androidx.compose.material.icons.filled.PlayArrow -import androidx.compose.material3.CircularProgressIndicator -import androidx.compose.material3.ExperimentalMaterial3Api -import androidx.compose.material3.Icon -import androidx.compose.material3.IconButton import androidx.compose.material3.MaterialTheme.colorScheme import androidx.compose.material3.Text import androidx.compose.material3.TextButton import androidx.compose.runtime.Composable import androidx.compose.runtime.getValue -import androidx.compose.runtime.mutableFloatStateOf +import androidx.compose.runtime.mutableStateOf import androidx.compose.runtime.remember import androidx.compose.runtime.setValue import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier -import androidx.compose.ui.graphics.toArgb import androidx.compose.ui.res.stringResource +import androidx.compose.ui.semantics.contentDescription +import androidx.compose.ui.semantics.semantics import androidx.compose.ui.unit.dp import com.nextcloud.talk.R +import com.nextcloud.talk.chat.audio.Waveforms import com.nextcloud.talk.chat.ui.model.ChatMessageUi -import com.nextcloud.talk.chat.ui.model.MessageTypeContent import com.nextcloud.talk.ui.ComposeWaveformSeekBar import com.nextcloud.talk.ui.WAVEFORM_SIZE -import com.nextcloud.talk.utils.AudioUtils -private const val SEEKBAR_MAX = 100 private const val START_WAVE_FORM_HEIGHT = 10 private const val END_WAVE_FORM_HEIGHT = 56 -@OptIn(ExperimentalMaterial3Api::class) -@Suppress("Detekt.LongMethod", "LongParameterList") +@Suppress("Detekt.LongMethod") @Composable fun VoiceMessage( - typeContent: MessageTypeContent.Voice, message: ChatMessageUi, isOneToOneConversation: Boolean = false, conversationThreadId: Long? = null, - onPlayPauseClick: (Int) -> Unit = {}, - onSeek: (messageId: Int, progress: Int) -> Unit = { _, _ -> }, - onSpeedClick: (messageId: Int) -> Unit = {} + callbacks: ChatMessageCallbacks = ChatMessageCallbacks() ) { MessageScaffold( uiMessage = message, @@ -64,77 +51,61 @@ fun VoiceMessage( conversationThreadId = conversationThreadId, forceTimeBelow = true, content = { - val inversePrimaryColor = colorScheme.inversePrimary - remember(inversePrimaryColor) { inversePrimaryColor.toArgb() } - val onPrimaryContainerColor = colorScheme.onPrimaryContainer - remember(onPrimaryContainerColor) { onPrimaryContainerColor.toArgb() } - val remainingSeconds = (typeContent.durationSeconds - typeContent.playedSeconds) - val waveformData = remember(typeContent.waveform) { - val floatArr = typeContent.waveform.toFloatArray() - if (floatArr.size < WAVEFORM_SIZE) { - FloatArray(WAVEFORM_SIZE) - } else { - AudioUtils.shrinkFloatArray(floatArr, WAVEFORM_SIZE) - } + val audioState = chatAudioMessageState(message) + val waveform = chatAudioWaveform(message) + val speed = chatAudioVoiceSpeed() + val waveformData = remember(waveform) { + waveform?.let { Waveforms.reduce(it.toFloatArray(), WAVEFORM_SIZE) } ?: FloatArray(WAVEFORM_SIZE) } - - val animValue by animateIntAsState( - if (waveformData.average() > 0) END_WAVE_FORM_HEIGHT else START_WAVE_FORM_HEIGHT, + val waveformHeight by animateIntAsState( + if (waveformData.any { it > 0f }) END_WAVE_FORM_HEIGHT else START_WAVE_FORM_HEIGHT, label = "size" ) - - val icon = if (typeContent.isPlaying) Icons.Filled.Pause else Icons.Filled.PlayArrow + var draggedProgress by remember { mutableStateOf(null) } + val seekDescription = stringResource(R.string.nc_audio_seek) Column { Row( verticalAlignment = Alignment.CenterVertically, modifier = Modifier.fillMaxWidth() ) { - if (typeContent.isDownloading) { - CircularProgressIndicator(modifier = Modifier.size(48.dp), strokeWidth = 2.dp) - } else { - IconButton( - onClick = { onPlayPauseClick(message.id) }, - modifier = Modifier.size(48.dp) - ) { - Icon( - imageVector = icon, - contentDescription = stringResource(R.string.play_pause_voice_message), - modifier = Modifier.size(40.dp) - ) - } - } - - var sliderValue by remember { mutableFloatStateOf(0f) } - sliderValue = typeContent.seekbarProgress * 1f / SEEKBAR_MAX - - ComposeWaveformSeekBar( - sliderValue, - { - val progressI = (it * SEEKBAR_MAX).toInt() - onSeek(message.id, progressI) - }, - modifier = Modifier - .height(animValue.dp) - .fillMaxWidth() - .padding(8.dp), // or weight(1f), - waveformData, - enabled = true + ChatAudioPlayPauseButton( + state = audioState, + contentDescription = stringResource(R.string.play_pause_voice_message), + onClick = { callbacks.onVoicePlayPauseClick(message.id) } ) + Box(modifier = Modifier.weight(1f)) { + ComposeWaveformSeekBar( + value = draggedProgress ?: audioState.progress, + onValueChange = { draggedProgress = it }, + modifier = Modifier + .height(waveformHeight.dp) + .fillMaxWidth() + .padding(8.dp), + waveData = waveformData, + enabled = audioState.canSeek, + onValueChangeFinished = { + draggedProgress?.let { callbacks.onVoiceSeek(message.id, it) } + draggedProgress = null + }, + sliderModifier = Modifier.semantics { contentDescription = seekDescription } + ) + } + TextButton( - onClick = { onSpeedClick(message.id) }, + onClick = { callbacks.onVoiceSpeedClick(message.id) }, modifier = Modifier.padding(start = 4.dp) ) { Text( - text = typeContent.playbackSpeed.label, + text = speed.label, color = colorScheme.onPrimaryContainer ) } } Text( - text = DateUtils.formatElapsedTime(remainingSeconds.toLong()), + text = formatAudioTime(audioState.displayedTimeMs), color = colorScheme.onPrimaryContainer, modifier = Modifier.padding(start = 4.dp) ) diff --git a/app/src/main/java/com/nextcloud/talk/utils/AudioUtils.kt b/app/src/main/java/com/nextcloud/talk/utils/AudioUtils.kt index 40b8ec81d67..680722fb138 100644 --- a/app/src/main/java/com/nextcloud/talk/utils/AudioUtils.kt +++ b/app/src/main/java/com/nextcloud/talk/utils/AudioUtils.kt @@ -8,229 +8,134 @@ package com.nextcloud.talk.utils import android.media.AudioFormat import android.media.MediaCodec -import android.media.MediaCodec.CodecException -import android.media.MediaCodecList import android.media.MediaExtractor import android.media.MediaFormat import android.os.SystemClock import android.util.Log -import androidx.lifecycle.DefaultLifecycleObserver -import androidx.lifecycle.LifecycleOwner -import com.nextcloud.talk.application.NextcloudTalkApplication +import androidx.annotation.WorkerThread +import com.nextcloud.talk.chat.audio.WaveformLevels import java.io.File import java.io.IOException +import java.nio.ByteBuffer import java.nio.ByteOrder -import kotlin.coroutines.resume -import kotlin.coroutines.suspendCoroutine -import kotlin.math.abs /** * AudioUtils are for processing raw audio using android's low level APIs, for more information read here * [MediaCodec documentation](https://developer.android.com/reference/android/media/MediaCodec) */ -object AudioUtils : DefaultLifecycleObserver { +object AudioUtils { private val TAG = AudioUtils::class.java.simpleName - private const val VALUE_10 = 10 - private const val TIME_LIMIT = 3000 - private const val DEFAULT_SIZE = 500 - private enum class LifeCycleFlag { - PAUSED, - RESUMED - } - private lateinit var currentLifeCycleFlag: LifeCycleFlag - - override fun onResume(owner: LifecycleOwner) { - super.onResume(owner) - currentLifeCycleFlag = LifeCycleFlag.RESUMED - } - - override fun onPause(owner: LifecycleOwner) { - super.onPause(owner) - currentLifeCycleFlag = LifeCycleFlag.PAUSED - } + private const val AUDIO_MIME_TYPE_PREFIX = "audio/" + private const val CODEC_TIMEOUT_US = 10_000L + private const val MAX_DECODING_TIME_MS = 15_000L + private const val PCM_16_BIT_FULL_SCALE = 32_768.0 /** - * Suspension function, returns a FloatArray of size 500, containing the values of an audio file squeezed between - * [0,1) + * Decodes the first audio track of [file] and returns the loudness of each decoded chunk, see + * [WaveformLevels], or null if the file cannot be decoded completely. Decoding gives up after a few seconds, + * as levels of only the beginning of a very long file would not match its playback progress. */ - @Throws(IOException::class) - suspend fun audioFileToFloatArray(file: File): FloatArray { - return suspendCoroutine { - // Used to keep track of the time it took to process the audio file - val startTime = SystemClock.elapsedRealtime() - - // Always a FloatArray of Size 500 - var result: MutableList? = mutableListOf() - - // Setting the file path to the audio file - val path = file.path - val mediaExtractor = MediaExtractor() - mediaExtractor.setDataSource(path) - - // Basically just boilerplate to set up meta data for the audio file - val mediaFormat = mediaExtractor.getTrackFormat(0) - // Frame rate is required for encoders, optional for decoders. So we set it to null here. - mediaFormat.setString(MediaFormat.KEY_FRAME_RATE, null) - mediaFormat.setInteger(MediaFormat.KEY_FRAME_RATE, 0) - - mediaExtractor.release() - - // More Boiler plate to set up the codec - val mediaCodecList = MediaCodecList(MediaCodecList.ALL_CODECS) - val codecName = mediaCodecList.findDecoderForFormat(mediaFormat) - val mediaCodec = MediaCodec.createByCodecName(codecName) - - /** - ************************************ Media Codec ******************************************* - * │ - * INPUT BUFFERS │ OUTPUT BUFFERS - * │ - * ┌────────────────┐ ┌───────┴────────┐ ┌─────────────────┐ - * │ │ Empty Buffer│ │ Filled Buffer│ │ - * │ │ [][][] │ │ [-][-][-] │ │ - * │ │ ◄───────────┤ ├────────────► │ │ - * │ Client │ │ Codec │ │ Client │ - * │ │ │ │ │ │ - * │ ├───────────► │ │ ◄────────────┤ │ - * │ │ [-][-][-] │ │ [][][] │ │ - * └────────────────┘Filled Buffer└───────┬────────┘Empty Buffer └─────────────────┘ - * │ - * Client provides │ Client consumes - * input Data │ output data - * - ******************************************************************************************** - */ - mediaCodec.setCallback(object : MediaCodec.Callback() { - private var extractor: MediaExtractor? = null - val tempList = mutableListOf() - init { - // Setting up the extractor to be guaranteed not null - extractor = MediaExtractor() - try { - extractor!!.setDataSource(path) - extractor!!.selectTrack(0) - } catch (e: IOException) { - NextcloudTalkApplication.sharedApplication?.logger?.e( - TAG, - "Failed to set up MediaExtractor for audio file", - e - ) - } - } - - override fun onInputBufferAvailable(codec: MediaCodec, index: Int) { - // Boiler plate, Extracts a buffer of encoded audio data to be sent to the codec for processing - val byteBuffer = codec.getInputBuffer(index) - if (byteBuffer != null && extractor != null) { - val sampleSize = extractor!!.readSampleData(byteBuffer, 0) - if (sampleSize > 0) { - val isOver = !extractor!!.advance() - codec.queueInputBuffer( - index, - 0, - sampleSize, - extractor!!.sampleTime, - if (isOver) MediaCodec.BUFFER_FLAG_END_OF_STREAM else 0 - ) - } - } - } - - override fun onOutputBufferAvailable(codec: MediaCodec, index: Int, info: MediaCodec.BufferInfo) { - // Boiler plate to get the audio data in a usable form - val outputBuffer = codec.getOutputBuffer(index) - val bufferFormat = codec.getOutputFormat(index) - val samples = outputBuffer!!.order(ByteOrder.nativeOrder()).asShortBuffer() - val numChannels = bufferFormat.getInteger(MediaFormat.KEY_CHANNEL_COUNT) - if (index < 0 || index >= numChannels) { - return - } - val sampleLength = (samples.remaining() / numChannels) - - // Squeezes the value of each sample between [0,1) using y = (x-1)/x - for (i in 0 until sampleLength) { - val x = abs(samples[i * numChannels + index].toInt()) / VALUE_10 - val y = (if (x > 0) ((x - 1) / x.toFloat()) else x.toFloat()) - tempList.add(y) - } - - codec.releaseOutputBuffer(index, false) - - // Cancels the process if it ends, exceeds the time limit, or the activity falls out of view - val currTime = SystemClock.elapsedRealtime() - startTime - if (info.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM > 0 || - currTime > TIME_LIMIT || - currentLifeCycleFlag == LifeCycleFlag.PAUSED - ) { - Log.d( - TAG, - "Processing ended with time: $currTime \n" + - "Is finished: ${info.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM > 0} \n" + - "Lifecycle state: $currentLifeCycleFlag" - ) - codec.stop() - codec.release() - extractor!!.release() - extractor = null - result = if (currTime < TIME_LIMIT) { - tempList - } else { - Log.e(TAG, "Error in MediaCodec Callback:\n\tonOutputBufferAvailable: Time limit exceeded") - null - } - } - } - - override fun onError(codec: MediaCodec, e: CodecException) { - Log.e(TAG, "Error in MediaCodec Callback: \n$e") - codec.stop() - codec.release() - extractor!!.release() - extractor = null - result = null - } - - override fun onOutputFormatChanged(codec: MediaCodec, format: MediaFormat) { - // unused atm - } - }) + @WorkerThread + fun extractWaveformLevels(file: File): FloatArray? { + val extractor = MediaExtractor() + var codec: MediaCodec? = null + return try { + extractor.setDataSource(file.path) + val track = findAudioTrack(extractor) + track?.let { (index, format) -> + extractor.selectTrack(index) + val decoder = MediaCodec.createDecoderByType(format.getString(MediaFormat.KEY_MIME).orEmpty()) + codec = decoder + decoder.configure(format, null, null, 0) + decoder.start() + decode(extractor, decoder) + } + } catch (e: IOException) { + Log.w(TAG, "Failed to decode audio file for its waveform", e) + null + } catch (e: IllegalStateException) { + Log.w(TAG, "Failed to decode audio file for its waveform", e) + null + } catch (e: IllegalArgumentException) { + Log.w(TAG, "Failed to decode audio file for its waveform", e) + null + } finally { + codec?.release() + extractor.release() + } + } - // More Boiler plate to start the codec - mediaFormat.setInteger(MediaFormat.KEY_PCM_ENCODING, AudioFormat.ENCODING_PCM_16BIT) - mediaCodec.configure(mediaFormat, null, null, 0) - mediaCodec.start() + private fun findAudioTrack(extractor: MediaExtractor): Pair? = + (0 until extractor.trackCount) + .map { index -> index to extractor.getTrackFormat(index) } + .firstOrNull { (_, format) -> + format.getString(MediaFormat.KEY_MIME)?.startsWith(AUDIO_MIME_TYPE_PREFIX) == true + } - // This runs until the codec finishes, the time limit is exceeded, or an error occurs - // If the time limit is exceed or an error occurs, the result should be null or empty - var currTime = SystemClock.elapsedRealtime() - startTime - while (result != null && - result!!.size <= 0 && - currTime < TIME_LIMIT // Guarantees Execution stops after 3 seconds - ) { - currTime = SystemClock.elapsedRealtime() - startTime - continue + private fun decode(extractor: MediaExtractor, codec: MediaCodec): FloatArray? { + val levels = WaveformLevels() + val info = MediaCodec.BufferInfo() + val deadline = SystemClock.elapsedRealtime() + MAX_DECODING_TIME_MS + var isFloatOutput = false + var isInputDone = false + var isOutputDone = false + while (!isOutputDone && SystemClock.elapsedRealtime() < deadline) { + if (!isInputDone) { + isInputDone = queueInput(extractor, codec) } + val outputIndex = codec.dequeueOutputBuffer(info, CODEC_TIMEOUT_US) + if (outputIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { + isFloatOutput = codec.outputFormat.isFloatPcm() + } else if (outputIndex >= 0) { + codec.getOutputBuffer(outputIndex)?.let { addLevel(it, isFloatOutput, levels) } + codec.releaseOutputBuffer(outputIndex, false) + isOutputDone = info.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM != 0 + } + } + return if (isOutputDone) levels.toArray() else null + } - if (result != null && result!!.size > DEFAULT_SIZE) { - it.resume(shrinkFloatArray(result!!.toFloatArray(), DEFAULT_SIZE)) + /** Feeds the next sample to the decoder and returns true once the end of the input was queued. */ + private fun queueInput(extractor: MediaExtractor, codec: MediaCodec): Boolean { + val inputIndex = codec.dequeueInputBuffer(CODEC_TIMEOUT_US) + val buffer = if (inputIndex >= 0) codec.getInputBuffer(inputIndex) else null + var isEndOfInput = false + if (buffer != null) { + val sampleSize = extractor.readSampleData(buffer, 0) + isEndOfInput = sampleSize < 0 + if (isEndOfInput) { + codec.queueInputBuffer(inputIndex, 0, 0, 0L, MediaCodec.BUFFER_FLAG_END_OF_STREAM) } else { - it.resume(FloatArray(DEFAULT_SIZE)) + codec.queueInputBuffer(inputIndex, 0, sampleSize, extractor.sampleTime, 0) + extractor.advance() } } + return isEndOfInput } - fun shrinkFloatArray(data: FloatArray, size: Int): FloatArray { - val result = FloatArray(size) - val scale = data.size / size - var begin = 0 - var end = scale - for (i in 0 until size) { - val arr = data.copyOfRange(begin, end) - result[i] = arr.average().toFloat() - begin += scale - end += scale + private fun addLevel(buffer: ByteBuffer, isFloatOutput: Boolean, levels: WaveformLevels) { + val pcm = buffer.order(ByteOrder.nativeOrder()) + var sumOfSquares = 0.0 + var sampleCount = 0 + if (isFloatOutput) { + val samples = pcm.asFloatBuffer() + while (samples.hasRemaining()) { + val sample = samples.get().toDouble() + sumOfSquares += sample * sample + sampleCount++ + } + } else { + val samples = pcm.asShortBuffer() + while (samples.hasRemaining()) { + val sample = samples.get() / PCM_16_BIT_FULL_SCALE + sumOfSquares += sample * sample + sampleCount++ + } } - - return result + levels.addChunk(sumOfSquares, sampleCount) } + + private fun MediaFormat.isFloatPcm(): Boolean = + containsKey(MediaFormat.KEY_PCM_ENCODING) && + getInteger(MediaFormat.KEY_PCM_ENCODING) == AudioFormat.ENCODING_PCM_FLOAT } diff --git a/app/src/main/java/com/nextcloud/talk/utils/preferences/AppPreferences.java b/app/src/main/java/com/nextcloud/talk/utils/preferences/AppPreferences.java index 440a83f37ef..af59e8e5262 100644 --- a/app/src/main/java/com/nextcloud/talk/utils/preferences/AppPreferences.java +++ b/app/src/main/java/com/nextcloud/talk/utils/preferences/AppPreferences.java @@ -181,10 +181,6 @@ public interface AppPreferences { String getSorting(); - void saveWaveFormForFile(String filename, Float[] array); - - Float[] getWaveFormFromFile(String filename); - void saveLastKnownId(String internalConversationId, int lastReadId); int getLastKnownId(String internalConversationId, int defaultValue); @@ -195,6 +191,18 @@ public interface AppPreferences { PlaybackSpeed getPreferredPlayback(String userId); + void saveAudioFilePlaybackSpeed(String userId, PlaybackSpeed speed); + + PlaybackSpeed getAudioFilePlaybackSpeed(String userId); + + boolean getChatAudioShuffle(); + + void setChatAudioShuffle(boolean value); + + String getChatAudioRepeatMode(); + + void setChatAudioRepeatMode(String value); + Long getNotificationWarningLastPostponedDate(); void setNotificationWarningLastPostponedDate(Long showNotificationWarning); diff --git a/app/src/main/java/com/nextcloud/talk/utils/preferences/AppPreferencesImpl.kt b/app/src/main/java/com/nextcloud/talk/utils/preferences/AppPreferencesImpl.kt index ccda4e3e032..03404c21322 100644 --- a/app/src/main/java/com/nextcloud/talk/utils/preferences/AppPreferencesImpl.kt +++ b/app/src/main/java/com/nextcloud/talk/utils/preferences/AppPreferencesImpl.kt @@ -553,18 +553,6 @@ class AppPreferencesImpl(val context: Context) : AppPreferences { return read.ifEmpty { default } } - override fun saveWaveFormForFile(filename: String, array: Array) = - runBlocking { - async { - writeString(filename, array.contentToString()) - } - } - - override fun getWaveFormFromFile(filename: String): Array { - val string = runBlocking { async { readString(filename).first() } }.getCompleted() - return if (string.isNotEmpty()) string.convertStringToArray() else floatArrayOf().toTypedArray() - } - override fun saveLastKnownId(internalConversationId: String, lastReadId: Int) { runBlocking { async { @@ -614,6 +602,46 @@ class AppPreferencesImpl(val context: Context) : AppPreferences { } }.getCompleted() + override fun saveAudioFilePlaybackSpeed(userId: String, speed: PlaybackSpeed) { + runBlocking { + async { + writeString(userId + AUDIO_FILE_PLAY_BACK, speed.name) + } + } + } + + override fun getAudioFilePlaybackSpeed(userId: String): PlaybackSpeed = + runBlocking { + async { + val name = readString(userId + AUDIO_FILE_PLAY_BACK).first() + return@async if (name == "") PlaybackSpeed.NORMAL else PlaybackSpeed.byName(name) + } + }.getCompleted() + + override fun getChatAudioShuffle(): Boolean = + runBlocking { + async { readBoolean(CHAT_AUDIO_SHUFFLE).first() } + }.getCompleted() + + override fun setChatAudioShuffle(value: Boolean) = + runBlocking { + async { + writeBoolean(CHAT_AUDIO_SHUFFLE, value) + } + } + + override fun getChatAudioRepeatMode(): String = + runBlocking { + async { readString(CHAT_AUDIO_REPEAT_MODE).first() } + }.getCompleted() + + override fun setChatAudioRepeatMode(value: String) = + runBlocking { + async { + writeString(CHAT_AUDIO_REPEAT_MODE, value) + } + } + override fun getNotificationWarningLastPostponedDate(): Long = runBlocking { async { readLong(LAST_NOTIFICATION_WARNING).first() } @@ -710,18 +738,13 @@ class AppPreferencesImpl(val context: Context) : AppPreferences { const val TYPING_STATUS = "typing_status" const val MESSAGE_QUEUE = "@message_queue" const val PLAY_BACK = "_playback" + const val AUDIO_FILE_PLAY_BACK = "_audio_file_playback" + const val CHAT_AUDIO_SHUFFLE = "chat_audio_shuffle" + const val CHAT_AUDIO_REPEAT_MODE = "chat_audio_repeat_mode" const val VOICE_MESSAGE_PLAYBACK_SPEEDS = "voice_message_playback_speeds" const val SHOW_REGULAR_NOTIFICATION_WARNING = "show_regular_notification_warning" const val LAST_NOTIFICATION_WARNING = "last_notification_warning" const val CONVERSATION_LIST_POSITION_OFFSET = "CONVERSATION_LIST_POSITION_OFFSET" const val CONVERSATION_LIST_LAST_USER_ID = "CONVERSATION_LIST_LAST_USER_ID" - private fun String.convertStringToArray(): Array { - var varString = this - val floatList = mutableListOf() - varString = varString.replace("\\[".toRegex(), "") - varString = varString.replace("]".toRegex(), "") - varString.split(",").forEach { floatList.add(it.toFloat()) } - return floatList.toTypedArray() - } } } diff --git a/app/src/main/res/layout/activity_chat.xml b/app/src/main/res/layout/activity_chat.xml index 16a10d70a5b..c1555d8edb6 100644 --- a/app/src/main/res/layout/activity_chat.xml +++ b/app/src/main/res/layout/activity_chat.xml @@ -62,6 +62,11 @@ android:layout_height="wrap_content" android:orientation="vertical"> + + Permission for audio recording is required Playback speed control Delete voice recording + Playback position + Retry playback + Play/pause + Stop playback + Show message + Open player + Previous + Next + Shuffle + Repeat off + Repeat all + Repeat one + Playlist + Could not load the audio. Check your connection. + The audio file is not available + The audio format is not supported + Could not play the audio Match contacts based on phone number to integrate Talk shortcut into system contacts app diff --git a/app/src/test/java/com/nextcloud/talk/chat/audio/ChatAudioCredentialsTest.kt b/app/src/test/java/com/nextcloud/talk/chat/audio/ChatAudioCredentialsTest.kt new file mode 100644 index 00000000000..81321889146 --- /dev/null +++ b/app/src/test/java/com/nextcloud/talk/chat/audio/ChatAudioCredentialsTest.kt @@ -0,0 +1,82 @@ +/* + * Nextcloud Talk - Android Client + * + * SPDX-FileCopyrightText: 2026 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: GPL-3.0-or-later + */ + +package com.nextcloud.talk.chat.audio + +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Test + +class ChatAudioCredentialsTest { + + private val baseUrl = "https://cloud.example.com" + + @Test + fun `files of the account are authenticated`() { + assertTrue( + ChatAudioCredentials.belongsToUser( + "https://cloud.example.com/remote.php/dav/files/alice/Talk/voice.mp3", + baseUrl, + "alice" + ) + ) + } + + @Test + fun `server installed in a sub directory is supported`() { + assertTrue( + ChatAudioCredentials.belongsToUser( + "https://example.com/nextcloud/remote.php/dav/files/alice/Talk/voice.mp3", + "https://example.com/nextcloud", + "alice" + ) + ) + } + + @Test + fun `other hosts never get the credentials`() { + val attacks = listOf( + "https://cloud.example.com.evil.org/remote.php/dav/files/alice/voice.mp3", + "https://cloud.example.com@evil.org/remote.php/dav/files/alice/voice.mp3", + "http://cloud.example.com/remote.php/dav/files/alice/voice.mp3", + "https://evil.org/https://cloud.example.com/remote.php/dav/files/alice/voice.mp3", + "https://cloud.example.com:8443/remote.php/dav/files/alice/voice.mp3" + ) + + attacks.forEach { url -> + assertFalse(url, ChatAudioCredentials.belongsToUser(url, baseUrl, "alice")) + } + } + + @Test + fun `files of other accounts or other endpoints are not authenticated`() { + assertFalse( + ChatAudioCredentials.belongsToUser( + "https://cloud.example.com/remote.php/dav/files/alice2/voice.mp3", + baseUrl, + "alice" + ) + ) + assertFalse( + ChatAudioCredentials.belongsToUser( + "https://cloud.example.com/ocs/v2.php/apps/spreed/api/v1/chat/abc", + baseUrl, + "alice" + ) + ) + } + + @Test + fun `incomplete accounts are never matched`() { + val url = "https://cloud.example.com/remote.php/dav/files/alice/voice.mp3" + + assertFalse(ChatAudioCredentials.belongsToUser(url, null, "alice")) + assertFalse(ChatAudioCredentials.belongsToUser(url, "", "alice")) + assertFalse(ChatAudioCredentials.belongsToUser(url, baseUrl, null)) + assertFalse(ChatAudioCredentials.belongsToUser(url, baseUrl, "")) + } +} diff --git a/app/src/test/java/com/nextcloud/talk/chat/audio/ChatAudioKeyTest.kt b/app/src/test/java/com/nextcloud/talk/chat/audio/ChatAudioKeyTest.kt new file mode 100644 index 00000000000..b4b670efd14 --- /dev/null +++ b/app/src/test/java/com/nextcloud/talk/chat/audio/ChatAudioKeyTest.kt @@ -0,0 +1,40 @@ +/* + * Nextcloud Talk - Android Client + * + * SPDX-FileCopyrightText: 2026 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: GPL-3.0-or-later + */ + +package com.nextcloud.talk.chat.audio + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNull +import org.junit.Test + +class ChatAudioKeyTest { + + @Test + fun `media id round trip restores the key`() { + val key = ChatAudioKey(internalUserId = 3L, roomToken = "abc123xy", messageId = 4711) + + assertEquals(key, ChatAudioKey.fromMediaId(key.mediaId)) + } + + @Test + fun `media ids of other sources are not parsed`() { + assertNull(ChatAudioKey.fromMediaId(null)) + assertNull(ChatAudioKey.fromMediaId("")) + assertNull(ChatAudioKey.fromMediaId("4711")) + assertNull(ChatAudioKey.fromMediaId("file:///sdcard/voice.mp3")) + } + + @Test + fun `malformed media ids are rejected`() { + assertNull(ChatAudioKey.fromMediaId("talk-audio:")) + assertNull(ChatAudioKey.fromMediaId("talk-audio:3/abc")) + assertNull(ChatAudioKey.fromMediaId("talk-audio:3//4711")) + assertNull(ChatAudioKey.fromMediaId("talk-audio:x/abc/4711")) + assertNull(ChatAudioKey.fromMediaId("talk-audio:3/abc/x")) + assertNull(ChatAudioKey.fromMediaId("talk-audio:3/a/b/4711")) + } +} diff --git a/app/src/test/java/com/nextcloud/talk/chat/audio/ChatAudioMetadataTest.kt b/app/src/test/java/com/nextcloud/talk/chat/audio/ChatAudioMetadataTest.kt new file mode 100644 index 00000000000..924d9d8af2d --- /dev/null +++ b/app/src/test/java/com/nextcloud/talk/chat/audio/ChatAudioMetadataTest.kt @@ -0,0 +1,53 @@ +/* + * Nextcloud Talk - Android Client + * + * SPDX-FileCopyrightText: 2026 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: GPL-3.0-or-later + */ + +package com.nextcloud.talk.chat.audio + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNull +import org.junit.Test + +class ChatAudioMetadataTest { + + private val longDuration = ChatAudioMetadata.LONG_AUDIO_DURATION_MS + 1000L + + @Test + fun `codec round trip keeps duration, waveform and the position of long recordings`() { + val metadata = ChatAudioMetadata( + durationMs = longDuration, + positionMs = 42_000L, + waveform = listOf(0f, 0.5f, 1f) + ) + + assertEquals(metadata, ChatAudioMetadataCodec.decode(ChatAudioMetadataCodec.encode(metadata))) + } + + @Test + fun `position of short recordings is not persisted`() { + val metadata = ChatAudioMetadata(durationMs = 20_000L, positionMs = 5_000L) + + val decoded = ChatAudioMetadataCodec.decode(ChatAudioMetadataCodec.encode(metadata)) + + assertEquals(ChatAudioMetadata(durationMs = 20_000L), decoded) + } + + @Test + fun `corrupt or foreign data is ignored`() { + assertNull(ChatAudioMetadataCodec.decode(ByteArray(0))) + assertNull(ChatAudioMetadataCodec.decode(byteArrayOf(0, 0, 0, 9))) + val truncated = ChatAudioMetadataCodec.encode(ChatAudioMetadata(1000L, 0L, listOf(1f, 1f))).dropLast(2) + assertNull(ChatAudioMetadataCodec.decode(truncated.toByteArray())) + } + + @Test + fun `resume position starts over at the end of a message`() { + assertEquals(0L, ChatAudioMetadata(durationMs = 10_000L, positionMs = 9_500L).resumePosition) + assertEquals(4_000L, ChatAudioMetadata(durationMs = 10_000L, positionMs = 4_000L).resumePosition) + assertEquals(4_000L, ChatAudioMetadata(durationMs = 0L, positionMs = 4_000L).resumePosition) + assertEquals(0L, ChatAudioMetadata().resumePosition) + } +} diff --git a/app/src/test/java/com/nextcloud/talk/chat/audio/ChatAudioPlaybackStateTest.kt b/app/src/test/java/com/nextcloud/talk/chat/audio/ChatAudioPlaybackStateTest.kt new file mode 100644 index 00000000000..2aab4108250 --- /dev/null +++ b/app/src/test/java/com/nextcloud/talk/chat/audio/ChatAudioPlaybackStateTest.kt @@ -0,0 +1,79 @@ +/* + * Nextcloud Talk - Android Client + * + * SPDX-FileCopyrightText: 2026 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: GPL-3.0-or-later + */ + +package com.nextcloud.talk.chat.audio + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Test + +class ChatAudioPlaybackStateTest { + + private val current = ChatAudioKey(1L, "room", 10) + private val other = ChatAudioKey(1L, "room", 11) + + private val playing = ChatAudioPlaybackState( + currentKey = current, + kind = ChatAudioKind.VOICE_MESSAGE, + isPlaying = true, + positionMs = 2_500L, + durationMs = 10_000L + ) + + @Test + fun `current message shows the live playback state`() { + val state = playing.messageState(current, ChatAudioMetadata(durationMs = 9_000L, positionMs = 1_000L)) + + assertTrue(state.isCurrent) + assertTrue(state.isPlaying) + assertEquals(2_500L, state.positionMs) + assertEquals(10_000L, state.durationMs) + assertEquals(0.25f, state.progress, DELTA) + assertEquals(2_500L, state.displayedTimeMs) + } + + @Test + fun `current message falls back to the known duration while the player does not know it yet`() { + val buffering = playing.copy(isBuffering = true, positionMs = 0L, durationMs = 0L) + + val state = buffering.messageState(current, ChatAudioMetadata(durationMs = 9_000L)) + + assertTrue(state.isBuffering) + assertEquals(9_000L, state.durationMs) + } + + @Test + fun `other messages show their remembered position and duration`() { + val state = playing.messageState(other, ChatAudioMetadata(durationMs = 8_000L, positionMs = 2_000L)) + + assertFalse(state.isCurrent) + assertFalse(state.isPlaying) + assertEquals(0.25f, state.progress, DELTA) + assertEquals(2_000L, state.displayedTimeMs) + } + + @Test + fun `unplayed messages show their duration or nothing if it is unknown`() { + assertEquals(8_000L, playing.messageState(other, ChatAudioMetadata(durationMs = 8_000L)).displayedTimeMs) + assertNull(playing.messageState(other, null).displayedTimeMs) + assertFalse(playing.messageState(other, null).canSeek) + } + + @Test + fun `errors are only reported for the current message`() { + val failed = playing.copy(isPlaying = false, error = ChatAudioError.NETWORK) + + assertEquals(ChatAudioError.NETWORK, failed.messageState(current, null).error) + assertNull(failed.messageState(other, null).error) + } + + companion object { + private const val DELTA = 0.0001f + } +} diff --git a/app/src/test/java/com/nextcloud/talk/chat/audio/ChatAudioQueueTest.kt b/app/src/test/java/com/nextcloud/talk/chat/audio/ChatAudioQueueTest.kt new file mode 100644 index 00000000000..9e57a017946 --- /dev/null +++ b/app/src/test/java/com/nextcloud/talk/chat/audio/ChatAudioQueueTest.kt @@ -0,0 +1,72 @@ +/* + * Nextcloud Talk - Android Client + * + * SPDX-FileCopyrightText: 2026 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: GPL-3.0-or-later + */ + +package com.nextcloud.talk.chat.audio + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNull +import org.junit.Test + +class ChatAudioQueueTest { + + private data class Message(val id: Int, val kind: ChatAudioKind?) + + private fun build(messages: List, startId: Int) = + ChatAudioQueue.build(messages, startId, { it.id }, { it.kind }) + ?.let { ChatAudioQueue.Selection(it.items.map { message -> message.id }, it.startIndex) } + + private val mixed = listOf( + Message(1, ChatAudioKind.VOICE_MESSAGE), + Message(2, ChatAudioKind.AUDIO_FILE), + Message(3, ChatAudioKind.VOICE_MESSAGE), + Message(4, null), + Message(5, ChatAudioKind.AUDIO_FILE), + Message(6, ChatAudioKind.VOICE_MESSAGE) + ) + + @Test + fun `voice messages continue with the newer voice messages only`() { + assertEquals(ChatAudioQueue.Selection(listOf(3, 6), 0), build(mixed, startId = 3)) + } + + @Test + fun `audio files are played among all audio files of the conversation`() { + assertEquals(ChatAudioQueue.Selection(listOf(2, 5), 1), build(mixed, startId = 5)) + assertEquals(ChatAudioQueue.Selection(listOf(2, 5), 0), build(mixed, startId = 2)) + } + + @Test + fun `unknown or unplayable start message results in no queue`() { + assertNull(build(mixed, startId = 42)) + assertNull(build(mixed, startId = 4)) + assertNull(build(emptyList(), startId = 1)) + } + + @Test + fun `voice message queue size is limited`() { + val messages = (1..ChatAudioQueue.MAX_SIZE * 2).map { Message(it, ChatAudioKind.VOICE_MESSAGE) } + + val queue = build(messages, startId = 1) + + assertEquals(ChatAudioQueue.Selection((1..ChatAudioQueue.MAX_SIZE).toList(), 0), queue) + } + + @Test + fun `audio file queue is limited to the files around the tapped one`() { + val messages = (1..ChatAudioQueue.MAX_SIZE * 2).map { Message(it, ChatAudioKind.AUDIO_FILE) } + val half = ChatAudioQueue.MAX_SIZE / 2 + + val middle = build(messages, startId = ChatAudioQueue.MAX_SIZE) + val newest = build(messages, startId = messages.size) + + assertEquals(ChatAudioQueue.MAX_SIZE, middle?.items?.size) + assertEquals(ChatAudioQueue.MAX_SIZE - half, middle?.items?.first()) + assertEquals(half, middle?.startIndex) + assertEquals(messages.size - ChatAudioQueue.MAX_SIZE + 1, newest?.items?.first()) + assertEquals(ChatAudioQueue.MAX_SIZE - 1, newest?.startIndex) + } +} diff --git a/app/src/test/java/com/nextcloud/talk/chat/audio/ChatAudioRepeatModeTest.kt b/app/src/test/java/com/nextcloud/talk/chat/audio/ChatAudioRepeatModeTest.kt new file mode 100644 index 00000000000..9b8bbde197f --- /dev/null +++ b/app/src/test/java/com/nextcloud/talk/chat/audio/ChatAudioRepeatModeTest.kt @@ -0,0 +1,28 @@ +/* + * Nextcloud Talk - Android Client + * + * SPDX-FileCopyrightText: 2026 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: GPL-3.0-or-later + */ + +package com.nextcloud.talk.chat.audio + +import org.junit.Assert.assertEquals +import org.junit.Test + +class ChatAudioRepeatModeTest { + + @Test + fun `repeat button switches from off to all to one and back to off`() { + assertEquals(ChatAudioRepeatMode.ALL, ChatAudioRepeatMode.OFF.next()) + assertEquals(ChatAudioRepeatMode.ONE, ChatAudioRepeatMode.ALL.next()) + assertEquals(ChatAudioRepeatMode.OFF, ChatAudioRepeatMode.ONE.next()) + } + + @Test + fun `stored names are restored and unknown ones mean off`() { + assertEquals(ChatAudioRepeatMode.ONE, ChatAudioRepeatMode.fromName("ONE")) + assertEquals(ChatAudioRepeatMode.OFF, ChatAudioRepeatMode.fromName("")) + assertEquals(ChatAudioRepeatMode.OFF, ChatAudioRepeatMode.fromName(null)) + } +} diff --git a/app/src/test/java/com/nextcloud/talk/chat/audio/ChatAudioStoreTest.kt b/app/src/test/java/com/nextcloud/talk/chat/audio/ChatAudioStoreTest.kt new file mode 100644 index 00000000000..10a03817002 --- /dev/null +++ b/app/src/test/java/com/nextcloud/talk/chat/audio/ChatAudioStoreTest.kt @@ -0,0 +1,145 @@ +/* + * Nextcloud Talk - Android Client + * + * SPDX-FileCopyrightText: 2026 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: GPL-3.0-or-later + */ + +package com.nextcloud.talk.chat.audio + +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.runBlocking +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Rule +import org.junit.Test +import org.junit.rules.TemporaryFolder + +class ChatAudioStoreTest { + + @get:Rule + val folder = TemporaryFolder() + + private val key = ChatAudioKey(1L, "room", 42) + private val longDuration = ChatAudioMetadata.LONG_AUDIO_DURATION_MS * 2 + + private fun newStore() = ChatAudioStore(folder.root, Dispatchers.Unconfined) + + @Test + fun `remembered values are available immediately`() { + val store = newStore() + + store.rememberDuration(key, 20_000L) + store.rememberPosition(key, 5_000L, 0L) + + assertEquals(ChatAudioMetadata(durationMs = 20_000L, positionMs = 5_000L), store.metadataFor(key)) + assertEquals(store.metadataFor(key), store.metadata.value[key]) + } + + @Test + fun `forgetting the position of an unknown message creates no entry`() { + val store = newStore() + + store.forgetPosition(key) + + assertNull(store.metadataFor(key)) + } + + @Test + fun `duration, waveform and position of long recordings are restored after a restart`() { + newStore().apply { + rememberDuration(key, longDuration) + rememberWaveform(key, floatArrayOf(0.5f, 1f)) + rememberPosition(key, 60_000L, longDuration) + } + + val restored = newStore().apply { load(key) }.metadataFor(key) + + assertEquals(ChatAudioMetadata(longDuration, 60_000L, listOf(0.5f, 1f)), restored) + } + + @Test + fun `short recordings start from the beginning after a restart`() { + newStore().rememberPosition(key, 5_000L, 20_000L) + + val restored = newStore().apply { load(key) }.metadataFor(key) + + assertEquals(ChatAudioMetadata(durationMs = 20_000L), restored) + } + + @Test + fun `a waveform that could not be computed is only kept until the app restarts`() { + newStore().apply { + rememberDuration(key, 20_000L) + rememberWaveform(key, FloatArray(0)) + assertEquals(emptyList(), metadataFor(key)?.waveform) + } + + val restored = newStore().apply { load(key) }.metadataFor(key) + + assertEquals(ChatAudioMetadata(durationMs = 20_000L), restored) + } + + @Test + fun `a position changed before the persisted one is loaded is kept`() { + newStore().rememberPosition(key, 60_000L, longDuration) + val store = newStore() + + store.forgetPosition(key) + store.load(key) + + assertEquals(ChatAudioMetadata(durationMs = longDuration), store.metadataFor(key)) + assertEquals(ChatAudioMetadata(durationMs = longDuration), newStore().apply { load(key) }.metadataFor(key)) + } + + @Test + fun `waiting for a load returns the persisted metadata`() { + newStore().rememberPosition(key, 60_000L, longDuration) + + val restored = runBlocking { newStore().loadNow(key) } + + assertEquals(ChatAudioMetadata(longDuration, 60_000L), restored) + } + + @Test + fun `forgetting an account deletes its metadata in memory and on disk`() { + val otherAccountKey = ChatAudioKey(12L, "room", 42) + val store = newStore().apply { + rememberDuration(key, longDuration) + rememberDuration(otherAccountKey, longDuration) + } + + store.forgetAccount(key.internalUserId) + + assertNull(store.metadataFor(key)) + assertEquals(ChatAudioMetadata(durationMs = longDuration), store.metadataFor(otherAccountKey)) + assertNull(newStore().apply { load(key) }.metadataFor(key)) + assertEquals( + ChatAudioMetadata(durationMs = longDuration), + newStore().apply { load(otherAccountKey) }.metadataFor(otherAccountKey) + ) + } + + @Test + fun `nothing is remembered about a forgotten account`() { + val store = newStore() + store.forgetAccount(key.internalUserId) + + store.rememberDuration(key, longDuration) + store.rememberPosition(key, 60_000L, longDuration) + + assertNull(store.metadataFor(key)) + assertTrue(folder.root.listFiles().orEmpty().isEmpty()) + } + + @Test + fun `corrupt files are ignored`() { + newStore().rememberDuration(key, 20_000L) + folder.root.listFiles().orEmpty().forEach { it.writeText("garbage") } + + val store = newStore().apply { load(key) } + + assertNull(store.metadataFor(key)) + } +} diff --git a/app/src/test/java/com/nextcloud/talk/chat/audio/ChatAudioTrackTest.kt b/app/src/test/java/com/nextcloud/talk/chat/audio/ChatAudioTrackTest.kt new file mode 100644 index 00000000000..d048f5dc6a7 --- /dev/null +++ b/app/src/test/java/com/nextcloud/talk/chat/audio/ChatAudioTrackTest.kt @@ -0,0 +1,137 @@ +/* + * Nextcloud Talk - Android Client + * + * SPDX-FileCopyrightText: 2026 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: GPL-3.0-or-later + */ + +package com.nextcloud.talk.chat.audio + +import android.app.Application +import androidx.media3.common.PlaybackException +import com.nextcloud.talk.chat.ui.model.ChatMessageUi +import com.nextcloud.talk.chat.ui.model.MessageStatusIcon +import com.nextcloud.talk.chat.ui.model.MessageTypeContent +import com.nextcloud.talk.data.user.model.User +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNotNull +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner +import org.robolectric.annotation.Config +import java.time.LocalDate + +@RunWith(RobolectricTestRunner::class) +@Config(application = Application::class, sdk = [33]) +class ChatAudioTrackTest { + + private val user = User( + id = 7L, + userId = "alice", + username = "alice", + baseUrl = "https://cloud.example.com", + token = "app-password" + ) + + private fun message( + content: MessageTypeContent?, + actorType: String = "users", + roomToken: String? = "room1", + path: String = "Talk/voice message.m4a" + ) = ChatMessageUi( + id = 42, + message = "{file}", + renderMarkdown = false, + actorDisplayName = "Bob", + actorType = actorType, + actorId = "bob", + isThread = false, + threadTitle = "", + threadReplies = 0, + incoming = true, + isDeleted = false, + avatarUrl = null, + statusIcon = MessageStatusIcon.SENT, + timestamp = 0L, + date = LocalDate.of(2026, 1, 1), + content = content, + roomToken = roomToken, + messageParameters = mapOf("file" to mapOf("path" to path, "id" to "123", "etag" to "abc")) + ) + + @Test + fun `voice message is streamed from the WebDAV files of the account`() { + val track = requireNotNull(message(MessageTypeContent.Voice).toChatAudioTrack(user, "Voice message")) + + assertEquals(ChatAudioKey(7L, "room1", 42), track.key) + assertEquals(ChatAudioKind.VOICE_MESSAGE, track.kind) + assertEquals("https://cloud.example.com/remote.php/dav/files/alice/Talk/voice%20message.m4a", track.uri) + assertEquals("https://cloud.example.com/123/abc", track.cacheKey) + assertEquals("Bob", track.title) + assertEquals("Voice message", track.artist) + assertTrue(track.artworkUri.orEmpty().startsWith("https://cloud.example.com/index.php/avatar/bob/")) + assertTrue(ChatAudioCredentials.belongsToUser(track.uri, user.baseUrl, user.userId)) + } + + @Test + fun `audio file is titled with its file name`() { + val track = message(MessageTypeContent.AudioFile("song.mp3")).toChatAudioTrack(user, "Voice message") + + assertEquals(ChatAudioKind.AUDIO_FILE, track?.kind) + assertEquals("song.mp3", track?.title) + assertEquals("Bob", track?.artist) + } + + @Test + fun `messages that cannot be played give no track`() { + assertNull(message(MessageTypeContent.RegularText).toChatAudioTrack(user, "Voice message")) + assertNull(message(MessageTypeContent.Voice, path = "").toChatAudioTrack(user, "Voice message")) + assertNull(message(MessageTypeContent.Voice, roomToken = null).toChatAudioTrack(user, "Voice message")) + assertNull(message(MessageTypeContent.Voice).toChatAudioTrack(User(), "Voice message")) + } + + @Test + fun `guests have no server avatar as artwork`() { + val track = message(MessageTypeContent.Voice, actorType = "guests").toChatAudioTrack(user, "Voice message") + + assertNotNull(track) + assertNull(track?.artworkUri) + } + + @Test + fun `media item keeps identity, cache key and kind`() { + val track = requireNotNull( + message(MessageTypeContent.AudioFile("song.mp3")).toChatAudioTrack(user, "Voice message") + ) + + val mediaItem = track.toMediaItem() + + assertEquals(track.key, ChatAudioKey.fromMediaId(mediaItem.mediaId)) + assertEquals(track.uri, mediaItem.localConfiguration?.uri.toString()) + assertEquals(track.cacheKey, mediaItem.localConfiguration?.customCacheKey) + assertEquals(ChatAudioKind.AUDIO_FILE, mediaItem.chatAudioKind()) + } + + @Test + fun `media item keeps the thread the message was played in`() { + val inThread = message(MessageTypeContent.Voice).toChatAudioTrack(user, "Voice message", threadId = 5L) + val inConversation = message(MessageTypeContent.Voice).toChatAudioTrack(user, "Voice message") + + assertEquals(5L, inThread?.toMediaItem()?.chatAudioThreadId()) + assertNull(inConversation?.toMediaItem()?.chatAudioThreadId()) + } + + @Test + fun `playback errors are mapped to what the user can do about them`() { + fun errorOf(errorCode: Int) = PlaybackException("test", null, errorCode).toChatAudioError() + + assertEquals(ChatAudioError.NETWORK, errorOf(PlaybackException.ERROR_CODE_IO_NETWORK_CONNECTION_FAILED)) + assertEquals(ChatAudioError.NETWORK, errorOf(PlaybackException.ERROR_CODE_IO_NETWORK_CONNECTION_TIMEOUT)) + assertEquals(ChatAudioError.UNAVAILABLE, errorOf(PlaybackException.ERROR_CODE_IO_BAD_HTTP_STATUS)) + assertEquals(ChatAudioError.UNAVAILABLE, errorOf(PlaybackException.ERROR_CODE_IO_FILE_NOT_FOUND)) + assertEquals(ChatAudioError.UNSUPPORTED, errorOf(PlaybackException.ERROR_CODE_DECODING_FORMAT_UNSUPPORTED)) + assertEquals(ChatAudioError.UNKNOWN, errorOf(PlaybackException.ERROR_CODE_UNSPECIFIED)) + } +} diff --git a/app/src/test/java/com/nextcloud/talk/chat/audio/WaveformsTest.kt b/app/src/test/java/com/nextcloud/talk/chat/audio/WaveformsTest.kt new file mode 100644 index 00000000000..1217fb83ef5 --- /dev/null +++ b/app/src/test/java/com/nextcloud/talk/chat/audio/WaveformsTest.kt @@ -0,0 +1,59 @@ +/* + * Nextcloud Talk - Android Client + * + * SPDX-FileCopyrightText: 2026 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: GPL-3.0-or-later + */ + +package com.nextcloud.talk.chat.audio + +import org.junit.Assert.assertArrayEquals +import org.junit.Assert.assertEquals +import org.junit.Test + +class WaveformsTest { + + private val delta = 0.0001f + + @Test + fun `levels are reduced to the maximum of each range and normalized`() { + val levels = floatArrayOf(0.1f, 0.2f, 0.4f, 0.1f, 0.05f, 0.1f) + + assertArrayEquals(floatArrayOf(0.5f, 1f, 0.25f), Waveforms.reduce(levels, 3), delta) + } + + @Test + fun `every level is taken into account, also if the sizes do not divide`() { + val levels = FloatArray(100) { 0.1f }.also { it[99] = 0.8f } + + val bars = Waveforms.reduce(levels, 30) + + assertEquals(30, bars.size) + assertEquals(1f, bars.last(), delta) + } + + @Test + fun `fewer levels than bars are stretched`() { + assertArrayEquals(floatArrayOf(0.5f, 0.5f, 1f, 1f), Waveforms.reduce(floatArrayOf(0.2f, 0.4f), 4), delta) + } + + @Test + fun `silence and empty input give flat bars`() { + assertArrayEquals(FloatArray(3), Waveforms.reduce(FloatArray(5), 3), delta) + assertArrayEquals(FloatArray(3), Waveforms.reduce(FloatArray(0), 3), delta) + assertEquals(0, Waveforms.reduce(floatArrayOf(1f), 0).size) + } + + @Test + fun `levels are the root mean square of each chunk`() { + val levels = WaveformLevels() + + levels.addChunk(sumOfSquares = 4 * 0.25, sampleCount = 4) + levels.addChunk(sumOfSquares = 1.0, sampleCount = 0) + repeat(300) { levels.addChunk(sumOfSquares = 0.0, sampleCount = 1) } + + assertEquals(301, levels.size) + assertEquals(0.5f, levels.toArray()[0], delta) + assertEquals(0f, levels.toArray()[300], delta) + } +}