V1.0.43 oss (#202)
* Added support for AI and Cloud credits in the Pro flavor. * Implemented credit-based authentication and authorization for AI features and Cloud TTS. * Updated AI feature access and purchase handling to a credit-based system. * Refactored and enhanced the Text-to-Speech (TTS) system with persistent caching and a redesigned UI. * Improved TTS cache management by organizing audio files by book title and adding a detailed cache storage UI. * Refactored the TTS service to use a WebSocket-based Gemini Live connection for cloud audio generation. * Removed the TTS cache settings tab and simplified voice sample playback by removing local caching logic. * Implemented a low-latency streaming mechanism for Cloud TTS using a custom `ConcurrentInputStream` and `ExoPlayer` data source. * Improved cloud TTS stability and prefetching logic in `TtsService` and `TtsPlaybackManager`. * Implemented AI summarization caching and cost tracking in the EPUB reader. * Enhanced chapter summary caching and UI feedback. * limit summaries for pro users to 10 per day * Implemented local caching for Cloud TTS audio chunks. * Removed the Free tier tab from `ProScreen` and simplified the subscription interface. Updated tab logic to focus on Pro and Credits, including a new cost breakdown section for AI and Cloud TTS features. * Refactored HTML parsing to include all child nodes during content chunking and semantic block parsing. * Improved image rendering consistency in epub pagination reader * Improved HTML parsing in `HtmlParser.kt` to better handle complex nested structures * Improved CSS styling support in the epub paginated reader for word spacing and text decorations. * Implemented scroll throttling in `epub_reader.js` to improve performance during scroll events * Improved CFI resolution and scrolling reliability in EPUB reader * Optimized PaginatedReader performance by caching text decorations. * Implemented batching for recent file database operations to handle large datasets and introduced `RecentFileSummary` to optimize data retrieval by excluding heavy JSON columns. * Improved navigation stability by wrapping `navController.navigate` and `popBackStack` calls in a try-catch block to handle `IllegalStateException` during concurrent transitions. Additionally, refined the backstack check for the main route to prevent redundant pops. * feat(tts): redesign TTS controls with overlay UI and cache management * Expanded and improved the TTS (Text-to-Speech) capabilities, particularly for Cloud voices. * Improved TTS playback control and cache management. * Integrated the TTS cache manager into the settings sheet and improved the TTS configuration UI. * Updated `DeviceVoicesTab` to respect the current TTS mode, disabling voice selection when not in `BASE` mode. * Improved error handling and state management for Cloud TTS in `TtsService` and `TtsPlaybackManager`. * Improved TTS voice selection UI and sample playback logic. * Updated `TtsUtils` and `TtsService` to remove `chunkIndex` from TTS cache filenames. Refined the cache file naming convention to rely on text and speaker hashes, and updated the cache file filter logic to correctly identify speakers in both legacy and new filename formats. * Optimized tile rendering and state propagation in PDF viewer * Added "Expand All", "Collapse All", and "Locate" functionality to the Table of Contents in both EPUB and PDF readers. * Added sign-in requirement for credit purchases and improved purchase migration logic. * Updated `EpubReaderTts` to support authenticated TTS requests by passing an auth token provider. The `ttsController.start` method now includes an `authToken` retrieved via `getAuthToken` and explicitly sets the `playbackSource` to "READER". * feat(ai): replace summarization popup with a comprehensive AI Hub Bottom Sheet * Improved locator logic and block traversal in `BookPaginator`. * Updated AI features and Cloud TTS logic. * Added manual clear and auto-reset functionality for AI summaries and recaps * Optimized file importing, EPUB parsing, and TTS playback concurrency. * Restricted TTS mode to BASE in OSS flavor and fixed TTS mode persistence in PDF viewer * Bump version to 1.0.43(44)
This commit is contained in:
Aryan
GitHub
parent
e8f6be2800
commit
46620fa71a
41 changed files with 4412 additions and 2406 deletions
|
|
@ -37,6 +37,8 @@ import androidx.media3.common.util.UnstableApi
|
|||
import androidx.media3.session.MediaController
|
||||
import androidx.media3.session.SessionToken
|
||||
import com.aryan.reader.BuildConfig
|
||||
import com.aryan.reader.epubreader.loadTtsPitch
|
||||
import com.aryan.reader.epubreader.loadTtsSpeechRate
|
||||
import com.aryan.reader.tts.TtsPlaybackManager.TtsState
|
||||
import com.google.common.util.concurrent.ListenableFuture
|
||||
import com.google.common.util.concurrent.MoreExecutors
|
||||
|
|
@ -65,15 +67,12 @@ private fun loadSpeaker(context: Context): String {
|
|||
}
|
||||
|
||||
@OptIn(UnstableApi::class)
|
||||
@Suppress("KotlinConstantConditions")
|
||||
fun loadTtsMode(context: Context): TtsPlaybackManager.TtsMode {
|
||||
val prefs = context.getSharedPreferences("reader_prefs", Context.MODE_PRIVATE)
|
||||
val savedModeName = prefs.getString("tts_mode", TtsPlaybackManager.TtsMode.BASE.name)
|
||||
?: TtsPlaybackManager.TtsMode.BASE.name
|
||||
|
||||
val isCloudAllowed = BuildConfig.DEBUG &&
|
||||
BuildConfig.IS_PRO &&
|
||||
BuildConfig.TTS_WORKER_URL.isNotBlank()
|
||||
val isCloudAllowed = BuildConfig.TTS_WORKER_URL.isNotBlank()
|
||||
|
||||
return if (isCloudAllowed) {
|
||||
try {
|
||||
|
|
@ -159,7 +158,8 @@ class TtsController(context: Context) : Player.Listener {
|
|||
chapterTitle: String?,
|
||||
coverImageUri: String?,
|
||||
ttsMode: TtsPlaybackManager.TtsMode,
|
||||
playbackSource: String = "READER"
|
||||
playbackSource: String = "READER",
|
||||
authToken: String? = null
|
||||
) {
|
||||
if (chunks.isEmpty()) {
|
||||
Timber.w("TtsController: start called with empty chunks!")
|
||||
|
|
@ -181,19 +181,12 @@ class TtsController(context: Context) : Player.Listener {
|
|||
putString(KEY_COVER_IMAGE_URI, coverImageUri)
|
||||
putString(KEY_TTS_MODE, ttsMode.name)
|
||||
putString(KEY_PLAYBACK_SOURCE, playbackSource)
|
||||
putString(KEY_AUTH_TOKEN, authToken)
|
||||
putFloat("playback_speed", loadTtsSpeechRate(context))
|
||||
putFloat("playback_pitch", loadTtsPitch(context))
|
||||
}
|
||||
Timber.tag("TTS_CLOUD_DIAG").d("TtsController sending START. Mode: $ttsMode, Chunks: ${chunks.size}, Token present: ${!authToken.isNullOrBlank()}")
|
||||
mediaController?.sendCustomCommand(START_TTS_COMMAND, args)
|
||||
|
||||
val metadataBuilder = androidx.media3.common.MediaMetadata.Builder()
|
||||
.setArtist(bookTitle)
|
||||
.setTitle(chapterTitle ?: "Reading Aloud")
|
||||
coverImageUri?.let { metadataBuilder.setArtworkUri(it.toUri()) }
|
||||
|
||||
val metadata = MediaItem.Builder()
|
||||
.setMediaId("tts_session")
|
||||
.setMediaMetadata(metadataBuilder.build())
|
||||
.build()
|
||||
mediaController?.setMediaItem(metadata)
|
||||
}
|
||||
|
||||
fun pause() {
|
||||
|
|
@ -224,6 +217,10 @@ class TtsController(context: Context) : Player.Listener {
|
|||
@Suppress("unused")
|
||||
fun changeTtsMode(mode: String) {
|
||||
Timber.d("UI sending CHANGE_TTS_MODE command.")
|
||||
val prefs = context.getSharedPreferences("reader_prefs", Context.MODE_PRIVATE)
|
||||
prefs.edit { putString("tts_mode", mode) }
|
||||
_ttsState.value = _ttsState.value.copy(ttsMode = mode)
|
||||
|
||||
val args = Bundle().apply {
|
||||
putString(KEY_TTS_MODE, mode)
|
||||
}
|
||||
|
|
@ -261,6 +258,7 @@ class TtsController(context: Context) : Player.Listener {
|
|||
val startOffset = mediaItemExtras?.getInt("startOffset", -1) ?: -1
|
||||
val currentWordSourceCfi = customState.getString("currentWordSourceCfi")
|
||||
val currentWordStartOffset = customState.getInt("currentWordStartOffset", -1)
|
||||
val serviceMode = customState.getString("ttsMode", _ttsState.value.ttsMode)
|
||||
|
||||
val currentState = _ttsState.value
|
||||
_ttsState.value = currentState.copy(
|
||||
|
|
@ -288,11 +286,22 @@ class TtsController(context: Context) : Player.Listener {
|
|||
currentWordSourceCfi = if (isPlaybackActive) currentWordSourceCfi else null,
|
||||
currentWordStartOffset = if (isPlaybackActive) currentWordStartOffset else -1,
|
||||
sessionFinished = sessionFinished,
|
||||
playbackSource = playbackSource
|
||||
playbackSource = playbackSource,
|
||||
ttsMode = serviceMode
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
fun setPlaybackParameters(speed: Float, pitch: Float) {
|
||||
val args = Bundle().apply {
|
||||
putFloat("speed", speed)
|
||||
putFloat("pitch", pitch)
|
||||
}
|
||||
mediaController?.sendCustomCommand(SET_PLAYBACK_PARAMS_COMMAND, args)
|
||||
}
|
||||
|
||||
fun release() {
|
||||
pollingJob?.cancel()
|
||||
scope.cancel()
|
||||
|
|
|
|||
|
|
@ -56,6 +56,7 @@ val FLUSH_PREFETCH_COMMAND = SessionCommand("com.aryan.reader.tts.FLUSH_PREFETCH
|
|||
private val STATE_UPDATE_COMMAND = SessionCommand("com.aryan.reader.tts.STATE_UPDATE", Bundle.EMPTY)
|
||||
val CHANGE_TTS_MODE_COMMAND = SessionCommand("com.aryan.reader.tts.CHANGE_MODE", Bundle.EMPTY)
|
||||
val SLICE_CURRENT_AND_RELOAD_COMMAND = SessionCommand("com.aryan.reader.tts.SLICE_AND_RELOAD", Bundle.EMPTY)
|
||||
val SET_PLAYBACK_PARAMS_COMMAND = SessionCommand("com.aryan.reader.tts.SET_PLAYBACK_PARAMS", Bundle.EMPTY)
|
||||
|
||||
const val KEY_TEXT_CHUNKS = "KEY_TEXT_CHUNKS"
|
||||
const val KEY_SOURCE_CFIS = "KEY_SOURCE_CFIS"
|
||||
|
|
@ -68,20 +69,27 @@ const val KEY_TTS_MODE = "KEY_TTS_MODE"
|
|||
const val KEY_WORD_TIMESTAMPS = "KEY_WORD_TIMESTAMPS"
|
||||
const val KEY_WORD_OFFSETS = "KEY_WORD_OFFSETS"
|
||||
const val KEY_PLAYBACK_SOURCE = "KEY_PLAYBACK_SOURCE"
|
||||
const val KEY_AUTH_TOKEN = "KEY_AUTH_TOKEN"
|
||||
|
||||
private const val PREFETCH_LOOKAHEAD = 2
|
||||
private const val PREFETCH_LOOKAHEAD = 3
|
||||
|
||||
@UnstableApi
|
||||
class TtsPlaybackManager(
|
||||
private val player: Player,
|
||||
private val generateAudioChunk: suspend (textChunk: String, speakerId: String, mode: TtsMode) -> TtsAudioData
|
||||
private val generateAudioChunk: suspend (bookTitle: String, chapterTitle: String?, chunkIndex: Int, totalChunks: Int, textChunk: String, speakerId: String, mode: TtsMode, authToken: String?) -> TtsAudioData,
|
||||
private val onResetContext: () -> Unit
|
||||
) : MediaSession.Callback, Player.Listener {
|
||||
|
||||
private val scope = CoroutineScope(Dispatchers.IO + SupervisorJob())
|
||||
private var mediaSession: MediaSession? = null
|
||||
private val prefetchingJobs = mutableMapOf<Int, Job>()
|
||||
private val prefetchingJobs = java.util.concurrent.ConcurrentHashMap<Int, Job>()
|
||||
private var wordTrackingJob: Job? = null
|
||||
private var preparationJob: Job? = null
|
||||
private var prefetchLoopJob: Job? = null
|
||||
private var lastPrefetchIndex = -1
|
||||
private var currentAuthToken: String? = null
|
||||
private val loadedChunks: MutableSet<Int> = java.util.Collections.newSetFromMap(java.util.concurrent.ConcurrentHashMap())
|
||||
private val chunkStreamIds = java.util.concurrent.ConcurrentHashMap<Int, String>()
|
||||
|
||||
enum class TtsMode {
|
||||
CLOUD, BASE
|
||||
|
|
@ -100,13 +108,14 @@ class TtsPlaybackManager(
|
|||
val currentWordSourceCfi: String? = null,
|
||||
val currentWordStartOffset: Int = -1,
|
||||
val sessionFinished: Boolean = false,
|
||||
val playbackSource: String? = null
|
||||
val playbackSource: String? = null,
|
||||
val ttsMode: String = TtsMode.CLOUD.name
|
||||
)
|
||||
|
||||
private val _ttsState = MutableStateFlow(TtsState())
|
||||
|
||||
private var textChunks: List<TtsChunk> = emptyList()
|
||||
private var audioFiles: MutableMap<Int, File> = mutableMapOf()
|
||||
private val audioFiles = java.util.concurrent.ConcurrentHashMap<Int, File>()
|
||||
private var currentSpeakerId = DEFAULT_SPEAKER_ID
|
||||
private var bookTitle: String? = null
|
||||
private var chapterTitle: String? = null
|
||||
|
|
@ -141,6 +150,7 @@ class TtsPlaybackManager(
|
|||
.add(CHANGE_TTS_MODE_COMMAND)
|
||||
.add(FLUSH_PREFETCH_COMMAND)
|
||||
.add(SLICE_CURRENT_AND_RELOAD_COMMAND)
|
||||
.add(SET_PLAYBACK_PARAMS_COMMAND)
|
||||
.build()
|
||||
val availablePlayerCommands = MediaSession.ConnectionResult.DEFAULT_PLAYER_COMMANDS.buildUpon()
|
||||
.remove(Player.COMMAND_SEEK_TO_NEXT_MEDIA_ITEM)
|
||||
|
|
@ -192,7 +202,9 @@ class TtsPlaybackManager(
|
|||
chunks.map { TtsChunk(it, "", -1) }
|
||||
}
|
||||
|
||||
handleStartTts(richChunks, speakerId, bookTitle, chapterTitle, coverImageUri, ttsMode, playbackSource)
|
||||
val authToken = args.getString(KEY_AUTH_TOKEN)
|
||||
Timber.tag("TTS_CLOUD_DIAG").d("TtsPlaybackManager received START. Token present: ${!authToken.isNullOrBlank()}")
|
||||
handleStartTts(richChunks, speakerId, bookTitle, chapterTitle, coverImageUri, ttsMode, playbackSource, args)
|
||||
}
|
||||
STOP_TTS_COMMAND -> {
|
||||
Timber.d("Received STOP command.")
|
||||
|
|
@ -209,23 +221,55 @@ class TtsPlaybackManager(
|
|||
}
|
||||
FLUSH_PREFETCH_COMMAND -> {
|
||||
Timber.d("Flushing prefetched TTS chunks for new parameters.")
|
||||
onResetContext()
|
||||
lastPrefetchIndex = -1
|
||||
prefetchLoopJob?.cancel()
|
||||
prefetchingJobs.values.forEach { it.cancel() }
|
||||
prefetchingJobs.clear()
|
||||
scope.launch(Dispatchers.IO) {
|
||||
val currentIdx = withContext(Dispatchers.Main) { player.currentMediaItemIndex }
|
||||
|
||||
scope.launch(Dispatchers.Main) {
|
||||
val currentIdx = player.currentMediaItemIndex
|
||||
if (currentIdx == C.INDEX_UNSET) return@launch
|
||||
val keysToRemove = audioFiles.keys.filter { it > currentIdx }
|
||||
keysToRemove.forEach { key ->
|
||||
audioFiles.remove(key)?.delete()
|
||||
|
||||
val keysToRemove = loadedChunks.filter { it > currentIdx }
|
||||
withContext(Dispatchers.IO) {
|
||||
keysToRemove.forEach { key ->
|
||||
loadedChunks.remove(key)
|
||||
val file = audioFiles.remove(key)
|
||||
deleteTempFile(file)
|
||||
val streamId = chunkStreamIds.remove(key)
|
||||
if (streamId != null) {
|
||||
StreamRegistry.remove(streamId)
|
||||
}
|
||||
}
|
||||
}
|
||||
withContext(Dispatchers.Main) {
|
||||
prefetchNextChunkAudio(currentIdx)
|
||||
|
||||
val itemsToRemove = mutableListOf<Int>()
|
||||
for (k in 0 until player.mediaItemCount) {
|
||||
val id = player.getMediaItemAt(k).mediaId.toIntOrNull() ?: -1
|
||||
if (id > currentIdx) {
|
||||
itemsToRemove.add(k)
|
||||
}
|
||||
}
|
||||
itemsToRemove.reversed().forEach {
|
||||
player.removeMediaItem(it)
|
||||
}
|
||||
|
||||
prefetchNextChunkAudio(currentIdx)
|
||||
}
|
||||
}
|
||||
SLICE_CURRENT_AND_RELOAD_COMMAND -> {
|
||||
handleSliceAndReload()
|
||||
}
|
||||
SET_PLAYBACK_PARAMS_COMMAND -> {
|
||||
val speed = args.getFloat("speed", 1f)
|
||||
val pitch = args.getFloat("pitch", 1f)
|
||||
if (currentTtsMode == TtsMode.CLOUD) {
|
||||
scope.launch(Dispatchers.Main) {
|
||||
player.playbackParameters = androidx.media3.common.PlaybackParameters(speed, pitch)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return Futures.immediateFuture(SessionResult(SessionResult.RESULT_SUCCESS))
|
||||
}
|
||||
|
|
@ -234,6 +278,11 @@ class TtsPlaybackManager(
|
|||
val currentIdx = player.currentMediaItemIndex
|
||||
if (currentIdx == C.INDEX_UNSET) return
|
||||
|
||||
player.pause()
|
||||
_ttsState.value = _ttsState.value.copy(isLoading = true)
|
||||
|
||||
onResetContext()
|
||||
|
||||
val offset = _ttsState.value.currentWordStartOffset
|
||||
val currentChunk = textChunks.getOrNull(currentIdx) ?: return
|
||||
|
||||
|
|
@ -241,11 +290,14 @@ class TtsPlaybackManager(
|
|||
wordTrackingJob?.cancel()
|
||||
player.stop()
|
||||
player.clearMediaItems()
|
||||
lastPrefetchIndex = -1
|
||||
prefetchLoopJob?.cancel()
|
||||
prefetchingJobs.values.forEach { it.cancel() }
|
||||
prefetchingJobs.clear()
|
||||
|
||||
preparationJob = scope.launch {
|
||||
clearAudioFiles()
|
||||
loadedChunks.clear()
|
||||
|
||||
if (offset == -1) {
|
||||
prepareAndPlayFirstChunk(startAtIndex = currentIdx, playWhenReady = false)
|
||||
|
|
@ -275,6 +327,7 @@ class TtsPlaybackManager(
|
|||
private fun handleChangeTtsMode(newMode: TtsMode) {
|
||||
if (currentTtsMode == newMode) return
|
||||
currentTtsMode = newMode
|
||||
_ttsState.value = _ttsState.value.copy(ttsMode = newMode.name)
|
||||
Timber.d("TTS Mode changed to $newMode (pending next start)")
|
||||
}
|
||||
|
||||
|
|
@ -285,12 +338,29 @@ class TtsPlaybackManager(
|
|||
chapterTitle: String?,
|
||||
coverImageUri: String?,
|
||||
ttsMode: TtsMode,
|
||||
playbackSource: String?
|
||||
playbackSource: String?,
|
||||
args: Bundle // Added this parameter
|
||||
) {
|
||||
if (chunks.isEmpty()) {
|
||||
_ttsState.value = _ttsState.value.copy(errorMessage = "No text to read.")
|
||||
return
|
||||
}
|
||||
|
||||
// --- YOUR SNIPPET START ---
|
||||
val authToken = args.getString(KEY_AUTH_TOKEN)
|
||||
val speed = args.getFloat("playback_speed", 1f)
|
||||
val pitch = args.getFloat("playback_pitch", 1f)
|
||||
|
||||
scope.launch(Dispatchers.Main) {
|
||||
if (ttsMode == TtsMode.CLOUD) {
|
||||
player.playbackParameters = androidx.media3.common.PlaybackParameters(speed, pitch)
|
||||
} else {
|
||||
player.playbackParameters = androidx.media3.common.PlaybackParameters(1f, 1f)
|
||||
}
|
||||
}
|
||||
|
||||
Timber.tag("TTS_CLOUD_DIAG").d("TtsPlaybackManager received START. Token present: ${!authToken.isNullOrBlank()}")
|
||||
|
||||
handleStopTts(clearState = false)
|
||||
textChunks = chunks
|
||||
currentSpeakerId = speakerId
|
||||
|
|
@ -298,13 +368,35 @@ class TtsPlaybackManager(
|
|||
this.bookTitle = bookTitle
|
||||
this.chapterTitle = chapterTitle
|
||||
this.coverImageUri = coverImageUri
|
||||
_ttsState.value = TtsState(isLoading = true, speakerId = speakerId, playbackSource = playbackSource)
|
||||
|
||||
onResetContext()
|
||||
loadedChunks.clear()
|
||||
lastPrefetchIndex = -1
|
||||
|
||||
_ttsState.value = TtsState(
|
||||
isLoading = true,
|
||||
speakerId = speakerId,
|
||||
playbackSource = playbackSource,
|
||||
ttsMode = ttsMode.name
|
||||
)
|
||||
|
||||
currentAuthToken = authToken
|
||||
preparationJob = scope.launch {
|
||||
prepareAndPlayFirstChunk()
|
||||
}
|
||||
}
|
||||
|
||||
fun forceStopWithError(errorMessage: String) {
|
||||
scope.launch(Dispatchers.Main) {
|
||||
_ttsState.value = _ttsState.value.copy(
|
||||
isLoading = false,
|
||||
isPlaying = false,
|
||||
errorMessage = errorMessage
|
||||
)
|
||||
handleStopTts(clearState = false)
|
||||
}
|
||||
}
|
||||
|
||||
private fun handleChangeSpeaker(newSpeakerId: String) {
|
||||
if (currentSpeakerId == newSpeakerId) return
|
||||
currentSpeakerId = newSpeakerId
|
||||
|
|
@ -319,27 +411,52 @@ class TtsPlaybackManager(
|
|||
return
|
||||
}
|
||||
|
||||
val ttsAudioData = generateAudioChunk(firstChunk.text, currentSpeakerId, currentTtsMode)
|
||||
val chunkStartTime = System.currentTimeMillis()
|
||||
Timber.tag("TTS_CLOUD_DIAG").i("Starting audio generation for first chunk (index=$startAtIndex).")
|
||||
|
||||
val ttsAudioData = generateAudioChunk(bookTitle ?: "Unknown Book", chapterTitle, startAtIndex, textChunks.size, firstChunk.text, currentSpeakerId, currentTtsMode, currentAuthToken)
|
||||
Timber.tag("TTS_CLOUD_DIAG").i("generateAudioChunk returned in ${System.currentTimeMillis() - chunkStartTime}ms")
|
||||
|
||||
if (ttsAudioData.error == "INSUFFICIENT_CREDITS") {
|
||||
withContext(Dispatchers.Main) {
|
||||
_ttsState.value = _ttsState.value.copy(isLoading = false, isPlaying = false, errorMessage = "INSUFFICIENT_CREDITS")
|
||||
handleStopTts(clearState = false)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
val audioFile = ttsAudioData.audioFile
|
||||
val streamUri = ttsAudioData.streamUri
|
||||
val serverText = ttsAudioData.serverText
|
||||
|
||||
if (audioFile != null && serverText != null) {
|
||||
audioFiles[startAtIndex] = audioFile
|
||||
if ((audioFile != null || streamUri != null) && serverText != null) {
|
||||
if (audioFile != null) {
|
||||
audioFiles[startAtIndex] = audioFile
|
||||
}
|
||||
loadedChunks.add(startAtIndex)
|
||||
|
||||
val updatedChunk = processWordTimings(firstChunk, serverText, ttsAudioData.wordTimings)
|
||||
val mutableChunks = textChunks.toMutableList()
|
||||
mutableChunks[startAtIndex] = updatedChunk
|
||||
textChunks = mutableChunks.toList()
|
||||
|
||||
val mediaItem = createMediaItem(serverText, audioFile.absolutePath, startAtIndex, updatedChunk)
|
||||
if (streamUri != null) {
|
||||
val uriStr = streamUri.toUri()
|
||||
val id = uriStr.host ?: uriStr.lastPathSegment
|
||||
if (id != null) chunkStreamIds[startAtIndex] = id
|
||||
}
|
||||
val pathToUse = streamUri ?: audioFile!!.absolutePath
|
||||
val mediaItem = createMediaItem(serverText, pathToUse, startAtIndex, updatedChunk)
|
||||
|
||||
withContext(Dispatchers.Main) {
|
||||
val prepStartTime = System.currentTimeMillis()
|
||||
player.setMediaItem(mediaItem)
|
||||
player.prepare()
|
||||
if (startAtPosition > 0) {
|
||||
player.seekTo(startAtPosition)
|
||||
}
|
||||
player.playWhenReady = playWhenReady
|
||||
Timber.tag("TTS_CLOUD_DIAG").i("ExoPlayer setMediaItem & prepare called in ${System.currentTimeMillis() - prepStartTime}ms")
|
||||
_ttsState.value = _ttsState.value.copy(
|
||||
isLoading = false,
|
||||
isPlaying = playWhenReady,
|
||||
|
|
@ -384,6 +501,8 @@ class TtsPlaybackManager(
|
|||
}
|
||||
|
||||
private fun handleStopTts(clearState: Boolean = true, userInitiated: Boolean = false) {
|
||||
Timber.tag("TTS_CLOUD_DIAG").d("handleStopTts called. clearState=$clearState, userInitiated=$userInitiated")
|
||||
onResetContext()
|
||||
preparationJob?.cancel()
|
||||
wordTrackingJob?.cancel()
|
||||
if (clearState) {
|
||||
|
|
@ -401,8 +520,11 @@ class TtsPlaybackManager(
|
|||
player.stop()
|
||||
player.clearMediaItems()
|
||||
textChunks = emptyList()
|
||||
lastPrefetchIndex = -1
|
||||
prefetchLoopJob?.cancel()
|
||||
prefetchingJobs.values.forEach { it.cancel() }
|
||||
prefetchingJobs.clear()
|
||||
loadedChunks.clear()
|
||||
|
||||
scope.launch {
|
||||
clearAudioFiles()
|
||||
|
|
@ -411,6 +533,7 @@ class TtsPlaybackManager(
|
|||
|
||||
override fun onMediaItemTransition(mediaItem: MediaItem?, reason: Int) {
|
||||
val newPlaylistIndex = player.currentMediaItemIndex
|
||||
Timber.tag("TTS_CLOUD_DIAG").d("onMediaItemTransition to playlistIndex: $newPlaylistIndex, mediaId: ${mediaItem?.mediaId}, reason: $reason")
|
||||
if (newPlaylistIndex == C.INDEX_UNSET) return
|
||||
|
||||
val currentChunkIndex = mediaItem?.mediaId?.toIntOrNull() ?: return
|
||||
|
|
@ -437,8 +560,14 @@ class TtsPlaybackManager(
|
|||
val previousChunkIndex = previousMediaItem.mediaId.toIntOrNull()
|
||||
|
||||
if (previousChunkIndex != null) {
|
||||
scope.launch {
|
||||
audioFiles.remove(previousChunkIndex)?.delete()
|
||||
scope.launch(Dispatchers.IO) {
|
||||
val file = audioFiles.remove(previousChunkIndex)
|
||||
deleteTempFile(file)
|
||||
loadedChunks.remove(previousChunkIndex)
|
||||
val streamId = chunkStreamIds.remove(previousChunkIndex)
|
||||
if (streamId != null) {
|
||||
StreamRegistry.remove(streamId)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -485,114 +614,193 @@ class TtsPlaybackManager(
|
|||
_ttsState.value = nextState
|
||||
|
||||
if (!isPlaying && player.playbackState == Player.STATE_IDLE) {
|
||||
if (!nextState.sessionEndedByStop) {
|
||||
if (!nextState.sessionEndedByStop && !nextState.isLoading && preparationJob?.isActive != true) {
|
||||
Timber.tag("TTS_CLOUD_DIAG").d("Auto-stopping TTS from onIsPlayingChanged (IDLE and not loading)")
|
||||
handleStopTts(userInitiated = true)
|
||||
} else {
|
||||
Timber.tag("TTS_CLOUD_DIAG").d("Ignoring STATE_IDLE in onIsPlayingChanged because isLoading=${nextState.isLoading}, preparationJob.isActive=${preparationJob?.isActive}")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
override fun onPlayerError(error: androidx.media3.common.PlaybackException) {
|
||||
Timber.e(error, "Player error: ${error.message}")
|
||||
Timber.tag("TTS_CLOUD_DIAG").e(error, "Player error: [${error.errorCodeName}] ${error.message}")
|
||||
_ttsState.value = _ttsState.value.copy(errorMessage = "Playback error: ${error.message}")
|
||||
handleStopTts(userInitiated = true)
|
||||
}
|
||||
|
||||
private fun prefetchNextChunkAudio(currentIndex: Int) {
|
||||
for (i in 1..PREFETCH_LOOKAHEAD) {
|
||||
val targetIndex = currentIndex + i
|
||||
if (targetIndex < textChunks.size) {
|
||||
if (prefetchingJobs.containsKey(targetIndex)) {
|
||||
continue
|
||||
}
|
||||
if (currentIndex == lastPrefetchIndex && prefetchLoopJob?.isActive == true) {
|
||||
return
|
||||
}
|
||||
lastPrefetchIndex = currentIndex
|
||||
|
||||
if (audioFiles.containsKey(targetIndex)) {
|
||||
continue
|
||||
}
|
||||
prefetchLoopJob?.cancel()
|
||||
prefetchLoopJob = scope.launch {
|
||||
for (i in 1..PREFETCH_LOOKAHEAD) {
|
||||
val targetIndex = currentIndex + i
|
||||
if (targetIndex < textChunks.size) {
|
||||
if (prefetchingJobs.containsKey(targetIndex)) continue
|
||||
if (audioFiles.containsKey(targetIndex)) continue
|
||||
if (loadedChunks.contains(targetIndex)) continue
|
||||
|
||||
Timber.d("PlaybackManager: Scheduling prefetch for chunk $targetIndex")
|
||||
Timber.d("PlaybackManager: Scheduling prefetch for chunk $targetIndex")
|
||||
|
||||
val job = scope.launch {
|
||||
val nextChunk = textChunks[targetIndex]
|
||||
val ttsAudioData = generateAudioChunk(nextChunk.text, currentSpeakerId, currentTtsMode)
|
||||
val audioFile = ttsAudioData.audioFile
|
||||
val serverText = ttsAudioData.serverText
|
||||
val job = launch {
|
||||
val nextChunk = textChunks[targetIndex]
|
||||
val prefetchStartTime = System.currentTimeMillis()
|
||||
Timber.tag("TTS_CLOUD_DIAG").i("Starting prefetch generation for chunk $targetIndex")
|
||||
|
||||
if (audioFile != null && serverText != null) {
|
||||
audioFiles[targetIndex] = audioFile
|
||||
val ttsAudioData = generateAudioChunk(bookTitle ?: "Unknown Book", chapterTitle, targetIndex, textChunks.size, nextChunk.text, currentSpeakerId, currentTtsMode, currentAuthToken)
|
||||
|
||||
val updatedChunk = processWordTimings(nextChunk, serverText, ttsAudioData.wordTimings)
|
||||
val mutableChunks = textChunks.toMutableList()
|
||||
mutableChunks[targetIndex] = updatedChunk
|
||||
textChunks = mutableChunks.toList()
|
||||
Timber.tag("TTS_CLOUD_DIAG").i("Prefetch audio setup for chunk $targetIndex took ${System.currentTimeMillis() - prefetchStartTime}ms")
|
||||
|
||||
val nextMediaItem = createMediaItem(serverText, audioFile.absolutePath, targetIndex, updatedChunk)
|
||||
withContext(Dispatchers.Main) {
|
||||
val wasLoading = _ttsState.value.isLoading
|
||||
|
||||
var exists = false
|
||||
for (k in 0 until player.mediaItemCount) {
|
||||
if (player.getMediaItemAt(k).mediaId == targetIndex.toString()) {
|
||||
exists = true
|
||||
break
|
||||
}
|
||||
if (ttsAudioData.error == "INSUFFICIENT_CREDITS") {
|
||||
withContext(Dispatchers.Main) {
|
||||
_ttsState.value = _ttsState.value.copy(isLoading = false, isPlaying = false, errorMessage = "INSUFFICIENT_CREDITS")
|
||||
handleStopTts(clearState = false)
|
||||
}
|
||||
return@launch
|
||||
}
|
||||
|
||||
if (!exists) {
|
||||
var insertPosition = player.mediaItemCount
|
||||
val audioFile = ttsAudioData.audioFile
|
||||
val streamUri = ttsAudioData.streamUri
|
||||
val serverText = ttsAudioData.serverText
|
||||
|
||||
if ((audioFile != null || streamUri != null) && serverText != null) {
|
||||
val updatedChunk = processWordTimings(nextChunk, serverText, ttsAudioData.wordTimings)
|
||||
val pathToUse = streamUri ?: audioFile!!.absolutePath
|
||||
val nextMediaItem = createMediaItem(serverText, pathToUse, targetIndex, updatedChunk)
|
||||
|
||||
withContext(Dispatchers.Main) {
|
||||
if (audioFile != null) {
|
||||
audioFiles[targetIndex] = audioFile
|
||||
}
|
||||
loadedChunks.add(targetIndex)
|
||||
|
||||
val mutableChunks = textChunks.toMutableList()
|
||||
mutableChunks[targetIndex] = updatedChunk
|
||||
textChunks = mutableChunks.toList()
|
||||
|
||||
if (streamUri != null) {
|
||||
val uriStr = streamUri.toUri()
|
||||
val id = uriStr.host ?: uriStr.lastPathSegment
|
||||
if (id != null) chunkStreamIds[targetIndex] = id
|
||||
}
|
||||
|
||||
val wasLoading = _ttsState.value.isLoading
|
||||
|
||||
var exists = false
|
||||
for (k in 0 until player.mediaItemCount) {
|
||||
val id = player.getMediaItemAt(k).mediaId.toIntOrNull() ?: -1
|
||||
if (id > targetIndex) {
|
||||
insertPosition = k
|
||||
if (player.getMediaItemAt(k).mediaId == targetIndex.toString()) {
|
||||
exists = true
|
||||
break
|
||||
}
|
||||
}
|
||||
player.addMediaItem(insertPosition, nextMediaItem)
|
||||
}
|
||||
|
||||
if (player.playbackState == Player.STATE_ENDED && player.playWhenReady && targetIndex == player.currentMediaItemIndex + 1) {
|
||||
player.seekToNextMediaItem()
|
||||
player.play()
|
||||
} else if (wasLoading && targetIndex == player.currentMediaItemIndex + 1) {
|
||||
_ttsState.value = _ttsState.value.copy(isLoading = false)
|
||||
if (!exists) {
|
||||
var insertPosition = player.mediaItemCount
|
||||
for (k in 0 until player.mediaItemCount) {
|
||||
val id = player.getMediaItemAt(k).mediaId.toIntOrNull() ?: -1
|
||||
if (id > targetIndex) {
|
||||
insertPosition = k
|
||||
break
|
||||
}
|
||||
}
|
||||
player.addMediaItem(insertPosition, nextMediaItem)
|
||||
}
|
||||
|
||||
if (player.playbackState == Player.STATE_ENDED && player.playWhenReady && targetIndex == player.currentMediaItemIndex + 1) {
|
||||
player.seekToNextMediaItem()
|
||||
player.play()
|
||||
} else if (wasLoading && targetIndex == player.currentMediaItemIndex + 1) {
|
||||
_ttsState.value = _ttsState.value.copy(isLoading = false)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
Timber.e("Prefetch: Failed to download chunk $targetIndex")
|
||||
}
|
||||
} else {
|
||||
Timber.e("Prefetch: Failed to download chunk $targetIndex")
|
||||
}
|
||||
}
|
||||
prefetchingJobs[targetIndex] = job
|
||||
job.invokeOnCompletion {
|
||||
prefetchingJobs.remove(targetIndex)
|
||||
prefetchingJobs[targetIndex] = job
|
||||
job.invokeOnCompletion {
|
||||
prefetchingJobs.remove(targetIndex)
|
||||
}
|
||||
|
||||
job.join()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private suspend fun trackWordByWord() {
|
||||
var loopCount = 0
|
||||
while (true) {
|
||||
val currentIdx = withContext(Dispatchers.Main) { player.currentMediaItemIndex }
|
||||
val currentMediaItem = withContext(Dispatchers.Main) { player.currentMediaItem } ?: break
|
||||
val playbackPosition = withContext(Dispatchers.Main) { player.currentPosition }
|
||||
|
||||
val extras = currentMediaItem.mediaMetadata.extras ?: break
|
||||
val timestamps = extras.getDoubleArray(KEY_WORD_TIMESTAMPS) ?: break
|
||||
val offsets = extras.getIntArray(KEY_WORD_OFFSETS) ?: break
|
||||
val sourceCfi = extras.getString("sourceCfi") ?: break
|
||||
if (loopCount % 20 == 0) {
|
||||
withContext(Dispatchers.Main) { player.playbackState }
|
||||
withContext(Dispatchers.Main) { player.isPlaying }
|
||||
}
|
||||
|
||||
val currentWordIndex = timestamps.indexOfLast { (it * 1000).toLong() <= playbackPosition }
|
||||
val uri = currentMediaItem.localConfiguration?.uri
|
||||
if (uri?.scheme == "ttsstream") {
|
||||
val streamId = uri.host ?: uri.lastPathSegment
|
||||
if (streamId != null) {
|
||||
val (isFinished, totalBytes) = StreamRegistry.getStreamMetadata(streamId)
|
||||
if (isFinished && totalBytes > 44) {
|
||||
val expectedDurationMs = (totalBytes - 44) / 48
|
||||
|
||||
if (currentWordIndex != -1) {
|
||||
val currentWordOffset = offsets[currentWordIndex]
|
||||
if (_ttsState.value.currentWordStartOffset != currentWordOffset || _ttsState.value.currentWordSourceCfi != sourceCfi) {
|
||||
_ttsState.value = _ttsState.value.copy(
|
||||
currentWordSourceCfi = sourceCfi,
|
||||
currentWordStartOffset = currentWordOffset
|
||||
)
|
||||
if (playbackPosition >= expectedDurationMs) {
|
||||
Timber.tag("TTS_CLOUD_DIAG").i("Stream finished naturally: pos=$playbackPosition, expected=$expectedDurationMs. Transitioning.")
|
||||
withContext(Dispatchers.Main) {
|
||||
if (player.currentMediaItemIndex == currentIdx) {
|
||||
if (player.hasNextMediaItem()) {
|
||||
player.seekToNextMediaItem()
|
||||
} else {
|
||||
player.stop()
|
||||
}
|
||||
}
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
delay(100)
|
||||
|
||||
val extras = currentMediaItem.mediaMetadata.extras ?: break
|
||||
val sourceCfi = extras.getString("sourceCfi") ?: break
|
||||
|
||||
val timestamps = extras.getDoubleArray(KEY_WORD_TIMESTAMPS)
|
||||
val offsets = extras.getIntArray(KEY_WORD_OFFSETS)
|
||||
|
||||
if (timestamps != null && offsets != null) {
|
||||
val currentWordIndex = timestamps.indexOfLast { (it * 1000).toLong() <= playbackPosition }
|
||||
if (currentWordIndex != -1) {
|
||||
val currentWordOffset = offsets[currentWordIndex]
|
||||
if (_ttsState.value.currentWordStartOffset != currentWordOffset || _ttsState.value.currentWordSourceCfi != sourceCfi) {
|
||||
_ttsState.value = _ttsState.value.copy(
|
||||
currentWordSourceCfi = sourceCfi,
|
||||
currentWordStartOffset = currentWordOffset
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
delay(50)
|
||||
loopCount++
|
||||
}
|
||||
}
|
||||
|
||||
override fun onPlayWhenReadyChanged(playWhenReady: Boolean, reason: Int) {
|
||||
Timber.tag("TTS_CLOUD_DIAG").d("onPlayWhenReadyChanged: playWhenReady=$playWhenReady, reason=$reason")
|
||||
}
|
||||
|
||||
override fun onPositionDiscontinuity(oldPosition: Player.PositionInfo, newPosition: Player.PositionInfo, reason: Int) {
|
||||
Timber.tag("TTS_CLOUD_DIAG").d("onPositionDiscontinuity: reason=$reason")
|
||||
}
|
||||
|
||||
private fun createMediaItem(text: String, path: String, index: Int, chunk: TtsChunk): MediaItem {
|
||||
val extras = Bundle().apply {
|
||||
putString("sourceCfi", chunk.sourceCfi)
|
||||
|
|
@ -615,17 +823,30 @@ class TtsPlaybackManager(
|
|||
.setExtras(extras)
|
||||
.build()
|
||||
|
||||
val uri = if (path.startsWith("ttsstream://")) path.toUri() else Uri.fromFile(File(path))
|
||||
|
||||
return MediaItem.Builder()
|
||||
.setUri(Uri.fromFile(File(path)))
|
||||
.setUri(uri)
|
||||
.setMediaId(index.toString())
|
||||
.setMediaMetadata(metadata)
|
||||
.build()
|
||||
}
|
||||
|
||||
private fun deleteTempFile(file: File?) {
|
||||
file?.let {
|
||||
if (it.name.startsWith("tts_audio_chunk_") || it.name.startsWith("base_tts_") || it.name.startsWith("tts_live_")) {
|
||||
it.delete()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private suspend fun clearAudioFiles() {
|
||||
withContext(Dispatchers.IO) {
|
||||
audioFiles.values.forEach { it.delete() }
|
||||
audioFiles.values.forEach { deleteTempFile(it) }
|
||||
audioFiles.clear()
|
||||
chunkStreamIds.values.forEach { StreamRegistry.remove(it) } // ADDED
|
||||
chunkStreamIds.clear() // ADDED
|
||||
loadedChunks.clear()
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -640,6 +861,7 @@ class TtsPlaybackManager(
|
|||
putInt("currentWordStartOffset", state.currentWordStartOffset)
|
||||
putBoolean("sessionFinished", state.sessionFinished)
|
||||
putString("playbackSource", state.playbackSource)
|
||||
putString("ttsMode", state.ttsMode)
|
||||
}
|
||||
return CommandButton.Builder()
|
||||
.setSessionCommand(STATE_UPDATE_COMMAND)
|
||||
|
|
@ -662,4 +884,15 @@ class TtsPlaybackManager(
|
|||
handleStopTts(userInitiated = true)
|
||||
Timber.d("TtsPlaybackManager released.")
|
||||
}
|
||||
|
||||
override fun onPlaybackStateChanged(playbackState: Int) {
|
||||
val stateName = when (playbackState) {
|
||||
Player.STATE_IDLE -> "STATE_IDLE"
|
||||
Player.STATE_BUFFERING -> "STATE_BUFFERING"
|
||||
Player.STATE_READY -> "STATE_READY"
|
||||
Player.STATE_ENDED -> "STATE_ENDED"
|
||||
else -> "UNKNOWN"
|
||||
}
|
||||
Timber.tag("TTS_CLOUD_DIAG").d("ExoPlayer playback state changed: $stateName")
|
||||
}
|
||||
}
|
||||
|
|
@ -23,8 +23,6 @@ import android.Manifest
|
|||
import android.content.Intent
|
||||
import android.content.pm.PackageManager
|
||||
import android.os.Build
|
||||
import android.util.Base64
|
||||
import timber.log.Timber
|
||||
import androidx.core.content.ContextCompat
|
||||
import androidx.media3.common.AudioAttributes
|
||||
import androidx.media3.common.C
|
||||
|
|
@ -33,23 +31,33 @@ import androidx.media3.exoplayer.ExoPlayer
|
|||
import androidx.media3.session.MediaSession
|
||||
import androidx.media3.session.MediaSessionService
|
||||
import com.aryan.reader.tts.TtsPlaybackManager.TtsMode
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.withContext
|
||||
import org.json.JSONObject
|
||||
import java.io.File
|
||||
import java.io.FileOutputStream
|
||||
import java.net.HttpURLConnection
|
||||
import java.net.URL
|
||||
import kotlinx.coroutines.CoroutineScope
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.SupervisorJob
|
||||
import kotlinx.coroutines.launch
|
||||
import org.json.JSONArray
|
||||
import org.json.JSONObject
|
||||
import timber.log.Timber
|
||||
import java.io.File
|
||||
import okhttp3.OkHttpClient
|
||||
import okhttp3.Request
|
||||
import okhttp3.Response
|
||||
import okhttp3.WebSocket
|
||||
import okhttp3.WebSocketListener
|
||||
import kotlinx.coroutines.CompletableDeferred
|
||||
import kotlinx.coroutines.cancel
|
||||
import kotlinx.coroutines.channels.Channel
|
||||
import kotlinx.coroutines.isActive
|
||||
import kotlinx.coroutines.sync.Mutex
|
||||
import kotlinx.coroutines.sync.withLock
|
||||
|
||||
data class WordTimingInfo(val word: String, val startTime: Double)
|
||||
|
||||
data class TtsAudioData(
|
||||
val audioFile: File?,
|
||||
val serverText: String?,
|
||||
val wordTimings: List<WordTimingInfo>?
|
||||
val wordTimings: List<WordTimingInfo>?,
|
||||
val error: String? = null,
|
||||
val streamUri: String? = null
|
||||
)
|
||||
|
||||
data class PageCharacterRange(
|
||||
|
|
@ -59,6 +67,165 @@ data class PageCharacterRange(
|
|||
val endOffset: Int
|
||||
)
|
||||
|
||||
class ConcurrentInputStream : java.io.InputStream() {
|
||||
private val queue = java.util.concurrent.LinkedBlockingQueue<ByteArray>()
|
||||
private var currentBuffer: ByteArray? = null
|
||||
private var bufferPos = 0
|
||||
private var eofReached = false
|
||||
|
||||
var isFinished = false
|
||||
private set
|
||||
|
||||
var isClosed = false
|
||||
private set
|
||||
|
||||
fun write(data: ByteArray) {
|
||||
if (!isClosed) queue.offer(data)
|
||||
}
|
||||
|
||||
override fun read(): Int {
|
||||
val b = ByteArray(1)
|
||||
val readCount = read(b, 0, 1)
|
||||
return if (readCount == -1) -1 else b[0].toInt() and 0xFF
|
||||
}
|
||||
|
||||
override fun read(b: ByteArray, off: Int, len: Int): Int {
|
||||
if (eofReached) {
|
||||
isFinished = true
|
||||
return -1
|
||||
}
|
||||
if (len == 0) return 0
|
||||
|
||||
if (currentBuffer == null || bufferPos >= currentBuffer!!.size) {
|
||||
try {
|
||||
// Blocks here safely until data arrives
|
||||
currentBuffer = queue.take()
|
||||
bufferPos = 0
|
||||
if (currentBuffer!!.isEmpty()) {
|
||||
eofReached = true
|
||||
isFinished = true
|
||||
return -1
|
||||
}
|
||||
} catch (_: InterruptedException) {
|
||||
Thread.currentThread().interrupt()
|
||||
return -1
|
||||
}
|
||||
}
|
||||
|
||||
val available = currentBuffer!!.size - bufferPos
|
||||
val toCopy = len.coerceAtMost(available)
|
||||
System.arraycopy(currentBuffer!!, bufferPos, b, off, toCopy)
|
||||
bufferPos += toCopy
|
||||
return toCopy
|
||||
}
|
||||
|
||||
override fun close() {
|
||||
if (!isClosed) {
|
||||
isClosed = true
|
||||
queue.offer(ByteArray(0)) // Send EOF marker
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
object StreamRegistry {
|
||||
private val streams = java.util.concurrent.ConcurrentHashMap<String, java.io.InputStream>()
|
||||
private val totalBytesMap = java.util.concurrent.ConcurrentHashMap<String, Long>()
|
||||
private val finishedMap = java.util.concurrent.ConcurrentHashMap<String, Boolean>()
|
||||
|
||||
fun register(id: String, stream: java.io.InputStream) {
|
||||
streams[id] = stream
|
||||
totalBytesMap[id] = 0L
|
||||
finishedMap[id] = false
|
||||
}
|
||||
fun get(id: String): java.io.InputStream? = streams[id]
|
||||
|
||||
fun markFinished(id: String, totalBytes: Long) {
|
||||
totalBytesMap[id] = totalBytes
|
||||
finishedMap[id] = true
|
||||
}
|
||||
fun getStreamMetadata(id: String): Pair<Boolean, Long> {
|
||||
return (finishedMap[id] ?: false) to (totalBytesMap[id] ?: 0L)
|
||||
}
|
||||
|
||||
fun remove(id: String) {
|
||||
streams.remove(id)?.let { try { it.close() } catch (_: Exception) {} }
|
||||
totalBytesMap.remove(id)
|
||||
finishedMap.remove(id)
|
||||
}
|
||||
fun clear() {
|
||||
streams.values.forEach { try { it.close() } catch (_: Exception) {} }
|
||||
streams.clear()
|
||||
}
|
||||
}
|
||||
|
||||
@UnstableApi
|
||||
class InputStreamDataSource : androidx.media3.datasource.BaseDataSource(true) {
|
||||
private var inputStream: java.io.InputStream? = null
|
||||
private var opened = false
|
||||
private var uri: android.net.Uri? = null
|
||||
private var bytesReadTotal: Long = 0
|
||||
|
||||
override fun open(dataSpec: androidx.media3.datasource.DataSpec): Long {
|
||||
uri = dataSpec.uri
|
||||
Timber.tag("TTS_CLOUD_DIAG").d("InputStreamDataSource.open called for $uri, position=${dataSpec.position}")
|
||||
|
||||
val streamId = uri?.host ?: uri?.lastPathSegment ?: throw java.io.IOException("No stream ID")
|
||||
val stream = StreamRegistry.get(streamId) ?: throw java.io.IOException("Stream not found")
|
||||
|
||||
if (stream is ConcurrentInputStream && stream.isFinished) {
|
||||
Timber.tag("TTS_CLOUD_DIAG").d("InputStreamDataSource.open returning 0 bytes for finished stream to prevent retry.")
|
||||
opened = true
|
||||
transferInitializing(dataSpec)
|
||||
transferStarted(dataSpec)
|
||||
return 0
|
||||
}
|
||||
|
||||
inputStream = stream
|
||||
opened = true
|
||||
transferInitializing(dataSpec)
|
||||
|
||||
if (dataSpec.position > bytesReadTotal) {
|
||||
val toSkip = dataSpec.position - bytesReadTotal
|
||||
var skipped = 0L
|
||||
while (skipped < toSkip) {
|
||||
val s = inputStream?.skip(toSkip - skipped) ?: 0L
|
||||
if (s <= 0L) break
|
||||
skipped += s
|
||||
}
|
||||
bytesReadTotal += skipped
|
||||
}
|
||||
|
||||
transferStarted(dataSpec)
|
||||
return C.LENGTH_UNSET.toLong()
|
||||
}
|
||||
|
||||
override fun read(buffer: ByteArray, offset: Int, length: Int): Int {
|
||||
if (length == 0) return 0
|
||||
return try {
|
||||
val bytesRead = inputStream?.read(buffer, offset, length) ?: -1
|
||||
if (bytesRead == -1) {
|
||||
Timber.tag("TTS_CLOUD_DIAG").d("InputStreamDataSource EOF reached for $uri")
|
||||
return C.RESULT_END_OF_INPUT
|
||||
}
|
||||
bytesReadTotal += bytesRead
|
||||
bytesTransferred(bytesRead)
|
||||
bytesRead
|
||||
} catch (e: java.io.IOException) {
|
||||
Timber.tag("TTS_CLOUD_DIAG").e(e, "Stream read interrupted/broken for $uri")
|
||||
C.RESULT_END_OF_INPUT
|
||||
}
|
||||
}
|
||||
|
||||
override fun getUri(): android.net.Uri? = uri
|
||||
|
||||
override fun close() {
|
||||
if (opened) {
|
||||
opened = false
|
||||
transferEnded()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@UnstableApi
|
||||
class TtsService : MediaSessionService() {
|
||||
private val scope = CoroutineScope(Dispatchers.Main + SupervisorJob())
|
||||
|
|
@ -66,6 +233,7 @@ class TtsService : MediaSessionService() {
|
|||
private lateinit var player: ExoPlayer
|
||||
private lateinit var playbackManager: TtsPlaybackManager
|
||||
private lateinit var baseTtsSynthesizer: BaseTtsSynthesizer
|
||||
private lateinit var cacheManager: TtsCacheManager
|
||||
|
||||
override fun onUpdateNotification(session: MediaSession, startInForegroundRequired: Boolean) {
|
||||
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU &&
|
||||
|
|
@ -81,116 +249,317 @@ class TtsService : MediaSessionService() {
|
|||
super.onUpdateNotification(session, startInForegroundRequired)
|
||||
}
|
||||
|
||||
/**
|
||||
* Generic function to download TTS audio from a server endpoint.
|
||||
* This is used for both the self-hosted server and the Google Cloud worker.
|
||||
*
|
||||
* @param chunkToSpeak The text to synthesize.
|
||||
* @param speakerId The identifier for the voice.
|
||||
* @param serverUrl The base URL of the TTS server.
|
||||
* @param audioFileExtension The file extension for the temporary audio file (e.g., ".flac", ".mp3").
|
||||
* @return A pair containing the temporary audio file and the text chunk returned by the server, or null if it fails.
|
||||
*/
|
||||
private suspend fun downloadFromTtsServer(
|
||||
chunkToSpeak: String,
|
||||
speakerId: String,
|
||||
serverUrl: String,
|
||||
audioFileExtension: String
|
||||
): TtsAudioData {
|
||||
if (chunkToSpeak.isBlank()) {
|
||||
return TtsAudioData(null, null, null)
|
||||
}
|
||||
return withContext(Dispatchers.IO) {
|
||||
var tempAudioFile: File? = null
|
||||
try {
|
||||
val url = URL(serverUrl)
|
||||
val connection = url.openConnection() as HttpURLConnection
|
||||
connection.requestMethod = "POST"
|
||||
connection.setRequestProperty("Content-Type", "application/json; charset=UTF-8")
|
||||
connection.setRequestProperty("Accept", "application/json")
|
||||
connection.connectTimeout = 15000
|
||||
connection.readTimeout = 60000
|
||||
connection.doOutput = true
|
||||
connection.doInput = true
|
||||
|
||||
val jsonPayload = JSONObject()
|
||||
jsonPayload.put("text", chunkToSpeak)
|
||||
jsonPayload.put("speaker", speakerId)
|
||||
val jsonInputString = jsonPayload.toString()
|
||||
connection.outputStream.use { os ->
|
||||
val input = jsonInputString.toByteArray(Charsets.UTF_8)
|
||||
os.write(input, 0, input.size)
|
||||
}
|
||||
|
||||
val responseCode = connection.responseCode
|
||||
if (responseCode != HttpURLConnection.HTTP_OK) {
|
||||
val errorBody = try { connection.errorStream?.bufferedReader()?.use { it.readText() } } catch (_: Exception) { "" }
|
||||
Timber.e("TTS Server request failed with code: $responseCode for URL: $serverUrl. Body: $errorBody")
|
||||
return@withContext TtsAudioData(null, null, null)
|
||||
}
|
||||
|
||||
val responseBody =
|
||||
connection.inputStream.bufferedReader(Charsets.UTF_8).use { it.readText() }
|
||||
val jsonResponse = JSONObject(responseBody)
|
||||
if (jsonResponse.has("audio_base64") && jsonResponse.has("text_chunk")) {
|
||||
val audioBase64 = jsonResponse.getString("audio_base64")
|
||||
val serverTextChunk = jsonResponse.getString("text_chunk")
|
||||
val audioBytes = Base64.decode(audioBase64, Base64.DEFAULT)
|
||||
|
||||
val wordTimings = mutableListOf<WordTimingInfo>()
|
||||
if (jsonResponse.has("word_timings")) {
|
||||
val timingsArray: JSONArray = jsonResponse.getJSONArray("word_timings")
|
||||
for (i in 0 until timingsArray.length()) {
|
||||
val timingObject = timingsArray.getJSONObject(i)
|
||||
wordTimings.add(
|
||||
WordTimingInfo(
|
||||
word = timingObject.getString("word"),
|
||||
startTime = timingObject.getDouble("startTime")
|
||||
)
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
tempAudioFile = File.createTempFile(
|
||||
"tts_audio_chunk_",
|
||||
audioFileExtension,
|
||||
applicationContext.cacheDir
|
||||
)
|
||||
FileOutputStream(tempAudioFile).use { output -> output.write(audioBytes) }
|
||||
TtsAudioData(tempAudioFile, serverTextChunk, wordTimings)
|
||||
} else {
|
||||
Timber.e("DownloadAudioChunk: 'audio_base64' or 'text_chunk' field missing."
|
||||
)
|
||||
TtsAudioData(null, null, null)
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
Timber.e(e, "DownloadAudioChunk: TTS Request Exception: ${e.message}")
|
||||
tempAudioFile?.delete()
|
||||
TtsAudioData(null, null, null)
|
||||
private val okHttpClient = OkHttpClient.Builder().build()
|
||||
private val liveClient by lazy {
|
||||
GeminiLiveClient(okHttpClient) { errorMsg ->
|
||||
if (::playbackManager.isInitialized) {
|
||||
playbackManager.forceStopWithError(errorMsg)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private val downloadAudioChunk: suspend (String, String) -> TtsAudioData =
|
||||
{ chunkToSpeak, speakerId ->
|
||||
downloadFromTtsServer(
|
||||
chunkToSpeak,
|
||||
speakerId,
|
||||
googleCloudWorkerTtsUrl,
|
||||
".mp3"
|
||||
)
|
||||
class GeminiLiveClient(
|
||||
private val client: OkHttpClient,
|
||||
private val onAsyncError: (String) -> Unit = {}
|
||||
) {
|
||||
private var webSocket: WebSocket? = null
|
||||
|
||||
private val connectionMutex = Mutex()
|
||||
private val generationMutex = Mutex()
|
||||
private var clientScope = CoroutineScope(Dispatchers.IO + SupervisorJob())
|
||||
|
||||
private var audioChannel = Channel<GeminiWsEvent>(Channel.UNLIMITED)
|
||||
private var setupDeferred = CompletableDeferred<Boolean>().apply { complete(false) }
|
||||
|
||||
var connectedSpeaker: String? = null
|
||||
|
||||
sealed class GeminiWsEvent {
|
||||
data class Audio(val bytes: ByteArray) : GeminiWsEvent()
|
||||
object TurnComplete : GeminiWsEvent()
|
||||
data class Error(val message: String) : GeminiWsEvent()
|
||||
}
|
||||
|
||||
suspend fun ensureConnected(serverUrl: String, speaker: String, authToken: String?) = connectionMutex.withLock {
|
||||
if (webSocket != null) {
|
||||
if (connectedSpeaker == speaker) {
|
||||
val isSetup = try { setupDeferred.await() } catch(_: Exception) { false }
|
||||
if (isSetup) return@withLock
|
||||
}
|
||||
Timber.tag("TTS_CLOUD_DIAG").d("Closing existing WS. Speaker changed or setup failed.")
|
||||
webSocket?.close(1000, "Reconnecting")
|
||||
webSocket = null
|
||||
}
|
||||
|
||||
val sanitizedUrl = serverUrl.removeSuffix("/")
|
||||
val wsUrlStr = sanitizedUrl.replace("https://", "wss://").replace("http://", "ws://")
|
||||
val url = "$wsUrlStr/live?speaker=$speaker&token=${authToken ?: ""}"
|
||||
|
||||
Timber.tag("TTS_CLOUD_DIAG").d("Connecting to WS: $url")
|
||||
val request = Request.Builder().url(url).build()
|
||||
val connectedDeferred = CompletableDeferred<Boolean>()
|
||||
|
||||
var connectionError: String? = null
|
||||
|
||||
setupDeferred = CompletableDeferred()
|
||||
connectedSpeaker = speaker
|
||||
|
||||
webSocket = client.newWebSocket(request, object : WebSocketListener() {
|
||||
override fun onOpen(webSocket: WebSocket, response: Response) {
|
||||
Timber.tag("TTS_CLOUD_DIAG").d("WS Opened. Sending Setup configuration to Gemini...")
|
||||
|
||||
val systemPrompt = """
|
||||
You are a professional audiobook narrator.
|
||||
Your ONLY task is to read the exact text provided to you, word for word, neutral emotion, and with good pacing.
|
||||
Do NOT add any conversational filler, acknowledgments, or extra words (e.g., do not say "Sure, here is the text").
|
||||
Do NOT skip any parts or summarize. Output ONLY the audio reading of the provided text. If you encounter unreadable, non-verbal, or non-linguistic content (e.g., symbols like "※▼◆", raw formatting markers, broken characters, or pure punctuation clusters with no readable words), silently skip it and continue reading.
|
||||
""".trimIndent()
|
||||
|
||||
val setupMsg = JSONObject().apply {
|
||||
put("setup", JSONObject().apply {
|
||||
put("model", "models/gemini-3.1-flash-live-preview")
|
||||
put("systemInstruction", JSONObject().apply {
|
||||
put("parts", org.json.JSONArray().apply {
|
||||
put(JSONObject().apply {
|
||||
put("text", systemPrompt)
|
||||
})
|
||||
})
|
||||
})
|
||||
put("generationConfig", JSONObject().apply {
|
||||
put("responseModalities", org.json.JSONArray().apply { put("AUDIO") })
|
||||
put("speechConfig", JSONObject().apply {
|
||||
put("voiceConfig", JSONObject().apply {
|
||||
put("prebuiltVoiceConfig", JSONObject().apply {
|
||||
put("voiceName", speaker)
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
}.toString()
|
||||
|
||||
webSocket.send(setupMsg)
|
||||
connectedDeferred.complete(true)
|
||||
}
|
||||
|
||||
override fun onMessage(webSocket: WebSocket, text: String) {
|
||||
try {
|
||||
val json = JSONObject(text)
|
||||
if (json.has("error")) {
|
||||
val errObj = json.opt("error")
|
||||
val errMsg = if (errObj is JSONObject) errObj.toString() else errObj?.toString() ?: "Unknown API Error"
|
||||
Timber.tag("TTS_CLOUD_DIAG").e("API ERROR RETURNED: $errMsg")
|
||||
audioChannel.trySend(GeminiWsEvent.Error(errMsg))
|
||||
setupDeferred.complete(false)
|
||||
return
|
||||
}
|
||||
if (json.has("setupComplete")) {
|
||||
setupDeferred.complete(true)
|
||||
}
|
||||
|
||||
val serverContent = json.optJSONObject("serverContent")
|
||||
if (serverContent != null) {
|
||||
val turnComplete = serverContent.optBoolean("turnComplete", false)
|
||||
val modelTurn = serverContent.optJSONObject("modelTurn")
|
||||
val parts = modelTurn?.optJSONArray("parts")
|
||||
|
||||
if (parts != null) {
|
||||
for (i in 0 until parts.length()) {
|
||||
val part = parts.getJSONObject(i)
|
||||
val inlineData = part.optJSONObject("inlineData")
|
||||
if (inlineData != null) {
|
||||
val b64 = inlineData.optString("data")
|
||||
if (b64.isNotEmpty()) {
|
||||
val bytes = android.util.Base64.decode(b64, android.util.Base64.DEFAULT)
|
||||
audioChannel.trySend(GeminiWsEvent.Audio(bytes))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (turnComplete) {
|
||||
audioChannel.trySend(GeminiWsEvent.TurnComplete)
|
||||
}
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
Timber.tag("TTS_CLOUD_DIAG").e(e, "Error parsing WS message text")
|
||||
}
|
||||
}
|
||||
|
||||
override fun onMessage(webSocket: WebSocket, bytes: okio.ByteString) {
|
||||
onMessage(webSocket, bytes.utf8())
|
||||
}
|
||||
|
||||
override fun onFailure(webSocket: WebSocket, t: Throwable, response: Response?) {
|
||||
connectionError = if (response?.code == 402) {
|
||||
"INSUFFICIENT_CREDITS"
|
||||
} else {
|
||||
"WS Failure: ${t.message} | Response: ${response?.code}"
|
||||
}
|
||||
Timber.tag("TTS_CLOUD_DIAG").e(t)
|
||||
audioChannel.trySend(GeminiWsEvent.Error(connectionError))
|
||||
this@GeminiLiveClient.webSocket = null
|
||||
connectedDeferred.complete(false)
|
||||
setupDeferred.complete(false)
|
||||
}
|
||||
|
||||
override fun onClosed(webSocket: WebSocket, code: Int, reason: String) {
|
||||
audioChannel.trySend(GeminiWsEvent.Error("Connection Closed: $reason"))
|
||||
this@GeminiLiveClient.webSocket = null
|
||||
setupDeferred.complete(false)
|
||||
}
|
||||
})
|
||||
|
||||
val isConnected = connectedDeferred.await()
|
||||
if (!isConnected) throw IllegalStateException(connectionError ?: "Failed to connect to proxy WebSocket")
|
||||
|
||||
val isSetup = try {
|
||||
kotlinx.coroutines.withTimeout(10000L) { setupDeferred.await() }
|
||||
} catch (_: Exception) { false }
|
||||
|
||||
if (!isSetup) {
|
||||
webSocket?.close(1000, "Setup failed")
|
||||
webSocket = null
|
||||
connectedSpeaker = null
|
||||
throw IllegalStateException("Failed to complete Gemini setup")
|
||||
} else {
|
||||
Timber.tag("TTS_CLOUD_DIAG").d("Gemini setup complete")
|
||||
}
|
||||
}
|
||||
|
||||
fun generateChunk(text: String, cacheFile: File?): TtsAudioData {
|
||||
if (text.isBlank()) return TtsAudioData(null, null, null, "Text is blank")
|
||||
|
||||
val streamId = java.util.UUID.randomUUID().toString()
|
||||
val concurrentStream = ConcurrentInputStream()
|
||||
StreamRegistry.register(streamId, concurrentStream)
|
||||
val header = createWavHeaderUnknownLength(24000)
|
||||
concurrentStream.write(header)
|
||||
|
||||
clientScope.launch {
|
||||
generationMutex.withLock {
|
||||
var fileOutputStream: java.io.FileOutputStream? = null
|
||||
var tempFile: File? = null
|
||||
|
||||
try {
|
||||
if (!isActive) return@launch
|
||||
|
||||
// Prepare cache temp file
|
||||
if (cacheFile != null) {
|
||||
tempFile = File(cacheFile.absolutePath + ".tmp")
|
||||
fileOutputStream = java.io.FileOutputStream(tempFile)
|
||||
fileOutputStream.write(header)
|
||||
}
|
||||
|
||||
Timber.tag("TTS_CLOUD_DIAG").d("Starting API generation task for chunk: ${text.take(15)}...")
|
||||
|
||||
audioChannel = Channel(Channel.UNLIMITED)
|
||||
val chunkGenStartTime = System.currentTimeMillis()
|
||||
var firstByteTime = -1L
|
||||
|
||||
val payload = JSONObject().apply {
|
||||
put("realtimeInput", JSONObject().apply {
|
||||
put("text", text)
|
||||
})
|
||||
}.toString()
|
||||
|
||||
val sent = webSocket?.send(payload) ?: false
|
||||
if (!sent) {
|
||||
Timber.tag("TTS_CLOUD_DIAG").e("Failed to send text payload over WS")
|
||||
return@launch
|
||||
}
|
||||
|
||||
var receivedAudioBytes = 0
|
||||
kotlinx.coroutines.withTimeout(30000L) {
|
||||
for (event in audioChannel) {
|
||||
when (event) {
|
||||
is GeminiWsEvent.Audio -> {
|
||||
if (firstByteTime == -1L) {
|
||||
firstByteTime = System.currentTimeMillis()
|
||||
Timber.tag("TTS_CLOUD_DIAG").i("TTFB: ${firstByteTime - chunkGenStartTime}ms")
|
||||
}
|
||||
concurrentStream.write(event.bytes)
|
||||
fileOutputStream?.write(event.bytes)
|
||||
receivedAudioBytes += event.bytes.size
|
||||
}
|
||||
is GeminiWsEvent.TurnComplete -> {
|
||||
Timber.tag("TTS_CLOUD_DIAG").i("Chunk generation complete. Bytes: $receivedAudioBytes")
|
||||
StreamRegistry.markFinished(streamId, receivedAudioBytes.toLong() + 44)
|
||||
|
||||
fileOutputStream?.close()
|
||||
fileOutputStream = null
|
||||
if (tempFile != null && cacheFile != null && receivedAudioBytes > 0) {
|
||||
patchWavHeader(tempFile, receivedAudioBytes)
|
||||
tempFile.renameTo(cacheFile)
|
||||
Timber.tag("TTS_CLOUD_DIAG").d("Successfully cached chunk to ${cacheFile.name}")
|
||||
}
|
||||
break
|
||||
}
|
||||
is GeminiWsEvent.Error -> {
|
||||
Timber.tag("TTS_CLOUD_DIAG").e("WS Error received: ${event.message}")
|
||||
onAsyncError(event.message)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (e: kotlinx.coroutines.TimeoutCancellationException) {
|
||||
Timber.tag("TTS_CLOUD_DIAG").e(e, "Timeout waiting for audio/TurnComplete")
|
||||
} catch (e: kotlinx.coroutines.CancellationException) {
|
||||
Timber.tag("TTS_CLOUD_DIAG").i(e, "Streaming job cancelled due to user skip/flush")
|
||||
} catch (e: Exception) {
|
||||
Timber.tag("TTS_CLOUD_DIAG").e(e, "Exception piping audio")
|
||||
} finally {
|
||||
Timber.tag("TTS_CLOUD_DIAG").d("Closing stream for ${text.take(15)}")
|
||||
concurrentStream.close()
|
||||
fileOutputStream?.close()
|
||||
if (cacheFile != null && !cacheFile.exists()) {
|
||||
tempFile?.delete()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return TtsAudioData(null, text, emptyList(), streamUri = "ttsstream://$streamId")
|
||||
}
|
||||
|
||||
fun close() {
|
||||
clientScope.cancel()
|
||||
clientScope = CoroutineScope(Dispatchers.IO + SupervisorJob())
|
||||
|
||||
webSocket?.close(1000, "Context Reset")
|
||||
webSocket = null
|
||||
connectedSpeaker = null
|
||||
setupDeferred = CompletableDeferred<Boolean>().apply { complete(false) }
|
||||
StreamRegistry.clear()
|
||||
}
|
||||
}
|
||||
|
||||
private val synthesizeBaseTtsChunk: suspend (String) -> TtsAudioData =
|
||||
{ chunkToSpeak ->
|
||||
val (file, text) = baseTtsSynthesizer.synthesizeToFile(chunkToSpeak)
|
||||
TtsAudioData(file, text, null)
|
||||
}
|
||||
|
||||
private val audioGenerator: suspend (text: String, speaker: String, mode: TtsMode) -> TtsAudioData =
|
||||
{ text, speaker, mode ->
|
||||
val audioGenerator: suspend (bookTitle: String, chapterTitle: String?, chunkIndex: Int, totalChunks: Int, text: String, speaker: String, mode: TtsMode, authToken: String?) -> TtsAudioData =
|
||||
{ bookTitle, chapterTitle, chunkIndex, totalChunks, text, speaker, mode, authToken ->
|
||||
cacheManager.saveTotalChunks(bookTitle, chapterTitle, totalChunks)
|
||||
when (mode) {
|
||||
TtsMode.CLOUD -> downloadAudioChunk(text, speaker)
|
||||
TtsMode.CLOUD -> {
|
||||
val cachedFile = cacheManager.getCacheFile(bookTitle, chapterTitle, text, speaker, mode)
|
||||
|
||||
if (cachedFile.exists() && cachedFile.length() > 44) {
|
||||
Timber.tag("TTS_CLOUD_DIAG").i("Using cached audio for chunk $chunkIndex")
|
||||
TtsAudioData(audioFile = cachedFile, serverText = text, wordTimings = emptyList(), error = null, streamUri = null)
|
||||
} else {
|
||||
try {
|
||||
liveClient.ensureConnected(googleCloudWorkerTtsUrl, speaker, authToken)
|
||||
liveClient.generateChunk(text, cachedFile)
|
||||
} catch (e: Exception) {
|
||||
Timber.tag("TTS_CLOUD_DIAG").e(e, "Cloud TTS generation failed")
|
||||
TtsAudioData(audioFile = null, serverText = null, wordTimings = null, error = e.message ?: "Failed to connect to TTS service")
|
||||
}
|
||||
}
|
||||
}
|
||||
TtsMode.BASE -> synthesizeBaseTtsChunk(text)
|
||||
}
|
||||
}
|
||||
|
|
@ -199,6 +568,8 @@ class TtsService : MediaSessionService() {
|
|||
super.onCreate()
|
||||
Timber.d("TtsService created.")
|
||||
|
||||
cacheManager = TtsCacheManager(this)
|
||||
|
||||
baseTtsSynthesizer = BaseTtsSynthesizer(this)
|
||||
scope.launch {
|
||||
try {
|
||||
|
|
@ -213,14 +584,49 @@ class TtsService : MediaSessionService() {
|
|||
.setUsage(C.USAGE_MEDIA)
|
||||
.build()
|
||||
|
||||
val defaultDataSourceFactory = androidx.media3.datasource.DefaultDataSource.Factory(this)
|
||||
val dataSourceFactory = androidx.media3.datasource.DataSource.Factory {
|
||||
object : androidx.media3.datasource.DataSource {
|
||||
private var dataSource: androidx.media3.datasource.DataSource? = null
|
||||
private val defaultDataSource = defaultDataSourceFactory.createDataSource()
|
||||
private val streamDataSource = InputStreamDataSource()
|
||||
|
||||
override fun addTransferListener(transferListener: androidx.media3.datasource.TransferListener) {
|
||||
defaultDataSource.addTransferListener(transferListener)
|
||||
streamDataSource.addTransferListener(transferListener)
|
||||
}
|
||||
|
||||
override fun open(dataSpec: androidx.media3.datasource.DataSpec): Long {
|
||||
dataSource = if (dataSpec.uri.scheme == "ttsstream") {
|
||||
streamDataSource
|
||||
} else {
|
||||
defaultDataSource
|
||||
}
|
||||
return dataSource!!.open(dataSpec)
|
||||
}
|
||||
|
||||
override fun read(buffer: ByteArray, offset: Int, length: Int): Int {
|
||||
return dataSource!!.read(buffer, offset, length)
|
||||
}
|
||||
|
||||
override fun getUri(): android.net.Uri? = dataSource?.uri
|
||||
|
||||
override fun close() {
|
||||
dataSource?.close()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
player = ExoPlayer.Builder(this)
|
||||
.setAudioAttributes(audioAttributes, true)
|
||||
.setHandleAudioBecomingNoisy(true)
|
||||
.setMediaSourceFactory(androidx.media3.exoplayer.source.DefaultMediaSourceFactory(this).setDataSourceFactory(dataSourceFactory))
|
||||
.build()
|
||||
|
||||
playbackManager = TtsPlaybackManager(
|
||||
player = player,
|
||||
generateAudioChunk = audioGenerator
|
||||
generateAudioChunk = audioGenerator,
|
||||
onResetContext = { liveClient.close() }
|
||||
)
|
||||
|
||||
mediaSession = MediaSession.Builder(this, player)
|
||||
|
|
|
|||
|
|
@ -21,36 +21,189 @@ package com.aryan.reader.tts
|
|||
|
||||
import android.content.Context
|
||||
import android.media.MediaPlayer
|
||||
import timber.log.Timber
|
||||
import androidx.annotation.OptIn
|
||||
import androidx.compose.runtime.getValue
|
||||
import androidx.compose.runtime.mutableStateOf
|
||||
import androidx.compose.runtime.setValue
|
||||
import androidx.core.net.toUri
|
||||
import androidx.media3.common.util.UnstableApi
|
||||
import com.aryan.reader.BuildConfig
|
||||
import kotlinx.coroutines.CoroutineScope
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.launch
|
||||
import kotlinx.coroutines.withContext
|
||||
import org.json.JSONObject
|
||||
import java.net.HttpURLConnection
|
||||
import java.net.URL
|
||||
import timber.log.Timber
|
||||
import java.io.File
|
||||
import java.io.RandomAccessFile
|
||||
import java.security.MessageDigest
|
||||
import kotlin.math.ln
|
||||
import kotlin.math.pow
|
||||
|
||||
const val googleCloudWorkerTtsUrl = BuildConfig.TTS_WORKER_URL
|
||||
|
||||
const val TTS_SAMPLE_TEXT = "The greater danger for most of us lies not in setting our aim too high and falling short; but in setting our aim too low, and achieving our mark."
|
||||
|
||||
const val TTS_CHUNK_MAX_LENGTH = 250
|
||||
const val DEFAULT_SPEAKER_ID = "Aoede"
|
||||
|
||||
const val DEFAULT_SPEAKER_ID = "en-US-Standard-F"
|
||||
data class GeminiVoice(val id: String, val name: String, val description: String)
|
||||
|
||||
@Suppress("unused")
|
||||
val GOOGLE_TTS_SPEAKERS = listOf(
|
||||
"US Female: F" to "en-US-Standard-F",
|
||||
"US Female: H" to "en-US-Standard-H",
|
||||
"US Male: I" to "en-US-Standard-I",
|
||||
"US Male: J" to "en-US-Standard-J"
|
||||
val GEMINI_TTS_SPEAKERS = listOf(
|
||||
GeminiVoice("Zephyr", "Zephyr", "Bright, Higher pitch"),
|
||||
GeminiVoice("Puck", "Puck", "Upbeat, Middle pitch"),
|
||||
GeminiVoice("Charon", "Charon", "Informative, Lower pitch"),
|
||||
GeminiVoice("Kore", "Kore", "Firm, Middle pitch"),
|
||||
GeminiVoice("Fenrir", "Fenrir", "Excitable, Lower middle pitch"),
|
||||
GeminiVoice("Leda", "Leda", "Youthful, Higher pitch"),
|
||||
GeminiVoice("Orus", "Orus", "Firm, Lower middle pitch"),
|
||||
GeminiVoice("Aoede", "Aoede", "Breezy, Middle pitch"),
|
||||
GeminiVoice("Callirrhoe", "Callirrhoe", "Easy-going, Middle pitch"),
|
||||
GeminiVoice("Autonoe", "Autonoe", "Bright, Middle pitch"),
|
||||
GeminiVoice("Enceladus", "Enceladus", "Breathy, Lower pitch"),
|
||||
GeminiVoice("Iapetus", "Iapetus", "Clear, Lower middle pitch"),
|
||||
GeminiVoice("Umbriel", "Umbriel", "Easy-going, Lower middle pitch"),
|
||||
GeminiVoice("Algieba", "Algieba", "Smooth, Lower pitch"),
|
||||
GeminiVoice("Despina", "Despina", "Smooth, Middle pitch"),
|
||||
GeminiVoice("Erinome", "Erinome", "Clear, Middle pitch"),
|
||||
GeminiVoice("Algenib", "Algenib", "Gravelly, Lower pitch"),
|
||||
GeminiVoice("Rasalgethi", "Rasalgethi", "Informative, Middle pitch"),
|
||||
GeminiVoice("Laomedeia", "Laomedeia", "Upbeat, Higher pitch"),
|
||||
GeminiVoice("Achernar", "Achernar", "Soft, Higher pitch"),
|
||||
GeminiVoice("Alnilam", "Alnilam", "Firm, Lower middle pitch"),
|
||||
GeminiVoice("Schedar", "Schedar", "Even, Lower middle pitch"),
|
||||
GeminiVoice("Gacrux", "Gacrux", "Mature, Middle pitch"),
|
||||
GeminiVoice("Pulcherrima", "Pulcherrima", "Forward, Middle pitch"),
|
||||
GeminiVoice("Achird", "Achird", "Friendly, Lower middle pitch"),
|
||||
GeminiVoice("Zubenelgenubi", "Zubenelgenubi", "Casual, Lower middle pitch"),
|
||||
GeminiVoice("Vindemiatrix", "Vindemiatrix", "Gentle, Middle pitch"),
|
||||
GeminiVoice("Sadachbia", "Sadachbia", "Lively, Lower pitch"),
|
||||
GeminiVoice("Sadaltager", "Sadaltager", "Lively, Lower pitch"),
|
||||
GeminiVoice("Sulafat", "Sulafat", "Warn, Middle pitch"),
|
||||
)
|
||||
|
||||
data class TtsChapterCacheInfo(
|
||||
val chapterTitle: String,
|
||||
val chunkCount: Int,
|
||||
val totalChunks: Int?,
|
||||
val sizeBytes: Long,
|
||||
val directory: File,
|
||||
val matchingFiles: List<File> = emptyList()
|
||||
)
|
||||
|
||||
fun formatBytes(bytes: Long): String {
|
||||
if (bytes < 1024) return "$bytes B"
|
||||
val exp = (ln(bytes.toDouble()) / ln(1024.0)).toInt()
|
||||
val pre = "KMGTPE"[exp - 1]
|
||||
return String.format("%.1f %cB", bytes / 1024.0.pow(exp.toDouble()), pre)
|
||||
}
|
||||
|
||||
class TtsCacheManager(private val context: Context) {
|
||||
private fun sanitize(name: String): String = name.replace(Regex("[^a-zA-Z0-9.-]"), "_")
|
||||
|
||||
private fun hash(input: String): String {
|
||||
val bytes = MessageDigest.getInstance("SHA-256").digest(input.toByteArray())
|
||||
return bytes.joinToString("") { "%02x".format(it) }.take(16)
|
||||
}
|
||||
|
||||
fun saveTotalChunks(bookTitle: String, chapterTitle: String?, totalChunks: Int) {
|
||||
val baseDir = File(context.filesDir, "TTS_Cache")
|
||||
val bookDir = File(baseDir, sanitize(bookTitle.take(50)))
|
||||
val chapterDir = File(bookDir, sanitize((chapterTitle ?: "Unknown_Chapter").take(50)))
|
||||
if (!chapterDir.exists()) chapterDir.mkdirs()
|
||||
val metaFile = File(chapterDir, "total_chunks.txt")
|
||||
metaFile.writeText(totalChunks.toString())
|
||||
}
|
||||
|
||||
@OptIn(UnstableApi::class)
|
||||
fun getCacheFile(
|
||||
bookTitle: String,
|
||||
chapterTitle: String?,
|
||||
text: String,
|
||||
speakerId: String,
|
||||
mode: TtsPlaybackManager.TtsMode
|
||||
): File {
|
||||
val baseDir = File(context.filesDir, "TTS_Cache")
|
||||
val bookDir = File(baseDir, sanitize(bookTitle.take(50)))
|
||||
val chapterDir = File(bookDir, sanitize((chapterTitle ?: "Unknown_Chapter").take(50)))
|
||||
if (!chapterDir.exists()) {
|
||||
chapterDir.mkdirs()
|
||||
}
|
||||
|
||||
val hashParams = hash(text + speakerId + mode.name)
|
||||
val safeSpeaker = sanitize(speakerId)
|
||||
|
||||
return File(chapterDir, "cached_chunk_${safeSpeaker}_$hashParams.wav")
|
||||
}
|
||||
|
||||
fun getBookCacheDir(bookTitle: String): File {
|
||||
val baseDir = File(context.filesDir, "TTS_Cache")
|
||||
return File(baseDir, sanitize(bookTitle.take(50)))
|
||||
}
|
||||
|
||||
fun getChapterCaches(bookTitle: String, speakerFilter: String? = null): List<TtsChapterCacheInfo> {
|
||||
val bookDir = getBookCacheDir(bookTitle)
|
||||
if (!bookDir.exists()) return emptyList()
|
||||
|
||||
return bookDir.listFiles()?.filter { it.isDirectory }?.mapNotNull { chapterDir ->
|
||||
val files = chapterDir.listFiles()?.filter { file ->
|
||||
if (!file.isFile || !file.name.endsWith(".wav")) return@filter false
|
||||
if (speakerFilter == null || speakerFilter == "All") return@filter true
|
||||
|
||||
val parts = file.name.split("_")
|
||||
|
||||
val speakerInName = if (parts.size >= 5 && parts[2].all { it.isDigit() }) {
|
||||
parts[3]
|
||||
} else if (parts.size >= 4) {
|
||||
parts[2]
|
||||
} else null
|
||||
|
||||
speakerInName == speakerFilter
|
||||
} ?: emptyList()
|
||||
|
||||
if (files.isEmpty()) null
|
||||
else {
|
||||
val size = files.sumOf { it.length() }
|
||||
val metaFile = File(chapterDir, "total_chunks.txt")
|
||||
val total = if (metaFile.exists()) metaFile.readText().toIntOrNull() else null
|
||||
|
||||
TtsChapterCacheInfo(
|
||||
chapterTitle = chapterDir.name,
|
||||
chunkCount = files.size,
|
||||
totalChunks = total,
|
||||
sizeBytes = size,
|
||||
directory = chapterDir,
|
||||
matchingFiles = files
|
||||
)
|
||||
}
|
||||
}?.sortedBy { it.chapterTitle } ?: emptyList()
|
||||
}
|
||||
|
||||
fun deleteChapterCache(chapterDir: File) {
|
||||
chapterDir.deleteRecursively()
|
||||
}
|
||||
|
||||
fun deleteSpecificFiles(files: List<File>, chapterDir: File) {
|
||||
files.forEach { it.delete() }
|
||||
if (chapterDir.listFiles()?.isEmpty() == true) {
|
||||
chapterDir.deleteRecursively()
|
||||
}
|
||||
}
|
||||
|
||||
fun clearBookCache(bookTitle: String) {
|
||||
getBookCacheDir(bookTitle).deleteRecursively()
|
||||
}
|
||||
}
|
||||
|
||||
fun patchWavHeader(file: File, pcmDataLength: Int) {
|
||||
try {
|
||||
RandomAccessFile(file, "rw").use { raf ->
|
||||
raf.seek(4)
|
||||
raf.writeInt(Integer.reverseBytes(36 + pcmDataLength))
|
||||
raf.seek(40)
|
||||
raf.writeInt(Integer.reverseBytes(pcmDataLength))
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
Timber.tag("TTS_CLOUD_DIAG").e(e, "Failed to patch WAV header for cached file")
|
||||
}
|
||||
}
|
||||
|
||||
fun splitTextIntoChunks(text: String, maxLengthPerChunk: Int = TTS_CHUNK_MAX_LENGTH): List<String> {
|
||||
if (text.isBlank()) return emptyList()
|
||||
val sentenceBoundaryRegex = Regex("""(?<!\w\.\w.)(?<![A-Z][a-z]\.)(?<=[.?!\n])\s+""")
|
||||
|
|
@ -88,31 +241,44 @@ fun splitTextIntoChunks(text: String, maxLengthPerChunk: Int = TTS_CHUNK_MAX_LEN
|
|||
return chunks
|
||||
}
|
||||
|
||||
@UnstableApi
|
||||
class SpeakerSamplePlayer(
|
||||
private val context: Context,
|
||||
private val scope: CoroutineScope
|
||||
private val scope: CoroutineScope,
|
||||
private val getAuthToken: suspend () -> String?
|
||||
) {
|
||||
private val sampleMediaPlayer = MediaPlayer()
|
||||
var loadingSpeakerId by mutableStateOf<String?>(null)
|
||||
var playingSpeakerId by mutableStateOf<String?>(null)
|
||||
|
||||
val cachedSpeakers = androidx.compose.runtime.mutableStateListOf<String>()
|
||||
|
||||
private val httpClient = okhttp3.OkHttpClient()
|
||||
@OptIn(UnstableApi::class)
|
||||
private val liveClient = TtsService.GeminiLiveClient(httpClient)
|
||||
|
||||
init {
|
||||
// Read initially existing files
|
||||
scope.launch(Dispatchers.IO) {
|
||||
val files = context.cacheDir.listFiles { _, name -> name.startsWith("sample_") && name.endsWith(".wav") }
|
||||
val ids = files?.map { it.name.removePrefix("sample_").removeSuffix(".wav") } ?: emptyList()
|
||||
withContext(Dispatchers.Main) {
|
||||
cachedSpeakers.addAll(ids)
|
||||
}
|
||||
}
|
||||
|
||||
sampleMediaPlayer.setOnErrorListener { mp, what, extra ->
|
||||
Timber.e("MediaPlayer error: what=$what, extra=$extra. Resetting.")
|
||||
playingSpeakerId = null
|
||||
loadingSpeakerId = null
|
||||
try {
|
||||
mp.reset()
|
||||
} catch (e: IllegalStateException) {
|
||||
Timber.e("Error resetting MediaPlayer: ${e.message}")
|
||||
}
|
||||
try { mp.reset() } catch (_: Exception) {}
|
||||
true
|
||||
}
|
||||
}
|
||||
|
||||
@Suppress("unused")
|
||||
fun playOrStop(speakerId: String) {
|
||||
scope.launch {
|
||||
liveClient.close()
|
||||
when {
|
||||
playingSpeakerId == speakerId -> {
|
||||
sampleMediaPlayer.stop()
|
||||
|
|
@ -122,51 +288,49 @@ class SpeakerSamplePlayer(
|
|||
loadingSpeakerId == speakerId -> {
|
||||
loadingSpeakerId = null
|
||||
}
|
||||
else -> playSample(speakerId)
|
||||
else -> {
|
||||
playSample(speakerId)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@OptIn(UnstableApi::class)
|
||||
private suspend fun playSample(speakerId: String) {
|
||||
if (sampleMediaPlayer.isPlaying) {
|
||||
sampleMediaPlayer.stop()
|
||||
}
|
||||
if (sampleMediaPlayer.isPlaying) sampleMediaPlayer.stop()
|
||||
sampleMediaPlayer.reset()
|
||||
loadingSpeakerId = speakerId
|
||||
playingSpeakerId = null
|
||||
|
||||
withContext(Dispatchers.IO) {
|
||||
val cacheFile = File(context.cacheDir, "sample_$speakerId.wav")
|
||||
try {
|
||||
val url = URL(googleCloudWorkerTtsUrl)
|
||||
val connection = url.openConnection() as HttpURLConnection
|
||||
connection.requestMethod = "POST"
|
||||
connection.setRequestProperty("Content-Type", "application/json; charset=UTF-8")
|
||||
connection.setRequestProperty("Accept", "application/json")
|
||||
connection.connectTimeout = 15000
|
||||
connection.readTimeout = 30000
|
||||
connection.doOutput = true
|
||||
connection.doInput = true
|
||||
if (!cacheFile.exists()) {
|
||||
val bucketName = "reader-9fc469d7.firebasestorage.app"
|
||||
val sampleUrl = "https://firebasestorage.googleapis.com/v0/b/$bucketName/o/samples%2Fsample_${speakerId}.wav?alt=media"
|
||||
|
||||
val jsonPayload = JSONObject().apply {
|
||||
put("text", TTS_SAMPLE_TEXT)
|
||||
put("speaker", speakerId)
|
||||
}
|
||||
connection.outputStream.use { os ->
|
||||
os.write(jsonPayload.toString().toByteArray(Charsets.UTF_8))
|
||||
}
|
||||
val request = okhttp3.Request.Builder()
|
||||
.url(sampleUrl)
|
||||
.build()
|
||||
|
||||
|
||||
if (connection.responseCode == HttpURLConnection.HTTP_OK) {
|
||||
val responseBody = connection.inputStream.bufferedReader().use { it.readText() }
|
||||
val audioBase64 = JSONObject(responseBody).getString("audio_base64")
|
||||
|
||||
val dataUri = "data:audio/mpeg;base64,$audioBase64"
|
||||
|
||||
withContext(Dispatchers.Main) {
|
||||
if (loadingSpeakerId != speakerId) {
|
||||
return@withContext
|
||||
val response = httpClient.newCall(request).execute()
|
||||
if (response.isSuccessful) {
|
||||
response.body?.byteStream()?.use { input ->
|
||||
cacheFile.outputStream().use { output ->
|
||||
input.copyTo(output)
|
||||
}
|
||||
}
|
||||
sampleMediaPlayer.setDataSource(context, dataUri.toUri())
|
||||
} else {
|
||||
Timber.e("Failed to download sample for $speakerId. HTTP ${response.code}")
|
||||
throw Exception("Failed to cache sample")
|
||||
}
|
||||
}
|
||||
|
||||
if (cacheFile.exists()) {
|
||||
withContext(Dispatchers.Main) {
|
||||
if (!cachedSpeakers.contains(speakerId)) cachedSpeakers.add(speakerId)
|
||||
if (loadingSpeakerId != speakerId) return@withContext
|
||||
sampleMediaPlayer.setDataSource(cacheFile.absolutePath)
|
||||
sampleMediaPlayer.setOnPreparedListener { mp ->
|
||||
if (loadingSpeakerId == speakerId) {
|
||||
mp.start()
|
||||
|
|
@ -180,16 +344,54 @@ class SpeakerSamplePlayer(
|
|||
sampleMediaPlayer.prepareAsync()
|
||||
}
|
||||
} else {
|
||||
Timber.e("Failed to fetch sample for $speakerId. Code: ${connection.responseCode}")
|
||||
withContext(Dispatchers.Main) { if (loadingSpeakerId == speakerId) loadingSpeakerId = null }
|
||||
throw Exception("Sample file missing after download attempt")
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
Timber.e(e, "Exception playing sample for $speakerId: ${e.message}")
|
||||
Timber.e(e, "Exception playing sample for $speakerId")
|
||||
withContext(Dispatchers.Main) { if (loadingSpeakerId == speakerId) loadingSpeakerId = null }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fun clearSamples() {
|
||||
scope.launch(Dispatchers.IO) {
|
||||
val files = context.cacheDir.listFiles { _, name -> name.startsWith("sample_") && name.endsWith(".wav") }
|
||||
files?.forEach { it.delete() }
|
||||
withContext(Dispatchers.Main) {
|
||||
cachedSpeakers.clear()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@OptIn(UnstableApi::class)
|
||||
fun release() {
|
||||
sampleMediaPlayer.release()
|
||||
liveClient.close()
|
||||
}
|
||||
}
|
||||
|
||||
fun createWavHeaderUnknownLength(sampleRate: Int): ByteArray {
|
||||
val numChannels = 1
|
||||
val bitsPerSample = 16
|
||||
val byteRate = sampleRate * numChannels * bitsPerSample / 8
|
||||
val blockAlign = numChannels * bitsPerSample / 8
|
||||
|
||||
val header = java.nio.ByteBuffer.allocate(44)
|
||||
header.order(java.nio.ByteOrder.LITTLE_ENDIAN)
|
||||
|
||||
header.put("RIFF".toByteArray(Charsets.US_ASCII))
|
||||
header.putInt(0x7FFFFFFF)
|
||||
header.put("WAVE".toByteArray(Charsets.US_ASCII))
|
||||
header.put("fmt ".toByteArray(Charsets.US_ASCII))
|
||||
header.putInt(16)
|
||||
header.putShort(1.toShort())
|
||||
header.putShort(numChannels.toShort())
|
||||
header.putInt(sampleRate)
|
||||
header.putInt(byteRate)
|
||||
header.putShort(blockAlign.toShort())
|
||||
header.putShort(bitsPerSample.toShort())
|
||||
header.put("data".toByteArray(Charsets.US_ASCII))
|
||||
header.putInt(0x7FFFFFFF - 36)
|
||||
|
||||
return header.array()
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue