d7a9cae9e1
* Add performance and stylus debugging logs * Refactor and decouple UI models from `MainViewModel` * Refactor library state management and projection logic * Implement desktop shell using Compose Multiplatform * Implement desktop shell using Compose Multiplatform * Implement desktop shell using Compose Multiplatform * Introduce ReaderEngine and enhance EPUB reader features in windows app * Move core paginated reader logic to a Kotlin Multiplatform `shared` module and introduce experimental desktop support. * Implement PDF rendering and text extraction for desktop using Pdfium * Add `NonReaderScreens.kt` and UI dependencies * Refactor and centralize library state management and models to improve cross-platform consistency * Implement JSON persistence for desktop library and enhance library management features including shelf CRUD, tagging, and metadata editing * Implement PDF annotation system and enhanced zoom controls for the desktop viewer * Implement WebView-based EPUB rendering for desktop using CEF and embedded resources * Optimize UI state projection, navigation state handling, and main screen pager performance * Implement Bring Your Own Key (BYOK) support for AI features in OSS version * Support Gemini-based Cloud TTS with BYOK support for OSS builds * Refactor table cell image sizing in `PaginatedReader` and improve `MobiParser` native library loading and error handling. * crash fixes * Enhance navigation stability with lifecycle-aware safety checks and update `navigation-compose` to 2.9.6 * Implement dynamic bottom padding for the page info bar to account for device rounded corners * Implement bidirectional jump history navigation and replace the jump-back pill with a dedicated `PdfJumpHistoryBar` * Optimize PDF tiling performance and refine pan-and-fling gesture handling * Implement customizable toolbars with drag-and-drop reordering and placement for PDF and EPUB readers * Updated UI for customize toolbar * Refine drag-and-drop reordering and section assignment for PDF and EPUB reader controls * restructure PDF viewer UI component hierarchy to fix verifier crash * Implement separate text dimming factors for light and dark themes * Synchronize Pdfium access and improve resource lifecycle safety across Kotlin and native layers * Enhance image alignment in paginated and EPUB readers through anchor detection and style-based positioning * Centralize file type resolution logic and implement HTML sanitization during import * Introduce vertical margin customization and configurable progress bar positioning * texture support in epub reader * Enhance TTS session management, progress tracking, and diagnostic logging * Optimize library state projection and folder synchronization performance by refactoring collection lookups and refining metadata extraction logic. * Refine TTS page mapping for PDF and overhaul TTS control UI * Implement natural session completion logic in `TtsPlaybackManager` for cloud tts * Replace Snackbar with `CustomTopBanner` for notifications in `PdfViewerScreen` * Refine TTS playback continuity across PDF pages and improve state management for session transitions * Implement global texture transparency and enhance textured theme support across PDF and EPUB readers. * Update reader themes and improve texture rendering in page animations, EPUB UI, and immersive mode * Add Support Project screen * Optimize library performance via projection caching, batch database updates, and scoped folder synchronization. * Enhance folder synchronization with fallback query mechanisms and refactor annotation sidecar importing logic * Bump version to 1.0.47 (51)
712 lines
31 KiB
Kotlin
712 lines
31 KiB
Kotlin
/*
|
|
* Episteme Reader - A native Android document reader.
|
|
* Copyright (C) 2026 Episteme
|
|
*
|
|
* This program is free software: you can redistribute it and/or modify
|
|
* it under the terms of the GNU Affero General Public License as
|
|
* published by the Free Software Foundation, either version 3 of the
|
|
* License, or (at your option) any later version.
|
|
*
|
|
* This program is distributed in the hope that it will be useful,
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
* GNU Affero General Public License for more details.
|
|
*
|
|
* You should have received a copy of the GNU Affero General Public License
|
|
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
*
|
|
* mail: epistemereader@gmail.com
|
|
*/
|
|
package com.aryan.reader.tts
|
|
|
|
import android.Manifest
|
|
import android.content.Intent
|
|
import android.content.pm.PackageManager
|
|
import android.os.Build
|
|
import androidx.core.content.ContextCompat
|
|
import androidx.media3.common.AudioAttributes
|
|
import androidx.media3.common.C
|
|
import androidx.media3.common.util.UnstableApi
|
|
import androidx.media3.exoplayer.ExoPlayer
|
|
import androidx.media3.session.MediaSession
|
|
import androidx.media3.session.MediaSessionService
|
|
import com.aryan.reader.GEMINI_CLOUD_TTS_MODEL
|
|
import com.aryan.reader.isByokCloudTtsAvailable
|
|
import com.aryan.reader.loadAiByokSettings
|
|
import com.aryan.reader.tts.TtsPlaybackManager.TtsMode
|
|
import kotlinx.coroutines.CoroutineScope
|
|
import kotlinx.coroutines.Dispatchers
|
|
import kotlinx.coroutines.SupervisorJob
|
|
import kotlinx.coroutines.launch
|
|
import org.json.JSONObject
|
|
import timber.log.Timber
|
|
import java.io.File
|
|
import okhttp3.OkHttpClient
|
|
import okhttp3.Request
|
|
import okhttp3.Response
|
|
import okhttp3.WebSocket
|
|
import okhttp3.WebSocketListener
|
|
import kotlinx.coroutines.CompletableDeferred
|
|
import kotlinx.coroutines.cancel
|
|
import kotlinx.coroutines.channels.Channel
|
|
import kotlinx.coroutines.isActive
|
|
import kotlinx.coroutines.sync.Mutex
|
|
import kotlinx.coroutines.sync.withLock
|
|
|
|
data class WordTimingInfo(val word: String, val startTime: Double)
|
|
|
|
data class TtsAudioData(
|
|
val audioFile: File?,
|
|
val serverText: String?,
|
|
val wordTimings: List<WordTimingInfo>?,
|
|
val error: String? = null,
|
|
val streamUri: String? = null
|
|
)
|
|
|
|
data class PageCharacterRange(
|
|
val pageInChapter: Int,
|
|
val cfi: String,
|
|
val startOffset: Int,
|
|
val endOffset: Int
|
|
)
|
|
|
|
class ConcurrentInputStream : java.io.InputStream() {
|
|
private val queue = java.util.concurrent.LinkedBlockingQueue<ByteArray>()
|
|
private var currentBuffer: ByteArray? = null
|
|
private var bufferPos = 0
|
|
private var eofReached = false
|
|
|
|
var isFinished = false
|
|
private set
|
|
|
|
var isClosed = false
|
|
private set
|
|
|
|
fun write(data: ByteArray) {
|
|
if (!isClosed) queue.offer(data)
|
|
}
|
|
|
|
override fun read(): Int {
|
|
val b = ByteArray(1)
|
|
val readCount = read(b, 0, 1)
|
|
return if (readCount == -1) -1 else b[0].toInt() and 0xFF
|
|
}
|
|
|
|
override fun read(b: ByteArray, off: Int, len: Int): Int {
|
|
if (eofReached) {
|
|
isFinished = true
|
|
return -1
|
|
}
|
|
if (len == 0) return 0
|
|
|
|
if (currentBuffer == null || bufferPos >= currentBuffer!!.size) {
|
|
try {
|
|
// Blocks here safely until data arrives
|
|
currentBuffer = queue.take()
|
|
bufferPos = 0
|
|
if (currentBuffer!!.isEmpty()) {
|
|
eofReached = true
|
|
isFinished = true
|
|
return -1
|
|
}
|
|
} catch (_: InterruptedException) {
|
|
Thread.currentThread().interrupt()
|
|
return -1
|
|
}
|
|
}
|
|
|
|
val available = currentBuffer!!.size - bufferPos
|
|
val toCopy = len.coerceAtMost(available)
|
|
System.arraycopy(currentBuffer!!, bufferPos, b, off, toCopy)
|
|
bufferPos += toCopy
|
|
return toCopy
|
|
}
|
|
|
|
override fun close() {
|
|
if (!isClosed) {
|
|
isClosed = true
|
|
queue.offer(ByteArray(0)) // Send EOF marker
|
|
}
|
|
}
|
|
}
|
|
|
|
object StreamRegistry {
|
|
private val streams = java.util.concurrent.ConcurrentHashMap<String, java.io.InputStream>()
|
|
private val totalBytesMap = java.util.concurrent.ConcurrentHashMap<String, Long>()
|
|
private val finishedMap = java.util.concurrent.ConcurrentHashMap<String, Boolean>()
|
|
|
|
fun register(id: String, stream: java.io.InputStream) {
|
|
streams[id] = stream
|
|
totalBytesMap[id] = 0L
|
|
finishedMap[id] = false
|
|
}
|
|
fun get(id: String): java.io.InputStream? = streams[id]
|
|
|
|
fun markFinished(id: String, totalBytes: Long) {
|
|
totalBytesMap[id] = totalBytes
|
|
finishedMap[id] = true
|
|
}
|
|
fun getStreamMetadata(id: String): Pair<Boolean, Long> {
|
|
return (finishedMap[id] ?: false) to (totalBytesMap[id] ?: 0L)
|
|
}
|
|
|
|
fun remove(id: String) {
|
|
streams.remove(id)?.let { try { it.close() } catch (_: Exception) {} }
|
|
totalBytesMap.remove(id)
|
|
finishedMap.remove(id)
|
|
}
|
|
fun clear() {
|
|
streams.values.forEach { try { it.close() } catch (_: Exception) {} }
|
|
streams.clear()
|
|
}
|
|
}
|
|
|
|
@UnstableApi
|
|
class InputStreamDataSource : androidx.media3.datasource.BaseDataSource(true) {
|
|
private var inputStream: java.io.InputStream? = null
|
|
private var opened = false
|
|
private var uri: android.net.Uri? = null
|
|
private var bytesReadTotal: Long = 0
|
|
|
|
override fun open(dataSpec: androidx.media3.datasource.DataSpec): Long {
|
|
uri = dataSpec.uri
|
|
Timber.tag("TTS_CLOUD_DIAG").d("InputStreamDataSource.open called for $uri, position=${dataSpec.position}")
|
|
|
|
val streamId = uri?.host ?: uri?.lastPathSegment ?: throw java.io.IOException("No stream ID")
|
|
val stream = StreamRegistry.get(streamId) ?: throw java.io.IOException("Stream not found")
|
|
|
|
if (stream is ConcurrentInputStream && stream.isFinished) {
|
|
Timber.tag("TTS_CLOUD_DIAG").d("InputStreamDataSource.open returning 0 bytes for finished stream to prevent retry.")
|
|
opened = true
|
|
transferInitializing(dataSpec)
|
|
transferStarted(dataSpec)
|
|
return 0
|
|
}
|
|
|
|
inputStream = stream
|
|
opened = true
|
|
transferInitializing(dataSpec)
|
|
|
|
if (dataSpec.position > bytesReadTotal) {
|
|
val toSkip = dataSpec.position - bytesReadTotal
|
|
var skipped = 0L
|
|
while (skipped < toSkip) {
|
|
val s = inputStream?.skip(toSkip - skipped) ?: 0L
|
|
if (s <= 0L) break
|
|
skipped += s
|
|
}
|
|
bytesReadTotal += skipped
|
|
}
|
|
|
|
transferStarted(dataSpec)
|
|
return C.LENGTH_UNSET.toLong()
|
|
}
|
|
|
|
override fun read(buffer: ByteArray, offset: Int, length: Int): Int {
|
|
if (length == 0) return 0
|
|
return try {
|
|
val bytesRead = inputStream?.read(buffer, offset, length) ?: -1
|
|
if (bytesRead == -1) {
|
|
Timber.tag("TTS_CLOUD_DIAG").d("InputStreamDataSource EOF reached for $uri")
|
|
return C.RESULT_END_OF_INPUT
|
|
}
|
|
bytesReadTotal += bytesRead
|
|
bytesTransferred(bytesRead)
|
|
bytesRead
|
|
} catch (e: java.io.IOException) {
|
|
Timber.tag("TTS_CLOUD_DIAG").e(e, "Stream read interrupted/broken for $uri")
|
|
C.RESULT_END_OF_INPUT
|
|
}
|
|
}
|
|
|
|
override fun getUri(): android.net.Uri? = uri
|
|
|
|
override fun close() {
|
|
if (opened) {
|
|
opened = false
|
|
transferEnded()
|
|
}
|
|
}
|
|
}
|
|
|
|
@UnstableApi
|
|
class TtsService : MediaSessionService() {
|
|
private val scope = CoroutineScope(Dispatchers.Main + SupervisorJob())
|
|
private var mediaSession: MediaSession? = null
|
|
private lateinit var player: ExoPlayer
|
|
private lateinit var playbackManager: TtsPlaybackManager
|
|
private lateinit var baseTtsSynthesizer: BaseTtsSynthesizer
|
|
private lateinit var cacheManager: TtsCacheManager
|
|
|
|
override fun onUpdateNotification(session: MediaSession, startInForegroundRequired: Boolean) {
|
|
val hasNotificationPermission = Build.VERSION.SDK_INT < Build.VERSION_CODES.TIRAMISU ||
|
|
ContextCompat.checkSelfPermission(this, Manifest.permission.POST_NOTIFICATIONS) == PackageManager.PERMISSION_GRANTED
|
|
val playerState = if (::player.isInitialized) {
|
|
"playbackState=${player.playbackState}, isPlaying=${player.isPlaying}, playWhenReady=${player.playWhenReady}, mediaItems=${player.mediaItemCount}, currentIndex=${player.currentMediaItemIndex}"
|
|
} else {
|
|
"player=uninitialized"
|
|
}
|
|
Timber.tag(TTS_NOTIFICATION_DIAG_TAG).i(
|
|
"onUpdateNotification called. startInForegroundRequired=$startInForegroundRequired, hasPostNotifications=$hasNotificationPermission, $playerState"
|
|
)
|
|
|
|
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU &&
|
|
ContextCompat.checkSelfPermission(this, Manifest.permission.POST_NOTIFICATIONS) != PackageManager.PERMISSION_GRANTED) {
|
|
|
|
if (startInForegroundRequired) {
|
|
Timber.tag(TTS_NOTIFICATION_DIAG_TAG).w("Notification permission missing while foreground is required. Calling stopSelf().")
|
|
stopSelf()
|
|
} else {
|
|
Timber.tag(TTS_NOTIFICATION_DIAG_TAG).w("Notification permission missing. Skipping notification update.")
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
Timber.tag(TTS_NOTIFICATION_DIAG_TAG).i("Delegating notification update to MediaSessionService.")
|
|
super.onUpdateNotification(session, startInForegroundRequired)
|
|
}
|
|
|
|
private val okHttpClient = OkHttpClient.Builder().build()
|
|
private val liveClient by lazy {
|
|
GeminiLiveClient(okHttpClient) { errorMsg ->
|
|
if (::playbackManager.isInitialized) {
|
|
playbackManager.forceStopWithError(errorMsg)
|
|
}
|
|
}
|
|
}
|
|
|
|
class GeminiLiveClient(
|
|
private val client: OkHttpClient,
|
|
private val onAsyncError: (String) -> Unit = {}
|
|
) {
|
|
private var webSocket: WebSocket? = null
|
|
|
|
private val connectionMutex = Mutex()
|
|
private val generationMutex = Mutex()
|
|
private var clientScope = CoroutineScope(Dispatchers.IO + SupervisorJob())
|
|
|
|
private var audioChannel = Channel<GeminiWsEvent>(Channel.UNLIMITED)
|
|
private var setupDeferred = CompletableDeferred<Boolean>().apply { complete(false) }
|
|
|
|
var connectedSpeaker: String? = null
|
|
|
|
sealed class GeminiWsEvent {
|
|
data class Audio(val bytes: ByteArray) : GeminiWsEvent()
|
|
object TurnComplete : GeminiWsEvent()
|
|
data class Error(val message: String) : GeminiWsEvent()
|
|
}
|
|
|
|
suspend fun ensureConnected(
|
|
serverUrl: String,
|
|
speaker: String,
|
|
authToken: String?,
|
|
directGeminiApiKey: String? = null
|
|
) = connectionMutex.withLock {
|
|
if (webSocket != null) {
|
|
if (connectedSpeaker == speaker) {
|
|
val isSetup = try { setupDeferred.await() } catch(_: Exception) { false }
|
|
if (isSetup) return@withLock
|
|
}
|
|
Timber.tag("TTS_CLOUD_DIAG").d("Closing existing WS. Speaker changed or setup failed.")
|
|
webSocket?.close(1000, "Reconnecting")
|
|
webSocket = null
|
|
}
|
|
|
|
val url = if (!directGeminiApiKey.isNullOrBlank()) {
|
|
"wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent?key=$directGeminiApiKey"
|
|
} else {
|
|
val sanitizedUrl = serverUrl.removeSuffix("/")
|
|
val wsUrlStr = sanitizedUrl.replace("https://", "wss://").replace("http://", "ws://")
|
|
"$wsUrlStr/live?speaker=$speaker&token=${authToken ?: ""}"
|
|
}
|
|
|
|
Timber.tag("TTS_CLOUD_DIAG").d("Connecting to WS: ${if (!directGeminiApiKey.isNullOrBlank()) "Gemini BYOK" else url}")
|
|
val request = Request.Builder().url(url).build()
|
|
val connectedDeferred = CompletableDeferred<Boolean>()
|
|
|
|
var connectionError: String? = null
|
|
|
|
setupDeferred = CompletableDeferred()
|
|
connectedSpeaker = speaker
|
|
|
|
webSocket = client.newWebSocket(request, object : WebSocketListener() {
|
|
override fun onOpen(webSocket: WebSocket, response: Response) {
|
|
Timber.tag("TTS_CLOUD_DIAG").d("WS Opened. Sending Setup configuration to Gemini...")
|
|
|
|
val systemPrompt = """
|
|
You are a professional audiobook narrator.
|
|
Your ONLY task is to read the exact text provided to you, word for word, neutral emotion, and with good pacing.
|
|
Do NOT add any conversational filler, acknowledgments, or extra words (e.g., do not say "Sure, here is the text").
|
|
Do NOT skip any parts or summarize. Output ONLY the audio reading of the provided text. If you encounter unreadable, non-verbal, or non-linguistic content (e.g., symbols like "※▼◆", raw formatting markers, broken characters, or pure punctuation clusters with no readable words), silently skip it and continue reading.
|
|
""".trimIndent()
|
|
|
|
val setupMsg = JSONObject().apply {
|
|
put("setup", JSONObject().apply {
|
|
put("model", "models/$GEMINI_CLOUD_TTS_MODEL")
|
|
put("systemInstruction", JSONObject().apply {
|
|
put("parts", org.json.JSONArray().apply {
|
|
put(JSONObject().apply {
|
|
put("text", systemPrompt)
|
|
})
|
|
})
|
|
})
|
|
put("generationConfig", JSONObject().apply {
|
|
put("responseModalities", org.json.JSONArray().apply { put("AUDIO") })
|
|
put("speechConfig", JSONObject().apply {
|
|
put("voiceConfig", JSONObject().apply {
|
|
put("prebuiltVoiceConfig", JSONObject().apply {
|
|
put("voiceName", speaker)
|
|
})
|
|
})
|
|
})
|
|
})
|
|
})
|
|
}.toString()
|
|
|
|
webSocket.send(setupMsg)
|
|
connectedDeferred.complete(true)
|
|
}
|
|
|
|
override fun onMessage(webSocket: WebSocket, text: String) {
|
|
try {
|
|
val json = JSONObject(text)
|
|
if (json.has("error")) {
|
|
val errObj = json.opt("error")
|
|
val errMsg = if (errObj is JSONObject) errObj.toString() else errObj?.toString() ?: "Unknown API Error"
|
|
Timber.tag("TTS_CLOUD_DIAG").e("API ERROR RETURNED: $errMsg")
|
|
audioChannel.trySend(GeminiWsEvent.Error(errMsg))
|
|
setupDeferred.complete(false)
|
|
return
|
|
}
|
|
if (json.has("setupComplete")) {
|
|
setupDeferred.complete(true)
|
|
}
|
|
|
|
val serverContent = json.optJSONObject("serverContent")
|
|
if (serverContent != null) {
|
|
val turnComplete = serverContent.optBoolean("turnComplete", false)
|
|
val modelTurn = serverContent.optJSONObject("modelTurn")
|
|
val parts = modelTurn?.optJSONArray("parts")
|
|
|
|
if (parts != null) {
|
|
for (i in 0 until parts.length()) {
|
|
val part = parts.getJSONObject(i)
|
|
val inlineData = part.optJSONObject("inlineData")
|
|
if (inlineData != null) {
|
|
val b64 = inlineData.optString("data")
|
|
if (b64.isNotEmpty()) {
|
|
val bytes = android.util.Base64.decode(b64, android.util.Base64.DEFAULT)
|
|
audioChannel.trySend(GeminiWsEvent.Audio(bytes))
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if (turnComplete) {
|
|
audioChannel.trySend(GeminiWsEvent.TurnComplete)
|
|
}
|
|
}
|
|
} catch (e: Exception) {
|
|
Timber.tag("TTS_CLOUD_DIAG").e(e, "Error parsing WS message text")
|
|
}
|
|
}
|
|
|
|
override fun onMessage(webSocket: WebSocket, bytes: okio.ByteString) {
|
|
onMessage(webSocket, bytes.utf8())
|
|
}
|
|
|
|
override fun onFailure(webSocket: WebSocket, t: Throwable, response: Response?) {
|
|
connectionError = if (response?.code == 402) {
|
|
"INSUFFICIENT_CREDITS"
|
|
} else {
|
|
"WS Failure: ${t.message} | Response: ${response?.code}"
|
|
}
|
|
Timber.tag("TTS_CLOUD_DIAG").e(t)
|
|
audioChannel.trySend(GeminiWsEvent.Error(connectionError))
|
|
this@GeminiLiveClient.webSocket = null
|
|
connectedDeferred.complete(false)
|
|
setupDeferred.complete(false)
|
|
}
|
|
|
|
override fun onClosed(webSocket: WebSocket, code: Int, reason: String) {
|
|
audioChannel.trySend(GeminiWsEvent.Error("Connection Closed: $reason"))
|
|
this@GeminiLiveClient.webSocket = null
|
|
setupDeferred.complete(false)
|
|
}
|
|
})
|
|
|
|
val isConnected = connectedDeferred.await()
|
|
if (!isConnected) throw IllegalStateException(connectionError ?: "Failed to connect to proxy WebSocket")
|
|
|
|
val isSetup = try {
|
|
kotlinx.coroutines.withTimeout(10000L) { setupDeferred.await() }
|
|
} catch (_: Exception) { false }
|
|
|
|
if (!isSetup) {
|
|
webSocket?.close(1000, "Setup failed")
|
|
webSocket = null
|
|
connectedSpeaker = null
|
|
throw IllegalStateException("Failed to complete Gemini setup")
|
|
} else {
|
|
Timber.tag("TTS_CLOUD_DIAG").d("Gemini setup complete")
|
|
}
|
|
}
|
|
|
|
fun generateChunk(text: String, cacheFile: File?): TtsAudioData {
|
|
if (text.isBlank()) return TtsAudioData(null, null, null, "Text is blank")
|
|
|
|
val streamId = java.util.UUID.randomUUID().toString()
|
|
val concurrentStream = ConcurrentInputStream()
|
|
StreamRegistry.register(streamId, concurrentStream)
|
|
val header = createWavHeaderUnknownLength(24000)
|
|
concurrentStream.write(header)
|
|
|
|
clientScope.launch {
|
|
generationMutex.withLock {
|
|
var fileOutputStream: java.io.FileOutputStream? = null
|
|
var tempFile: File? = null
|
|
|
|
try {
|
|
if (!isActive) return@launch
|
|
|
|
// Prepare cache temp file
|
|
if (cacheFile != null) {
|
|
tempFile = File(cacheFile.absolutePath + ".tmp")
|
|
fileOutputStream = java.io.FileOutputStream(tempFile)
|
|
fileOutputStream.write(header)
|
|
}
|
|
|
|
Timber.tag("TTS_CLOUD_DIAG").d("Starting API generation task for chunk: ${text.take(15)}...")
|
|
|
|
audioChannel = Channel(Channel.UNLIMITED)
|
|
val chunkGenStartTime = System.currentTimeMillis()
|
|
var firstByteTime = -1L
|
|
|
|
val payload = JSONObject().apply {
|
|
put("realtimeInput", JSONObject().apply {
|
|
put("text", text)
|
|
})
|
|
}.toString()
|
|
|
|
val sent = webSocket?.send(payload) ?: false
|
|
if (!sent) {
|
|
Timber.tag("TTS_CLOUD_DIAG").e("Failed to send text payload over WS")
|
|
return@launch
|
|
}
|
|
|
|
var receivedAudioBytes = 0
|
|
kotlinx.coroutines.withTimeout(30000L) {
|
|
for (event in audioChannel) {
|
|
when (event) {
|
|
is GeminiWsEvent.Audio -> {
|
|
if (firstByteTime == -1L) {
|
|
firstByteTime = System.currentTimeMillis()
|
|
Timber.tag("TTS_CLOUD_DIAG").i("TTFB: ${firstByteTime - chunkGenStartTime}ms")
|
|
}
|
|
concurrentStream.write(event.bytes)
|
|
fileOutputStream?.write(event.bytes)
|
|
receivedAudioBytes += event.bytes.size
|
|
}
|
|
is GeminiWsEvent.TurnComplete -> {
|
|
Timber.tag("TTS_CLOUD_DIAG").i("Chunk generation complete. Bytes: $receivedAudioBytes")
|
|
StreamRegistry.markFinished(streamId, receivedAudioBytes.toLong() + 44)
|
|
|
|
fileOutputStream?.close()
|
|
fileOutputStream = null
|
|
if (tempFile != null && cacheFile != null && receivedAudioBytes > 0) {
|
|
patchWavHeader(tempFile, receivedAudioBytes)
|
|
tempFile.renameTo(cacheFile)
|
|
Timber.tag("TTS_CLOUD_DIAG").d("Successfully cached chunk to ${cacheFile.name}")
|
|
}
|
|
break
|
|
}
|
|
is GeminiWsEvent.Error -> {
|
|
Timber.tag("TTS_CLOUD_DIAG").e("WS Error received: ${event.message}")
|
|
onAsyncError(event.message)
|
|
break
|
|
}
|
|
}
|
|
}
|
|
}
|
|
} catch (e: kotlinx.coroutines.TimeoutCancellationException) {
|
|
Timber.tag("TTS_CLOUD_DIAG").e(e, "Timeout waiting for audio/TurnComplete")
|
|
} catch (e: kotlinx.coroutines.CancellationException) {
|
|
Timber.tag("TTS_CLOUD_DIAG").i(e, "Streaming job cancelled due to user skip/flush")
|
|
} catch (e: Exception) {
|
|
Timber.tag("TTS_CLOUD_DIAG").e(e, "Exception piping audio")
|
|
} finally {
|
|
Timber.tag("TTS_CLOUD_DIAG").d("Closing stream for ${text.take(15)}")
|
|
concurrentStream.close()
|
|
fileOutputStream?.close()
|
|
if (cacheFile != null && !cacheFile.exists()) {
|
|
tempFile?.delete()
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
return TtsAudioData(null, text, emptyList(), streamUri = "ttsstream://$streamId")
|
|
}
|
|
|
|
fun close() {
|
|
clientScope.cancel()
|
|
clientScope = CoroutineScope(Dispatchers.IO + SupervisorJob())
|
|
|
|
webSocket?.close(1000, "Context Reset")
|
|
webSocket = null
|
|
connectedSpeaker = null
|
|
setupDeferred = CompletableDeferred<Boolean>().apply { complete(false) }
|
|
StreamRegistry.clear()
|
|
}
|
|
}
|
|
|
|
private val synthesizeBaseTtsChunk: suspend (String) -> TtsAudioData =
|
|
{ chunkToSpeak ->
|
|
val (file, text) = baseTtsSynthesizer.synthesizeToFile(chunkToSpeak)
|
|
TtsAudioData(file, text, null)
|
|
}
|
|
|
|
val audioGenerator: suspend (bookTitle: String, chapterTitle: String?, chunkIndex: Int, totalChunks: Int, text: String, speaker: String, mode: TtsMode, authToken: String?) -> TtsAudioData =
|
|
{ bookTitle, chapterTitle, chunkIndex, totalChunks, text, speaker, mode, authToken ->
|
|
cacheManager.saveTotalChunks(bookTitle, chapterTitle, totalChunks)
|
|
when (mode) {
|
|
TtsMode.CLOUD -> {
|
|
val cachedFile = cacheManager.getCacheFile(bookTitle, chapterTitle, text, speaker, mode)
|
|
|
|
if (cachedFile.exists() && cachedFile.length() > 44) {
|
|
Timber.tag("TTS_CLOUD_DIAG").i("Using cached audio for chunk $chunkIndex")
|
|
TtsAudioData(audioFile = cachedFile, serverText = text, wordTimings = emptyList(), error = null, streamUri = null)
|
|
} else {
|
|
try {
|
|
val directGeminiApiKey = if (isByokCloudTtsAvailable(this@TtsService)) {
|
|
loadAiByokSettings(this@TtsService).geminiKey
|
|
} else {
|
|
null
|
|
}
|
|
if (directGeminiApiKey.isNullOrBlank() && googleCloudWorkerTtsUrl.isBlank()) {
|
|
TtsAudioData(audioFile = null, serverText = null, wordTimings = null, error = "Cloud TTS is not configured.")
|
|
} else {
|
|
liveClient.ensureConnected(googleCloudWorkerTtsUrl, speaker, authToken, directGeminiApiKey)
|
|
liveClient.generateChunk(text, cachedFile)
|
|
}
|
|
} catch (e: Exception) {
|
|
Timber.tag("TTS_CLOUD_DIAG").e(e, "Cloud TTS generation failed")
|
|
TtsAudioData(audioFile = null, serverText = null, wordTimings = null, error = e.message ?: "Failed to connect to TTS service")
|
|
}
|
|
}
|
|
}
|
|
TtsMode.BASE -> synthesizeBaseTtsChunk(text)
|
|
}
|
|
}
|
|
|
|
override fun onCreate() {
|
|
super.onCreate()
|
|
Timber.d("TtsService created.")
|
|
val hasNotificationPermission = Build.VERSION.SDK_INT < Build.VERSION_CODES.TIRAMISU ||
|
|
ContextCompat.checkSelfPermission(this, Manifest.permission.POST_NOTIFICATIONS) == PackageManager.PERMISSION_GRANTED
|
|
Timber.tag(TTS_NOTIFICATION_DIAG_TAG).i(
|
|
"TtsService onCreate. sdk=${Build.VERSION.SDK_INT}, hasPostNotifications=$hasNotificationPermission"
|
|
)
|
|
|
|
cacheManager = TtsCacheManager(this)
|
|
|
|
baseTtsSynthesizer = BaseTtsSynthesizer(this)
|
|
scope.launch {
|
|
try {
|
|
baseTtsSynthesizer.initialize()
|
|
} catch (e: Exception) {
|
|
Timber.e(e, "Base TTS synthesizer failed to initialize")
|
|
}
|
|
}
|
|
|
|
val audioAttributes = AudioAttributes.Builder()
|
|
.setContentType(C.AUDIO_CONTENT_TYPE_SPEECH)
|
|
.setUsage(C.USAGE_MEDIA)
|
|
.build()
|
|
|
|
val defaultDataSourceFactory = androidx.media3.datasource.DefaultDataSource.Factory(this)
|
|
val dataSourceFactory = androidx.media3.datasource.DataSource.Factory {
|
|
object : androidx.media3.datasource.DataSource {
|
|
private var dataSource: androidx.media3.datasource.DataSource? = null
|
|
private val defaultDataSource = defaultDataSourceFactory.createDataSource()
|
|
private val streamDataSource = InputStreamDataSource()
|
|
|
|
override fun addTransferListener(transferListener: androidx.media3.datasource.TransferListener) {
|
|
defaultDataSource.addTransferListener(transferListener)
|
|
streamDataSource.addTransferListener(transferListener)
|
|
}
|
|
|
|
override fun open(dataSpec: androidx.media3.datasource.DataSpec): Long {
|
|
dataSource = if (dataSpec.uri.scheme == "ttsstream") {
|
|
streamDataSource
|
|
} else {
|
|
defaultDataSource
|
|
}
|
|
return dataSource!!.open(dataSpec)
|
|
}
|
|
|
|
override fun read(buffer: ByteArray, offset: Int, length: Int): Int {
|
|
return dataSource!!.read(buffer, offset, length)
|
|
}
|
|
|
|
override fun getUri(): android.net.Uri? = dataSource?.uri
|
|
|
|
override fun close() {
|
|
dataSource?.close()
|
|
}
|
|
}
|
|
}
|
|
|
|
player = ExoPlayer.Builder(this)
|
|
.setAudioAttributes(audioAttributes, true)
|
|
.setHandleAudioBecomingNoisy(true)
|
|
.setMediaSourceFactory(androidx.media3.exoplayer.source.DefaultMediaSourceFactory(this).setDataSourceFactory(dataSourceFactory))
|
|
.build()
|
|
Timber.tag(TTS_NOTIFICATION_DIAG_TAG).i("ExoPlayer created for TTS service.")
|
|
|
|
playbackManager = TtsPlaybackManager(
|
|
player = player,
|
|
generateAudioChunk = audioGenerator,
|
|
onResetContext = { liveClient.close() }
|
|
)
|
|
|
|
mediaSession = MediaSession.Builder(this, player)
|
|
.setCallback(playbackManager)
|
|
.build()
|
|
|
|
mediaSession?.let { playbackManager.setMediaSession(it) }
|
|
Timber.tag(TTS_NOTIFICATION_DIAG_TAG).i("MediaSession created and attached to playback manager. sessionAvailable=${mediaSession != null}")
|
|
}
|
|
|
|
override fun onTaskRemoved(rootIntent: Intent?) {
|
|
Timber.tag(TTS_NOTIFICATION_DIAG_TAG).i(
|
|
"onTaskRemoved. playWhenReady=${if (::player.isInitialized) player.playWhenReady else null}, isPlaying=${if (::player.isInitialized) player.isPlaying else null}"
|
|
)
|
|
if (!player.playWhenReady) {
|
|
Timber.tag(TTS_NOTIFICATION_DIAG_TAG).w("Task removed while player is not playWhenReady. Calling stopSelf().")
|
|
stopSelf()
|
|
}
|
|
Timber.d("onTaskRemoved called, stopping service.")
|
|
}
|
|
|
|
override fun onGetSession(controllerInfo: MediaSession.ControllerInfo): MediaSession? {
|
|
Timber.tag(TTS_NOTIFICATION_DIAG_TAG).i(
|
|
"onGetSession. package=${controllerInfo.packageName}, sessionAvailable=${mediaSession != null}"
|
|
)
|
|
return mediaSession
|
|
}
|
|
|
|
override fun onDestroy() {
|
|
Timber.d("TtsService is being destroyed.")
|
|
Timber.tag(TTS_NOTIFICATION_DIAG_TAG).w("TtsService onDestroy.")
|
|
baseTtsSynthesizer.shutdown()
|
|
playbackManager.release()
|
|
mediaSession?.run {
|
|
player.release()
|
|
release()
|
|
mediaSession = null
|
|
}
|
|
super.onDestroy()
|
|
}
|
|
}
|