Pdf text reflow (#36)

* Implemented PDF reflow mode by introducing a mechanism to convert PDF content to Markdown/HTML for viewing in the EPUB reader.

Specific changes include:
- Added `PdfReflowGenerator` and `PdfToMarkdownGenerator` to handle PDF text extraction and conversion to reflowable formats.
- Updated `MainViewModel` with `toggleReflowMode` logic to switch between original PDF and reflowed views.
- Modified `RecentFileEntity` and `RecentFileDao` to persist user reflow preferences, including a Room database migration (v12 to v13).
- Updated `PdfViewerScreen` and `EpubReaderControls` to include UI options for toggling reflow mode.
- Integrated reflow preference check into the book opening workflow to automatically load the preferred view.
- Updated `AppNavigation` and `EpubReaderScreen` to support the new view switching state.

* Implemented background processing and incremental loading for PDF reflow mode.

- Added `reflowProgress` to `MainViewModel` to track and display PDF-to-Markdown conversion progress in the UI.
- Refactored `PdfToMarkdownGenerator` to generate a skeleton EPUB structure immediately while processing page content (text and images) asynchronously.
- Switched PDF text extraction to use `PDFBox` with optimized memory settings and JPEG compression for images.
- Implemented priority page processing in reflow mode, starting with the user's current page.
- Added "Clear Reflow Cache" debug option to the Home Screen.
- Enhanced `BookPaginator` to support lazy loading of chapter content from disk and improved cache hit detection.

* Refactored PDF Reflow Mode to generate standalone Markdown files instead of temporary EPUB books.

* perf(reflow): optimize PDF-to-Markdown conversion and fix viewing lag

- Re-architected PdfToMarkdownGenerator to use a single-pass stream (O(N) complexity), fixing performance bottlenecks and timeouts on large PDFs.
- Implemented "Virtual Chaptering" in SingleFileImporter for Markdown files to split content into page-level HTML files, eliminating UI lag during reading.
- Simplified ReflowWorker to delegate progress tracking and looping to the generator.
- Enhanced PdfViewerScreen with a prominent top-bar progress indicator and a completion snackbar with an "OPEN" action.

* Optimized EPUB parsing performance and fixed PDF viewer UI layout.

- Optimized `EpubParser` by implementing parallel chapter parsing using coroutines and a semaphore to limit concurrency.
- Reduced memory usage in `EpubParser` and `SingleFileImporter` by no longer storing full HTML content in memory for chapters.
- Updated `EpubXMLFileParser` to support an existing `Document` object to avoid redundant Jsoup parsing.
- Fixed an issue in `PdfViewerScreen` where the snackbar was appearing under the bottom app bar.
This commit is contained in:
Aryan 2026-03-07 16:10:34 +05:30 committed by GitHub
parent 8f52549c19
commit 1e879eb604
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
21 changed files with 936 additions and 165 deletions

View file

@ -0,0 +1,135 @@
package com.aryan.reader.pdf
import android.content.Context
import com.aryan.reader.epub.EpubBook
import com.aryan.reader.epub.EpubChapter
import com.aryan.reader.pdf.data.PdfTextRepository
import io.legere.pdfiumandroid.suspend.PdfDocumentKt
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.withContext
import java.io.File
import java.util.UUID
object PdfReflowGenerator {
suspend fun generateReflowBook(
context: Context,
bookId: String,
document: PdfDocumentKt,
repository: PdfTextRepository,
totalPages: Int
): EpubBook = withContext(Dispatchers.Default) {
val cacheDir = File(context.cacheDir, "reflow_cache/$bookId")
if (cacheDir.exists()) {
cacheDir.deleteRecursively()
}
cacheDir.mkdirs()
val chapters = mutableListOf<EpubChapter>()
val css = """
body { font-family: sans-serif; line-height: 1.6; padding: 1em; }
p { margin-bottom: 1em; }
h1, h2 { color: #333; margin-top: 1.5em; }
.page-marker { color: #888; font-size: 0.8em; margin-bottom: 2em; border-bottom: 1px solid #eee; }
""".trimIndent()
// We generate a chapter for every page to keep sync simple
for (i in 0 until totalPages) {
val rawText = repository.getOrExtractText(bookId, document, i)
val cleanedHtml = processTextToHtml(rawText, i + 1)
val fileName = "page_$i.html"
val file = File(cacheDir, fileName)
val fullHtml = """
<!DOCTYPE html>
<html>
<head>
<title>Page ${i + 1}</title>
<style>$css</style>
</head>
<body>
$cleanedHtml
</body>
</html>
""".trimIndent()
file.writeText(fullHtml)
chapters.add(
EpubChapter(
chapterId = "${bookId}_page_$i",
absPath = fileName,
title = "Page ${i + 1}",
htmlFilePath = fileName,
plainTextContent = rawText, // Raw text for search/TTS
htmlContent = fullHtml,
depth = 0,
isInToc = true
)
)
}
EpubBook(
fileName = "Reflow_Session",
title = document.getDocumentMeta().title ?: "Reflow View",
author = document.getDocumentMeta().author ?: "",
language = "en",
coverImage = null,
chapters = chapters,
chaptersForPagination = chapters,
images = emptyList(),
pageList = emptyList(),
extractionBasePath = cacheDir.absolutePath,
css = emptyMap()
)
}
private fun processTextToHtml(rawText: String, pageNumber: Int): String {
if (rawText.isBlank()) return "<p><i>(No text on this page)</i></p>"
val lines = rawText.split('\n')
val sb = StringBuilder()
sb.append("<div class='page-marker'>Page $pageNumber</div>")
var currentParagraph = StringBuilder()
for (line in lines) {
val trimmed = line.trim()
if (trimmed.isEmpty()) {
if (currentParagraph.isNotEmpty()) {
sb.append("<p>${currentParagraph.toString()}</p>")
currentParagraph.clear()
}
continue
}
// Heuristic: Header detection (All caps, short line, no punctuation at end)
val isHeader = trimmed.length < 50 && trimmed.all { it.isUpperCase() || !it.isLetter() } && !trimmed.endsWith(".")
if (isHeader) {
if (currentParagraph.isNotEmpty()) {
sb.append("<p>${currentParagraph.toString()}</p>")
currentParagraph.clear()
}
sb.append("<h2>$trimmed</h2>")
continue
}
if (currentParagraph.isNotEmpty()) {
currentParagraph.append(" ")
}
currentParagraph.append(trimmed)
if (trimmed.endsWith(".") || trimmed.endsWith("?") || trimmed.endsWith("!") || trimmed.endsWith(":")) {
}
}
if (currentParagraph.isNotEmpty()) {
sb.append("<p>${currentParagraph.toString()}</p>")
}
return sb.toString()
}
}

View file

@ -0,0 +1,134 @@
// PdfToMarkdownGenerator.kt
package com.aryan.reader.pdf
import android.content.Context
import android.net.Uri
import com.tom_roush.pdfbox.io.MemoryUsageSetting
import com.tom_roush.pdfbox.pdmodel.PDDocument
import com.tom_roush.pdfbox.pdmodel.PDPage
import com.tom_roush.pdfbox.text.PDFTextStripper
import com.tom_roush.pdfbox.text.TextPosition
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.withContext
import timber.log.Timber
import java.io.File
import kotlin.math.roundToInt
object PdfToMarkdownGenerator {
// Unique delimiter to split pages reliably
const val PAGE_DELIMITER = "\n\n[[PAGE_BREAK]]\n\n"
suspend fun generateMarkdownFile(
context: Context,
pdfUri: Uri,
destFile: File,
startPage: Int = 1,
onProgress: (Float) -> Unit
): Boolean = withContext(Dispatchers.IO) {
try {
context.contentResolver.openInputStream(pdfUri)?.use { inputStream ->
// Setup mixed memory usage to handle larger files without OOM
PDDocument.load(inputStream, MemoryUsageSetting.setupMixed(50 * 1024 * 1024)).use { doc ->
val totalPages = doc.numberOfPages
// Configure stripper for linear processing
val stripper = MarkdownStripper(totalPages, onProgress)
stripper.startPage = startPage
stripper.endPage = totalPages
// Write directly to file stream (O(N) complexity)
destFile.bufferedWriter().use { writer ->
stripper.writeText(doc, writer)
}
}
}
return@withContext true
} catch (e: Exception) {
Timber.e(e, "Failed to generate Markdown from PDF")
return@withContext false
}
}
private class MarkdownStripper(
private val totalPages: Int,
private val onProgress: (Float) -> Unit
) : PDFTextStripper() {
private var currentPageBaseFontSize = 0f
init {
sortByPosition = true
suppressDuplicateOverlappingText = true
paragraphStart = ""
paragraphEnd = "\n\n"
}
// Override endPage to update progress and insert delimiter
override fun endPage(page: PDPage?) {
super.endPage(page)
try {
// Insert our custom delimiter so importer can split chapters
output.write(PAGE_DELIMITER)
// Update progress
val current = currentPageNo // inherited from PDFTextStripper
if (totalPages > 0) {
onProgress(current.toFloat() / totalPages.toFloat())
}
} catch (e: Exception) {
Timber.e(e, "Error writing page delimiter")
}
}
override fun startPage(page: PDPage?) {
currentPageBaseFontSize = 0f
super.startPage(page)
}
private fun calculateBaseFontSize(textPositions: List<TextPosition>) {
val sizeCounts = mutableMapOf<Float, Int>()
textPositions.forEach { pos ->
val size = pos.fontSizeInPt.roundToInt().toFloat()
sizeCounts[size] = (sizeCounts[size] ?: 0) + 1
}
currentPageBaseFontSize = sizeCounts.maxByOrNull { it.value }?.key ?: 12f
}
override fun writeString(text: String?, textPositions: MutableList<TextPosition>?) {
if (text.isNullOrBlank() || textPositions.isNullOrEmpty()) return
if (currentPageBaseFontSize == 0f) {
calculateBaseFontSize(textPositions)
}
val firstPos = textPositions[0]
val fontSize = firstPos.fontSizeInPt
val fontDescriptor = firstPos.font?.fontDescriptor
val isBold = fontDescriptor?.isForceBold == true ||
(firstPos.font?.name?.contains("Bold", ignoreCase = true) == true)
val isItalic = fontDescriptor?.isItalic == true ||
(firstPos.font?.name?.contains("Italic", ignoreCase = true) == true)
// Header detection logic
val isHeader = fontSize > currentPageBaseFontSize * 1.2
val isBigHeader = fontSize > currentPageBaseFontSize * 1.5
val sb = StringBuilder()
if (isBigHeader) sb.append("## ")
else if (isHeader) sb.append("### ")
if (isBold && !isHeader) sb.append("**")
if (isItalic) sb.append("*")
text.forEach { char -> sb.append(char) }
if (isItalic) sb.append("*")
if (isBold && !isHeader) sb.append("**")
writeString(sb.toString())
}
}
}

View file

@ -29,6 +29,8 @@ import android.content.Context
import android.content.pm.PackageManager
import android.graphics.Bitmap
import android.graphics.RectF
import androidx.compose.material3.SnackbarDuration
import androidx.compose.material3.SnackbarResult
import android.net.Uri
import android.os.Build
import android.os.ParcelFileDescriptor
@ -220,6 +222,7 @@ import androidx.paging.compose.LazyPagingItems
import androidx.paging.compose.collectAsLazyPagingItems
import androidx.paging.compose.itemContentType
import androidx.paging.compose.itemKey
import androidx.work.WorkInfo
import com.aryan.reader.AiDefinitionPopup
import com.aryan.reader.AiDefinitionResult
import com.aryan.reader.BuildConfig
@ -673,7 +676,16 @@ fun PdfViewerScreen(
var isBackgroundIndexing by remember { mutableStateOf(false) }
var backgroundIndexingProgress by remember { mutableFloatStateOf(0f) }
var currentBookId by remember { mutableStateOf<String?>(null) }
val bookId = currentBookId ?: pdfUri.toString().hashCode().toString()
val uiState by viewModel.uiState.collectAsState()
val reflowBookId = remember(bookId) { "${bookId}_reflow" }
val hasReflowFile by remember(uiState.recentFiles, reflowBookId) {
derivedStateOf {
uiState.recentFiles.any { it.bookId == reflowBookId && !it.isDeleted }
}
}
val originalFileName by remember(uiState.recentFiles, pdfUri) {
derivedStateOf {
uiState.recentFiles.find { it.uriString == pdfUri.toString() }?.displayName
@ -737,9 +749,6 @@ fun PdfViewerScreen(
derivedStateOf { isEditMode && !isDockMinimized }
}
var currentBookId by remember { mutableStateOf<String?>(null) }
val bookId = currentBookId ?: pdfUri.toString().hashCode().toString()
var isAutoScrollLocal by remember { mutableStateOf(loadPdfAutoScrollLocalMode(context, bookId)) }
LaunchedEffect(bookId) {
@ -1602,6 +1611,23 @@ fun PdfViewerScreen(
}
}
val reflowInfo by viewModel.reflowWorkInfo.collectAsState(initial = null)
val isReflowingThisBook by remember(reflowInfo, bookId) {
derivedStateOf {
reflowInfo?.tags?.contains("book_$bookId") == true &&
(reflowInfo?.state == WorkInfo.State.RUNNING || reflowInfo?.state == WorkInfo.State.ENQUEUED)
}
}
val reflowProgressValue by remember(reflowInfo, isReflowingThisBook) {
derivedStateOf {
if (isReflowingThisBook) {
reflowInfo?.progress?.getFloat(ReflowWorker.KEY_PROGRESS, 0f) ?: 0f
} else 0f
}
}
val onBookmarkClick: () -> Unit = {
val currentPage = if (displayMode == DisplayMode.PAGINATION) {
pagerState.currentPage
@ -1611,6 +1637,25 @@ fun PdfViewerScreen(
onToggleBookmark(currentPage)
}
LaunchedEffect(reflowInfo) {
if (reflowInfo?.state == WorkInfo.State.SUCCEEDED &&
reflowInfo?.tags?.contains("book_$bookId") == true) {
val result = snackbarHostState.showSnackbar(
message = "Text View generation complete!",
actionLabel = "OPEN",
duration = SnackbarDuration.Long
)
if (result == SnackbarResult.ActionPerformed) {
val item = uiState.recentFiles.find { it.bookId == reflowBookId }
if (item != null) {
viewModel.onRecentFileClicked(item)
}
}
}
}
LaunchedEffect(pdfUri) { debugPdfLinks(context, pdfUri, pdfiumCore, this) }
LaunchedEffect(currentBookId) {
@ -2837,6 +2882,12 @@ fun PdfViewerScreen(
}
}
val showStandardBars = showBars && !isEditMode
val snackbarPadding by animateDpAsState(
targetValue = if (showStandardBars && !searchState.isSearchActive) 56.dp else 0.dp,
label = "SnackbarPadding"
)
ModalNavigationDrawer(
drawerState = drawerState, gesturesEnabled = drawerState.isOpen, drawerContent = {
ModalDrawerSheet(modifier = Modifier.windowInsetsPadding(WindowInsets.statusBars)) {
@ -3100,7 +3151,14 @@ fun PdfViewerScreen(
}
}
}) {
Scaffold(snackbarHost = { SnackbarHost(snackbarHostState) }) { paddingValues ->
Scaffold(
snackbarHost = {
SnackbarHost(
hostState = snackbarHostState,
modifier = Modifier.padding(bottom = snackbarPadding)
)
}
) { paddingValues ->
BoxWithConstraints(modifier = Modifier
.fillMaxSize()
.padding(paddingValues)) {
@ -4083,8 +4141,6 @@ fun PdfViewerScreen(
}
}
val showStandardBars = showBars && !isEditMode
// Custom Top Bar
AnimatedVisibility(
visible = showStandardBars,
@ -4298,7 +4354,7 @@ fun PdfViewerScreen(
)
}
)
if (BuildConfig.DEBUG) {
DropdownMenuItem(
text = { Text("TTS Settings (Debug)") },
@ -4345,6 +4401,44 @@ fun PdfViewerScreen(
)
)
}
HorizontalDivider()
DropdownMenuItem(
text = {
Text(
when {
isReflowingThisBook -> "Generating... ${(reflowProgressValue * 100).toInt()}%"
hasReflowFile -> "Open Text View"
else -> "Generate Text View"
}
)
},
enabled = pdfDocument != null && !isReflowingThisBook,
onClick = {
showMoreMenu = false
if (hasReflowFile) {
val item = uiState.recentFiles.find { it.bookId == reflowBookId }
if (item != null) {
viewModel.onRecentFileClicked(item)
}
} else {
viewModel.generateAndImportReflowFile(
pdfBookId = bookId,
pdfUri = pdfUri,
originalTitle = originalFileName
)
}
},
leadingIcon = {
Icon(
painter = painterResource(id = R.drawable.format_size),
contentDescription = null,
modifier = Modifier.size(20.dp)
)
}
)
HorizontalDivider()
DropdownMenuItem(text = { Text("Share") }, onClick = {
@ -4373,6 +4467,50 @@ fun PdfViewerScreen(
}
}
AnimatedVisibility(
visible = showStandardBars && isReflowingThisBook,
enter = fadeIn() + slideInVertically(),
exit = fadeOut() + slideOutVertically(),
modifier = Modifier
.align(Alignment.TopCenter)
.padding(top = 56.dp)
.fillMaxWidth()
.padding(horizontal = 8.dp)
) {
Surface(
color = MaterialTheme.colorScheme.surfaceVariant,
modifier = Modifier.fillMaxWidth(),
shape = RoundedCornerShape(bottomStart = 8.dp, bottomEnd = 8.dp),
shadowElevation = 4.dp
) {
Column(modifier = Modifier.padding(12.dp)) {
Row(
verticalAlignment = Alignment.CenterVertically,
modifier = Modifier.fillMaxWidth()
) {
Text(
text = "Generating Text View...",
style = MaterialTheme.typography.titleSmall,
color = MaterialTheme.colorScheme.onSurfaceVariant,
modifier = Modifier.weight(1f)
)
Text(
text = "${(reflowProgressValue * 100).toInt()}%",
style = MaterialTheme.typography.labelSmall,
fontWeight = FontWeight.Bold,
color = MaterialTheme.colorScheme.primary
)
}
Spacer(Modifier.height(8.dp))
androidx.compose.material3.LinearProgressIndicator(
progress = { reflowProgressValue },
modifier = Modifier.fillMaxWidth().height(6.dp),
trackColor = MaterialTheme.colorScheme.surfaceVariant.copy(alpha = 0.5f),
)
}
}
}
// Search Results Panel
AnimatedVisibility(
visible = searchState.isSearchActive && searchState.showSearchResultsPanel,
@ -5772,6 +5910,7 @@ fun PdfViewerScreen(
}
}
}
val autoScrollPadding by animateDpAsState(
targetValue = if (showBars) (56.dp + 16.dp) else 16.dp,
label = "AutoScrollPadding"

View file

@ -0,0 +1,84 @@
// ReflowWorker.kt
package com.aryan.reader.pdf
import android.content.Context
import androidx.core.net.toUri
import androidx.work.CoroutineWorker
import androidx.work.WorkerParameters
import androidx.work.workDataOf
import com.aryan.reader.FileType
import com.aryan.reader.data.RecentFileItem
import com.aryan.reader.data.RecentFilesRepository
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.withContext
import timber.log.Timber
import java.io.File
class ReflowWorker(
context: Context,
params: WorkerParameters
) : CoroutineWorker(context, params) {
override suspend fun doWork(): Result = withContext(Dispatchers.IO) {
val bookId = inputData.getString(KEY_BOOK_ID) ?: return@withContext Result.failure()
val pdfUriString = inputData.getString(KEY_PDF_URI) ?: return@withContext Result.failure()
val originalTitle = inputData.getString(KEY_ORIGINAL_TITLE) ?: "Document"
val reflowBookId = "${bookId}_reflow"
val destFile = File(applicationContext.filesDir, "${bookId}_reflow.md")
val pdfUri = pdfUriString.toUri()
Timber.tag("ReflowWorker").d("Starting background reflow for $originalTitle.")
// Delegate entire process to Generator (it now handles the loop and progress)
val success = PdfToMarkdownGenerator.generateMarkdownFile(
applicationContext,
pdfUri,
destFile,
startPage = 1 // Always start from beginning for full regeneration
) { progress ->
// Report progress
setProgressAsync(workDataOf(KEY_PROGRESS to progress))
}
if (success && destFile.exists()) {
Timber.tag("ReflowWorker").d("Reflow complete. Importing to database.")
val repo = RecentFilesRepository(applicationContext)
val newItem = RecentFileItem(
bookId = reflowBookId,
uriString = destFile.toUri().toString(),
type = FileType.MD,
displayName = "$originalTitle (Text View)",
timestamp = System.currentTimeMillis(),
coverImagePath = null,
title = "$originalTitle (Reflow)",
author = "Generated",
isAvailable = true,
isRecent = true,
lastModifiedTimestamp = System.currentTimeMillis(),
isDeleted = false,
sourceFolderUri = null
)
repo.addRecentFile(newItem)
// 100% Progress
setProgressAsync(workDataOf(KEY_PROGRESS to 1.0f))
return@withContext Result.success()
} else {
Timber.e("Reflow failed or was incomplete.")
return@withContext Result.failure()
}
}
companion object {
const val WORK_NAME = "reflow_work"
const val KEY_BOOK_ID = "book_id"
const val KEY_PDF_URI = "pdf_uri"
const val KEY_ORIGINAL_TITLE = "original_title"
const val KEY_PROGRESS = "progress"
}
}