Optimized HTML importing and PDF core management: (#82)
- Refactored `SingleFileImporter` to use a streaming approach for HTML imports, improving memory efficiency for large files. - Implemented `writeHtmlChapter` helper to handle segmented HTML processing and chapter generation. - Introduced `PdfiumCoreProvider` to provide a singleton instance of `PdfiumCoreKt`. - Updated `PdfToHtmlGenerator` and `PdfViewerScreen` to use the shared `PdfiumCoreProvider` instance.
This commit is contained in:
Aryan
GitHub
parent
00d6931b3a
commit
93f03941b2
3 changed files with 152 additions and 55 deletions
|
|
@ -366,68 +366,120 @@ class SingleFileImporter(private val context: Context) {
|
||||||
|
|
||||||
val parseStart = System.currentTimeMillis()
|
val parseStart = System.currentTimeMillis()
|
||||||
Timber.tag("FileOpenPerf").d("[HTML] parseHtml START | file=$originalBookNameHint")
|
Timber.tag("FileOpenPerf").d("[HTML] parseHtml START | file=$originalBookNameHint")
|
||||||
Timber.d("Importing HTML: $originalBookNameHint")
|
Timber.d("Importing HTML (Streaming): $originalBookNameHint")
|
||||||
|
|
||||||
val content = inputStream.bufferedReader().use { it.readText() }
|
var title = originalBookNameHint.substringBeforeLast(".")
|
||||||
val doc = Jsoup.parse(content)
|
var author = "Unknown"
|
||||||
val title = doc.title().takeIf { it.isNotBlank() } ?: originalBookNameHint.substringBeforeLast(".")
|
val cssBuilder = java.lang.StringBuilder()
|
||||||
Timber.tag("FileOpenPerf").d("[HTML] parseHtml: Read ${content.length} chars | elapsed=${System.currentTimeMillis() - parseStart}ms")
|
val chapters = mutableListOf<EpubChapter>()
|
||||||
|
|
||||||
val author = doc.select("meta[name=author]").attr("content").takeIf { it.isNotBlank() }
|
inputStream.bufferedReader().use { reader ->
|
||||||
?: doc.select("meta[property=article:author]").attr("content").takeIf { it.isNotBlank() }
|
var inStyle = false
|
||||||
|
var inBody = false
|
||||||
|
var pageNum = 1
|
||||||
|
val currentChapterBuilder = java.lang.StringBuilder()
|
||||||
|
|
||||||
val cssStyle = doc.select("style").html()
|
var line: String?
|
||||||
val bodyHtml = doc.body().html()
|
while (reader.readLine().also { line = it } != null) {
|
||||||
|
val trimmed = line!!.trim()
|
||||||
|
|
||||||
val rawChapters = if (bodyHtml.contains("<page-break></page-break>")) {
|
if (!inBody) {
|
||||||
bodyHtml.split("<page-break></page-break>")
|
if (trimmed.startsWith("<title", ignoreCase = true)) {
|
||||||
} else {
|
val t = trimmed.substringAfter(">").substringBefore("</title>")
|
||||||
listOf(bodyHtml)
|
if (t.isNotBlank()) title = t
|
||||||
|
}
|
||||||
|
val authorMatch = Regex("<meta[^>]+name=\"author\"[^>]+content=\"([^\"]+)\"").find(
|
||||||
|
line
|
||||||
|
)
|
||||||
|
?: Regex("<meta[^>]+property=\"article:author\"[^>]+content=\"([^\"]+)\"").find(
|
||||||
|
line
|
||||||
|
)
|
||||||
|
if (authorMatch != null) {
|
||||||
|
author = authorMatch.groupValues[1]
|
||||||
|
}
|
||||||
|
|
||||||
|
if (trimmed.startsWith("<style", ignoreCase = true)) {
|
||||||
|
inStyle = true
|
||||||
|
val styleContent = line.substringAfter(">").substringBefore("</style>")
|
||||||
|
if (styleContent.isNotBlank()) cssBuilder.append(styleContent).append("\n")
|
||||||
|
if (trimmed.contains("</style>")) {
|
||||||
|
inStyle = false
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if (inStyle) {
|
||||||
|
if (trimmed.contains("</style>")) {
|
||||||
|
cssBuilder.append(line.substringBefore("</style>")).append("\n")
|
||||||
|
inStyle = false
|
||||||
|
} else {
|
||||||
|
cssBuilder.append(line).append("\n")
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if (trimmed.equals("<body>", ignoreCase = true)) {
|
||||||
|
inBody = true
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if (trimmed.startsWith("<body ", ignoreCase = true)) {
|
||||||
|
inBody = true
|
||||||
|
val afterBody = line.substringAfter(">", "")
|
||||||
|
if (afterBody.isNotBlank()) currentChapterBuilder.append(afterBody).append("\n")
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if (trimmed.startsWith("<p") || trimmed.startsWith("<div") ||
|
||||||
|
trimmed.startsWith("<h") || trimmed.startsWith("<section") ||
|
||||||
|
trimmed.contains("<page-break>") ||
|
||||||
|
(trimmed.isNotBlank() && !trimmed.startsWith("<") && !trimmed.startsWith("<!"))) {
|
||||||
|
inBody = true
|
||||||
|
currentChapterBuilder.append(line).append("\n")
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
if (trimmed.equals("</body>", ignoreCase = true) || trimmed.equals("</html>", ignoreCase = true)) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if (line.contains("<page-break></page-break>")) {
|
||||||
|
val parts = line.split("<page-break></page-break>")
|
||||||
|
for (i in parts.indices) {
|
||||||
|
currentChapterBuilder.append(parts[i]).append("\n")
|
||||||
|
if (i < parts.size - 1) {
|
||||||
|
val chapterHtml = currentChapterBuilder.toString()
|
||||||
|
if (chapterHtml.isNotBlank()) {
|
||||||
|
chapters.add(writeHtmlChapter(extractionDir, bookId, pageNum++, title, cssBuilder.toString(), chapterHtml))
|
||||||
|
}
|
||||||
|
currentChapterBuilder.clear() // Clean memory allocation
|
||||||
|
}
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
currentChapterBuilder.append(line).append("\n")
|
||||||
|
|
||||||
|
if (currentChapterBuilder.length > 2_000_000) {
|
||||||
|
chapters.add(writeHtmlChapter(extractionDir, bookId, pageNum++, title, cssBuilder.toString(), currentChapterBuilder.toString()))
|
||||||
|
currentChapterBuilder.clear()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
val finalChapterHtml = currentChapterBuilder.toString()
|
||||||
|
if (finalChapterHtml.isNotBlank()) {
|
||||||
|
chapters.add(writeHtmlChapter(extractionDir, bookId, pageNum++, title, cssBuilder.toString(), finalChapterHtml))
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
val chapters = rawChapters.mapIndexed { index, rawText ->
|
if (chapters.isEmpty()) {
|
||||||
async(Dispatchers.Default) {
|
chapters.add(writeHtmlChapter(extractionDir, bookId, 1, title, cssBuilder.toString(), "<p>(Empty File)</p>"))
|
||||||
if (rawText.isBlank()) return@async null
|
}
|
||||||
|
|
||||||
val pageNum = index + 1
|
Timber.tag("FileOpenPerf").d("[HTML] parseHtml COMPLETE | chapters=${chapters.size} | elapsed=${System.currentTimeMillis() - parseStart}ms")
|
||||||
val chapterTitle = if (rawChapters.size > 1) "Page $pageNum" else title
|
|
||||||
val fileName = "page_$pageNum.html"
|
|
||||||
val file = File(extractionDir, fileName)
|
|
||||||
|
|
||||||
val fullHtml = """
|
|
||||||
<!DOCTYPE html>
|
|
||||||
<html xmlns="http://www.w3.org/1999/xhtml">
|
|
||||||
<head>
|
|
||||||
<title>${title.replace("\"", """)}</title>
|
|
||||||
<style>${cssStyle}</style>
|
|
||||||
</head>
|
|
||||||
<body>
|
|
||||||
${rawText.trim()}
|
|
||||||
</body>
|
|
||||||
</html>
|
|
||||||
""".trimIndent()
|
|
||||||
|
|
||||||
file.writeText(fullHtml)
|
|
||||||
|
|
||||||
EpubChapter(
|
|
||||||
chapterId = "${bookId}_$pageNum",
|
|
||||||
absPath = fileName,
|
|
||||||
title = chapterTitle,
|
|
||||||
htmlFilePath = fileName,
|
|
||||||
plainTextContent = Jsoup.parse(fullHtml).text(),
|
|
||||||
htmlContent = "",
|
|
||||||
depth = 0,
|
|
||||||
isInToc = true
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}.awaitAll().filterNotNull()
|
|
||||||
|
|
||||||
Timber.tag("FileOpenPerf").d("[HTML] parseHtml COMPLETE | elapsed=${System.currentTimeMillis() - parseStart}ms")
|
|
||||||
|
|
||||||
val book = EpubBook(
|
val book = EpubBook(
|
||||||
fileName = originalBookNameHint,
|
fileName = originalBookNameHint,
|
||||||
title = title,
|
title = title,
|
||||||
author = author ?: "Unknown",
|
author = author,
|
||||||
language = "en",
|
language = "en",
|
||||||
coverImage = null,
|
coverImage = null,
|
||||||
chapters = chapters,
|
chapters = chapters,
|
||||||
|
|
@ -446,4 +498,45 @@ class SingleFileImporter(private val context: Context) {
|
||||||
|
|
||||||
return@withContext book
|
return@withContext book
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private fun writeHtmlChapter(
|
||||||
|
extractionDir: File,
|
||||||
|
bookId: String,
|
||||||
|
pageNum: Int,
|
||||||
|
title: String,
|
||||||
|
cssStyle: String,
|
||||||
|
bodyContent: String
|
||||||
|
): EpubChapter {
|
||||||
|
val chapterTitle = if (pageNum > 1 || bodyContent.contains("<page-break")) "Page $pageNum" else title
|
||||||
|
val fileName = "page_$pageNum.html"
|
||||||
|
val file = File(extractionDir, fileName)
|
||||||
|
|
||||||
|
val fullHtml = """
|
||||||
|
<!DOCTYPE html>
|
||||||
|
<html xmlns="http://www.w3.org/1999/xhtml">
|
||||||
|
<head>
|
||||||
|
<title>${title.replace("\"", """)}</title>
|
||||||
|
<style>${cssStyle}</style>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
${bodyContent.trim()}
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
|
""".trimIndent()
|
||||||
|
|
||||||
|
file.writeText(fullHtml)
|
||||||
|
|
||||||
|
val plainText = Jsoup.parse(fullHtml).text()
|
||||||
|
|
||||||
|
return EpubChapter(
|
||||||
|
chapterId = "${bookId}_$pageNum",
|
||||||
|
absPath = fileName,
|
||||||
|
title = chapterTitle,
|
||||||
|
htmlFilePath = fileName,
|
||||||
|
plainTextContent = plainText,
|
||||||
|
htmlContent = "",
|
||||||
|
depth = 0,
|
||||||
|
isInToc = true
|
||||||
|
)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -29,7 +29,7 @@ object PdfToHtmlGenerator {
|
||||||
val t0 = System.currentTimeMillis()
|
val t0 = System.currentTimeMillis()
|
||||||
Timber.tag(TAG).d("generateHtmlFile START | uri=$pdfUri | startPage=$startPage")
|
Timber.tag(TAG).d("generateHtmlFile START | uri=$pdfUri | startPage=$startPage")
|
||||||
|
|
||||||
val pdfiumCore = PdfiumCoreKt(Dispatchers.Default)
|
val pdfiumCore = PdfiumCoreProvider.core
|
||||||
val pfd = context.contentResolver.openFileDescriptor(pdfUri, "r") ?: run {
|
val pfd = context.contentResolver.openFileDescriptor(pdfUri, "r") ?: run {
|
||||||
Timber.tag(TAG).e("Failed to open ParcelFileDescriptor")
|
Timber.tag(TAG).e("Failed to open ParcelFileDescriptor")
|
||||||
return@withContext false
|
return@withContext false
|
||||||
|
|
|
||||||
|
|
@ -344,6 +344,12 @@ private const val PREF_EXTERNAL_DICT_PKG = "external_dictionary_package"
|
||||||
private const val PREF_EXTERNAL_TRANSLATE_PKG = "external_translate_package"
|
private const val PREF_EXTERNAL_TRANSLATE_PKG = "external_translate_package"
|
||||||
private const val PREF_EXTERNAL_SEARCH_PKG = "external_search_package"
|
private const val PREF_EXTERNAL_SEARCH_PKG = "external_search_package"
|
||||||
|
|
||||||
|
object PdfiumCoreProvider {
|
||||||
|
val core: PdfiumCoreKt by lazy {
|
||||||
|
PdfiumCoreKt(Dispatchers.Default)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private fun loadUseOnlineDict(context: Context): Boolean {
|
private fun loadUseOnlineDict(context: Context): Boolean {
|
||||||
@Suppress("KotlinConstantConditions") if (BuildConfig.FLAVOR == "oss") return false
|
@Suppress("KotlinConstantConditions") if (BuildConfig.FLAVOR == "oss") return false
|
||||||
val prefs = context.getSharedPreferences(SETTINGS_PREFS_NAME, Context.MODE_PRIVATE)
|
val prefs = context.getSharedPreferences(SETTINGS_PREFS_NAME, Context.MODE_PRIVATE)
|
||||||
|
|
@ -1426,7 +1432,7 @@ fun PdfViewerScreen(
|
||||||
var selectedTextBoxId by rememberSaveable { mutableStateOf<String?>(null) }
|
var selectedTextBoxId by rememberSaveable { mutableStateOf<String?>(null) }
|
||||||
val userHighlights = remember { mutableStateListOf<PdfUserHighlight>() }
|
val userHighlights = remember { mutableStateListOf<PdfUserHighlight>() }
|
||||||
val drawingState = remember { PdfDrawingState() }
|
val drawingState = remember { PdfDrawingState() }
|
||||||
val pdfiumCore = remember(context) { PdfiumCoreKt(Dispatchers.Default) }
|
val pdfiumCore = remember { PdfiumCoreProvider.core }
|
||||||
val verticalReaderState = rememberVerticalPdfReaderState()
|
val verticalReaderState = rememberVerticalPdfReaderState()
|
||||||
var virtualPages by remember { mutableStateOf<List<VirtualPage>>(emptyList()) }
|
var virtualPages by remember { mutableStateOf<List<VirtualPage>>(emptyList()) }
|
||||||
val totalDisplayPages by remember(virtualPages, totalPages) {
|
val totalDisplayPages by remember(virtualPages, totalPages) {
|
||||||
|
|
@ -2331,8 +2337,6 @@ fun PdfViewerScreen(
|
||||||
onToggleBookmark(currentPage)
|
onToggleBookmark(currentPage)
|
||||||
}
|
}
|
||||||
|
|
||||||
LaunchedEffect(pdfUri) { debugPdfLinks(context, pdfUri, pdfiumCore, this) }
|
|
||||||
|
|
||||||
LaunchedEffect(currentBookId) {
|
LaunchedEffect(currentBookId) {
|
||||||
if (currentBookId != null) {
|
if (currentBookId != null) {
|
||||||
val loaded = annotationRepository.loadAnnotations(currentBookId!!)
|
val loaded = annotationRepository.loadAnnotations(currentBookId!!)
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue