From cd2718b8d36215861c31a480964d62ade69a6929 Mon Sep 17 00:00:00 2001 From: Acclorite <140836141+Acclorite@users.noreply.github.com> Date: Sat, 4 Jan 2025 20:03:11 +0200 Subject: [PATCH] =?UTF-8?q?=F0=9F=9A=80=20Improve=20DocumentParser?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Directly embedded MarkdownParser logic inside DocumentParser, which allows to reduce iteration of the text by one (therefore increasing loading speed) --- .../book_story/data/parser/DocumentParser.kt | 61 +++++++++++++---- .../book_story/data/parser/TextParserImpl.kt | 65 ++++++++++--------- .../data/parser/epub/EpubTextParser.kt | 62 +++++------------- .../data/parser/html/HtmlTextParser.kt | 34 +--------- 4 files changed, 103 insertions(+), 119 deletions(-) diff --git a/app/src/main/java/ua/acclorite/book_story/data/parser/DocumentParser.kt b/app/src/main/java/ua/acclorite/book_story/data/parser/DocumentParser.kt index 3c76fcd4..d082cc46 100644 --- a/app/src/main/java/ua/acclorite/book_story/data/parser/DocumentParser.kt +++ b/app/src/main/java/ua/acclorite/book_story/data/parser/DocumentParser.kt @@ -2,11 +2,15 @@ package ua.acclorite.book_story.data.parser import kotlinx.coroutines.yield import org.jsoup.nodes.Document +import ua.acclorite.book_story.domain.reader.ReaderText +import ua.acclorite.book_story.presentation.core.util.clearAllMarkdown import ua.acclorite.book_story.presentation.core.util.clearMarkdown import ua.acclorite.book_story.presentation.core.util.containsVisibleText import javax.inject.Inject -class DocumentParser @Inject constructor() { +class DocumentParser @Inject constructor( + private val markdownParser: MarkdownParser +) { /** * Parses document to get it's text. * Fixes issues such as manual line breaking in

. @@ -14,17 +18,17 @@ class DocumentParser @Inject constructor() { * * @return Parsed text line by line with Markdown(all lines are not blank). */ - suspend fun Document.parseDocument(): List { - val lines = mutableListOf() - + suspend fun Document.parseDocument(includeChapter: Boolean = true): List { yield() + val readerText = mutableListOf() + var chapterAdded = false + body().apply { - // Remove manual line breaks from all

- select("p").forEach { element -> + // Remove manual line breaks from all

, + select("p, a").forEach { element -> yield() element.html(element.html().replace(Regex("\\n+"), " ")) - element.append("\n") } // Remove 's title @@ -42,10 +46,9 @@ class DocumentParser @Inject constructor() { val link = element.attr("href") if (!link.startsWith("http") || element.wholeText().isBlank()) return@forEach - element.prepend("[") - element.append("](${element.attr("href")})") + element.prepend("[")/* text in between */.append("](${element.attr("href")})") } - }.wholeText().lines().forEach { line -> + }.wholeText().lines().forEachIndexed { index, line -> yield() val formattedLine = line.replace( @@ -56,13 +59,45 @@ class DocumentParser @Inject constructor() { Regex("""_\s*(.*?)\s*_"""), "_$1_" ).trim() - if (formattedLine.clearMarkdown().containsVisibleText()) { - lines.add(formattedLine) + if (line.containsVisibleText()) { + when (line) { + "***", "---" -> readerText.add(ReaderText.Separator) + + else -> { + if ( + !chapterAdded && + formattedLine.clearAllMarkdown().containsVisibleText() && + includeChapter + ) { + readerText.add( + 0, ReaderText.Chapter( + title = formattedLine.clearAllMarkdown() + ) + ) + chapterAdded = true + } else if ( + formattedLine.clearMarkdown().containsVisibleText() + ) { + readerText.add( + ReaderText.Text( + line = markdownParser.parse(formattedLine) + ) + ) + } + } + } } } yield() - return lines + if ( + readerText.filterIsInstance().isEmpty() || + (includeChapter && readerText.filterIsInstance().isEmpty()) + ) { + return emptyList() + } + + return readerText } } \ No newline at end of file diff --git a/app/src/main/java/ua/acclorite/book_story/data/parser/TextParserImpl.kt b/app/src/main/java/ua/acclorite/book_story/data/parser/TextParserImpl.kt index 1b97e632..cfbd4d06 100644 --- a/app/src/main/java/ua/acclorite/book_story/data/parser/TextParserImpl.kt +++ b/app/src/main/java/ua/acclorite/book_story/data/parser/TextParserImpl.kt @@ -1,6 +1,8 @@ package ua.acclorite.book_story.data.parser import android.util.Log +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext import ua.acclorite.book_story.data.parser.epub.EpubTextParser import ua.acclorite.book_story.data.parser.fb2.Fb2TextParser import ua.acclorite.book_story.data.parser.html.HtmlTextParser @@ -13,10 +15,13 @@ import javax.inject.Inject private const val TEXT_PARSER = "Text Parser" class TextParserImpl @Inject constructor( + // Markdown parser (Markdown) private val txtTextParser: TxtTextParser, private val pdfTextParser: PdfTextParser, - private val epubTextParser: EpubTextParser, private val fb2TextParser: Fb2TextParser, + + // Document parser (HTML+Markdown) + private val epubTextParser: EpubTextParser, private val htmlTextParser: HtmlTextParser ) : TextParser { override suspend fun parse(file: File): List { @@ -26,42 +31,44 @@ class TextParserImpl @Inject constructor( } val fileFormat = ".${file.extension}".lowercase().trim() - return when (fileFormat) { - ".pdf" -> { - pdfTextParser.parse(file) - } + return withContext(Dispatchers.IO) { + when (fileFormat) { + ".pdf" -> { + pdfTextParser.parse(file) + } - ".epub" -> { - epubTextParser.parse(file) - } + ".epub" -> { + epubTextParser.parse(file) + } - ".txt" -> { - txtTextParser.parse(file) - } + ".txt" -> { + txtTextParser.parse(file) + } - ".fb2" -> { - fb2TextParser.parse(file) - } + ".fb2" -> { + fb2TextParser.parse(file) + } - ".zip" -> { - epubTextParser.parse(file) - } + ".zip" -> { + epubTextParser.parse(file) + } - ".html" -> { - htmlTextParser.parse(file) - } + ".html" -> { + htmlTextParser.parse(file) + } - ".htm" -> { - htmlTextParser.parse(file) - } + ".htm" -> { + htmlTextParser.parse(file) + } - ".md" -> { - htmlTextParser.parse(file) - } + ".md" -> { + htmlTextParser.parse(file) + } - else -> { - Log.e(TEXT_PARSER, "Wrong file format, could not find supported extension.") - emptyList() + else -> { + Log.e(TEXT_PARSER, "Wrong file format, could not find supported extension.") + emptyList() + } } } } diff --git a/app/src/main/java/ua/acclorite/book_story/data/parser/epub/EpubTextParser.kt b/app/src/main/java/ua/acclorite/book_story/data/parser/epub/EpubTextParser.kt index 57b93c7e..1be3dcca 100644 --- a/app/src/main/java/ua/acclorite/book_story/data/parser/epub/EpubTextParser.kt +++ b/app/src/main/java/ua/acclorite/book_story/data/parser/epub/EpubTextParser.kt @@ -13,12 +13,10 @@ import kotlinx.coroutines.withContext import kotlinx.coroutines.yield import org.jsoup.Jsoup import ua.acclorite.book_story.data.parser.DocumentParser -import ua.acclorite.book_story.data.parser.MarkdownParser import ua.acclorite.book_story.data.parser.TextParser import ua.acclorite.book_story.domain.reader.ReaderText import ua.acclorite.book_story.presentation.core.util.addAll -import ua.acclorite.book_story.presentation.core.util.clearAllMarkdown -import ua.acclorite.book_story.presentation.core.util.clearMarkdown +import ua.acclorite.book_story.presentation.core.util.containsVisibleText import java.io.File import java.util.concurrent.ConcurrentLinkedQueue import java.util.zip.ZipEntry @@ -32,7 +30,6 @@ private typealias Title = String private val dispatcher = Dispatchers.IO.limitedParallelism(2) class EpubTextParser @Inject constructor( - private val markdownParser: MarkdownParser, private val documentParser: DocumentParser ) : TextParser { @@ -150,56 +147,32 @@ class EpubTextParser @Inject constructor( ) { // Getting all text val content = zip.getInputStream(entry).bufferedReader().use { it.readText() } - var text = documentParser.run { - Jsoup.parse(content).parseDocument() - } - val readerText = mutableListOf() + var readerText = documentParser.run { + Jsoup.parse(content).parseDocument(includeChapter = false) + }.toMutableList() // Adding chapter title from TOC if found - var chapterAdded = false getChapterTitleFromToc( chapterSource = entry.name, chapterTitleMap = chapterTitleMap ).apply { - if (this == null) return@apply + val chapterTitle = this ?: run { + val firstVisibleText = readerText.firstOrNull { line -> + line is ReaderText.Text && line.line.text.containsVisibleText() + } as? ReaderText.Text ?: return + firstVisibleText.line.text + } + + readerText = readerText.dropWhile { line -> + (line is ReaderText.Text && line.line.text.lowercase() == chapterTitle.lowercase()) + }.toMutableList() + readerText.add( + 0, ReaderText.Chapter( - title = this + title = chapterTitle ) ) - chapterAdded = true - - text = text.dropWhile { line -> - line.clearMarkdown().lowercase() == this.lowercase() - } - } - - // Format and add text - text.forEach { line -> - yield() - - if (line.isNotBlank()) { - when (line) { - "***", "---" -> readerText.add( - ReaderText.Separator - ) - - else -> { - if (!chapterAdded && line.clearAllMarkdown().isNotBlank()) { - readerText.add( - 0, ReaderText.Chapter( - title = line.clearAllMarkdown() - ) - ) - chapterAdded = true - } else readerText.add( - ReaderText.Text( - line = markdownParser.parse(line) - ) - ) - } - } - } } if ( @@ -260,6 +233,7 @@ class EpubTextParser @Inject constructor( .getOrElse(chapterSource.substringAfterLast("/")) { null } ?.joinToString(separator = " / ") ?.ifBlank { null } + ?.trim() } /** diff --git a/app/src/main/java/ua/acclorite/book_story/data/parser/html/HtmlTextParser.kt b/app/src/main/java/ua/acclorite/book_story/data/parser/html/HtmlTextParser.kt index bc591c9c..d18341ad 100644 --- a/app/src/main/java/ua/acclorite/book_story/data/parser/html/HtmlTextParser.kt +++ b/app/src/main/java/ua/acclorite/book_story/data/parser/html/HtmlTextParser.kt @@ -4,17 +4,14 @@ import android.util.Log import kotlinx.coroutines.yield import org.jsoup.Jsoup import ua.acclorite.book_story.data.parser.DocumentParser -import ua.acclorite.book_story.data.parser.MarkdownParser import ua.acclorite.book_story.data.parser.TextParser import ua.acclorite.book_story.domain.reader.ReaderText -import ua.acclorite.book_story.presentation.core.util.clearAllMarkdown import java.io.File import javax.inject.Inject private const val HTML_TAG = "HTML Parser" class HtmlTextParser @Inject constructor( - private val markdownParser: MarkdownParser, private val documentParser: DocumentParser ) : TextParser { @@ -22,38 +19,9 @@ class HtmlTextParser @Inject constructor( Log.i(HTML_TAG, "Started HTML parsing: ${file.name}.") return try { - var chapterAdded = false - val documentLines = documentParser.run { + val readerText = documentParser.run { Jsoup.parse(file).parseDocument() } - val readerText = mutableListOf() - - for (line in documentLines) { - yield() - - if (line.isNotBlank()) { - when (line) { - "***", "---" -> readerText.add( - ReaderText.Separator - ) - - else -> { - if (!chapterAdded && line.clearAllMarkdown().isNotBlank()) { - readerText.add( - 0, ReaderText.Chapter( - title = line.clearAllMarkdown() - ) - ) - chapterAdded = true - } else readerText.add( - ReaderText.Text( - line = markdownParser.parse(line) - ) - ) - } - } - } - } yield()