Atte149/book-story
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

🚀 Improve DocumentParser

Browse source 
* Directly embedded MarkdownParser logic inside DocumentParser, which allows to reduce iteration of the text by one (therefore increasing loading speed)
This commit is contained in:
Acclorite 2025-01-04 20:03:11 +02:00
parent d22dc17c64
commit cd2718b8d3
4 changed files with 103 additions and 119 deletions
Download patch file Download diff file Expand all files Collapse all files

61
app/src/main/java/ua/acclorite/book_story/data/parser/DocumentParser.kt
View file

@ -2,11 +2,15 @@ package ua.acclorite.book_story.data.parser
import kotlinx.coroutines.yield
import org.jsoup.nodes.Document
import ua.acclorite.book_story.domain.reader.ReaderText
import ua.acclorite.book_story.presentation.core.util.clearAllMarkdown
import ua.acclorite.book_story.presentation.core.util.clearMarkdown
import ua.acclorite.book_story.presentation.core.util.containsVisibleText
import javax.inject.Inject
class DocumentParser @Inject constructor() {
class DocumentParser @Inject constructor(
private val markdownParser: MarkdownParser
) {
/**
* Parses document to get it's text.
* Fixes issues such as manual line breaking in <p>.
@ -14,17 +18,17 @@ class DocumentParser @Inject constructor() {
*
* @return Parsed text line by line with Markdown(all lines are not blank).
*/
suspend fun Document.parseDocument(): List<String> {
val lines = mutableListOf<String>()
suspend fun Document.parseDocument(includeChapter: Boolean = true): List<ReaderText> {
yield()
val readerText = mutableListOf<ReaderText>()
var chapterAdded = false
body().apply {
// Remove manual line breaks from all <p>
select("p").forEach { element ->
// Remove manual line breaks from all <p>, <a>
select("p, a").forEach { element ->
yield()
element.html(element.html().replace(Regex("\\n+"), " "))
element.append("\n")
}
// Remove <head>'s title
@ -42,10 +46,9 @@ class DocumentParser @Inject constructor() {
val link = element.attr("href")
if (!link.startsWith("http") || element.wholeText().isBlank()) return@forEach
element.prepend("[")
element.append("](${element.attr("href")})")
element.prepend("[")/* text in between */.append("](${element.attr("href")})")
}
}.wholeText().lines().forEach { line ->
}.wholeText().lines().forEachIndexed { index, line ->
yield()
val formattedLine = line.replace(
@ -56,13 +59,45 @@ class DocumentParser @Inject constructor() {
Regex("""_\s*(.*?)\s*_"""), "_$1_"
).trim()
if (formattedLine.clearMarkdown().containsVisibleText()) {
lines.add(formattedLine)
if (line.containsVisibleText()) {
when (line) {
"***", "---" -> readerText.add(ReaderText.Separator)
else -> {
if (
!chapterAdded &&
formattedLine.clearAllMarkdown().containsVisibleText() &&
includeChapter
) {
readerText.add(
0, ReaderText.Chapter(
title = formattedLine.clearAllMarkdown()
)
)
chapterAdded = true
} else if (
formattedLine.clearMarkdown().containsVisibleText()
) {
readerText.add(
ReaderText.Text(
line = markdownParser.parse(formattedLine)
)
)
}
}
}
}
}
yield()
return lines
if (
readerText.filterIsInstance<ReaderText.Text>().isEmpty() ||
(includeChapter && readerText.filterIsInstance<ReaderText.Chapter>().isEmpty())
) {
return emptyList()
}
return readerText
}
}

11
app/src/main/java/ua/acclorite/book_story/data/parser/TextParserImpl.kt
View file

@ -1,6 +1,8 @@
package ua.acclorite.book_story.data.parser
import android.util.Log
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.withContext
import ua.acclorite.book_story.data.parser.epub.EpubTextParser
import ua.acclorite.book_story.data.parser.fb2.Fb2TextParser
import ua.acclorite.book_story.data.parser.html.HtmlTextParser
@ -13,10 +15,13 @@ import javax.inject.Inject
private const val TEXT_PARSER = "Text Parser"
class TextParserImpl @Inject constructor(
// Markdown parser (Markdown)
private val txtTextParser: TxtTextParser,
private val pdfTextParser: PdfTextParser,
private val epubTextParser: EpubTextParser,
private val fb2TextParser: Fb2TextParser,
// Document parser (HTML+Markdown)
private val epubTextParser: EpubTextParser,
private val htmlTextParser: HtmlTextParser
) : TextParser {
override suspend fun parse(file: File): List<ReaderText> {
@ -26,7 +31,8 @@ class TextParserImpl @Inject constructor(
}
val fileFormat = ".${file.extension}".lowercase().trim()
return when (fileFormat) {
return withContext(Dispatchers.IO) {
when (fileFormat) {
".pdf" -> {
pdfTextParser.parse(file)
}
@ -66,3 +72,4 @@ class TextParserImpl @Inject constructor(
}
}
}
}

62
app/src/main/java/ua/acclorite/book_story/data/parser/epub/EpubTextParser.kt
View file

@ -13,12 +13,10 @@ import kotlinx.coroutines.withContext
import kotlinx.coroutines.yield
import org.jsoup.Jsoup
import ua.acclorite.book_story.data.parser.DocumentParser
import ua.acclorite.book_story.data.parser.MarkdownParser
import ua.acclorite.book_story.data.parser.TextParser
import ua.acclorite.book_story.domain.reader.ReaderText
import ua.acclorite.book_story.presentation.core.util.addAll
import ua.acclorite.book_story.presentation.core.util.clearAllMarkdown
import ua.acclorite.book_story.presentation.core.util.clearMarkdown
import ua.acclorite.book_story.presentation.core.util.containsVisibleText
import java.io.File
import java.util.concurrent.ConcurrentLinkedQueue
import java.util.zip.ZipEntry
@ -32,7 +30,6 @@ private typealias Title = String
private val dispatcher = Dispatchers.IO.limitedParallelism(2)
class EpubTextParser @Inject constructor(
private val markdownParser: MarkdownParser,
private val documentParser: DocumentParser
) : TextParser {
@ -150,56 +147,32 @@ class EpubTextParser @Inject constructor(
) {
// Getting all text
val content = zip.getInputStream(entry).bufferedReader().use { it.readText() }
var text = documentParser.run {
Jsoup.parse(content).parseDocument()
}
val readerText = mutableListOf<ReaderText>()
var readerText = documentParser.run {
Jsoup.parse(content).parseDocument(includeChapter = false)
}.toMutableList()
// Adding chapter title from TOC if found
var chapterAdded = false
getChapterTitleFromToc(
chapterSource = entry.name,
chapterTitleMap = chapterTitleMap
).apply {
if (this == null) return@apply
val chapterTitle = this ?: run {
val firstVisibleText = readerText.firstOrNull { line ->
line is ReaderText.Text && line.line.text.containsVisibleText()
} as? ReaderText.Text ?: return
firstVisibleText.line.text
}
readerText = readerText.dropWhile { line ->
(line is ReaderText.Text && line.line.text.lowercase() == chapterTitle.lowercase())
}.toMutableList()
readerText.add(
0,
ReaderText.Chapter(
title = this
title = chapterTitle
)
)
chapterAdded = true
text = text.dropWhile { line ->
line.clearMarkdown().lowercase() == this.lowercase()
}
}
// Format and add text
text.forEach { line ->
yield()
if (line.isNotBlank()) {
when (line) {
"***", "---" -> readerText.add(
ReaderText.Separator
)
else -> {
if (!chapterAdded && line.clearAllMarkdown().isNotBlank()) {
readerText.add(
0, ReaderText.Chapter(
title = line.clearAllMarkdown()
)
)
chapterAdded = true
} else readerText.add(
ReaderText.Text(
line = markdownParser.parse(line)
)
)
}
}
}
}
if (
@ -260,6 +233,7 @@ class EpubTextParser @Inject constructor(
.getOrElse(chapterSource.substringAfterLast("/")) { null }
?.joinToString(separator = " / ")
?.ifBlank { null }
?.trim()
}
/**

34
app/src/main/java/ua/acclorite/book_story/data/parser/html/HtmlTextParser.kt
View file

@ -4,17 +4,14 @@ import android.util.Log
import kotlinx.coroutines.yield
import org.jsoup.Jsoup
import ua.acclorite.book_story.data.parser.DocumentParser
import ua.acclorite.book_story.data.parser.MarkdownParser
import ua.acclorite.book_story.data.parser.TextParser
import ua.acclorite.book_story.domain.reader.ReaderText
import ua.acclorite.book_story.presentation.core.util.clearAllMarkdown
import java.io.File
import javax.inject.Inject
private const val HTML_TAG = "HTML Parser"
class HtmlTextParser @Inject constructor(
private val markdownParser: MarkdownParser,
private val documentParser: DocumentParser
) : TextParser {
@ -22,38 +19,9 @@ class HtmlTextParser @Inject constructor(
Log.i(HTML_TAG, "Started HTML parsing: ${file.name}.")
return try {
var chapterAdded = false
val documentLines = documentParser.run {
val readerText = documentParser.run {
Jsoup.parse(file).parseDocument()
}
val readerText = mutableListOf<ReaderText>()
for (line in documentLines) {
yield()
if (line.isNotBlank()) {
when (line) {
"***", "---" -> readerText.add(
ReaderText.Separator
)
else -> {
if (!chapterAdded && line.clearAllMarkdown().isNotBlank()) {
readerText.add(
0, ReaderText.Chapter(
title = line.clearAllMarkdown()
)
)
chapterAdded = true
} else readerText.add(
ReaderText.Text(
line = markdownParser.parse(line)
)
)
}
}
}
}
yield()