Improved text parsers, fixed some issues: Epub, Pdf (v1.0.0)

app/src/main/java/ua/acclorite/book_story/data/parser/epub/EpubTextParser.kt

@@ -4,6 +4,8 @@ import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.withContext
 import nl.siegmann.epublib.epub.EpubReader
 import org.jsoup.Jsoup
+import org.jsoup.nodes.Document.OutputSettings
+import org.jsoup.safety.Safelist
 import ua.acclorite.book_story.R
 import ua.acclorite.book_story.data.parser.TextParser
 import ua.acclorite.book_story.domain.model.StringWithId
@@ -16,6 +18,7 @@ import java.io.InputStreamReader
 import java.nio.charset.Charset
 import javax.inject.Inject
 
+
 class EpubTextParser @Inject constructor() : TextParser {
 
     override suspend fun parse(file: File): Resource<List<StringWithId>> {
@@ -44,9 +47,8 @@ class EpubTextParser @Inject constructor() : TextParser {
                     inputStream.close()
                 }
 
-                while (withContext(Dispatchers.IO) {
-                        reader.readLine()
-                    }.also { line = it } != null) {
+                while (withContext(Dispatchers.IO) { reader.readLine() }
+                        .also { line = it } != null) {
                     unformattedText.append(line).append("\n")
                 }
 
@@ -57,7 +59,24 @@ class EpubTextParser @Inject constructor() : TextParser {
 
             val stringWithIds = mutableListOf<StringWithId>()
 
-            Jsoup.parse(unformattedText.toString()).wholeText().split("\n").forEach {
+            val parsedText = Jsoup.parse(unformattedText.toString())
+            parsedText.outputSettings(OutputSettings().prettyPrint(false))
+            parsedText.select("br").append("\n")
+            parsedText.select("p").prepend("\n")
+            parsedText.select("em").append(" ").prepend("")
+
+            val formattedText = Jsoup.clean(
+                parsedText.html(),
+                "",
+                Safelist.none(),
+                OutputSettings().prettyPrint(false)
+            )
+
+            formattedText
+                .replace("&nbsp;", "")
+                .replace("\u00a0", "")
+                .split("\n")
+                .forEach {
                     if (it.isNotBlank()) {
                         stringWithIds.add(StringWithId(it.trim()))
                     }

app/src/main/java/ua/acclorite/book_story/data/parser/pdf/PdfTextParser.kt

@@ -46,12 +46,62 @@ class PdfTextParser @Inject constructor(private val application: Application) :
 
             val lines = mutableListOf<String>()
             unformattedLines.forEachIndexed { index, string ->
-                if (string.trim().first().isLowerCase() && index > 0) {
-                    lines[lines.lastIndex] += " ${string.trim()}"
+                try {
+                    val line = string.trim()
+
+                    if (index == 0) {
+                        lines.add(line)
+                        return@forEachIndexed
+                    }
+
+                    if (
+                        line.all {
+                            if (it == ' ') {
+                                true
+                            } else if (it.isUpperCase() || it.isDigit()) {
+                                true
                             } else {
-                    lines.add(string.trim())
+                                false
                             }
                         }
+                    ) {
+                        return@forEachIndexed
+                    }
+
+                    if (line.all { it.isDigit() }) {
+                        return@forEachIndexed
+                    }
+
+                    if (line.first().isLowerCase()) {
+                        val currentLine = lines[lines.lastIndex]
+
+                        if (currentLine.last() == '-') {
+                            if (currentLine[currentLine.lastIndex - 1].isLowerCase()) {
+                                lines[lines.lastIndex] = currentLine.dropLast(1) + line
+                                return@forEachIndexed
+                            }
+                        }
+
+                        lines[lines.lastIndex] += " $line"
+                        return@forEachIndexed
+                    }
+
+                    if (line.first().isUpperCase() || line.first().isDigit()) {
+                        lines.add(line)
+                        return@forEachIndexed
+                    }
+
+                    if (line.first().isLetter()) {
+                        lines[lines.lastIndex] += " $line"
+                        return@forEachIndexed
+                    }
+
+                } catch (e: Exception) {
+                    e.printStackTrace()
+                    return@forEachIndexed
+                }
+            }
+
             lines.forEach { line ->
                 stringWithIds.add(StringWithId(line.trim()))
             }
@@ -62,6 +112,7 @@ class PdfTextParser @Inject constructor(private val application: Application) :
 
             return Resource.Success(stringWithIds)
         } catch (e: Exception) {
+            e.printStackTrace()
             return Resource.Error(
                 UIText.StringResource(
                     R.string.error_query,