Enhancement/message parsing (#369)

* Test custom regex

* Remove statics, use lists, and add amp (#366)

* Remove statics, use lists, and add amp

* Remove another jvmstatic

* Update parser for messages

* Update dependencies

* Update travis

* Use parsing for background im fetcher

* Update changelog

* Update changelog

7 files changed, 179 insertions, 74 deletions

diff --git a/app/src/main/kotlin/com/pitchedapps/frost/facebook/FbUrlFormatter.kt b/app/src/main/kotlin/com/pitchedapps/frost/facebook/FbUrlFormatter.kt
index 542277b8..4879e68b 100644
--- a/app/src/main/kotlin/com/pitchedapps/frost/facebook/FbUrlFormatter.kt
+++ b/app/src/main/kotlin/com/pitchedapps/frost/facebook/FbUrlFormatter.kt
@@ -7,10 +7,13 @@ import com.pitchedapps.frost.utils.L
  *
  * Custom url builder so we can easily test it without the Android framework
  */
-val String.formattedFbUrl: String
-    get() = FbUrlFormatter(this).toString()
+inline val String.formattedFbUrl: String
+    get() = FbUrlFormatter(this, false).toString()
 
-class FbUrlFormatter(url: String) {
+inline val String.formattedFbUrlCss: String
+    get() = FbUrlFormatter(this, true).toString()
+
+class FbUrlFormatter(url: String, css: Boolean = false) {
     private val queries = mutableMapOf<String, String>()
     private val cleaned: String
 
@@ -30,6 +33,8 @@ class FbUrlFormatter(url: String) {
             discardable.forEach { cleanedUrl = cleanedUrl.replace(it, "", true) }
             val changed = cleanedUrl != url //note that discardables strip away the first '?'
             converter.forEach { (k, v) -> cleanedUrl = cleanedUrl.replace(k, v, true) }
+            //must decode for css
+            if (css) decoder.forEach { (k, v) -> cleanedUrl = cleanedUrl.replace(k, v, true) }
             val qm = cleanedUrl.indexOf(if (changed) "&" else "?")
             if (qm > -1) {
                 cleanedUrl.substring(qm + 1).split("&").forEach {
@@ -39,8 +44,10 @@ class FbUrlFormatter(url: String) {
                 cleanedUrl = cleanedUrl.substring(0, qm)
             }
             //only decode non query portion of the url
-            decoder.forEach { (k, v) -> cleanedUrl = cleanedUrl.replace(k, v, true) }
+            if (!css) decoder.forEach { (k, v) -> cleanedUrl = cleanedUrl.replace(k, v, true) }
             discardableQueries.forEach { queries.remove(it) }
+            //final cleanup
+            misc.forEach { (k, v) -> cleanedUrl = cleanedUrl.replace(k, v, true) }
             if (cleanedUrl.startsWith("#!")) cleanedUrl = cleanedUrl.substring(2)
             if (cleanedUrl.startsWith("/")) cleanedUrl = FB_URL_BASE + cleanedUrl.substring(1)
             cleanedUrl = cleanedUrl.replaceFirst(".facebook.com//", ".facebook.com/") //sometimes we are given a bad url
@@ -72,7 +79,6 @@ class FbUrlFormatter(url: String) {
          * Taken from FaceSlim
          * https://github.com/indywidualny/FaceSlim/blob/master/app/src/main/java/org/indywidualni/fblite/util/Miscellany.java
          */
-        @JvmStatic
         val discardable = arrayOf(
                 "http://lm.facebook.com/l.php?u=",
                 "https://lm.facebook.com/l.php?u=",
@@ -83,11 +89,13 @@ class FbUrlFormatter(url: String) {
                 "/video_redirect/?src="
         )
 
-        @JvmStatic
+        val misc = listOf(
+                "&amp;" to "&"
+        )
+
         val discardableQueries = arrayOf("ref", "refid")
 
-        @JvmStatic
-        val decoder = mapOf(
+        val decoder = listOf(
                 "%3C" to "<", "%3E" to ">", "%23" to "#", "%25" to "%",
                 "%7B" to "{", "%7D" to "}", "%7C" to "|", "%5C" to "\\",
                 "%5E" to "^", "%7E" to "~", "%5B" to "[", "%5D" to "]",
@@ -97,8 +105,7 @@ class FbUrlFormatter(url: String) {
                 "%20" to " "
         )
 
-        @JvmStatic
-        val cssDecoder = mapOf(
+        val cssDecoder = listOf(
                 "\\3C " to "<", "\\3E " to ">", "\\23 " to "#", "\\25 " to "%",
                 "\\7B " to "{", "\\7D " to "}", "\\7C " to "|", "\\5C " to "\\",
                 "\\5E " to "^", "\\7E " to "~", "\\5B " to "[", "\\5D " to "]",
@@ -108,8 +115,7 @@ class FbUrlFormatter(url: String) {
                 "%20" to " "
         )
 
-        @JvmStatic
-        val converter = mapOf(
+        val converter = listOf(
                 "\\3C " to "%3C", "\\3E " to "%3E", "\\23 " to "%23", "\\25 " to "%25",
                 "\\7B " to "%7B", "\\7D " to "%7D", "\\7C " to "%7C", "\\5C " to "%5C",
                 "\\5E " to "%5E", "\\7E " to "%7E", "\\5B " to "%5B", "\\5D " to "%5D",
diff --git a/app/src/main/kotlin/com/pitchedapps/frost/parsers/FrostParser.kt b/app/src/main/kotlin/com/pitchedapps/frost/parsers/FrostParser.kt
new file mode 100644
index 00000000..86b280a8
--- /dev/null
+++ b/app/src/main/kotlin/com/pitchedapps/frost/parsers/FrostParser.kt
@@ -0,0 +1,44 @@
+package com.pitchedapps.frost.parsers
+
+/**
+ * Created by Allan Wang on 2017-10-06.
+ *
+ * Interface for a given parser
+ * Use cases should be attached as delegates to objects that implement this interface
+ */
+interface FrostParser<T> {
+    fun parse(text: String?): T?
+    fun debug(text: String?): String
+}
+
+internal abstract class FrostParserBase<T> : FrostParser<T> {
+    override final fun parse(text: String?): T?
+            = if (text == null) null else parseImpl(text)
+
+    protected abstract fun parseImpl(text: String): T?
+
+    override final fun debug(text: String?): String {
+        val result = mutableListOf<String>()
+        result.add("Testing parser for ${this::class.java.simpleName}")
+        if (text == null) {
+            result.add("Input is null")
+            return result.joinToString("\n")
+        }
+        val output = parseImpl(text)
+        if (output == null) {
+            result.add("Output is null")
+            return result.joinToString("\n")
+        }
+        debugImpl(output, result)
+        return result.joinToString("\n")
+    }
+
+    protected abstract fun debugImpl(data: T, result: MutableList<String>)
+}
+
+object FrostRegex {
+    val epoch = Regex(":([0-9]+)")
+    val notifId = Regex("notif_id\":([0-9]+)")
+    val messageNotifId = Regex("thread_fbid_([0-9]+)")
+    val profilePicture = Regex("url\\(\"(.*?)\"\\)")
+}
\ No newline at end of file
diff --git a/app/src/main/kotlin/com/pitchedapps/frost/parsers/MessageParser.kt b/app/src/main/kotlin/com/pitchedapps/frost/parsers/MessageParser.kt
new file mode 100644
index 00000000..dbe2c0bb
--- /dev/null
+++ b/app/src/main/kotlin/com/pitchedapps/frost/parsers/MessageParser.kt
@@ -0,0 +1,85 @@
+package com.pitchedapps.frost.parsers
+
+import com.pitchedapps.frost.facebook.formattedFbUrl
+import com.pitchedapps.frost.facebook.formattedFbUrlCss
+import com.pitchedapps.frost.utils.L
+import org.apache.commons.text.StringEscapeUtils
+import org.jsoup.Jsoup
+import org.jsoup.nodes.Element
+
+/**
+ * Created by Allan Wang on 2017-10-06.
+ *
+ * In Facebook, messages are passed through scripts and loaded into view via react afterwards
+ * We can parse out the content we want directly and load it ourselves
+ *
+ */
+object MessageParser : FrostParser<Triple<List<FrostThread>, FrostLink?, List<FrostLink>>> by MessageParserImpl()
+
+data class FrostThread(val id: Int, val img: String, val title: String, val time: Long, val url: String, val unread: Boolean, val content: String?)
+
+data class FrostLink(val text: String, val href: String)
+
+private class MessageParserImpl : FrostParserBase<Triple<List<FrostThread>, FrostLink?, List<FrostLink>>>() {
+
+    override fun parseImpl(text: String): Triple<List<FrostThread>, FrostLink?, List<FrostLink>>? {
+        var content = StringEscapeUtils.unescapeEcmaScript(text)
+        val begin = content.indexOf("id=\"threadlist_rows\"")
+        if (begin <= 0) {
+            L.d("Threadlist not found")
+            return null
+        }
+        content = content.substring(begin)
+        val end = content.indexOf("</script>")
+        if (end <= 0) {
+            L.d("Script tail not found")
+            return null
+        }
+        content = content.substring(0, end).substringBeforeLast("</div>")
+        val body = Jsoup.parseBodyFragment("<div $content")
+        val threadList = body.getElementById("threadlist_rows")
+        val threads: List<FrostThread> = threadList.getElementsByAttributeValueContaining("id", "thread_fbid_")
+                .mapNotNull { parseMessage(it) }
+        val seeMore = parseLink(body.getElementById("see_older_threads"))
+        val extraLinks = threadList.nextElementSibling().select("a")
+                .mapNotNull { parseLink(it) }
+        return Triple(threads, seeMore, extraLinks)
+    }
+
+    private fun parseMessage(element: Element): FrostThread? {
+        val a = element.getElementsByTag("a").first() ?: return null
+        val abbr = element.getElementsByTag("abbr")
+        println(abbr.attr("data-store"))
+        val epoch = FrostRegex.epoch.find(abbr.attr("data-store"))
+                ?.groupValues?.getOrNull(1)?.toLongOrNull() ?: -1L
+        //fetch id
+        val id = FrostRegex.messageNotifId.find(element.id())
+                ?.groupValues?.getOrNull(1)?.toLongOrNull() ?: System.currentTimeMillis()
+        val content = element.select("span.snippet").firstOrNull()?.text()?.trim()
+        //fetch convo pic
+        val p = element.select("i.img[style*=url]")
+        val pUrl = FrostRegex.profilePicture.find(p.attr("style"))?.groups?.get(1)?.value?.formattedFbUrl ?: ""
+        L.v("url", a.attr("href"))
+        return FrostThread(
+                id = id.toInt(),
+                img = pUrl.formattedFbUrlCss,
+                title = a.text(),
+                time = epoch,
+                url = a.attr("href").formattedFbUrlCss,
+                unread = !element.hasClass("acw"),
+                content = content
+        )
+    }
+
+    private fun parseLink(element: Element?): FrostLink? {
+        val a = element?.getElementsByTag("a")?.first() ?: return null
+        return FrostLink(a.text(), a.attr("href"))
+    }
+
+    override fun debugImpl(data: Triple<List<FrostThread>, FrostLink?, List<FrostLink>>, result: MutableList<String>) {
+        result.addAll(data.first.map { it.toString() })
+        result.add("See more link:")
+        result.add("\t${data.second}")
+        result.addAll(data.third.map { it.toString() })
+    }
+}
diff --git a/app/src/main/kotlin/com/pitchedapps/frost/services/FrostNotifications.kt b/app/src/main/kotlin/com/pitchedapps/frost/services/FrostNotifications.kt
index b892af91..b1f26d99 100644
--- a/app/src/main/kotlin/com/pitchedapps/frost/services/FrostNotifications.kt
+++ b/app/src/main/kotlin/com/pitchedapps/frost/services/FrostNotifications.kt
@@ -24,6 +24,7 @@ import com.pitchedapps.frost.dbflow.CookieModel
 import com.pitchedapps.frost.enums.OverlayContext
 import com.pitchedapps.frost.facebook.FbItem
 import com.pitchedapps.frost.facebook.formattedFbUrl
+import com.pitchedapps.frost.parsers.FrostThread
 import com.pitchedapps.frost.utils.*
 import org.jetbrains.anko.runOnUiThread
 
@@ -159,7 +160,10 @@ data class NotificationContent(val data: CookieModel,
                                val title: String? = null,
                                val text: String,
                                val timestamp: Long,
-                               val profileUrl: String)
+                               val profileUrl: String) {
+    constructor(data:CookieModel, thread: FrostThread)
+            :this(data, thread.id, thread.url, thread.title, thread.content ?: "", thread.time, thread.img)
+}
 
 const val NOTIFICATION_PERIODIC_JOB = 7
 
diff --git a/app/src/main/kotlin/com/pitchedapps/frost/services/NotificationService.kt b/app/src/main/kotlin/com/pitchedapps/frost/services/NotificationService.kt
index d481e941..ac3c89dd 100644
--- a/app/src/main/kotlin/com/pitchedapps/frost/services/NotificationService.kt
+++ b/app/src/main/kotlin/com/pitchedapps/frost/services/NotificationService.kt
@@ -9,18 +9,15 @@ import com.pitchedapps.frost.BuildConfig
 import com.pitchedapps.frost.R
 import com.pitchedapps.frost.dbflow.CookieModel
 import com.pitchedapps.frost.dbflow.lastNotificationTime
-import com.pitchedapps.frost.dbflow.loadFbCookie
 import com.pitchedapps.frost.dbflow.loadFbCookiesSync
 import com.pitchedapps.frost.facebook.FACEBOOK_COM
 import com.pitchedapps.frost.facebook.FbItem
 import com.pitchedapps.frost.facebook.USER_AGENT_BASIC
 import com.pitchedapps.frost.facebook.formattedFbUrl
-import com.pitchedapps.frost.injectors.JsAssets
+import com.pitchedapps.frost.parsers.MessageParser
 import com.pitchedapps.frost.utils.L
 import com.pitchedapps.frost.utils.Prefs
 import com.pitchedapps.frost.utils.frostAnswersCustom
-import com.pitchedapps.frost.web.launchHeadlessHtmlExtractor
-import io.reactivex.schedulers.Schedulers
 import org.jetbrains.anko.doAsync
 import org.jsoup.Jsoup
 import org.jsoup.nodes.Element
@@ -75,25 +72,14 @@ class NotificationService : JobService() {
     override fun onStartJob(params: JobParameters?): Boolean {
         L.i("Fetching notifications")
         future = doAsync {
-            if (Prefs.notificationAllAccounts) {
-                val cookies = loadFbCookiesSync()
-                cookies.forEach { fetchGeneralNotifications(it) }
-            } else {
-                val currentCookie = loadFbCookie(Prefs.userId)
-                if (currentCookie != null) {
-                    fetchGeneralNotifications(currentCookie)
-                }
-            }
-            L.d("Finished main notifications")
-            if (Prefs.notificationsInstantMessages) {
-                val currentCookie = loadFbCookie(Prefs.userId)
-                if (currentCookie != null) {
-                    fetchMessageNotifications(currentCookie) {
-                        L.i("Notif IM fetching finished ${if (it) "succesfully" else "unsuccessfully"}")
-                        finish(params)
-                    }
-                    return@doAsync
-                }
+            val currentId = Prefs.userId
+            val cookies = loadFbCookiesSync()
+            cookies.forEach {
+                val current = it.id == currentId
+                if (current || Prefs.notificationAllAccounts)
+                    fetchGeneralNotifications(it)
+                if (Prefs.notificationsInstantMessages && (current || Prefs.notificationsImAllAccounts))
+                    fetchMessageNotifications(it)
             }
             finish(params)
         }
@@ -161,33 +147,22 @@ class NotificationService : JobService() {
      * ----------------------------------------------------------------
      */
 
-    inline fun fetchMessageNotifications(data: CookieModel, crossinline callback: (success: Boolean) -> Unit) {
-        launchHeadlessHtmlExtractor(FbItem.MESSAGES.url, JsAssets.NOTIF_MSG) {
-            it.observeOn(Schedulers.newThread()).subscribe { (html, errorRes) ->
-                L.d("Notf IM html received")
-                if (errorRes != -1) return@subscribe callback(false)
-                fetchMessageNotifications(data, html)
-                callback(true)
-            }
-        }
-    }
-
-    fun fetchMessageNotifications(data: CookieModel, html: String) {
+    fun fetchMessageNotifications(data: CookieModel) {
         L.d("Notif IM fetch", data.toString())
-        val doc = Jsoup.parseBodyFragment(html)
-        val unreadNotifications = (doc.getElementById("threadlist_rows") ?: return L.eThrow("Notification messages not found")).getElementsByClass("aclb")
+        val doc = Jsoup.connect(FbItem.MESSAGES.url).cookie(FACEBOOK_COM, data.cookie).userAgent(USER_AGENT_BASIC).get()
+        val (threads, _, _) = MessageParser.parse(doc.toString()) ?: return L.e("Could not parse IM")
+
         var notifCount = 0
         val prevNotifTime = lastNotificationTime(data.id)
         val prevLatestEpoch = prevNotifTime.epochIm
         L.v("Notif Prev Latest Im Epoch $prevLatestEpoch")
         var newLatestEpoch = prevLatestEpoch
-        unreadNotifications.forEach unread@ { elem ->
-            val notif = parseMessageNotification(data, elem) ?: return@unread
-            L.v("Notif im timestamp ${notif.timestamp}")
-            if (notif.timestamp <= prevLatestEpoch) return@unread
-            NotificationType.MESSAGE.createNotification(this, notif, notifCount == 0)
-            if (notif.timestamp > newLatestEpoch)
-                newLatestEpoch = notif.timestamp
+        threads.filter { it.unread }.forEach { notif ->
+            L.v("Notif Im timestamp ${notif.time}")
+            if (notif.time <= prevLatestEpoch) return@forEach
+            NotificationType.MESSAGE.createNotification(this, NotificationContent(data, notif), notifCount == 0)
+            if (notif.time > newLatestEpoch)
+                newLatestEpoch = notif.time
             notifCount++
         }
         if (newLatestEpoch != prevLatestEpoch) prevNotifTime.copy(epochIm = newLatestEpoch).save()
@@ -195,22 +170,6 @@ class NotificationService : JobService() {
         NotificationType.MESSAGE.summaryNotification(this, data.id, notifCount)
     }
 
-    fun parseMessageNotification(data: CookieModel, element: Element): NotificationContent? {
-        val a = element.getElementsByTag("a").first() ?: return null
-        val abbr = element.getElementsByTag("abbr")
-        val epoch = epochMatcher.find(abbr.attr("data-store"))?.groups?.get(1)?.value?.toLong() ?: return logNotif("No epoch")
-        val thread = element.getElementsByAttributeValueContaining("id", "thread_fbid_").first() ?: return null
-        //fetch id
-        val notifId = messageNotifIdMatcher.find(thread.id())?.groups?.get(1)?.value?.toLong() ?: System.currentTimeMillis()
-        val text = element.select("span.snippet").firstOrNull()?.text()?.trim() ?: getString(R.string.new_message)
-        if (Prefs.notificationKeywords.any { text.contains(it, ignoreCase = true) }) return null //notification filtered out
-        //fetch convo pic
-        val p = element.select("i.img[style*=url]")
-        val pUrl = profMatcher.find(p.attr("style"))?.groups?.get(1)?.value?.formattedFbUrl ?: ""
-        L.v("url", a.attr("href"))
-        return NotificationContent(data, notifId.toInt(), a.attr("href"), a.text(), text, epoch, pUrl)
-    }
-
     private fun Context.debugNotification(text: String) {
         if (!BuildConfig.DEBUG) return
         val notifBuilder = frostNotification.withDefaults()
diff --git a/app/src/main/kotlin/com/pitchedapps/frost/settings/Notifications.kt b/app/src/main/kotlin/com/pitchedapps/frost/settings/Notifications.kt
index 0f2f8638..f46517ac 100644
--- a/app/src/main/kotlin/com/pitchedapps/frost/settings/Notifications.kt
+++ b/app/src/main/kotlin/com/pitchedapps/frost/settings/Notifications.kt
@@ -58,10 +58,15 @@ fun SettingsActivity.getNotificationPrefs(): KPrefAdapterBuilder.() -> Unit = {
         descRes = R.string.notification_all_accounts_desc
     }
 
-    checkbox(R.string.notification_messages, { Prefs.notificationsInstantMessages }, { Prefs.notificationsInstantMessages = it }) {
+    checkbox(R.string.notification_messages, { Prefs.notificationsInstantMessages }, { Prefs.notificationsInstantMessages = it; reloadByTitle(R.string.notification_messages_all_accounts) }) {
         descRes = R.string.notification_messages_desc
     }
 
+    checkbox(R.string.notification_messages_all_accounts, { Prefs.notificationsImAllAccounts }, { Prefs.notificationsImAllAccounts = it }) {
+        descRes = R.string.notification_messages_all_accounts_desc
+        enabler = { Prefs.notificationsInstantMessages }
+    }
+
     checkbox(R.string.notification_sound, { Prefs.notificationSound }, { Prefs.notificationSound = it; reloadByTitle(R.string.notification_ringtone, R.string.message_ringtone) })
 
     fun KPrefText.KPrefTextContract<String>.ringtone(code: Int) {
diff --git a/app/src/main/kotlin/com/pitchedapps/frost/utils/Prefs.kt b/app/src/main/kotlin/com/pitchedapps/frost/utils/Prefs.kt
index 2897cf3d..68b4b80d 100644
--- a/app/src/main/kotlin/com/pitchedapps/frost/utils/Prefs.kt
+++ b/app/src/main/kotlin/com/pitchedapps/frost/utils/Prefs.kt
@@ -102,6 +102,8 @@ object Prefs : KPref() {
 
     var notificationsInstantMessages: Boolean by kpref("notification_im", false)
 
+    var notificationsImAllAccounts: Boolean by kpref("notification_im_all_accounts", false)
+
     var notificationVibrate: Boolean by kpref("notification_vibrate", true)
 
     var notificationSound: Boolean by kpref("notification_sound", true)