aboutsummaryrefslogtreecommitdiff
path: root/app/src/main/kotlin/com
diff options
context:
space:
mode:
authorAllan Wang <me@allanwang.ca>2017-10-07 01:00:24 -0400
committerGitHub <noreply@github.com>2017-10-07 01:00:24 -0400
commit60cc50d8f6785d33adf4dafd456c836c96a9e3de (patch)
treea887c1dc931f0c434e912c9209c8fa91c2d003e9 /app/src/main/kotlin/com
parent5382d8222915727a47c36f0e7ac280c701dbfcdf (diff)
downloadfrost-60cc50d8f6785d33adf4dafd456c836c96a9e3de.tar.gz
frost-60cc50d8f6785d33adf4dafd456c836c96a9e3de.tar.bz2
frost-60cc50d8f6785d33adf4dafd456c836c96a9e3de.zip
Enhancement/message parsing (#369)
* Test custom regex * Remove statics, use lists, and add amp (#366) * Remove statics, use lists, and add amp * Remove another jvmstatic * Update parser for messages * Update dependencies * Update travis * Use parsing for background im fetcher * Update changelog * Update changelog
Diffstat (limited to 'app/src/main/kotlin/com')
-rw-r--r--app/src/main/kotlin/com/pitchedapps/frost/facebook/FbUrlFormatter.kt30
-rw-r--r--app/src/main/kotlin/com/pitchedapps/frost/parsers/FrostParser.kt44
-rw-r--r--app/src/main/kotlin/com/pitchedapps/frost/parsers/MessageParser.kt85
-rw-r--r--app/src/main/kotlin/com/pitchedapps/frost/services/FrostNotifications.kt6
-rw-r--r--app/src/main/kotlin/com/pitchedapps/frost/services/NotificationService.kt79
-rw-r--r--app/src/main/kotlin/com/pitchedapps/frost/settings/Notifications.kt7
-rw-r--r--app/src/main/kotlin/com/pitchedapps/frost/utils/Prefs.kt2
7 files changed, 179 insertions, 74 deletions
diff --git a/app/src/main/kotlin/com/pitchedapps/frost/facebook/FbUrlFormatter.kt b/app/src/main/kotlin/com/pitchedapps/frost/facebook/FbUrlFormatter.kt
index 542277b8..4879e68b 100644
--- a/app/src/main/kotlin/com/pitchedapps/frost/facebook/FbUrlFormatter.kt
+++ b/app/src/main/kotlin/com/pitchedapps/frost/facebook/FbUrlFormatter.kt
@@ -7,10 +7,13 @@ import com.pitchedapps.frost.utils.L
*
* Custom url builder so we can easily test it without the Android framework
*/
-val String.formattedFbUrl: String
- get() = FbUrlFormatter(this).toString()
+inline val String.formattedFbUrl: String
+ get() = FbUrlFormatter(this, false).toString()
-class FbUrlFormatter(url: String) {
+inline val String.formattedFbUrlCss: String
+ get() = FbUrlFormatter(this, true).toString()
+
+class FbUrlFormatter(url: String, css: Boolean = false) {
private val queries = mutableMapOf<String, String>()
private val cleaned: String
@@ -30,6 +33,8 @@ class FbUrlFormatter(url: String) {
discardable.forEach { cleanedUrl = cleanedUrl.replace(it, "", true) }
val changed = cleanedUrl != url //note that discardables strip away the first '?'
converter.forEach { (k, v) -> cleanedUrl = cleanedUrl.replace(k, v, true) }
+ //must decode for css
+ if (css) decoder.forEach { (k, v) -> cleanedUrl = cleanedUrl.replace(k, v, true) }
val qm = cleanedUrl.indexOf(if (changed) "&" else "?")
if (qm > -1) {
cleanedUrl.substring(qm + 1).split("&").forEach {
@@ -39,8 +44,10 @@ class FbUrlFormatter(url: String) {
cleanedUrl = cleanedUrl.substring(0, qm)
}
//only decode non query portion of the url
- decoder.forEach { (k, v) -> cleanedUrl = cleanedUrl.replace(k, v, true) }
+ if (!css) decoder.forEach { (k, v) -> cleanedUrl = cleanedUrl.replace(k, v, true) }
discardableQueries.forEach { queries.remove(it) }
+ //final cleanup
+ misc.forEach { (k, v) -> cleanedUrl = cleanedUrl.replace(k, v, true) }
if (cleanedUrl.startsWith("#!")) cleanedUrl = cleanedUrl.substring(2)
if (cleanedUrl.startsWith("/")) cleanedUrl = FB_URL_BASE + cleanedUrl.substring(1)
cleanedUrl = cleanedUrl.replaceFirst(".facebook.com//", ".facebook.com/") //sometimes we are given a bad url
@@ -72,7 +79,6 @@ class FbUrlFormatter(url: String) {
* Taken from FaceSlim
* https://github.com/indywidualny/FaceSlim/blob/master/app/src/main/java/org/indywidualni/fblite/util/Miscellany.java
*/
- @JvmStatic
val discardable = arrayOf(
"http://lm.facebook.com/l.php?u=",
"https://lm.facebook.com/l.php?u=",
@@ -83,11 +89,13 @@ class FbUrlFormatter(url: String) {
"/video_redirect/?src="
)
- @JvmStatic
+ val misc = listOf(
+ "&amp;" to "&"
+ )
+
val discardableQueries = arrayOf("ref", "refid")
- @JvmStatic
- val decoder = mapOf(
+ val decoder = listOf(
"%3C" to "<", "%3E" to ">", "%23" to "#", "%25" to "%",
"%7B" to "{", "%7D" to "}", "%7C" to "|", "%5C" to "\\",
"%5E" to "^", "%7E" to "~", "%5B" to "[", "%5D" to "]",
@@ -97,8 +105,7 @@ class FbUrlFormatter(url: String) {
"%20" to " "
)
- @JvmStatic
- val cssDecoder = mapOf(
+ val cssDecoder = listOf(
"\\3C " to "<", "\\3E " to ">", "\\23 " to "#", "\\25 " to "%",
"\\7B " to "{", "\\7D " to "}", "\\7C " to "|", "\\5C " to "\\",
"\\5E " to "^", "\\7E " to "~", "\\5B " to "[", "\\5D " to "]",
@@ -108,8 +115,7 @@ class FbUrlFormatter(url: String) {
"%20" to " "
)
- @JvmStatic
- val converter = mapOf(
+ val converter = listOf(
"\\3C " to "%3C", "\\3E " to "%3E", "\\23 " to "%23", "\\25 " to "%25",
"\\7B " to "%7B", "\\7D " to "%7D", "\\7C " to "%7C", "\\5C " to "%5C",
"\\5E " to "%5E", "\\7E " to "%7E", "\\5B " to "%5B", "\\5D " to "%5D",
diff --git a/app/src/main/kotlin/com/pitchedapps/frost/parsers/FrostParser.kt b/app/src/main/kotlin/com/pitchedapps/frost/parsers/FrostParser.kt
new file mode 100644
index 00000000..86b280a8
--- /dev/null
+++ b/app/src/main/kotlin/com/pitchedapps/frost/parsers/FrostParser.kt
@@ -0,0 +1,44 @@
+package com.pitchedapps.frost.parsers
+
+/**
+ * Created by Allan Wang on 2017-10-06.
+ *
+ * Interface for a given parser
+ * Use cases should be attached as delegates to objects that implement this interface
+ */
+interface FrostParser<T> {
+ fun parse(text: String?): T?
+ fun debug(text: String?): String
+}
+
+internal abstract class FrostParserBase<T> : FrostParser<T> {
+ override final fun parse(text: String?): T?
+ = if (text == null) null else parseImpl(text)
+
+ protected abstract fun parseImpl(text: String): T?
+
+ override final fun debug(text: String?): String {
+ val result = mutableListOf<String>()
+ result.add("Testing parser for ${this::class.java.simpleName}")
+ if (text == null) {
+ result.add("Input is null")
+ return result.joinToString("\n")
+ }
+ val output = parseImpl(text)
+ if (output == null) {
+ result.add("Output is null")
+ return result.joinToString("\n")
+ }
+ debugImpl(output, result)
+ return result.joinToString("\n")
+ }
+
+ protected abstract fun debugImpl(data: T, result: MutableList<String>)
+}
+
+object FrostRegex {
+ val epoch = Regex(":([0-9]+)")
+ val notifId = Regex("notif_id\":([0-9]+)")
+ val messageNotifId = Regex("thread_fbid_([0-9]+)")
+ val profilePicture = Regex("url\\(\"(.*?)\"\\)")
+} \ No newline at end of file
diff --git a/app/src/main/kotlin/com/pitchedapps/frost/parsers/MessageParser.kt b/app/src/main/kotlin/com/pitchedapps/frost/parsers/MessageParser.kt
new file mode 100644
index 00000000..dbe2c0bb
--- /dev/null
+++ b/app/src/main/kotlin/com/pitchedapps/frost/parsers/MessageParser.kt
@@ -0,0 +1,85 @@
+package com.pitchedapps.frost.parsers
+
+import com.pitchedapps.frost.facebook.formattedFbUrl
+import com.pitchedapps.frost.facebook.formattedFbUrlCss
+import com.pitchedapps.frost.utils.L
+import org.apache.commons.text.StringEscapeUtils
+import org.jsoup.Jsoup
+import org.jsoup.nodes.Element
+
+/**
+ * Created by Allan Wang on 2017-10-06.
+ *
+ * In Facebook, messages are passed through scripts and loaded into view via react afterwards
+ * We can parse out the content we want directly and load it ourselves
+ *
+ */
+object MessageParser : FrostParser<Triple<List<FrostThread>, FrostLink?, List<FrostLink>>> by MessageParserImpl()
+
+data class FrostThread(val id: Int, val img: String, val title: String, val time: Long, val url: String, val unread: Boolean, val content: String?)
+
+data class FrostLink(val text: String, val href: String)
+
+private class MessageParserImpl : FrostParserBase<Triple<List<FrostThread>, FrostLink?, List<FrostLink>>>() {
+
+ override fun parseImpl(text: String): Triple<List<FrostThread>, FrostLink?, List<FrostLink>>? {
+ var content = StringEscapeUtils.unescapeEcmaScript(text)
+ val begin = content.indexOf("id=\"threadlist_rows\"")
+ if (begin <= 0) {
+ L.d("Threadlist not found")
+ return null
+ }
+ content = content.substring(begin)
+ val end = content.indexOf("</script>")
+ if (end <= 0) {
+ L.d("Script tail not found")
+ return null
+ }
+ content = content.substring(0, end).substringBeforeLast("</div>")
+ val body = Jsoup.parseBodyFragment("<div $content")
+ val threadList = body.getElementById("threadlist_rows")
+ val threads: List<FrostThread> = threadList.getElementsByAttributeValueContaining("id", "thread_fbid_")
+ .mapNotNull { parseMessage(it) }
+ val seeMore = parseLink(body.getElementById("see_older_threads"))
+ val extraLinks = threadList.nextElementSibling().select("a")
+ .mapNotNull { parseLink(it) }
+ return Triple(threads, seeMore, extraLinks)
+ }
+
+ private fun parseMessage(element: Element): FrostThread? {
+ val a = element.getElementsByTag("a").first() ?: return null
+ val abbr = element.getElementsByTag("abbr")
+ println(abbr.attr("data-store"))
+ val epoch = FrostRegex.epoch.find(abbr.attr("data-store"))
+ ?.groupValues?.getOrNull(1)?.toLongOrNull() ?: -1L
+ //fetch id
+ val id = FrostRegex.messageNotifId.find(element.id())
+ ?.groupValues?.getOrNull(1)?.toLongOrNull() ?: System.currentTimeMillis()
+ val content = element.select("span.snippet").firstOrNull()?.text()?.trim()
+ //fetch convo pic
+ val p = element.select("i.img[style*=url]")
+ val pUrl = FrostRegex.profilePicture.find(p.attr("style"))?.groups?.get(1)?.value?.formattedFbUrl ?: ""
+ L.v("url", a.attr("href"))
+ return FrostThread(
+ id = id.toInt(),
+ img = pUrl.formattedFbUrlCss,
+ title = a.text(),
+ time = epoch,
+ url = a.attr("href").formattedFbUrlCss,
+ unread = !element.hasClass("acw"),
+ content = content
+ )
+ }
+
+ private fun parseLink(element: Element?): FrostLink? {
+ val a = element?.getElementsByTag("a")?.first() ?: return null
+ return FrostLink(a.text(), a.attr("href"))
+ }
+
+ override fun debugImpl(data: Triple<List<FrostThread>, FrostLink?, List<FrostLink>>, result: MutableList<String>) {
+ result.addAll(data.first.map { it.toString() })
+ result.add("See more link:")
+ result.add("\t${data.second}")
+ result.addAll(data.third.map { it.toString() })
+ }
+}
diff --git a/app/src/main/kotlin/com/pitchedapps/frost/services/FrostNotifications.kt b/app/src/main/kotlin/com/pitchedapps/frost/services/FrostNotifications.kt
index b892af91..b1f26d99 100644
--- a/app/src/main/kotlin/com/pitchedapps/frost/services/FrostNotifications.kt
+++ b/app/src/main/kotlin/com/pitchedapps/frost/services/FrostNotifications.kt
@@ -24,6 +24,7 @@ import com.pitchedapps.frost.dbflow.CookieModel
import com.pitchedapps.frost.enums.OverlayContext
import com.pitchedapps.frost.facebook.FbItem
import com.pitchedapps.frost.facebook.formattedFbUrl
+import com.pitchedapps.frost.parsers.FrostThread
import com.pitchedapps.frost.utils.*
import org.jetbrains.anko.runOnUiThread
@@ -159,7 +160,10 @@ data class NotificationContent(val data: CookieModel,
val title: String? = null,
val text: String,
val timestamp: Long,
- val profileUrl: String)
+ val profileUrl: String) {
+ constructor(data:CookieModel, thread: FrostThread)
+ :this(data, thread.id, thread.url, thread.title, thread.content ?: "", thread.time, thread.img)
+}
const val NOTIFICATION_PERIODIC_JOB = 7
diff --git a/app/src/main/kotlin/com/pitchedapps/frost/services/NotificationService.kt b/app/src/main/kotlin/com/pitchedapps/frost/services/NotificationService.kt
index d481e941..ac3c89dd 100644
--- a/app/src/main/kotlin/com/pitchedapps/frost/services/NotificationService.kt
+++ b/app/src/main/kotlin/com/pitchedapps/frost/services/NotificationService.kt
@@ -9,18 +9,15 @@ import com.pitchedapps.frost.BuildConfig
import com.pitchedapps.frost.R
import com.pitchedapps.frost.dbflow.CookieModel
import com.pitchedapps.frost.dbflow.lastNotificationTime
-import com.pitchedapps.frost.dbflow.loadFbCookie
import com.pitchedapps.frost.dbflow.loadFbCookiesSync
import com.pitchedapps.frost.facebook.FACEBOOK_COM
import com.pitchedapps.frost.facebook.FbItem
import com.pitchedapps.frost.facebook.USER_AGENT_BASIC
import com.pitchedapps.frost.facebook.formattedFbUrl
-import com.pitchedapps.frost.injectors.JsAssets
+import com.pitchedapps.frost.parsers.MessageParser
import com.pitchedapps.frost.utils.L
import com.pitchedapps.frost.utils.Prefs
import com.pitchedapps.frost.utils.frostAnswersCustom
-import com.pitchedapps.frost.web.launchHeadlessHtmlExtractor
-import io.reactivex.schedulers.Schedulers
import org.jetbrains.anko.doAsync
import org.jsoup.Jsoup
import org.jsoup.nodes.Element
@@ -75,25 +72,14 @@ class NotificationService : JobService() {
override fun onStartJob(params: JobParameters?): Boolean {
L.i("Fetching notifications")
future = doAsync {
- if (Prefs.notificationAllAccounts) {
- val cookies = loadFbCookiesSync()
- cookies.forEach { fetchGeneralNotifications(it) }
- } else {
- val currentCookie = loadFbCookie(Prefs.userId)
- if (currentCookie != null) {
- fetchGeneralNotifications(currentCookie)
- }
- }
- L.d("Finished main notifications")
- if (Prefs.notificationsInstantMessages) {
- val currentCookie = loadFbCookie(Prefs.userId)
- if (currentCookie != null) {
- fetchMessageNotifications(currentCookie) {
- L.i("Notif IM fetching finished ${if (it) "succesfully" else "unsuccessfully"}")
- finish(params)
- }
- return@doAsync
- }
+ val currentId = Prefs.userId
+ val cookies = loadFbCookiesSync()
+ cookies.forEach {
+ val current = it.id == currentId
+ if (current || Prefs.notificationAllAccounts)
+ fetchGeneralNotifications(it)
+ if (Prefs.notificationsInstantMessages && (current || Prefs.notificationsImAllAccounts))
+ fetchMessageNotifications(it)
}
finish(params)
}
@@ -161,33 +147,22 @@ class NotificationService : JobService() {
* ----------------------------------------------------------------
*/
- inline fun fetchMessageNotifications(data: CookieModel, crossinline callback: (success: Boolean) -> Unit) {
- launchHeadlessHtmlExtractor(FbItem.MESSAGES.url, JsAssets.NOTIF_MSG) {
- it.observeOn(Schedulers.newThread()).subscribe { (html, errorRes) ->
- L.d("Notf IM html received")
- if (errorRes != -1) return@subscribe callback(false)
- fetchMessageNotifications(data, html)
- callback(true)
- }
- }
- }
-
- fun fetchMessageNotifications(data: CookieModel, html: String) {
+ fun fetchMessageNotifications(data: CookieModel) {
L.d("Notif IM fetch", data.toString())
- val doc = Jsoup.parseBodyFragment(html)
- val unreadNotifications = (doc.getElementById("threadlist_rows") ?: return L.eThrow("Notification messages not found")).getElementsByClass("aclb")
+ val doc = Jsoup.connect(FbItem.MESSAGES.url).cookie(FACEBOOK_COM, data.cookie).userAgent(USER_AGENT_BASIC).get()
+ val (threads, _, _) = MessageParser.parse(doc.toString()) ?: return L.e("Could not parse IM")
+
var notifCount = 0
val prevNotifTime = lastNotificationTime(data.id)
val prevLatestEpoch = prevNotifTime.epochIm
L.v("Notif Prev Latest Im Epoch $prevLatestEpoch")
var newLatestEpoch = prevLatestEpoch
- unreadNotifications.forEach unread@ { elem ->
- val notif = parseMessageNotification(data, elem) ?: return@unread
- L.v("Notif im timestamp ${notif.timestamp}")
- if (notif.timestamp <= prevLatestEpoch) return@unread
- NotificationType.MESSAGE.createNotification(this, notif, notifCount == 0)
- if (notif.timestamp > newLatestEpoch)
- newLatestEpoch = notif.timestamp
+ threads.filter { it.unread }.forEach { notif ->
+ L.v("Notif Im timestamp ${notif.time}")
+ if (notif.time <= prevLatestEpoch) return@forEach
+ NotificationType.MESSAGE.createNotification(this, NotificationContent(data, notif), notifCount == 0)
+ if (notif.time > newLatestEpoch)
+ newLatestEpoch = notif.time
notifCount++
}
if (newLatestEpoch != prevLatestEpoch) prevNotifTime.copy(epochIm = newLatestEpoch).save()
@@ -195,22 +170,6 @@ class NotificationService : JobService() {
NotificationType.MESSAGE.summaryNotification(this, data.id, notifCount)
}
- fun parseMessageNotification(data: CookieModel, element: Element): NotificationContent? {
- val a = element.getElementsByTag("a").first() ?: return null
- val abbr = element.getElementsByTag("abbr")
- val epoch = epochMatcher.find(abbr.attr("data-store"))?.groups?.get(1)?.value?.toLong() ?: return logNotif("No epoch")
- val thread = element.getElementsByAttributeValueContaining("id", "thread_fbid_").first() ?: return null
- //fetch id
- val notifId = messageNotifIdMatcher.find(thread.id())?.groups?.get(1)?.value?.toLong() ?: System.currentTimeMillis()
- val text = element.select("span.snippet").firstOrNull()?.text()?.trim() ?: getString(R.string.new_message)
- if (Prefs.notificationKeywords.any { text.contains(it, ignoreCase = true) }) return null //notification filtered out
- //fetch convo pic
- val p = element.select("i.img[style*=url]")
- val pUrl = profMatcher.find(p.attr("style"))?.groups?.get(1)?.value?.formattedFbUrl ?: ""
- L.v("url", a.attr("href"))
- return NotificationContent(data, notifId.toInt(), a.attr("href"), a.text(), text, epoch, pUrl)
- }
-
private fun Context.debugNotification(text: String) {
if (!BuildConfig.DEBUG) return
val notifBuilder = frostNotification.withDefaults()
diff --git a/app/src/main/kotlin/com/pitchedapps/frost/settings/Notifications.kt b/app/src/main/kotlin/com/pitchedapps/frost/settings/Notifications.kt
index 0f2f8638..f46517ac 100644
--- a/app/src/main/kotlin/com/pitchedapps/frost/settings/Notifications.kt
+++ b/app/src/main/kotlin/com/pitchedapps/frost/settings/Notifications.kt
@@ -58,10 +58,15 @@ fun SettingsActivity.getNotificationPrefs(): KPrefAdapterBuilder.() -> Unit = {
descRes = R.string.notification_all_accounts_desc
}
- checkbox(R.string.notification_messages, { Prefs.notificationsInstantMessages }, { Prefs.notificationsInstantMessages = it }) {
+ checkbox(R.string.notification_messages, { Prefs.notificationsInstantMessages }, { Prefs.notificationsInstantMessages = it; reloadByTitle(R.string.notification_messages_all_accounts) }) {
descRes = R.string.notification_messages_desc
}
+ checkbox(R.string.notification_messages_all_accounts, { Prefs.notificationsImAllAccounts }, { Prefs.notificationsImAllAccounts = it }) {
+ descRes = R.string.notification_messages_all_accounts_desc
+ enabler = { Prefs.notificationsInstantMessages }
+ }
+
checkbox(R.string.notification_sound, { Prefs.notificationSound }, { Prefs.notificationSound = it; reloadByTitle(R.string.notification_ringtone, R.string.message_ringtone) })
fun KPrefText.KPrefTextContract<String>.ringtone(code: Int) {
diff --git a/app/src/main/kotlin/com/pitchedapps/frost/utils/Prefs.kt b/app/src/main/kotlin/com/pitchedapps/frost/utils/Prefs.kt
index 2897cf3d..68b4b80d 100644
--- a/app/src/main/kotlin/com/pitchedapps/frost/utils/Prefs.kt
+++ b/app/src/main/kotlin/com/pitchedapps/frost/utils/Prefs.kt
@@ -102,6 +102,8 @@ object Prefs : KPref() {
var notificationsInstantMessages: Boolean by kpref("notification_im", false)
+ var notificationsImAllAccounts: Boolean by kpref("notification_im_all_accounts", false)
+
var notificationVibrate: Boolean by kpref("notification_vibrate", true)
var notificationSound: Boolean by kpref("notification_sound", true)