diff options
Diffstat (limited to 'app/src')
12 files changed, 245 insertions, 77 deletions
diff --git a/app/src/main/kotlin/com/pitchedapps/frost/facebook/FbUrlFormatter.kt b/app/src/main/kotlin/com/pitchedapps/frost/facebook/FbUrlFormatter.kt index 542277b8..4879e68b 100644 --- a/app/src/main/kotlin/com/pitchedapps/frost/facebook/FbUrlFormatter.kt +++ b/app/src/main/kotlin/com/pitchedapps/frost/facebook/FbUrlFormatter.kt @@ -7,10 +7,13 @@ import com.pitchedapps.frost.utils.L * * Custom url builder so we can easily test it without the Android framework */ -val String.formattedFbUrl: String - get() = FbUrlFormatter(this).toString() +inline val String.formattedFbUrl: String + get() = FbUrlFormatter(this, false).toString() -class FbUrlFormatter(url: String) { +inline val String.formattedFbUrlCss: String + get() = FbUrlFormatter(this, true).toString() + +class FbUrlFormatter(url: String, css: Boolean = false) { private val queries = mutableMapOf<String, String>() private val cleaned: String @@ -30,6 +33,8 @@ class FbUrlFormatter(url: String) { discardable.forEach { cleanedUrl = cleanedUrl.replace(it, "", true) } val changed = cleanedUrl != url //note that discardables strip away the first '?' converter.forEach { (k, v) -> cleanedUrl = cleanedUrl.replace(k, v, true) } + //must decode for css + if (css) decoder.forEach { (k, v) -> cleanedUrl = cleanedUrl.replace(k, v, true) } val qm = cleanedUrl.indexOf(if (changed) "&" else "?") if (qm > -1) { cleanedUrl.substring(qm + 1).split("&").forEach { @@ -39,8 +44,10 @@ class FbUrlFormatter(url: String) { cleanedUrl = cleanedUrl.substring(0, qm) } //only decode non query portion of the url - decoder.forEach { (k, v) -> cleanedUrl = cleanedUrl.replace(k, v, true) } + if (!css) decoder.forEach { (k, v) -> cleanedUrl = cleanedUrl.replace(k, v, true) } discardableQueries.forEach { queries.remove(it) } + //final cleanup + misc.forEach { (k, v) -> cleanedUrl = cleanedUrl.replace(k, v, true) } if (cleanedUrl.startsWith("#!")) cleanedUrl = cleanedUrl.substring(2) if (cleanedUrl.startsWith("/")) cleanedUrl = FB_URL_BASE + cleanedUrl.substring(1) cleanedUrl = cleanedUrl.replaceFirst(".facebook.com//", ".facebook.com/") //sometimes we are given a bad url @@ -72,7 +79,6 @@ class FbUrlFormatter(url: String) { * Taken from FaceSlim * https://github.com/indywidualny/FaceSlim/blob/master/app/src/main/java/org/indywidualni/fblite/util/Miscellany.java */ - @JvmStatic val discardable = arrayOf( "http://lm.facebook.com/l.php?u=", "https://lm.facebook.com/l.php?u=", @@ -83,11 +89,13 @@ class FbUrlFormatter(url: String) { "/video_redirect/?src=" ) - @JvmStatic + val misc = listOf( + "&" to "&" + ) + val discardableQueries = arrayOf("ref", "refid") - @JvmStatic - val decoder = mapOf( + val decoder = listOf( "%3C" to "<", "%3E" to ">", "%23" to "#", "%25" to "%", "%7B" to "{", "%7D" to "}", "%7C" to "|", "%5C" to "\\", "%5E" to "^", "%7E" to "~", "%5B" to "[", "%5D" to "]", @@ -97,8 +105,7 @@ class FbUrlFormatter(url: String) { "%20" to " " ) - @JvmStatic - val cssDecoder = mapOf( + val cssDecoder = listOf( "\\3C " to "<", "\\3E " to ">", "\\23 " to "#", "\\25 " to "%", "\\7B " to "{", "\\7D " to "}", "\\7C " to "|", "\\5C " to "\\", "\\5E " to "^", "\\7E " to "~", "\\5B " to "[", "\\5D " to "]", @@ -108,8 +115,7 @@ class FbUrlFormatter(url: String) { "%20" to " " ) - @JvmStatic - val converter = mapOf( + val converter = listOf( "\\3C " to "%3C", "\\3E " to "%3E", "\\23 " to "%23", "\\25 " to "%25", "\\7B " to "%7B", "\\7D " to "%7D", "\\7C " to "%7C", "\\5C " to "%5C", "\\5E " to "%5E", "\\7E " to "%7E", "\\5B " to "%5B", "\\5D " to "%5D", diff --git a/app/src/main/kotlin/com/pitchedapps/frost/parsers/FrostParser.kt b/app/src/main/kotlin/com/pitchedapps/frost/parsers/FrostParser.kt new file mode 100644 index 00000000..86b280a8 --- /dev/null +++ b/app/src/main/kotlin/com/pitchedapps/frost/parsers/FrostParser.kt @@ -0,0 +1,44 @@ +package com.pitchedapps.frost.parsers + +/** + * Created by Allan Wang on 2017-10-06. + * + * Interface for a given parser + * Use cases should be attached as delegates to objects that implement this interface + */ +interface FrostParser<T> { + fun parse(text: String?): T? + fun debug(text: String?): String +} + +internal abstract class FrostParserBase<T> : FrostParser<T> { + override final fun parse(text: String?): T? + = if (text == null) null else parseImpl(text) + + protected abstract fun parseImpl(text: String): T? + + override final fun debug(text: String?): String { + val result = mutableListOf<String>() + result.add("Testing parser for ${this::class.java.simpleName}") + if (text == null) { + result.add("Input is null") + return result.joinToString("\n") + } + val output = parseImpl(text) + if (output == null) { + result.add("Output is null") + return result.joinToString("\n") + } + debugImpl(output, result) + return result.joinToString("\n") + } + + protected abstract fun debugImpl(data: T, result: MutableList<String>) +} + +object FrostRegex { + val epoch = Regex(":([0-9]+)") + val notifId = Regex("notif_id\":([0-9]+)") + val messageNotifId = Regex("thread_fbid_([0-9]+)") + val profilePicture = Regex("url\\(\"(.*?)\"\\)") +}
\ No newline at end of file diff --git a/app/src/main/kotlin/com/pitchedapps/frost/parsers/MessageParser.kt b/app/src/main/kotlin/com/pitchedapps/frost/parsers/MessageParser.kt new file mode 100644 index 00000000..dbe2c0bb --- /dev/null +++ b/app/src/main/kotlin/com/pitchedapps/frost/parsers/MessageParser.kt @@ -0,0 +1,85 @@ +package com.pitchedapps.frost.parsers + +import com.pitchedapps.frost.facebook.formattedFbUrl +import com.pitchedapps.frost.facebook.formattedFbUrlCss +import com.pitchedapps.frost.utils.L +import org.apache.commons.text.StringEscapeUtils +import org.jsoup.Jsoup +import org.jsoup.nodes.Element + +/** + * Created by Allan Wang on 2017-10-06. + * + * In Facebook, messages are passed through scripts and loaded into view via react afterwards + * We can parse out the content we want directly and load it ourselves + * + */ +object MessageParser : FrostParser<Triple<List<FrostThread>, FrostLink?, List<FrostLink>>> by MessageParserImpl() + +data class FrostThread(val id: Int, val img: String, val title: String, val time: Long, val url: String, val unread: Boolean, val content: String?) + +data class FrostLink(val text: String, val href: String) + +private class MessageParserImpl : FrostParserBase<Triple<List<FrostThread>, FrostLink?, List<FrostLink>>>() { + + override fun parseImpl(text: String): Triple<List<FrostThread>, FrostLink?, List<FrostLink>>? { + var content = StringEscapeUtils.unescapeEcmaScript(text) + val begin = content.indexOf("id=\"threadlist_rows\"") + if (begin <= 0) { + L.d("Threadlist not found") + return null + } + content = content.substring(begin) + val end = content.indexOf("</script>") + if (end <= 0) { + L.d("Script tail not found") + return null + } + content = content.substring(0, end).substringBeforeLast("</div>") + val body = Jsoup.parseBodyFragment("<div $content") + val threadList = body.getElementById("threadlist_rows") + val threads: List<FrostThread> = threadList.getElementsByAttributeValueContaining("id", "thread_fbid_") + .mapNotNull { parseMessage(it) } + val seeMore = parseLink(body.getElementById("see_older_threads")) + val extraLinks = threadList.nextElementSibling().select("a") + .mapNotNull { parseLink(it) } + return Triple(threads, seeMore, extraLinks) + } + + private fun parseMessage(element: Element): FrostThread? { + val a = element.getElementsByTag("a").first() ?: return null + val abbr = element.getElementsByTag("abbr") + println(abbr.attr("data-store")) + val epoch = FrostRegex.epoch.find(abbr.attr("data-store")) + ?.groupValues?.getOrNull(1)?.toLongOrNull() ?: -1L + //fetch id + val id = FrostRegex.messageNotifId.find(element.id()) + ?.groupValues?.getOrNull(1)?.toLongOrNull() ?: System.currentTimeMillis() + val content = element.select("span.snippet").firstOrNull()?.text()?.trim() + //fetch convo pic + val p = element.select("i.img[style*=url]") + val pUrl = FrostRegex.profilePicture.find(p.attr("style"))?.groups?.get(1)?.value?.formattedFbUrl ?: "" + L.v("url", a.attr("href")) + return FrostThread( + id = id.toInt(), + img = pUrl.formattedFbUrlCss, + title = a.text(), + time = epoch, + url = a.attr("href").formattedFbUrlCss, + unread = !element.hasClass("acw"), + content = content + ) + } + + private fun parseLink(element: Element?): FrostLink? { + val a = element?.getElementsByTag("a")?.first() ?: return null + return FrostLink(a.text(), a.attr("href")) + } + + override fun debugImpl(data: Triple<List<FrostThread>, FrostLink?, List<FrostLink>>, result: MutableList<String>) { + result.addAll(data.first.map { it.toString() }) + result.add("See more link:") + result.add("\t${data.second}") + result.addAll(data.third.map { it.toString() }) + } +} diff --git a/app/src/main/kotlin/com/pitchedapps/frost/services/FrostNotifications.kt b/app/src/main/kotlin/com/pitchedapps/frost/services/FrostNotifications.kt index b892af91..b1f26d99 100644 --- a/app/src/main/kotlin/com/pitchedapps/frost/services/FrostNotifications.kt +++ b/app/src/main/kotlin/com/pitchedapps/frost/services/FrostNotifications.kt @@ -24,6 +24,7 @@ import com.pitchedapps.frost.dbflow.CookieModel import com.pitchedapps.frost.enums.OverlayContext import com.pitchedapps.frost.facebook.FbItem import com.pitchedapps.frost.facebook.formattedFbUrl +import com.pitchedapps.frost.parsers.FrostThread import com.pitchedapps.frost.utils.* import org.jetbrains.anko.runOnUiThread @@ -159,7 +160,10 @@ data class NotificationContent(val data: CookieModel, val title: String? = null, val text: String, val timestamp: Long, - val profileUrl: String) + val profileUrl: String) { + constructor(data:CookieModel, thread: FrostThread) + :this(data, thread.id, thread.url, thread.title, thread.content ?: "", thread.time, thread.img) +} const val NOTIFICATION_PERIODIC_JOB = 7 diff --git a/app/src/main/kotlin/com/pitchedapps/frost/services/NotificationService.kt b/app/src/main/kotlin/com/pitchedapps/frost/services/NotificationService.kt index d481e941..ac3c89dd 100644 --- a/app/src/main/kotlin/com/pitchedapps/frost/services/NotificationService.kt +++ b/app/src/main/kotlin/com/pitchedapps/frost/services/NotificationService.kt @@ -9,18 +9,15 @@ import com.pitchedapps.frost.BuildConfig import com.pitchedapps.frost.R import com.pitchedapps.frost.dbflow.CookieModel import com.pitchedapps.frost.dbflow.lastNotificationTime -import com.pitchedapps.frost.dbflow.loadFbCookie import com.pitchedapps.frost.dbflow.loadFbCookiesSync import com.pitchedapps.frost.facebook.FACEBOOK_COM import com.pitchedapps.frost.facebook.FbItem import com.pitchedapps.frost.facebook.USER_AGENT_BASIC import com.pitchedapps.frost.facebook.formattedFbUrl -import com.pitchedapps.frost.injectors.JsAssets +import com.pitchedapps.frost.parsers.MessageParser import com.pitchedapps.frost.utils.L import com.pitchedapps.frost.utils.Prefs import com.pitchedapps.frost.utils.frostAnswersCustom -import com.pitchedapps.frost.web.launchHeadlessHtmlExtractor -import io.reactivex.schedulers.Schedulers import org.jetbrains.anko.doAsync import org.jsoup.Jsoup import org.jsoup.nodes.Element @@ -75,25 +72,14 @@ class NotificationService : JobService() { override fun onStartJob(params: JobParameters?): Boolean { L.i("Fetching notifications") future = doAsync { - if (Prefs.notificationAllAccounts) { - val cookies = loadFbCookiesSync() - cookies.forEach { fetchGeneralNotifications(it) } - } else { - val currentCookie = loadFbCookie(Prefs.userId) - if (currentCookie != null) { - fetchGeneralNotifications(currentCookie) - } - } - L.d("Finished main notifications") - if (Prefs.notificationsInstantMessages) { - val currentCookie = loadFbCookie(Prefs.userId) - if (currentCookie != null) { - fetchMessageNotifications(currentCookie) { - L.i("Notif IM fetching finished ${if (it) "succesfully" else "unsuccessfully"}") - finish(params) - } - return@doAsync - } + val currentId = Prefs.userId + val cookies = loadFbCookiesSync() + cookies.forEach { + val current = it.id == currentId + if (current || Prefs.notificationAllAccounts) + fetchGeneralNotifications(it) + if (Prefs.notificationsInstantMessages && (current || Prefs.notificationsImAllAccounts)) + fetchMessageNotifications(it) } finish(params) } @@ -161,33 +147,22 @@ class NotificationService : JobService() { * ---------------------------------------------------------------- */ - inline fun fetchMessageNotifications(data: CookieModel, crossinline callback: (success: Boolean) -> Unit) { - launchHeadlessHtmlExtractor(FbItem.MESSAGES.url, JsAssets.NOTIF_MSG) { - it.observeOn(Schedulers.newThread()).subscribe { (html, errorRes) -> - L.d("Notf IM html received") - if (errorRes != -1) return@subscribe callback(false) - fetchMessageNotifications(data, html) - callback(true) - } - } - } - - fun fetchMessageNotifications(data: CookieModel, html: String) { + fun fetchMessageNotifications(data: CookieModel) { L.d("Notif IM fetch", data.toString()) - val doc = Jsoup.parseBodyFragment(html) - val unreadNotifications = (doc.getElementById("threadlist_rows") ?: return L.eThrow("Notification messages not found")).getElementsByClass("aclb") + val doc = Jsoup.connect(FbItem.MESSAGES.url).cookie(FACEBOOK_COM, data.cookie).userAgent(USER_AGENT_BASIC).get() + val (threads, _, _) = MessageParser.parse(doc.toString()) ?: return L.e("Could not parse IM") + var notifCount = 0 val prevNotifTime = lastNotificationTime(data.id) val prevLatestEpoch = prevNotifTime.epochIm L.v("Notif Prev Latest Im Epoch $prevLatestEpoch") var newLatestEpoch = prevLatestEpoch - unreadNotifications.forEach unread@ { elem -> - val notif = parseMessageNotification(data, elem) ?: return@unread - L.v("Notif im timestamp ${notif.timestamp}") - if (notif.timestamp <= prevLatestEpoch) return@unread - NotificationType.MESSAGE.createNotification(this, notif, notifCount == 0) - if (notif.timestamp > newLatestEpoch) - newLatestEpoch = notif.timestamp + threads.filter { it.unread }.forEach { notif -> + L.v("Notif Im timestamp ${notif.time}") + if (notif.time <= prevLatestEpoch) return@forEach + NotificationType.MESSAGE.createNotification(this, NotificationContent(data, notif), notifCount == 0) + if (notif.time > newLatestEpoch) + newLatestEpoch = notif.time notifCount++ } if (newLatestEpoch != prevLatestEpoch) prevNotifTime.copy(epochIm = newLatestEpoch).save() @@ -195,22 +170,6 @@ class NotificationService : JobService() { NotificationType.MESSAGE.summaryNotification(this, data.id, notifCount) } - fun parseMessageNotification(data: CookieModel, element: Element): NotificationContent? { - val a = element.getElementsByTag("a").first() ?: return null - val abbr = element.getElementsByTag("abbr") - val epoch = epochMatcher.find(abbr.attr("data-store"))?.groups?.get(1)?.value?.toLong() ?: return logNotif("No epoch") - val thread = element.getElementsByAttributeValueContaining("id", "thread_fbid_").first() ?: return null - //fetch id - val notifId = messageNotifIdMatcher.find(thread.id())?.groups?.get(1)?.value?.toLong() ?: System.currentTimeMillis() - val text = element.select("span.snippet").firstOrNull()?.text()?.trim() ?: getString(R.string.new_message) - if (Prefs.notificationKeywords.any { text.contains(it, ignoreCase = true) }) return null //notification filtered out - //fetch convo pic - val p = element.select("i.img[style*=url]") - val pUrl = profMatcher.find(p.attr("style"))?.groups?.get(1)?.value?.formattedFbUrl ?: "" - L.v("url", a.attr("href")) - return NotificationContent(data, notifId.toInt(), a.attr("href"), a.text(), text, epoch, pUrl) - } - private fun Context.debugNotification(text: String) { if (!BuildConfig.DEBUG) return val notifBuilder = frostNotification.withDefaults() diff --git a/app/src/main/kotlin/com/pitchedapps/frost/settings/Notifications.kt b/app/src/main/kotlin/com/pitchedapps/frost/settings/Notifications.kt index 0f2f8638..f46517ac 100644 --- a/app/src/main/kotlin/com/pitchedapps/frost/settings/Notifications.kt +++ b/app/src/main/kotlin/com/pitchedapps/frost/settings/Notifications.kt @@ -58,10 +58,15 @@ fun SettingsActivity.getNotificationPrefs(): KPrefAdapterBuilder.() -> Unit = { descRes = R.string.notification_all_accounts_desc } - checkbox(R.string.notification_messages, { Prefs.notificationsInstantMessages }, { Prefs.notificationsInstantMessages = it }) { + checkbox(R.string.notification_messages, { Prefs.notificationsInstantMessages }, { Prefs.notificationsInstantMessages = it; reloadByTitle(R.string.notification_messages_all_accounts) }) { descRes = R.string.notification_messages_desc } + checkbox(R.string.notification_messages_all_accounts, { Prefs.notificationsImAllAccounts }, { Prefs.notificationsImAllAccounts = it }) { + descRes = R.string.notification_messages_all_accounts_desc + enabler = { Prefs.notificationsInstantMessages } + } + checkbox(R.string.notification_sound, { Prefs.notificationSound }, { Prefs.notificationSound = it; reloadByTitle(R.string.notification_ringtone, R.string.message_ringtone) }) fun KPrefText.KPrefTextContract<String>.ringtone(code: Int) { diff --git a/app/src/main/kotlin/com/pitchedapps/frost/utils/Prefs.kt b/app/src/main/kotlin/com/pitchedapps/frost/utils/Prefs.kt index 2897cf3d..68b4b80d 100644 --- a/app/src/main/kotlin/com/pitchedapps/frost/utils/Prefs.kt +++ b/app/src/main/kotlin/com/pitchedapps/frost/utils/Prefs.kt @@ -102,6 +102,8 @@ object Prefs : KPref() { var notificationsInstantMessages: Boolean by kpref("notification_im", false) + var notificationsImAllAccounts: Boolean by kpref("notification_im_all_accounts", false) + var notificationVibrate: Boolean by kpref("notification_vibrate", true) var notificationSound: Boolean by kpref("notification_sound", true) diff --git a/app/src/main/res/values/strings_pref_notifications.xml b/app/src/main/res/values/strings_pref_notifications.xml index 9530cdaa..61c8c598 100644 --- a/app/src/main/res/values/strings_pref_notifications.xml +++ b/app/src/main/res/values/strings_pref_notifications.xml @@ -12,6 +12,8 @@ <string name="notification_all_accounts_desc">Get notifications for every account that is logged in. Disabling this will only fetch notifications form the currently selected account.</string> <string name="notification_messages">Enable message notifications</string> <string name="notification_messages_desc">Get instant message notifications for your current account.</string> + <string name="notification_messages_all_accounts">Notify messages from all accounts</string> + <string name="notification_messages_all_accounts_desc">Get instant message notifications from all accounts</string> <string name="notification_fetch_now">Fetch Notifications Now</string> <string name="notification_fetch_now_desc">Trigger the notification fetcher once. Note that fetching instant messages takes time.</string> <string name="notification_fetch_success">Fetching Notifications…</string> diff --git a/app/src/main/res/xml/frost_changelog.xml b/app/src/main/res/xml/frost_changelog.xml index b3bf851a..8cf6b685 100644 --- a/app/src/main/res/xml/frost_changelog.xml +++ b/app/src/main/res/xml/frost_changelog.xml @@ -5,15 +5,18 @@ <version title="v"/> <item text="" /> --> + + <version title="v1.5.6"/> + <item text="Greatly improve background notification fetcher" /> + <item text="Support fetching messages from all accounts" /> + <item text="Update theme" /> <version title="v1.5.5"/> <item text="Numerous bug fixes in KAU" /> - <item text="Set background back to white on non facebook pages" /> + <item text="Set background back to white on non facebook pages" /> <item text="Make read notification/message colors more obvious" /> <item text="Clean up and small bug fixes" /> <item text="Fix facebook link parsing issue for many links" /> - <item text="" /> - <item text="" /> <version title="v1.5.2"/> <item text="Add default download manager to download all files" /> diff --git a/app/src/test/kotlin/com/pitchedapps/frost/Base.java b/app/src/test/kotlin/com/pitchedapps/frost/Base.java new file mode 100644 index 00000000..42a7da48 --- /dev/null +++ b/app/src/test/kotlin/com/pitchedapps/frost/Base.java @@ -0,0 +1,10 @@ +package com.pitchedapps.frost; + +/** + * Created by Allan Wang on 2017-10-06. + * + * Empty class to hold a reference to the target output + */ + +public class Base { +} diff --git a/app/src/test/kotlin/com/pitchedapps/frost/parsers/MessageParserTest.kt b/app/src/test/kotlin/com/pitchedapps/frost/parsers/MessageParserTest.kt new file mode 100644 index 00000000..dfdf8cc2 --- /dev/null +++ b/app/src/test/kotlin/com/pitchedapps/frost/parsers/MessageParserTest.kt @@ -0,0 +1,30 @@ +package com.pitchedapps.frost.parsers + +import com.pitchedapps.frost.facebook.formattedFbUrlCss +import org.junit.Test +import kotlin.test.assertEquals + +/** + * Created by Allan Wang on 2017-10-06. + */ +class MessageParserTest { + + @Test + fun basic() { + val content = getResource("priv/messages.html") ?: return + println(MessageParser.debug(content)) + } + + @Test + fun parseEpoch() { + val input = "{\"time\":1507301642,\"short\":true,\"forceseconds\":false}" + assertEquals(1507301642, FrostRegex.epoch.find(input)!!.groupValues[1].toLong()) + } + + @Test + fun parseImage() { + var input = "https\\3a //scontent.fyhu1-1.fna.fbcdn.net/v/t1.0-1/cp0/e15/q65/p100x100/12994387_243040309382307_4586627375882013710_n.jpg?efg\\3d eyJpIjoidCJ9\\26 oh\\3d b9ae0d7a1298989fe24873e2ee4054b6\\26 oe\\3d 5A3A7FE1" + input = input.formattedFbUrlCss + println(input) + } +}
\ No newline at end of file diff --git a/app/src/test/kotlin/com/pitchedapps/frost/parsers/ParserTestHelper.kt b/app/src/test/kotlin/com/pitchedapps/frost/parsers/ParserTestHelper.kt new file mode 100644 index 00000000..78050439 --- /dev/null +++ b/app/src/test/kotlin/com/pitchedapps/frost/parsers/ParserTestHelper.kt @@ -0,0 +1,18 @@ +package com.pitchedapps.frost.parsers + +import com.pitchedapps.frost.Base +import java.net.URL +import java.nio.file.Paths + +/** + * Created by Allan Wang on 2017-10-06. + */ + fun <T : Any> T.getResource(path: String): String? { + Paths.get("src/test/resources/${path.trimStart('/')}") + val resource: URL? = Base::class.java.classLoader.getResource(path) + if (resource == null) { + println("Resource at $path could not be found") + return null + } + return resource.readText() +}
\ No newline at end of file |