From 7460935f32748b10f6b3fedf9e77a373a9010d05 Mon Sep 17 00:00:00 2001 From: Allan Wang Date: Thu, 27 Sep 2018 18:03:53 -0400 Subject: Move parsers to facebook folder (#1109) --- .../frost/activities/BaseMainActivity.kt | 4 +- .../frost/facebook/parsers/FrostParser.kt | 128 +++++++++++++++++++++ .../frost/facebook/parsers/MessageParser.kt | 126 ++++++++++++++++++++ .../frost/facebook/parsers/NotifParser.kt | 100 ++++++++++++++++ .../frost/facebook/parsers/SearchParser.kt | 77 +++++++++++++ .../frost/facebook/requests/FbRequest.kt | 6 +- .../frost/fragments/RecyclerFragmentBase.kt | 4 +- .../frost/fragments/RecyclerFragments.kt | 6 +- .../pitchedapps/frost/iitems/NotificationIItem.kt | 2 +- .../com/pitchedapps/frost/parsers/FrostParser.kt | 128 --------------------- .../com/pitchedapps/frost/parsers/MessageParser.kt | 126 -------------------- .../com/pitchedapps/frost/parsers/NotifParser.kt | 100 ---------------- .../com/pitchedapps/frost/parsers/SearchParser.kt | 77 ------------- .../frost/services/FrostNotifications.kt | 8 +- .../kotlin/com/pitchedapps/frost/settings/Debug.kt | 8 +- 15 files changed, 450 insertions(+), 450 deletions(-) create mode 100644 app/src/main/kotlin/com/pitchedapps/frost/facebook/parsers/FrostParser.kt create mode 100644 app/src/main/kotlin/com/pitchedapps/frost/facebook/parsers/MessageParser.kt create mode 100644 app/src/main/kotlin/com/pitchedapps/frost/facebook/parsers/NotifParser.kt create mode 100644 app/src/main/kotlin/com/pitchedapps/frost/facebook/parsers/SearchParser.kt delete mode 100644 app/src/main/kotlin/com/pitchedapps/frost/parsers/FrostParser.kt delete mode 100644 app/src/main/kotlin/com/pitchedapps/frost/parsers/MessageParser.kt delete mode 100644 app/src/main/kotlin/com/pitchedapps/frost/parsers/NotifParser.kt delete mode 100644 app/src/main/kotlin/com/pitchedapps/frost/parsers/SearchParser.kt (limited to 'app/src/main/kotlin/com/pitchedapps') diff --git a/app/src/main/kotlin/com/pitchedapps/frost/activities/BaseMainActivity.kt b/app/src/main/kotlin/com/pitchedapps/frost/activities/BaseMainActivity.kt index db49d994..9ed51652 100644 --- a/app/src/main/kotlin/com/pitchedapps/frost/activities/BaseMainActivity.kt +++ b/app/src/main/kotlin/com/pitchedapps/frost/activities/BaseMainActivity.kt @@ -55,8 +55,8 @@ import com.pitchedapps.frost.facebook.FbItem import com.pitchedapps.frost.facebook.profilePictureUrl import com.pitchedapps.frost.fragments.BaseFragment import com.pitchedapps.frost.fragments.WebFragment -import com.pitchedapps.frost.parsers.FrostSearch -import com.pitchedapps.frost.parsers.SearchParser +import com.pitchedapps.frost.facebook.parsers.FrostSearch +import com.pitchedapps.frost.facebook.parsers.SearchParser import com.pitchedapps.frost.utils.* import com.pitchedapps.frost.views.BadgedIcon import com.pitchedapps.frost.views.FrostVideoViewer diff --git a/app/src/main/kotlin/com/pitchedapps/frost/facebook/parsers/FrostParser.kt b/app/src/main/kotlin/com/pitchedapps/frost/facebook/parsers/FrostParser.kt new file mode 100644 index 00000000..5d023023 --- /dev/null +++ b/app/src/main/kotlin/com/pitchedapps/frost/facebook/parsers/FrostParser.kt @@ -0,0 +1,128 @@ +package com.pitchedapps.frost.facebook.parsers + +import com.pitchedapps.frost.dbflow.CookieModel +import com.pitchedapps.frost.facebook.FB_CSS_URL_MATCHER +import com.pitchedapps.frost.facebook.formattedFbUrl +import com.pitchedapps.frost.facebook.get +import com.pitchedapps.frost.services.NotificationContent +import com.pitchedapps.frost.utils.frostJsoup +import org.jsoup.Jsoup +import org.jsoup.nodes.Document +import org.jsoup.nodes.Element +import org.jsoup.select.Elements + +/** + * Created by Allan Wang on 2017-10-06. + * + * Interface for a given parser + * Use cases should be attached as delegates to objects that implement this interface + * + * In all cases, parsing will be done from a JSoup document + * Variants accepting strings are also permitted, and they will be converted to documents accordingly + * The return type must be nonnull if no parsing errors occurred, as null signifies a parse error + * If null really must be allowed, use Optionals + */ +interface FrostParser { + + /** + * Name associated to parser + * Purely for display + */ + var nameRes: Int + + /** + * Url to request from + */ + val url: String + + /** + * Call parsing with default implementation using cookie + */ + fun parse(cookie: String?): ParseResponse? + + /** + * Call parsing with given document + */ + fun parse(cookie: String?, document: Document): ParseResponse? + + /** + * Call parsing using jsoup to fetch from given url + */ + fun parseFromUrl(cookie: String?, url: String): ParseResponse? + + /** + * Call parsing with given data + */ + fun parseFromData(cookie: String?, text: String): ParseResponse? + +} + +const val FALLBACK_TIME_MOD = 1000000 + +data class FrostLink(val text: String, val href: String) + +data class ParseResponse(val cookie: String, val data: T) { + override fun toString() = "ParseResponse\ncookie: $cookie\ndata:\n$data" +} + +interface ParseNotification { + fun getUnreadNotifications(data: CookieModel): List +} + +internal fun List.toJsonString(tag: String, indent: Int) = StringBuilder().apply { + val tabs = "\t".repeat(indent) + append("$tabs$tag: [\n\t$tabs") + append(this@toJsonString.joinToString("\n\t$tabs")) + append("\n$tabs]\n") +}.toString() + +/** + * T should have a readable toString() function + * [redirectToText] dictates whether all data should be converted to text then back to document before parsing + */ +internal abstract class FrostParserBase(private val redirectToText: Boolean) : FrostParser { + + final override fun parse(cookie: String?) = parseFromUrl(cookie, url) + + final override fun parseFromData(cookie: String?, text: String): ParseResponse? { + cookie ?: return null + val doc = textToDoc(text) ?: return null + val data = parseImpl(doc) ?: return null + return ParseResponse(cookie, data) + } + + final override fun parseFromUrl(cookie: String?, url: String): ParseResponse? = + parse(cookie, frostJsoup(cookie, url)) + + override fun parse(cookie: String?, document: Document): ParseResponse? { + cookie ?: return null + if (redirectToText) + return parseFromData(cookie, document.toString()) + val data = parseImpl(document) ?: return null + return ParseResponse(cookie, data) + } + + protected abstract fun parseImpl(doc: Document): T? + + // protected abstract fun parse(doc: Document): T? + + /** + * Attempts to find inner element with some style containing a url + * Returns the formatted url, or an empty string if nothing was found + */ + protected fun Element.getInnerImgStyle() = + select("i.img[style*=url]").getStyleUrl() + + protected fun Elements.getStyleUrl() = + FB_CSS_URL_MATCHER.find(attr("style"))[1]?.formattedFbUrl + + protected open fun textToDoc(text: String) = if (!redirectToText) + Jsoup.parse(text) + else + throw RuntimeException("${this::class.java.simpleName} requires text redirect but did not implement textToDoc") + + protected fun parseLink(element: Element?): FrostLink? { + val a = element?.getElementsByTag("a")?.first() ?: return null + return FrostLink(a.text(), a.attr("href")) + } +} \ No newline at end of file diff --git a/app/src/main/kotlin/com/pitchedapps/frost/facebook/parsers/MessageParser.kt b/app/src/main/kotlin/com/pitchedapps/frost/facebook/parsers/MessageParser.kt new file mode 100644 index 00000000..f32c3452 --- /dev/null +++ b/app/src/main/kotlin/com/pitchedapps/frost/facebook/parsers/MessageParser.kt @@ -0,0 +1,126 @@ +package com.pitchedapps.frost.facebook.parsers + +import com.pitchedapps.frost.dbflow.CookieModel +import com.pitchedapps.frost.facebook.* +import com.pitchedapps.frost.services.NotificationContent +import com.pitchedapps.frost.utils.L +import org.apache.commons.text.StringEscapeUtils +import org.jsoup.Jsoup +import org.jsoup.nodes.Document +import org.jsoup.nodes.Element + +/** + * Created by Allan Wang on 2017-10-06. + * + * In Facebook, messages are passed through scripts and loaded into view via react afterwards + * We can parse out the content we want directly and load it ourselves + * + */ +object MessageParser : FrostParser by MessageParserImpl() { + + fun queryUser(cookie: String?, name: String) = parseFromUrl(cookie, "${FbItem.MESSAGES.url}/?q=$name") + +} + +data class FrostMessages(val threads: List, + val seeMore: FrostLink?, + val extraLinks: List +) : ParseNotification { + override fun toString() = StringBuilder().apply { + append("FrostMessages {\n") + append(threads.toJsonString("threads", 1)) + append("\tsee more: $seeMore\n") + append(extraLinks.toJsonString("extra links", 1)) + append("}") + }.toString() + + override fun getUnreadNotifications(data: CookieModel) = + threads.filter(FrostThread::unread).map { + with(it) { + NotificationContent( + data = data, + id = id, + href = url, + title = title, + text = content ?: "", + timestamp = time, + profileUrl = img + ) + } + } +} + +/** + * [id] user/thread id, or current time fallback + * [img] parsed url for profile img + * [time] time of message + * [url] link to thread + * [unread] true if image is unread, false otherwise + * [content] optional string for thread + */ +data class FrostThread(val id: Long, + val img: String?, + val title: String, + val time: Long, + val url: String, + val unread: Boolean, + val content: String?, + val contentImgUrl: String?) + +private class MessageParserImpl : FrostParserBase(true) { + + override var nameRes = FbItem.MESSAGES.titleId + + override val url = FbItem.MESSAGES.url + + override fun textToDoc(text: String): Document? { + var content = StringEscapeUtils.unescapeEcmaScript(text) + val begin = content.indexOf("id=\"threadlist_rows\"") + if (begin <= 0) { + L.d { "Threadlist not found" } + return null + } + content = content.substring(begin) + val end = content.indexOf("") + if (end <= 0) { + L.d { "Script tail not found" } + return null + } + content = content.substring(0, end).substringBeforeLast("") + return Jsoup.parseBodyFragment("
= threadList.getElementsByAttributeValueContaining("id", "thread_fbid_") + .mapNotNull(this::parseMessage) + val seeMore = parseLink(doc.getElementById("see_older_threads")) + val extraLinks = threadList.nextElementSibling().select("a") + .mapNotNull(this::parseLink) + return FrostMessages(threads, seeMore, extraLinks) + } + + private fun parseMessage(element: Element): FrostThread? { + val a = element.getElementsByTag("a").first() ?: return null + val abbr = element.getElementsByTag("abbr") + val epoch = FB_EPOCH_MATCHER.find(abbr.attr("data-store"))[1]?.toLongOrNull() ?: -1L + //fetch id + val id = FB_MESSAGE_NOTIF_ID_MATCHER.find(element.id())[1]?.toLongOrNull() + ?: System.currentTimeMillis() % FALLBACK_TIME_MOD + val snippet = element.select("span.snippet").firstOrNull() + val content = snippet?.text()?.trim() + val contentImg = snippet?.select("i[style*=url]")?.getStyleUrl() + val img = element.getInnerImgStyle() + return FrostThread( + id = id, + img = img, + title = a.text(), + time = epoch, + url = a.attr("href").formattedFbUrl, + unread = !element.hasClass("acw"), + content = content, + contentImgUrl = contentImg + ) + } + +} diff --git a/app/src/main/kotlin/com/pitchedapps/frost/facebook/parsers/NotifParser.kt b/app/src/main/kotlin/com/pitchedapps/frost/facebook/parsers/NotifParser.kt new file mode 100644 index 00000000..03b913c7 --- /dev/null +++ b/app/src/main/kotlin/com/pitchedapps/frost/facebook/parsers/NotifParser.kt @@ -0,0 +1,100 @@ +package com.pitchedapps.frost.facebook.parsers + +import com.pitchedapps.frost.dbflow.CookieModel +import com.pitchedapps.frost.facebook.* +import com.pitchedapps.frost.services.NotificationContent +import org.jsoup.nodes.Document +import org.jsoup.nodes.Element + +/** + * Created by Allan Wang on 2017-12-25. + * + */ +object NotifParser : FrostParser by NotifParserImpl() + +data class FrostNotifs( + val notifs: List, + val seeMore: FrostLink? +) : ParseNotification { + override fun toString() = StringBuilder().apply { + append("FrostNotifs {\n") + append(notifs.toJsonString("notifs", 1)) + append("\tsee more: $seeMore\n") + append("}") + }.toString() + + override fun getUnreadNotifications(data: CookieModel) = + notifs.filter(FrostNotif::unread).map { + with(it) { + NotificationContent( + data = data, + id = id, + href = url, + title = null, + text = content, + timestamp = time, + profileUrl = img + ) + } + } +} + +/** + * [id] notif id, or current time fallback + * [img] parsed url for profile img + * [time] time of message + * [url] link to thread + * [unread] true if image is unread, false otherwise + * [content] optional string for thread + * [timeString] text version of time from Facebook + * [thumbnailUrl] optional thumbnail url if existent + */ +data class FrostNotif(val id: Long, + val img: String?, + val time: Long, + val url: String, + val unread: Boolean, + val content: String, + val timeString: String, + val thumbnailUrl: String?) + +private class NotifParserImpl : FrostParserBase(false) { + + override var nameRes = FbItem.NOTIFICATIONS.titleId + + override val url = FbItem.NOTIFICATIONS.url + + override fun parseImpl(doc: Document): FrostNotifs? { + val notificationList = doc.getElementById("notifications_list") ?: return null + val notifications = notificationList + .getElementsByAttributeValueContaining("id", "list_notif_") + .mapNotNull(this::parseNotif) + val seeMore = parseLink(doc.getElementsByAttributeValue("href", "/notifications.php?more").first()) + return FrostNotifs(notifications, seeMore) + } + + private fun parseNotif(element: Element): FrostNotif? { + val a = element.getElementsByTag("a").first() ?: return null + val abbr = element.getElementsByTag("abbr") + val epoch = FB_EPOCH_MATCHER.find(abbr.attr("data-store"))[1]?.toLongOrNull() ?: -1L + //fetch id + val id = FB_NOTIF_ID_MATCHER.find(element.id())[1]?.toLongOrNull() + ?: System.currentTimeMillis() % FALLBACK_TIME_MOD + val img = element.getInnerImgStyle() + val timeString = abbr.text() + val content = a.text().replace("\u00a0", " ").removeSuffix(timeString).trim() //remove   + val thumbnail = element.selectFirst("img.thumbnail")?.attr("src") + return FrostNotif( + id = id, + img = img, + time = epoch, + url = a.attr("href").formattedFbUrl, + unread = !element.hasClass("acw"), + content = content, + timeString = timeString, + thumbnailUrl = if (thumbnail?.isNotEmpty() == true) thumbnail else null + ) + } + + +} diff --git a/app/src/main/kotlin/com/pitchedapps/frost/facebook/parsers/SearchParser.kt b/app/src/main/kotlin/com/pitchedapps/frost/facebook/parsers/SearchParser.kt new file mode 100644 index 00000000..d3367514 --- /dev/null +++ b/app/src/main/kotlin/com/pitchedapps/frost/facebook/parsers/SearchParser.kt @@ -0,0 +1,77 @@ +package com.pitchedapps.frost.facebook.parsers + +import ca.allanwang.kau.searchview.SearchItem +import com.pitchedapps.frost.facebook.FbItem +import com.pitchedapps.frost.facebook.formattedFbUrl +import com.pitchedapps.frost.facebook.parsers.FrostSearch.Companion.create +import com.pitchedapps.frost.utils.L +import org.jsoup.nodes.Document +import org.jsoup.nodes.Element + +/** + * Created by Allan Wang on 2017-10-09. + */ +object SearchParser : FrostParser by SearchParserImpl() { + fun query(cookie: String?, input: String): ParseResponse? { + val url = "${FbItem._SEARCH.url}?q=${if (input.isNotBlank()) input else "a"}" + L._i { "Search Query $url" } + return parseFromUrl(cookie, url) + } +} + +enum class SearchKeys(val key: String) { + USERS("keywords_users"), + EVENTS("keywords_events") +} + +data class FrostSearches(val results: List) { + + override fun toString() = StringBuilder().apply { + append("FrostSearches {\n") + append(results.toJsonString("results", 1)) + append("}") + }.toString() +} + +/** + * As far as I'm aware, all links are independent, so the queries don't matter + * A lot of it is tracking information, which I'll strip away + * Other text items are formatted for safety + * + * Note that it's best to create search results from [create] + */ +data class FrostSearch(val href: String, val title: String, val description: String?) { + + fun toSearchItem() = SearchItem(href, title, description) + + companion object { + fun create(href: String, title: String, description: String?) = FrostSearch( + with(href.indexOf("?")) { if (this == -1) href else href.substring(0, this) }, + title.format(), + description?.format() + ) + } +} + +private class SearchParserImpl : FrostParserBase(false) { + + override var nameRes = FbItem._SEARCH.titleId + + override val url = "${FbItem._SEARCH.url}?q=a" + + override fun parseImpl(doc: Document): FrostSearches? { + val container: Element = doc.getElementById("BrowseResultsContainer") + ?: doc.getElementById("root") + ?: return null + /** + * + * Removed [data-store*=result_id] + */ + return FrostSearches(container.select("a.touchable[href]").filter(Element::hasText).map { + FrostSearch.create(it.attr("href").formattedFbUrl, + it.select("._uoi").first()?.text() ?: "", + it.select("._1tcc").first()?.text()) + }.filter { it.title.isNotBlank() }) + } + +} \ No newline at end of file diff --git a/app/src/main/kotlin/com/pitchedapps/frost/facebook/requests/FbRequest.kt b/app/src/main/kotlin/com/pitchedapps/frost/facebook/requests/FbRequest.kt index 692312a1..a4b0a347 100644 --- a/app/src/main/kotlin/com/pitchedapps/frost/facebook/requests/FbRequest.kt +++ b/app/src/main/kotlin/com/pitchedapps/frost/facebook/requests/FbRequest.kt @@ -52,7 +52,7 @@ data class RequestAuth(val userId: Long = -1, val cookie: String = "", val fb_dtsg: String = "", val rev: String = "") { - val isValid + val isComplete get() = userId > 0 && cookie.isNotEmpty() && fb_dtsg.isNotEmpty() && rev.isNotEmpty() } @@ -121,13 +121,13 @@ fun String.getAuth(): RequestAuth { val fb_dtsg = FB_DTSG_MATCHER.find(text)[1] if (fb_dtsg != null) { auth = auth.copy(fb_dtsg = fb_dtsg) - if (auth.isValid) return auth + if (auth.isComplete) return auth } val rev = FB_REV_MATCHER.find(text)[1] if (rev != null) { auth = auth.copy(rev = rev) - if (auth.isValid) return auth + if (auth.isComplete) return auth } } } diff --git a/app/src/main/kotlin/com/pitchedapps/frost/fragments/RecyclerFragmentBase.kt b/app/src/main/kotlin/com/pitchedapps/frost/fragments/RecyclerFragmentBase.kt index 51df4606..e3b8f3d3 100644 --- a/app/src/main/kotlin/com/pitchedapps/frost/fragments/RecyclerFragmentBase.kt +++ b/app/src/main/kotlin/com/pitchedapps/frost/fragments/RecyclerFragmentBase.kt @@ -8,8 +8,8 @@ import com.mikepenz.fastadapter.adapters.ModelAdapter import com.mikepenz.fastadapter_extensions.items.ProgressItem import com.pitchedapps.frost.R import com.pitchedapps.frost.facebook.FbCookie -import com.pitchedapps.frost.parsers.FrostParser -import com.pitchedapps.frost.parsers.ParseResponse +import com.pitchedapps.frost.facebook.parsers.FrostParser +import com.pitchedapps.frost.facebook.parsers.ParseResponse import com.pitchedapps.frost.utils.L import com.pitchedapps.frost.utils.frostJsoup import com.pitchedapps.frost.views.FrostRecyclerView diff --git a/app/src/main/kotlin/com/pitchedapps/frost/fragments/RecyclerFragments.kt b/app/src/main/kotlin/com/pitchedapps/frost/fragments/RecyclerFragments.kt index ca2912e8..512ea82c 100644 --- a/app/src/main/kotlin/com/pitchedapps/frost/fragments/RecyclerFragments.kt +++ b/app/src/main/kotlin/com/pitchedapps/frost/fragments/RecyclerFragments.kt @@ -5,9 +5,9 @@ import com.pitchedapps.frost.facebook.FbCookie import com.pitchedapps.frost.facebook.FbItem import com.pitchedapps.frost.facebook.requests.* import com.pitchedapps.frost.iitems.* -import com.pitchedapps.frost.parsers.FrostNotifs -import com.pitchedapps.frost.parsers.NotifParser -import com.pitchedapps.frost.parsers.ParseResponse +import com.pitchedapps.frost.facebook.parsers.FrostNotifs +import com.pitchedapps.frost.facebook.parsers.NotifParser +import com.pitchedapps.frost.facebook.parsers.ParseResponse import com.pitchedapps.frost.utils.frostJsoup import com.pitchedapps.frost.views.FrostRecyclerView import org.jetbrains.anko.doAsync diff --git a/app/src/main/kotlin/com/pitchedapps/frost/iitems/NotificationIItem.kt b/app/src/main/kotlin/com/pitchedapps/frost/iitems/NotificationIItem.kt index e5dcd8a4..185da4fd 100644 --- a/app/src/main/kotlin/com/pitchedapps/frost/iitems/NotificationIItem.kt +++ b/app/src/main/kotlin/com/pitchedapps/frost/iitems/NotificationIItem.kt @@ -16,7 +16,7 @@ import com.mikepenz.fastadapter.commons.utils.DiffCallback import com.pitchedapps.frost.R import com.pitchedapps.frost.glide.FrostGlide import com.pitchedapps.frost.glide.GlideApp -import com.pitchedapps.frost.parsers.FrostNotif +import com.pitchedapps.frost.facebook.parsers.FrostNotif import com.pitchedapps.frost.services.FrostRunnable import com.pitchedapps.frost.utils.Prefs import com.pitchedapps.frost.utils.launchWebOverlay diff --git a/app/src/main/kotlin/com/pitchedapps/frost/parsers/FrostParser.kt b/app/src/main/kotlin/com/pitchedapps/frost/parsers/FrostParser.kt deleted file mode 100644 index 03e6209e..00000000 --- a/app/src/main/kotlin/com/pitchedapps/frost/parsers/FrostParser.kt +++ /dev/null @@ -1,128 +0,0 @@ -package com.pitchedapps.frost.parsers - -import com.pitchedapps.frost.dbflow.CookieModel -import com.pitchedapps.frost.facebook.FB_CSS_URL_MATCHER -import com.pitchedapps.frost.facebook.formattedFbUrl -import com.pitchedapps.frost.facebook.get -import com.pitchedapps.frost.services.NotificationContent -import com.pitchedapps.frost.utils.frostJsoup -import org.jsoup.Jsoup -import org.jsoup.nodes.Document -import org.jsoup.nodes.Element -import org.jsoup.select.Elements - -/** - * Created by Allan Wang on 2017-10-06. - * - * Interface for a given parser - * Use cases should be attached as delegates to objects that implement this interface - * - * In all cases, parsing will be done from a JSoup document - * Variants accepting strings are also permitted, and they will be converted to documents accordingly - * The return type must be nonnull if no parsing errors occurred, as null signifies a parse error - * If null really must be allowed, use Optionals - */ -interface FrostParser { - - /** - * Name associated to parser - * Purely for display - */ - var nameRes: Int - - /** - * Url to request from - */ - val url: String - - /** - * Call parsing with default implementation using cookie - */ - fun parse(cookie: String?): ParseResponse? - - /** - * Call parsing with given document - */ - fun parse(cookie: String?, document: Document): ParseResponse? - - /** - * Call parsing using jsoup to fetch from given url - */ - fun parseFromUrl(cookie: String?, url: String): ParseResponse? - - /** - * Call parsing with given data - */ - fun parseFromData(cookie: String?, text: String): ParseResponse? - -} - -const val FALLBACK_TIME_MOD = 1000000 - -data class FrostLink(val text: String, val href: String) - -data class ParseResponse(val cookie: String, val data: T) { - override fun toString() = "ParseResponse\ncookie: $cookie\ndata:\n$data" -} - -interface ParseNotification { - fun getUnreadNotifications(data: CookieModel): List -} - -internal fun List.toJsonString(tag: String, indent: Int) = StringBuilder().apply { - val tabs = "\t".repeat(indent) - append("$tabs$tag: [\n\t$tabs") - append(this@toJsonString.joinToString("\n\t$tabs")) - append("\n$tabs]\n") -}.toString() - -/** - * T should have a readable toString() function - * [redirectToText] dictates whether all data should be converted to text then back to document before parsing - */ -internal abstract class FrostParserBase(private val redirectToText: Boolean) : FrostParser { - - final override fun parse(cookie: String?) = parseFromUrl(cookie, url) - - final override fun parseFromData(cookie: String?, text: String): ParseResponse? { - cookie ?: return null - val doc = textToDoc(text) ?: return null - val data = parseImpl(doc) ?: return null - return ParseResponse(cookie, data) - } - - final override fun parseFromUrl(cookie: String?, url: String): ParseResponse? = - parse(cookie, frostJsoup(cookie, url)) - - override fun parse(cookie: String?, document: Document): ParseResponse? { - cookie ?: return null - if (redirectToText) - return parseFromData(cookie, document.toString()) - val data = parseImpl(document) ?: return null - return ParseResponse(cookie, data) - } - - protected abstract fun parseImpl(doc: Document): T? - - // protected abstract fun parse(doc: Document): T? - - /** - * Attempts to find inner element with some style containing a url - * Returns the formatted url, or an empty string if nothing was found - */ - protected fun Element.getInnerImgStyle() = - select("i.img[style*=url]").getStyleUrl() - - protected fun Elements.getStyleUrl() = - FB_CSS_URL_MATCHER.find(attr("style"))[1]?.formattedFbUrl - - protected open fun textToDoc(text: String) = if (!redirectToText) - Jsoup.parse(text) - else - throw RuntimeException("${this::class.java.simpleName} requires text redirect but did not implement textToDoc") - - protected fun parseLink(element: Element?): FrostLink? { - val a = element?.getElementsByTag("a")?.first() ?: return null - return FrostLink(a.text(), a.attr("href")) - } -} \ No newline at end of file diff --git a/app/src/main/kotlin/com/pitchedapps/frost/parsers/MessageParser.kt b/app/src/main/kotlin/com/pitchedapps/frost/parsers/MessageParser.kt deleted file mode 100644 index 697cbbe8..00000000 --- a/app/src/main/kotlin/com/pitchedapps/frost/parsers/MessageParser.kt +++ /dev/null @@ -1,126 +0,0 @@ -package com.pitchedapps.frost.parsers - -import com.pitchedapps.frost.dbflow.CookieModel -import com.pitchedapps.frost.facebook.* -import com.pitchedapps.frost.services.NotificationContent -import com.pitchedapps.frost.utils.L -import org.apache.commons.text.StringEscapeUtils -import org.jsoup.Jsoup -import org.jsoup.nodes.Document -import org.jsoup.nodes.Element - -/** - * Created by Allan Wang on 2017-10-06. - * - * In Facebook, messages are passed through scripts and loaded into view via react afterwards - * We can parse out the content we want directly and load it ourselves - * - */ -object MessageParser : FrostParser by MessageParserImpl() { - - fun queryUser(cookie: String?, name: String) = parseFromUrl(cookie, "${FbItem.MESSAGES.url}/?q=$name") - -} - -data class FrostMessages(val threads: List, - val seeMore: FrostLink?, - val extraLinks: List -) : ParseNotification { - override fun toString() = StringBuilder().apply { - append("FrostMessages {\n") - append(threads.toJsonString("threads", 1)) - append("\tsee more: $seeMore\n") - append(extraLinks.toJsonString("extra links", 1)) - append("}") - }.toString() - - override fun getUnreadNotifications(data: CookieModel) = - threads.filter(FrostThread::unread).map { - with(it) { - NotificationContent( - data = data, - id = id, - href = url, - title = title, - text = content ?: "", - timestamp = time, - profileUrl = img - ) - } - } -} - -/** - * [id] user/thread id, or current time fallback - * [img] parsed url for profile img - * [time] time of message - * [url] link to thread - * [unread] true if image is unread, false otherwise - * [content] optional string for thread - */ -data class FrostThread(val id: Long, - val img: String?, - val title: String, - val time: Long, - val url: String, - val unread: Boolean, - val content: String?, - val contentImgUrl: String?) - -private class MessageParserImpl : FrostParserBase(true) { - - override var nameRes = FbItem.MESSAGES.titleId - - override val url = FbItem.MESSAGES.url - - override fun textToDoc(text: String): Document? { - var content = StringEscapeUtils.unescapeEcmaScript(text) - val begin = content.indexOf("id=\"threadlist_rows\"") - if (begin <= 0) { - L.d { "Threadlist not found" } - return null - } - content = content.substring(begin) - val end = content.indexOf("") - if (end <= 0) { - L.d { "Script tail not found" } - return null - } - content = content.substring(0, end).substringBeforeLast("
") - return Jsoup.parseBodyFragment("
= threadList.getElementsByAttributeValueContaining("id", "thread_fbid_") - .mapNotNull(this::parseMessage) - val seeMore = parseLink(doc.getElementById("see_older_threads")) - val extraLinks = threadList.nextElementSibling().select("a") - .mapNotNull(this::parseLink) - return FrostMessages(threads, seeMore, extraLinks) - } - - private fun parseMessage(element: Element): FrostThread? { - val a = element.getElementsByTag("a").first() ?: return null - val abbr = element.getElementsByTag("abbr") - val epoch = FB_EPOCH_MATCHER.find(abbr.attr("data-store"))[1]?.toLongOrNull() ?: -1L - //fetch id - val id = FB_MESSAGE_NOTIF_ID_MATCHER.find(element.id())[1]?.toLongOrNull() - ?: System.currentTimeMillis() % FALLBACK_TIME_MOD - val snippet = element.select("span.snippet").firstOrNull() - val content = snippet?.text()?.trim() - val contentImg = snippet?.select("i[style*=url]")?.getStyleUrl() - val img = element.getInnerImgStyle() - return FrostThread( - id = id, - img = img, - title = a.text(), - time = epoch, - url = a.attr("href").formattedFbUrl, - unread = !element.hasClass("acw"), - content = content, - contentImgUrl = contentImg - ) - } - -} diff --git a/app/src/main/kotlin/com/pitchedapps/frost/parsers/NotifParser.kt b/app/src/main/kotlin/com/pitchedapps/frost/parsers/NotifParser.kt deleted file mode 100644 index 812f12e3..00000000 --- a/app/src/main/kotlin/com/pitchedapps/frost/parsers/NotifParser.kt +++ /dev/null @@ -1,100 +0,0 @@ -package com.pitchedapps.frost.parsers - -import com.pitchedapps.frost.dbflow.CookieModel -import com.pitchedapps.frost.facebook.* -import com.pitchedapps.frost.services.NotificationContent -import org.jsoup.nodes.Document -import org.jsoup.nodes.Element - -/** - * Created by Allan Wang on 2017-12-25. - * - */ -object NotifParser : FrostParser by NotifParserImpl() - -data class FrostNotifs( - val notifs: List, - val seeMore: FrostLink? -) : ParseNotification { - override fun toString() = StringBuilder().apply { - append("FrostNotifs {\n") - append(notifs.toJsonString("notifs", 1)) - append("\tsee more: $seeMore\n") - append("}") - }.toString() - - override fun getUnreadNotifications(data: CookieModel) = - notifs.filter(FrostNotif::unread).map { - with(it) { - NotificationContent( - data = data, - id = id, - href = url, - title = null, - text = content, - timestamp = time, - profileUrl = img - ) - } - } -} - -/** - * [id] notif id, or current time fallback - * [img] parsed url for profile img - * [time] time of message - * [url] link to thread - * [unread] true if image is unread, false otherwise - * [content] optional string for thread - * [timeString] text version of time from Facebook - * [thumbnailUrl] optional thumbnail url if existent - */ -data class FrostNotif(val id: Long, - val img: String?, - val time: Long, - val url: String, - val unread: Boolean, - val content: String, - val timeString: String, - val thumbnailUrl: String?) - -private class NotifParserImpl : FrostParserBase(false) { - - override var nameRes = FbItem.NOTIFICATIONS.titleId - - override val url = FbItem.NOTIFICATIONS.url - - override fun parseImpl(doc: Document): FrostNotifs? { - val notificationList = doc.getElementById("notifications_list") ?: return null - val notifications = notificationList - .getElementsByAttributeValueContaining("id", "list_notif_") - .mapNotNull(this::parseNotif) - val seeMore = parseLink(doc.getElementsByAttributeValue("href", "/notifications.php?more").first()) - return FrostNotifs(notifications, seeMore) - } - - private fun parseNotif(element: Element): FrostNotif? { - val a = element.getElementsByTag("a").first() ?: return null - val abbr = element.getElementsByTag("abbr") - val epoch = FB_EPOCH_MATCHER.find(abbr.attr("data-store"))[1]?.toLongOrNull() ?: -1L - //fetch id - val id = FB_NOTIF_ID_MATCHER.find(element.id())[1]?.toLongOrNull() - ?: System.currentTimeMillis() % FALLBACK_TIME_MOD - val img = element.getInnerImgStyle() - val timeString = abbr.text() - val content = a.text().replace("\u00a0", " ").removeSuffix(timeString).trim() //remove   - val thumbnail = element.selectFirst("img.thumbnail")?.attr("src") - return FrostNotif( - id = id, - img = img, - time = epoch, - url = a.attr("href").formattedFbUrl, - unread = !element.hasClass("acw"), - content = content, - timeString = timeString, - thumbnailUrl = if (thumbnail?.isNotEmpty() == true) thumbnail else null - ) - } - - -} diff --git a/app/src/main/kotlin/com/pitchedapps/frost/parsers/SearchParser.kt b/app/src/main/kotlin/com/pitchedapps/frost/parsers/SearchParser.kt deleted file mode 100644 index 5300bf11..00000000 --- a/app/src/main/kotlin/com/pitchedapps/frost/parsers/SearchParser.kt +++ /dev/null @@ -1,77 +0,0 @@ -package com.pitchedapps.frost.parsers - -import ca.allanwang.kau.searchview.SearchItem -import com.pitchedapps.frost.facebook.FbItem -import com.pitchedapps.frost.facebook.formattedFbUrl -import com.pitchedapps.frost.parsers.FrostSearch.Companion.create -import com.pitchedapps.frost.utils.L -import org.jsoup.nodes.Document -import org.jsoup.nodes.Element - -/** - * Created by Allan Wang on 2017-10-09. - */ -object SearchParser : FrostParser by SearchParserImpl() { - fun query(cookie: String?, input: String): ParseResponse? { - val url = "${FbItem._SEARCH.url}?q=${if (input.isNotBlank()) input else "a"}" - L._i { "Search Query $url" } - return parseFromUrl(cookie, url) - } -} - -enum class SearchKeys(val key: String) { - USERS("keywords_users"), - EVENTS("keywords_events") -} - -data class FrostSearches(val results: List) { - - override fun toString() = StringBuilder().apply { - append("FrostSearches {\n") - append(results.toJsonString("results", 1)) - append("}") - }.toString() -} - -/** - * As far as I'm aware, all links are independent, so the queries don't matter - * A lot of it is tracking information, which I'll strip away - * Other text items are formatted for safety - * - * Note that it's best to create search results from [create] - */ -data class FrostSearch(val href: String, val title: String, val description: String?) { - - fun toSearchItem() = SearchItem(href, title, description) - - companion object { - fun create(href: String, title: String, description: String?) = FrostSearch( - with(href.indexOf("?")) { if (this == -1) href else href.substring(0, this) }, - title.format(), - description?.format() - ) - } -} - -private class SearchParserImpl : FrostParserBase(false) { - - override var nameRes = FbItem._SEARCH.titleId - - override val url = "${FbItem._SEARCH.url}?q=a" - - override fun parseImpl(doc: Document): FrostSearches? { - val container: Element = doc.getElementById("BrowseResultsContainer") - ?: doc.getElementById("root") - ?: return null - /** - * - * Removed [data-store*=result_id] - */ - return FrostSearches(container.select("a.touchable[href]").filter(Element::hasText).map { - FrostSearch.create(it.attr("href").formattedFbUrl, - it.select("._uoi").first()?.text() ?: "", - it.select("._1tcc").first()?.text()) - }.filter { it.title.isNotBlank() }) - } - -} \ No newline at end of file diff --git a/app/src/main/kotlin/com/pitchedapps/frost/services/FrostNotifications.kt b/app/src/main/kotlin/com/pitchedapps/frost/services/FrostNotifications.kt index bc2e66a5..279b4027 100644 --- a/app/src/main/kotlin/com/pitchedapps/frost/services/FrostNotifications.kt +++ b/app/src/main/kotlin/com/pitchedapps/frost/services/FrostNotifications.kt @@ -21,10 +21,10 @@ import com.pitchedapps.frost.enums.OverlayContext import com.pitchedapps.frost.facebook.FbItem import com.pitchedapps.frost.glide.FrostGlide import com.pitchedapps.frost.glide.GlideApp -import com.pitchedapps.frost.parsers.FrostParser -import com.pitchedapps.frost.parsers.MessageParser -import com.pitchedapps.frost.parsers.NotifParser -import com.pitchedapps.frost.parsers.ParseNotification +import com.pitchedapps.frost.facebook.parsers.FrostParser +import com.pitchedapps.frost.facebook.parsers.MessageParser +import com.pitchedapps.frost.facebook.parsers.NotifParser +import com.pitchedapps.frost.facebook.parsers.ParseNotification import com.pitchedapps.frost.utils.ARG_USER_ID import com.pitchedapps.frost.utils.L import com.pitchedapps.frost.utils.Prefs diff --git a/app/src/main/kotlin/com/pitchedapps/frost/settings/Debug.kt b/app/src/main/kotlin/com/pitchedapps/frost/settings/Debug.kt index 7ca7d778..91dac242 100644 --- a/app/src/main/kotlin/com/pitchedapps/frost/settings/Debug.kt +++ b/app/src/main/kotlin/com/pitchedapps/frost/settings/Debug.kt @@ -12,10 +12,10 @@ import com.pitchedapps.frost.activities.SettingsActivity.Companion.ACTIVITY_REQU import com.pitchedapps.frost.debugger.OfflineWebsite import com.pitchedapps.frost.facebook.FB_URL_BASE import com.pitchedapps.frost.facebook.FbCookie -import com.pitchedapps.frost.parsers.FrostParser -import com.pitchedapps.frost.parsers.MessageParser -import com.pitchedapps.frost.parsers.NotifParser -import com.pitchedapps.frost.parsers.SearchParser +import com.pitchedapps.frost.facebook.parsers.FrostParser +import com.pitchedapps.frost.facebook.parsers.MessageParser +import com.pitchedapps.frost.facebook.parsers.NotifParser +import com.pitchedapps.frost.facebook.parsers.SearchParser import com.pitchedapps.frost.utils.L import com.pitchedapps.frost.utils.frostUriFromFile import com.pitchedapps.frost.utils.sendFrostEmail -- cgit v1.2.3