From 041bafcceadbd5203e95f2692899ac903dd2e883 Mon Sep 17 00:00:00 2001 From: Allan Wang Date: Fri, 29 Dec 2017 23:37:10 -0500 Subject: Feature/image retrieval (#581) * Refactor * Attempt new content * Clean up to make compile friendly * Update docs --- app/build.gradle | 2 + .../com/pitchedapps/frost/facebook/FbItem.kt | 1 + .../com/pitchedapps/frost/facebook/FbRegex.kt | 1 + .../com/pitchedapps/frost/facebook/FbRequest.kt | 160 --------------------- .../frost/facebook/requests/FbRequest.kt | 154 ++++++++++++++++++++ .../pitchedapps/frost/facebook/requests/Images.kt | 67 +++++++++ .../frost/facebook/requests/Notifications.kt | 27 ++++ .../frost/services/FrostRequestService.kt | 6 +- .../test/kotlin/com/pitchedapps/frost/MiscTest.kt | 2 +- .../com/pitchedapps/frost/facebook/FbRegexTest.kt | 6 + .../pitchedapps/frost/facebook/FbRequestTest.kt | 11 ++ .../com/pitchedapps/frost/internal/Internal.kt | 2 + gradle.properties | 1 + 13 files changed, 276 insertions(+), 164 deletions(-) delete mode 100644 app/src/main/kotlin/com/pitchedapps/frost/facebook/FbRequest.kt create mode 100644 app/src/main/kotlin/com/pitchedapps/frost/facebook/requests/FbRequest.kt create mode 100644 app/src/main/kotlin/com/pitchedapps/frost/facebook/requests/Images.kt create mode 100644 app/src/main/kotlin/com/pitchedapps/frost/facebook/requests/Notifications.kt diff --git a/app/build.gradle b/app/build.gradle index 1c06cee9..a5b5cbc3 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -170,6 +170,8 @@ dependencies { implementation"com.mikepenz:fastadapter-extensions:${FAST_ADAPTER_EXTENSIONS}@aar" + implementation "com.github.bumptech.glide:okhttp3-integration:${GLIDE}" + //noinspection GradleDependency releaseImplementation "com.squareup.leakcanary:leakcanary-android-no-op:${LEAK_CANARY}" //noinspection GradleDependency diff --git a/app/src/main/kotlin/com/pitchedapps/frost/facebook/FbItem.kt b/app/src/main/kotlin/com/pitchedapps/frost/facebook/FbItem.kt index e2e9d9e5..ad180023 100644 --- a/app/src/main/kotlin/com/pitchedapps/frost/facebook/FbItem.kt +++ b/app/src/main/kotlin/com/pitchedapps/frost/facebook/FbItem.kt @@ -20,6 +20,7 @@ enum class FbItem( relativeUrl: String, val fragmentCreator: () -> BaseFragment = ::WebFragment ) : EnumBundle { + ACTIVITY_LOG(R.string.activity_log, GoogleMaterial.Icon.gmd_list, "me/allactivity"), BIRTHDAYS(R.string.birthdays, GoogleMaterial.Icon.gmd_cake, "events/birthdays"), CHAT(R.string.chat, GoogleMaterial.Icon.gmd_chat, "buddylist"), diff --git a/app/src/main/kotlin/com/pitchedapps/frost/facebook/FbRegex.kt b/app/src/main/kotlin/com/pitchedapps/frost/facebook/FbRegex.kt index 24f685be..acc23cad 100644 --- a/app/src/main/kotlin/com/pitchedapps/frost/facebook/FbRegex.kt +++ b/app/src/main/kotlin/com/pitchedapps/frost/facebook/FbRegex.kt @@ -24,6 +24,7 @@ val FB_EPOCH_MATCHER: Regex by lazy { Regex(":([0-9]+)") } val FB_NOTIF_ID_MATCHER: Regex by lazy { Regex("notif_([0-9]+)") } val FB_MESSAGE_NOTIF_ID_MATCHER: Regex by lazy { Regex("[thread|user]_fbid_([0-9]+)") } val FB_CSS_URL_MATCHER: Regex by lazy { Regex("url\\([\"|']?(.*?)[\"|']?\\)") } +val FB_JSON_URL_MATCHER: Regex by lazy { Regex("\"(http.*?)\"") } operator fun MatchResult?.get(groupIndex: Int) = this?.groupValues?.get(groupIndex) diff --git a/app/src/main/kotlin/com/pitchedapps/frost/facebook/FbRequest.kt b/app/src/main/kotlin/com/pitchedapps/frost/facebook/FbRequest.kt deleted file mode 100644 index 51e14097..00000000 --- a/app/src/main/kotlin/com/pitchedapps/frost/facebook/FbRequest.kt +++ /dev/null @@ -1,160 +0,0 @@ -package com.pitchedapps.frost.facebook - -import com.pitchedapps.frost.BuildConfig -import com.pitchedapps.frost.utils.L -import io.reactivex.Single -import io.reactivex.schedulers.Schedulers -import okhttp3.* -import okhttp3.logging.HttpLoggingInterceptor -import org.apache.commons.text.StringEscapeUtils - -/** - * Created by Allan Wang on 21/12/17. - */ -private val authMap: MutableMap = mutableMapOf() - -fun String.fbRequest(action: RequestAuth.() -> Unit) { - val savedAuth = authMap[this] - if (savedAuth != null) { - savedAuth.action() - } else { - val auth = getAuth() - if (!auth.isValid) { - L.e("Attempted fbrequest with invalid auth") - return - } - authMap.put(this, auth) - L.i(null, "Found auth $auth") - auth.action() - } -} - -data class RequestAuth(val userId: Long = -1, - val cookie: String = "", - val fb_dtsg: String = "", - val rev: String = "") { - val isValid - get() = userId > 0 && cookie.isNotEmpty() && fb_dtsg.isNotEmpty() && rev.isNotEmpty() -} - -/** - * Request container with the execution call - */ -class FrostRequest(val call: Call, private val invoke: (Call) -> T) { - fun invoke() = invoke(call) -} - -private inline fun RequestAuth.frostRequest( - noinline invoke: (Call) -> T, - builder: Request.Builder.() -> Request.Builder // to ensure we don't do anything extra at the end -): FrostRequest { - val request = cookie.requestBuilder() - request.builder() - return FrostRequest(request.call(), invoke) -} - -private val client: OkHttpClient by lazy { - val builder = OkHttpClient.Builder() - if (BuildConfig.DEBUG) - builder.addInterceptor(HttpLoggingInterceptor() - .setLevel(HttpLoggingInterceptor.Level.BASIC)) - builder.build() -} - -private fun List>.toForm(): FormBody { - val builder = FormBody.Builder() - forEach { (key, value) -> - val v = value?.toString() ?: "" - builder.add(key, v) - } - return builder.build() -} - -private fun List>.withEmptyData(vararg key: String): List> { - val newList = toMutableList() - newList.addAll(key.map { it to null }) - return newList -} - -private fun String.requestBuilder() = Request.Builder() - .header("Cookie", this) - .header("User-Agent", USER_AGENT_BASIC) - .cacheControl(CacheControl.FORCE_NETWORK) - -private fun Request.Builder.call() = client.newCall(build()) - -fun String.getAuth(): RequestAuth { - var auth = RequestAuth(cookie = this) - val id = FB_USER_MATCHER.find(this)[1]?.toLong() ?: return auth - auth = auth.copy(userId = id) - val call = this.requestBuilder() - .url(FB_URL_BASE) - .get() - .call() - call.execute().body()?.charStream()?.useLines { - it.forEach { - val text = StringEscapeUtils.unescapeEcmaScript(it) - val fb_dtsg = FB_DTSG_MATCHER.find(text)[1] - if (fb_dtsg != null) { - L.d(null, "fb_dtsg for ${auth.userId}: $fb_dtsg") - auth = auth.copy(fb_dtsg = fb_dtsg) - if (auth.isValid) return auth - } - - val rev = FB_REV_MATCHER.find(text)[1] - if (rev != null) { - L.d(null, "rev for ${auth.userId}: $rev") - auth = auth.copy(rev = rev) - if (auth.isValid) return auth - } - } - } - - return auth -} - -fun RequestAuth.markNotificationRead(notifId: Long): FrostRequest { - - val body = listOf( - "click_type" to "notification_click", - "id" to notifId, - "target_id" to "null", - "fb_dtsg" to fb_dtsg, - "__user" to userId - ).withEmptyData("m_sess", "__dyn", "__req", "__ajax__") - - return frostRequest(::executeForNoError) { - url("${FB_URL_BASE}a/jewel_notifications_log.php") - post(body.toForm()) - } -} - -inline fun Array.zip(crossinline mapper: (List) -> O, - crossinline caller: (T) -> R): Single { - val singles = map { Single.fromCallable { caller(it) }.subscribeOn(Schedulers.io()) } - return Single.zip(singles) { - val results = it.mapNotNull { it as? R } - mapper(results) - } -} - -fun RequestAuth.markNotificationsRead(vararg notifId: Long) = - notifId.toTypedArray().zip( - { it.all { it } }, - { markNotificationRead(it).invoke() }) - -/** - * Execute the call and attempt to check validity - * Valid = not blank & no "error" instance - */ -fun executeForNoError(call: Call): Boolean { - val body = call.execute().body() ?: return false - var empty = true - body.charStream().useLines { - it.forEach { - if (it.contains("error")) return false - if (empty && it.isNotEmpty()) empty = false - } - } - return !empty -} diff --git a/app/src/main/kotlin/com/pitchedapps/frost/facebook/requests/FbRequest.kt b/app/src/main/kotlin/com/pitchedapps/frost/facebook/requests/FbRequest.kt new file mode 100644 index 00000000..e3e77c5c --- /dev/null +++ b/app/src/main/kotlin/com/pitchedapps/frost/facebook/requests/FbRequest.kt @@ -0,0 +1,154 @@ +package com.pitchedapps.frost.facebook.requests + +import com.pitchedapps.frost.BuildConfig +import com.pitchedapps.frost.facebook.* +import com.pitchedapps.frost.utils.L +import io.reactivex.Single +import io.reactivex.schedulers.Schedulers +import okhttp3.* +import okhttp3.logging.HttpLoggingInterceptor +import org.apache.commons.text.StringEscapeUtils + +/** + * Created by Allan Wang on 21/12/17. + */ +private val authMap: MutableMap = mutableMapOf() + +/** + * Synchronously fetch [RequestAuth] from cookie + * [action] will only be called if a valid auth is found. + * Otherwise, [fail] will be called + */ +fun String.fbRequest(fail: () -> Unit = {}, action: RequestAuth.() -> Unit) { + val savedAuth = authMap[this] + if (savedAuth != null) { + savedAuth.action() + } else { + val auth = getAuth() + if (!auth.isValid) { + L.e("Attempted fbrequest with invalid auth") + return fail() + } + authMap.put(this, auth) + L.i(null, "Found auth $auth") + auth.action() + } +} + +/** + * Underlying container for all fb requests + */ +data class RequestAuth(val userId: Long = -1, + val cookie: String = "", + val fb_dtsg: String = "", + val rev: String = "") { + val isValid + get() = userId > 0 && cookie.isNotEmpty() && fb_dtsg.isNotEmpty() && rev.isNotEmpty() +} + +/** + * Request container with the execution call + */ +class FrostRequest(val call: Call, private val invoke: (Call) -> T) { + fun invoke() = invoke(call) +} + +internal inline fun RequestAuth.frostRequest( + noinline invoke: (Call) -> T, + builder: Request.Builder.() -> Request.Builder // to ensure we don't do anything extra at the end +): FrostRequest { + val request = cookie.requestBuilder() + request.builder() + return FrostRequest(request.call(), invoke) +} + +private val client: OkHttpClient by lazy { + val builder = OkHttpClient.Builder() + if (BuildConfig.DEBUG) + builder.addInterceptor(HttpLoggingInterceptor() + .setLevel(HttpLoggingInterceptor.Level.BASIC)) + builder.build() +} + +internal fun List>.toForm(): FormBody { + val builder = FormBody.Builder() + forEach { (key, value) -> + val v = value?.toString() ?: "" + builder.add(key, v) + } + return builder.build() +} + +internal fun List>.withEmptyData(vararg key: String): List> { + val newList = toMutableList() + newList.addAll(key.map { it to null }) + return newList +} + +private fun String.requestBuilder() = Request.Builder() + .header("Cookie", this) + .header("User-Agent", USER_AGENT_BASIC) + .cacheControl(CacheControl.FORCE_NETWORK) + +fun Request.Builder.call() = client.newCall(build())!! + +fun String.getAuth(): RequestAuth { + var auth = RequestAuth(cookie = this) + val id = FB_USER_MATCHER.find(this)[1]?.toLong() ?: return auth + auth = auth.copy(userId = id) + val call = this.requestBuilder() + .url(FB_URL_BASE) + .get() + .call() + call.execute().body()?.charStream()?.useLines { + it.forEach { + val text = StringEscapeUtils.unescapeEcmaScript(it) + val fb_dtsg = FB_DTSG_MATCHER.find(text)[1] + if (fb_dtsg != null) { + L.d(null, "fb_dtsg for ${auth.userId}: $fb_dtsg") + auth = auth.copy(fb_dtsg = fb_dtsg) + if (auth.isValid) return auth + } + + val rev = FB_REV_MATCHER.find(text)[1] + if (rev != null) { + L.d(null, "rev for ${auth.userId}: $rev") + auth = auth.copy(rev = rev) + if (auth.isValid) return auth + } + } + } + + return auth +} + +inline fun Array.zip(crossinline mapper: (List) -> O, + crossinline caller: (T) -> R): Single { + val singles = map { Single.fromCallable { caller(it) }.subscribeOn(Schedulers.io()) } + return Single.zip(singles) { + val results = it.mapNotNull { it as? R } + mapper(results) + } +} + +/** + * Execute the call and attempt to check validity + * Valid = not blank & no "error" instance + */ +fun executeForNoError(call: Call): Boolean { + val body = call.execute().body() ?: return false + var empty = true + body.charStream().useLines { + it.forEach { + if (it.contains("error")) return false + if (empty && it.isNotEmpty()) empty = false + } + } + return !empty +} + +fun getJsonUrl(call: Call): String? { + val body = call.execute().body() ?: return null + val url = FB_JSON_URL_MATCHER.find(body.string())[1] ?: return null + return StringEscapeUtils.unescapeEcmaScript(url) +} diff --git a/app/src/main/kotlin/com/pitchedapps/frost/facebook/requests/Images.kt b/app/src/main/kotlin/com/pitchedapps/frost/facebook/requests/Images.kt new file mode 100644 index 00000000..61a94ac5 --- /dev/null +++ b/app/src/main/kotlin/com/pitchedapps/frost/facebook/requests/Images.kt @@ -0,0 +1,67 @@ +package com.pitchedapps.frost.facebook.requests + +import com.bumptech.glide.Priority +import com.bumptech.glide.load.DataSource +import com.bumptech.glide.load.data.DataFetcher +import com.pitchedapps.frost.facebook.FB_URL_BASE +import okhttp3.Call +import okhttp3.Request +import java.io.IOException +import java.io.InputStream + +/** + * Created by Allan Wang on 29/12/17. + */ +fun RequestAuth.getFullSizedImage(fbid: Long) = frostRequest(::getJsonUrl) { + url("${FB_URL_BASE}photo/view_full_size/?fbid=$fbid&__ajax__=&__user=$userId") + get() +} + +class ImageFbidFetcher(private val fbid: Long, + private val cookie: String) : DataFetcher { + + @Volatile private var cancelled: Boolean = false + private var urlCall: Call? = null + private var inputStream: InputStream? = null + + private fun DataFetcher.DataCallback.fail(msg: String) { + onLoadFailed(RuntimeException(msg)) + } + + override fun getDataClass(): Class = InputStream::class.java + + override fun getDataSource(): DataSource = DataSource.REMOTE + + override fun loadData(priority: Priority, callback: DataFetcher.DataCallback) { + cookie.fbRequest(fail = { callback.fail("Invalid auth") }) { + if (cancelled) return@fbRequest callback.fail("Cancelled") + val url = getFullSizedImage(fbid).invoke() ?: return@fbRequest callback.fail("Null url") + if (cancelled) return@fbRequest callback.fail("Cancelled") + urlCall = Request.Builder().url(url).get().call() + + inputStream = try { + urlCall?.execute()?.body()?.byteStream() + } catch (e: IOException) { + null + } + + callback.onDataReady(inputStream) + } + } + + override fun cleanup() { + try { + inputStream?.close() + } catch (e: IOException) { + } finally { + inputStream = null + } + } + + override fun cancel() { + cancelled = true + urlCall?.cancel() + urlCall = null + cleanup() + } +} diff --git a/app/src/main/kotlin/com/pitchedapps/frost/facebook/requests/Notifications.kt b/app/src/main/kotlin/com/pitchedapps/frost/facebook/requests/Notifications.kt new file mode 100644 index 00000000..82a9364b --- /dev/null +++ b/app/src/main/kotlin/com/pitchedapps/frost/facebook/requests/Notifications.kt @@ -0,0 +1,27 @@ +package com.pitchedapps.frost.facebook.requests + +import com.pitchedapps.frost.facebook.FB_URL_BASE + +/** + * Created by Allan Wang on 29/12/17. + */ +fun RequestAuth.markNotificationRead(notifId: Long): FrostRequest { + + val body = listOf( + "click_type" to "notification_click", + "id" to notifId, + "target_id" to "null", + "fb_dtsg" to fb_dtsg, + "__user" to userId + ).withEmptyData("m_sess", "__dyn", "__req", "__ajax__") + + return frostRequest(::executeForNoError) { + url("${FB_URL_BASE}a/jewel_notifications_log.php") + post(body.toForm()) + } +} + +fun RequestAuth.markNotificationsRead(vararg notifId: Long) = + notifId.toTypedArray().zip( + { it.all { it } }, + { markNotificationRead(it).invoke() }) \ No newline at end of file diff --git a/app/src/main/kotlin/com/pitchedapps/frost/services/FrostRequestService.kt b/app/src/main/kotlin/com/pitchedapps/frost/services/FrostRequestService.kt index 74a8b98d..2b407b7d 100644 --- a/app/src/main/kotlin/com/pitchedapps/frost/services/FrostRequestService.kt +++ b/app/src/main/kotlin/com/pitchedapps/frost/services/FrostRequestService.kt @@ -9,9 +9,9 @@ import android.content.Context import android.content.Intent import android.os.BaseBundle import android.os.PersistableBundle -import com.pitchedapps.frost.facebook.RequestAuth -import com.pitchedapps.frost.facebook.fbRequest -import com.pitchedapps.frost.facebook.markNotificationRead +import com.pitchedapps.frost.facebook.requests.RequestAuth +import com.pitchedapps.frost.facebook.requests.fbRequest +import com.pitchedapps.frost.facebook.requests.markNotificationRead import com.pitchedapps.frost.utils.EnumBundle import com.pitchedapps.frost.utils.EnumBundleCompanion import com.pitchedapps.frost.utils.EnumCompanion diff --git a/app/src/test/kotlin/com/pitchedapps/frost/MiscTest.kt b/app/src/test/kotlin/com/pitchedapps/frost/MiscTest.kt index 54792086..a565aa7d 100644 --- a/app/src/test/kotlin/com/pitchedapps/frost/MiscTest.kt +++ b/app/src/test/kotlin/com/pitchedapps/frost/MiscTest.kt @@ -1,6 +1,6 @@ package com.pitchedapps.frost -import com.pitchedapps.frost.facebook.zip +import com.pitchedapps.frost.facebook.requests.zip import com.pitchedapps.frost.injectors.CssHider import org.junit.Test import kotlin.test.assertTrue diff --git a/app/src/test/kotlin/com/pitchedapps/frost/facebook/FbRegexTest.kt b/app/src/test/kotlin/com/pitchedapps/frost/facebook/FbRegexTest.kt index da815b34..a79ccf3f 100644 --- a/app/src/test/kotlin/com/pitchedapps/frost/facebook/FbRegexTest.kt +++ b/app/src/test/kotlin/com/pitchedapps/frost/facebook/FbRegexTest.kt @@ -41,6 +41,12 @@ class FbRegexTest { assertEquals(id, FB_MESSAGE_NOTIF_ID_MATCHER.find(data)[1]?.toLong(), "thread_fbid mismatch") val userData = "threadlist_row_other_user_fbid_${id}thread_fbid_" assertEquals(id, FB_MESSAGE_NOTIF_ID_MATCHER.find(userData)[1]?.toLong(), "user_fbid mismatch") + } + @Test + fun jsonUrlRegex() { + val url = "https://www.hello.world" + val data = "\"uri\":\"$url\"}" + assertEquals(url, FB_JSON_URL_MATCHER.find(data)[1]) } } \ No newline at end of file diff --git a/app/src/test/kotlin/com/pitchedapps/frost/facebook/FbRequestTest.kt b/app/src/test/kotlin/com/pitchedapps/frost/facebook/FbRequestTest.kt index c3b19727..93f09fc6 100644 --- a/app/src/test/kotlin/com/pitchedapps/frost/facebook/FbRequestTest.kt +++ b/app/src/test/kotlin/com/pitchedapps/frost/facebook/FbRequestTest.kt @@ -1,5 +1,8 @@ package com.pitchedapps.frost.facebook +import com.pitchedapps.frost.facebook.requests.getAuth +import com.pitchedapps.frost.facebook.requests.getFullSizedImage +import com.pitchedapps.frost.facebook.requests.markNotificationRead import com.pitchedapps.frost.internal.AUTH import com.pitchedapps.frost.internal.COOKIE import com.pitchedapps.frost.internal.USER_ID @@ -48,4 +51,12 @@ class FbRequestTest { AUTH.markNotificationRead(notifId).call.assertNoError() } + @Test + fun fullSizeImage() { + val fbid = 10155966932992838L // google's current cover photo + val url = AUTH.getFullSizedImage(fbid).invoke() + println(url) + assertTrue(url?.startsWith("https://scontent") == true) + } + } \ No newline at end of file diff --git a/app/src/test/kotlin/com/pitchedapps/frost/internal/Internal.kt b/app/src/test/kotlin/com/pitchedapps/frost/internal/Internal.kt index ed88453a..fb2b2a45 100644 --- a/app/src/test/kotlin/com/pitchedapps/frost/internal/Internal.kt +++ b/app/src/test/kotlin/com/pitchedapps/frost/internal/Internal.kt @@ -1,6 +1,8 @@ package com.pitchedapps.frost.internal import com.pitchedapps.frost.facebook.* +import com.pitchedapps.frost.facebook.requests.RequestAuth +import com.pitchedapps.frost.facebook.requests.getAuth import com.pitchedapps.frost.utils.frostJsoup import org.junit.Assume import java.io.File diff --git a/gradle.properties b/gradle.properties index adbd6c10..1e7f0d8c 100644 --- a/gradle.properties +++ b/gradle.properties @@ -26,6 +26,7 @@ CRASHLYTICS=2.8.0 DBFLOW=4.2.3 EXOMEDIA=4.1.0 FAST_ADAPTER_EXTENSIONS=3.0.3 +GLIDE=4.4.0 IAB=1.0.44 IICON_COMMUNITY=2.0.46.1 IICON_MATERIAL=2.2.0.4 -- cgit v1.2.3