From 894c1c4d2c7568d26165baf0d9d192bdf3b288ef Mon Sep 17 00:00:00 2001 From: Allan Wang Date: Sat, 12 Aug 2017 19:38:26 -0700 Subject: Feature/jsoup debugger (#152) * Create debugger * Update debugger content * Create debugging logic * Finalize and test debugger * Add reload listener --- .../main/kotlin/com/pitchedapps/frost/FrostApp.kt | 4 +- .../pitchedapps/frost/activities/AboutActivity.kt | 20 +++ .../frost/activities/SettingsActivity.kt | 16 ++- .../com/pitchedapps/frost/injectors/JsActions.kt | 3 +- .../kotlin/com/pitchedapps/frost/settings/Debug.kt | 156 +++++++++++++++++++++ .../com/pitchedapps/frost/utils/JsoupCleaner.kt | 34 +++++ .../kotlin/com/pitchedapps/frost/utils/Prefs.kt | 2 + .../pitchedapps/frost/web/HeadlessHtmlExtractor.kt | 88 ++++++++++++ app/src/main/res/values/strings_pref_debug.xml | 18 +++ app/src/main/res/values/strings_preferences.xml | 2 + .../pitchedapps/frost/utils/JsoupCleanerTest.kt | 56 ++++++++ gradle.properties | 2 +- 12 files changed, 390 insertions(+), 11 deletions(-) create mode 100644 app/src/main/kotlin/com/pitchedapps/frost/settings/Debug.kt create mode 100644 app/src/main/kotlin/com/pitchedapps/frost/utils/JsoupCleaner.kt create mode 100644 app/src/main/kotlin/com/pitchedapps/frost/web/HeadlessHtmlExtractor.kt create mode 100644 app/src/main/res/values/strings_pref_debug.xml create mode 100644 app/src/test/kotlin/com/pitchedapps/frost/utils/JsoupCleanerTest.kt diff --git a/app/src/main/kotlin/com/pitchedapps/frost/FrostApp.kt b/app/src/main/kotlin/com/pitchedapps/frost/FrostApp.kt index 17ef972e..4fabf8b8 100644 --- a/app/src/main/kotlin/com/pitchedapps/frost/FrostApp.kt +++ b/app/src/main/kotlin/com/pitchedapps/frost/FrostApp.kt @@ -77,11 +77,11 @@ class FrostApp : Application() { L.d("Activity ${activity.localClassName} destroyed") } - override fun onActivitySaveInstanceState(activity: Activity, outState: Bundle) {} + override fun onActivitySaveInstanceState(activity: Activity, outState: Bundle?) {} override fun onActivityStopped(activity: Activity) {} - override fun onActivityCreated(activity: Activity, savedInstanceState: Bundle) { + override fun onActivityCreated(activity: Activity, savedInstanceState: Bundle?) { L.d("Activity ${activity.localClassName} created") } }) diff --git a/app/src/main/kotlin/com/pitchedapps/frost/activities/AboutActivity.kt b/app/src/main/kotlin/com/pitchedapps/frost/activities/AboutActivity.kt index 4514eb10..304e4182 100644 --- a/app/src/main/kotlin/com/pitchedapps/frost/activities/AboutActivity.kt +++ b/app/src/main/kotlin/com/pitchedapps/frost/activities/AboutActivity.kt @@ -22,6 +22,7 @@ import com.mikepenz.google_material_typeface_library.GoogleMaterial import com.mikepenz.iconics.typeface.IIcon import com.pitchedapps.frost.BuildConfig import com.pitchedapps.frost.R +import com.pitchedapps.frost.utils.L import com.pitchedapps.frost.utils.Prefs @@ -62,6 +63,9 @@ class AboutActivity : AboutActivityBase(null, { return l } + var lastClick = -1L + var clickCount = 0 + override fun postInflateMainPage(adapter: FastItemThemedAdapter>) { /** * Frost may not be a library but we're conveying the same info @@ -79,6 +83,22 @@ class AboutActivity : AboutActivityBase(null, { } } adapter.add(LibraryIItem(frost)).add(AboutLinks()) + adapter.withOnClickListener { _, _, item, _ -> + if (item is LibraryIItem) { + val now = System.currentTimeMillis() + if (now - lastClick > 500) + clickCount = 0 + else + clickCount++ + lastClick = now + if (clickCount == 7 && !Prefs.debugSettings) { + Prefs.debugSettings = true + L.d("Debugging section enabled") + toast(R.string.debug_toast_enabled) + } + } + false + } } class AboutLinks : AbstractItem(), ThemableIItem by ThemableIItemDelegate() { diff --git a/app/src/main/kotlin/com/pitchedapps/frost/activities/SettingsActivity.kt b/app/src/main/kotlin/com/pitchedapps/frost/activities/SettingsActivity.kt index 3fd3e3b5..196aa461 100644 --- a/app/src/main/kotlin/com/pitchedapps/frost/activities/SettingsActivity.kt +++ b/app/src/main/kotlin/com/pitchedapps/frost/activities/SettingsActivity.kt @@ -6,16 +6,12 @@ import android.content.Intent import android.os.Bundle import android.view.Menu import android.view.MenuItem -import ca.allanwang.kau.about.kauLaunchAbout import ca.allanwang.kau.kpref.activity.CoreAttributeContract import ca.allanwang.kau.kpref.activity.KPrefActivity import ca.allanwang.kau.kpref.activity.KPrefAdapterBuilder import ca.allanwang.kau.kpref.activity.items.KPrefItemBase import ca.allanwang.kau.ui.views.RippleCanvas -import ca.allanwang.kau.utils.finishSlideOut -import ca.allanwang.kau.utils.setMenuIcons -import ca.allanwang.kau.utils.string -import ca.allanwang.kau.utils.tint +import ca.allanwang.kau.utils.* import ca.allanwang.kau.xml.showChangelog import com.mikepenz.community_material_typeface_library.CommunityMaterial import com.mikepenz.google_material_typeface_library.GoogleMaterial @@ -38,7 +34,7 @@ class SettingsActivity : KPrefActivity(), FrostBilling by IABSettings() { override fun onActivityResult(requestCode: Int, resultCode: Int, data: Intent?) { if (!onActivityResultBilling(requestCode, resultCode, data)) super.onActivityResult(requestCode, resultCode, data) - reload() + reloadList() } override fun kPrefCoreAttributes(): CoreAttributeContract.() -> Unit = { @@ -86,7 +82,7 @@ class SettingsActivity : KPrefActivity(), FrostBilling by IABSettings() { plainText(R.string.about_frost) { descRes = R.string.about_frost_desc iicon = GoogleMaterial.Icon.gmd_info - onClick = { _, _, _ -> kauLaunchAbout(AboutActivity::class.java); true } + onClick = { _, _, _ -> startActivityForResult(AboutActivity::class.java, 9, true); true } } plainText(R.string.replay_intro) { @@ -94,6 +90,12 @@ class SettingsActivity : KPrefActivity(), FrostBilling by IABSettings() { onClick = { _, _, _ -> launchIntroActivity(cookies()); true } } + subItems(R.string.debug_frost, getDebugPrefs()) { + descRes = R.string.debug_frost_desc + iicon = CommunityMaterial.Icon.cmd_android_debug_bridge + visible = { Prefs.debugSettings } + } + if (BuildConfig.DEBUG) { checkbox(R.string.custom_pro, { Prefs.debugPro }, { Prefs.debugPro = it }) } diff --git a/app/src/main/kotlin/com/pitchedapps/frost/injectors/JsActions.kt b/app/src/main/kotlin/com/pitchedapps/frost/injectors/JsActions.kt index fae1846b..3fa03bcc 100644 --- a/app/src/main/kotlin/com/pitchedapps/frost/injectors/JsActions.kt +++ b/app/src/main/kotlin/com/pitchedapps/frost/injectors/JsActions.kt @@ -14,7 +14,8 @@ enum class JsActions(body: String) : InjectorContract { * see [com.pitchedapps.frost.web.FrostJSI.loadLogin] */ LOGIN_CHECK("document.getElementById('signup-button')&&Frost.loadLogin();"), - BASE_HREF("document.write(\"\");"), + BASE_HREF("""document.write("");"""), + FETCH_BODY("""setTimeout(function(){var e=document.querySelector("main");e||(e=document.querySelector("body")),Frost.handleHtml(e.outerHTML)},1e2);"""), /** * Used as a pseudoinjector for maybe functions */ diff --git a/app/src/main/kotlin/com/pitchedapps/frost/settings/Debug.kt b/app/src/main/kotlin/com/pitchedapps/frost/settings/Debug.kt new file mode 100644 index 00000000..d98a50c3 --- /dev/null +++ b/app/src/main/kotlin/com/pitchedapps/frost/settings/Debug.kt @@ -0,0 +1,156 @@ +package com.pitchedapps.frost.settings + +import android.content.Context +import android.support.annotation.UiThread +import ca.allanwang.kau.email.sendEmail +import ca.allanwang.kau.kpref.activity.KPrefAdapterBuilder +import ca.allanwang.kau.utils.string +import com.afollestad.materialdialogs.MaterialDialog +import com.pitchedapps.frost.R +import com.pitchedapps.frost.activities.SettingsActivity +import com.pitchedapps.frost.facebook.FACEBOOK_COM +import com.pitchedapps.frost.facebook.FbCookie +import com.pitchedapps.frost.facebook.FbTab +import com.pitchedapps.frost.facebook.USER_AGENT_BASIC +import com.pitchedapps.frost.injectors.InjectorContract +import com.pitchedapps.frost.injectors.JsActions +import com.pitchedapps.frost.utils.L +import com.pitchedapps.frost.utils.cleanHtml +import com.pitchedapps.frost.utils.materialDialogThemed +import com.pitchedapps.frost.web.launchHeadlessHtmlExtractor +import com.pitchedapps.frost.web.query +import io.reactivex.disposables.Disposable +import org.jetbrains.anko.AnkoAsyncContext +import org.jetbrains.anko.doAsync +import org.jetbrains.anko.runOnUiThread +import org.jetbrains.anko.uiThread +import org.jsoup.Jsoup +import org.jsoup.nodes.Document + +/** + * Created by Allan Wang on 2017-06-30. + * + * A sub pref section that is enabled through a hidden preference + * Each category will load a page, extract the contents, remove private info, and create a report + */ +fun SettingsActivity.getDebugPrefs(): KPrefAdapterBuilder.() -> Unit = { + + plainText(R.string.experimental_disclaimer) { + descRes = R.string.debug_disclaimer_info + } + + Debugger.values().forEach { + plainText(it.data.titleId) { + iicon = it.data.icon + onClick = { itemView, _, _ -> it.debug(itemView.context); true } + } + } + +} + +private enum class Debugger(val data: FbTab, val injector: InjectorContract?, vararg query: String) { + NOTIFICATIONS(FbTab.NOTIFICATIONS, null, "#notifications_list"), + SEARCH(FbTab.SEARCH, JsActions.FETCH_BODY); + + val query = if (query.isNotEmpty()) arrayOf(*query, "#root", "main", "body") else emptyArray() + + fun debug(context: Context) { + val dialog = context.materialDialogThemed { + title("Debugging") + progress(true, 0) + canceledOnTouchOutside(false) + positiveText(R.string.kau_cancel) + onPositive { dialog, _ -> dialog.cancel() } + } + if (injector != null) dialog.extractHtml(injector) + else dialog.debugAsync { + loadJsoup() + } + } + + fun MaterialDialog.debugAsync(task: AnkoAsyncContext.() -> Unit) { + doAsync({ t: Throwable -> + val msg = t.message + L.e("Debugger failed: $msg") + context.runOnUiThread { + cancel() + context.materialDialogThemed { + title(R.string.debug_incomplete) + if (msg != null) content(msg) + } + } + }, task) + } + + /** + * Wait for html to be returned from headless webview + * + * from [debug] to [simplifyJsoup] if [query] is not empty, or [createReport] otherwise + */ + @UiThread + private fun MaterialDialog.extractHtml(injector: InjectorContract) { + setContent("Fetching webpage") + var disposable: Disposable? = null + setOnCancelListener { disposable?.dispose() } + context.launchHeadlessHtmlExtractor(data.url, injector) { + disposable = it.subscribe { + (html, errorRes) -> + debugAsync { + if (errorRes == -1) { + L.i("Debug report successful", html) + if (query.isNotEmpty()) simplifyJsoup(Jsoup.parseBodyFragment(html)) + else createReport(html) + } else { + throw Throwable(context.string(errorRes)) + } + } + } + } + } + + /** + * Get data directly from the link and search for our queries, returning the outerHTML + * of the first query found + * + * from [debug] to [simplifyJsoup] + */ + private fun AnkoAsyncContext.loadJsoup() { + uiThread { + it.setContent("Load Jsoup") + it.setOnCancelListener(null) + it.debugAsync { + val connection = Jsoup.connect(data.url).cookie(FACEBOOK_COM, FbCookie.webCookie).userAgent(USER_AGENT_BASIC) + val doc = connection.get() + simplifyJsoup(doc) + } + } + } + + /** + * Takes snippet of given document that matches the first query in the [query] items + * before sending it to [createReport] + */ + private fun AnkoAsyncContext.simplifyJsoup(doc: Document) { + weakRef.get() ?: return + val q = query.first { doc.select(it).isNotEmpty() } + createReport(doc.select(q).outerHtml()) + } + + private fun AnkoAsyncContext.createReport(html: String) { + uiThread { + it.context.materialDialogThemed { + title(R.string.debug_report_completed) + content(R.string.debug_report_completed_content) + positiveText(R.string.kau_yes) + negativeText(R.string.kau_no) + onPositive { dialog, _ -> + dialog.context.sendEmail(R.string.dev_email, R.string.kau_cancel) { + addItem("Query List", query.contentToString()) + footer = html.cleanHtml() + } + } + } + it.dismiss() + } + } +} \ No newline at end of file diff --git a/app/src/main/kotlin/com/pitchedapps/frost/utils/JsoupCleaner.kt b/app/src/main/kotlin/com/pitchedapps/frost/utils/JsoupCleaner.kt new file mode 100644 index 00000000..da8672f4 --- /dev/null +++ b/app/src/main/kotlin/com/pitchedapps/frost/utils/JsoupCleaner.kt @@ -0,0 +1,34 @@ +package com.pitchedapps.frost.utils + +import org.jsoup.Jsoup +import org.jsoup.nodes.Attribute +import org.jsoup.nodes.Element +import org.jsoup.safety.Whitelist + +/** + * Created by Allan Wang on 2017-08-10. + * + * Parses html with Jsoup and cleans the data, emitting just the frame containing debugging info + * + * Removes text, removes unnecessary nodes + */ +fun String.cleanHtml() = cleanText().cleanJsoup() + +internal fun String.cleanText(): String = replace(Regex(">(?s).+?<"), "><") + +internal fun String.cleanJsoup(): String = Jsoup.clean(this, PrivacyWhitelist()) + +class PrivacyWhitelist : Whitelist() { + + val blacklistAttrs = arrayOf("style", "aria-label", "rel") + val blacklistTags = arrayOf("body", "html", "head", "i", "b", "u", "style", "script", + "br", "p", "span", "ul", "ol", "li") + + override fun isSafeAttribute(tagName: String, el: Element, attr: Attribute): Boolean { + val key = attr.key + if (key == "href") attr.setValue("-") + return key !in blacklistAttrs + } + + override fun isSafeTag(tag: String) = tag !in blacklistTags +} diff --git a/app/src/main/kotlin/com/pitchedapps/frost/utils/Prefs.kt b/app/src/main/kotlin/com/pitchedapps/frost/utils/Prefs.kt index e52109c4..de759862 100644 --- a/app/src/main/kotlin/com/pitchedapps/frost/utils/Prefs.kt +++ b/app/src/main/kotlin/com/pitchedapps/frost/utils/Prefs.kt @@ -130,4 +130,6 @@ object Prefs : KPref() { var loadMediaOnMeteredNetwork: Boolean by kpref("media_on_metered_network", true) + var debugSettings: Boolean by kpref("debug_settings", false) + } diff --git a/app/src/main/kotlin/com/pitchedapps/frost/web/HeadlessHtmlExtractor.kt b/app/src/main/kotlin/com/pitchedapps/frost/web/HeadlessHtmlExtractor.kt new file mode 100644 index 00000000..a756aaa0 --- /dev/null +++ b/app/src/main/kotlin/com/pitchedapps/frost/web/HeadlessHtmlExtractor.kt @@ -0,0 +1,88 @@ +package com.pitchedapps.frost.web + +import android.annotation.SuppressLint +import android.content.Context +import android.webkit.JavascriptInterface +import android.webkit.WebView +import ca.allanwang.kau.utils.gone +import com.pitchedapps.frost.R +import com.pitchedapps.frost.facebook.USER_AGENT_BASIC +import com.pitchedapps.frost.injectors.InjectorContract +import com.pitchedapps.frost.utils.L +import io.reactivex.Single +import io.reactivex.SingleEmitter +import io.reactivex.android.schedulers.AndroidSchedulers +import io.reactivex.schedulers.Schedulers +import org.jetbrains.anko.runOnUiThread +import java.util.concurrent.TimeUnit + +/** + * Created by Allan Wang on 2017-08-12. + * + * Launches a headless html request and returns a result pair + * When successful, the pair will contain the html content and -1 + * When unsuccessful, the pair will contain an empty string and a StringRes for the given error + * + * All errors are rerouted to success calls, so no exceptions should occur. + * The headless extractor will also destroy itself on cancellation or when the request is finished + */ +fun Context.launchHeadlessHtmlExtractor(url: String, injector: InjectorContract, action: (Single>) -> Unit) { + val single = Single.create> { e: SingleEmitter> -> + val extractor = HeadlessHtmlExtractor(this, url, injector, e) + e.setCancellable { + runOnUiThread { extractor.destroy() } + e.onSuccess("" to R.string.debug_request_cancelled) + } + }.subscribeOn(AndroidSchedulers.mainThread()) + .timeout(20, TimeUnit.SECONDS, Schedulers.io(), { it.onSuccess("" to R.string.debug_request_timeout) }) + .onErrorReturn { "" to R.string.debug_request_error } + action(single) +} + +/** + * Given a link and some javascript, will load the link and load the JS on completion + * The JS is expected to call [HeadlessHtmlExtractor.HtmlJSI.handleHtml], which will be sent + * to the [emitter] + */ +@SuppressLint("ViewConstructor") +private class HeadlessHtmlExtractor( + context: Context, url: String, val injector: InjectorContract, val emitter: SingleEmitter> +) : WebView(context) { + + val startTime = System.currentTimeMillis() + + init { + L.v("Created HeadlessHtmlExtractor for $url") + gone() + setupWebview(url) + } + + @SuppressLint("SetJavaScriptEnabled") + private fun setupWebview(url: String) { + settings.javaScriptEnabled = true + settings.userAgentString = USER_AGENT_BASIC + webViewClient = HeadlessWebViewClient(url, injector) // basic client that loads our JS once the page has loaded + webChromeClient = QuietChromeClient() // basic client that disables logging + addJavascriptInterface(HtmlJSI(), "Frost") + loadUrl(url) + } + + inner class HtmlJSI { + @JavascriptInterface + fun handleHtml(html: String?) { + val time = System.currentTimeMillis() - startTime + emitter.onSuccess((html ?: "") to -1) + post { + L.d("HeadlessHtmlExtractor fetched $url in $time ms") + settings.javaScriptEnabled = false + settings.blockNetworkLoads = true + destroy() + } + } + } + + override fun destroy() { + super.destroy() + L.d("HeadlessHtmlExtractor destroyed") + } +} \ No newline at end of file diff --git a/app/src/main/res/values/strings_pref_debug.xml b/app/src/main/res/values/strings_pref_debug.xml new file mode 100644 index 00000000..fb52f2b4 --- /dev/null +++ b/app/src/main/res/values/strings_pref_debug.xml @@ -0,0 +1,18 @@ + + + + Debugging section is enabled! Go back to settings. + + Though most private content is automatically removed in the report, some sensitive info may still be visible. + \nPlease have a look at the debug report before sending it. + \n\nClicking one of the options below will prepare an email response with the web page data. + + + Incomplete report + An error occurred in the debugger. + The request has been cancelled. + The request has timed out. + + Debug report completed + Would you like to send the report? + \ No newline at end of file diff --git a/app/src/main/res/values/strings_preferences.xml b/app/src/main/res/values/strings_preferences.xml index 3cacf684..05335345 100644 --- a/app/src/main/res/values/strings_preferences.xml +++ b/app/src/main/res/values/strings_preferences.xml @@ -22,6 +22,8 @@ About Frost for Facebook Version, Credits, and FAQs + Frost Debugger + Send html data to help with debugging. Replay Introduction \ No newline at end of file diff --git a/app/src/test/kotlin/com/pitchedapps/frost/utils/JsoupCleanerTest.kt b/app/src/test/kotlin/com/pitchedapps/frost/utils/JsoupCleanerTest.kt new file mode 100644 index 00000000..4ec09ea6 --- /dev/null +++ b/app/src/test/kotlin/com/pitchedapps/frost/utils/JsoupCleanerTest.kt @@ -0,0 +1,56 @@ +package com.pitchedapps.frost.utils + +import org.junit.Test +import kotlin.test.assertEquals + +/** + * Created by Allan Wang on 2017-08-10. + */ +class JsoupCleanerTest { + + val whitespaceRegex = Regex("\\s+") + + fun String.cleanWhitespace() = replace("\n", "").replace(whitespaceRegex, " ").replace("> <", "><") + + private fun String.assertCleanHtml(expected: String) { + assertEquals(expected.cleanWhitespace(), cleanHtml().cleanWhitespace()) + } + + private fun String.assertCleanJsoup(expected: String) { + assertEquals(expected.cleanWhitespace(), cleanJsoup().cleanWhitespace()) + } + + private fun String.assertCleanText(expected: String) { + assertEquals(expected.cleanWhitespace(), cleanText().cleanWhitespace()) + } + + @Test + fun noChange() { + " HI ".assertCleanJsoup(" HI ") + } + + @Test + fun basicText() { + """
Hello world
""".assertCleanHtml("""
""") + } + + @Test + fun multiLineText() { + """
Hello + world
""".assertCleanHtml("""
""") + } + + @Test + fun textRemoval() { + """
HelloWorld
""".assertCleanText("
") + } + + @Test + fun kau() { + val html = """
KAU

An extensive collection of Kotlin Android Utils

Github Page
KAUclose
  • Huge package of one line extension functions
  • Custom UI views
  • Adapter items and animators
  • SearchView
  • Custom delegates
""" + val expected = """
""" + html.assertCleanHtml(expected) + } + +} + diff --git a/gradle.properties b/gradle.properties index c71f8b82..59f1af52 100644 --- a/gradle.properties +++ b/gradle.properties @@ -17,7 +17,7 @@ MIN_SDK=21 TARGET_SDK=26 BUILD_TOOLS=26.0.1 -KAU=3.3.1 +KAU=6d9201e KOTLIN=1.1.3-2 CRASHLYTICS=2.6.8 -- cgit v1.2.3