diff options
| -rw-r--r-- | libre/iceweasel/PKGBUILD | 30 | ||||
| -rw-r--r-- | libre/iceweasel/libre-0001-always-sync-remote-settings-with-local-dump.patch | 987 |
2 files changed, 1016 insertions, 1 deletions
diff --git a/libre/iceweasel/PKGBUILD b/libre/iceweasel/PKGBUILD index fe53ac039..fffe5070b 100644 --- a/libre/iceweasel/PKGBUILD +++ b/libre/iceweasel/PKGBUILD @@ -27,6 +27,9 @@ # - Modify the addons pages to use GNU IceCat plugins sources, rather # than addons.mozilla.org, which hosts non-free addons # - Disable EME, which is implemented via the non-free libWideVine CDM +# - Disable Normandy that let Mozilla push messages with recommendations +# of nonfree software +# - Make Remote Settings work completely offline using local data # - Rebrand to Iceweasel, per the mozilla trademark policy, # due to these FSDG changes # @@ -74,6 +77,7 @@ source=(https://archive.mozilla.org/pub/firefox/releases/$pkgver/source/firefox- source+=(https://repo.parabola.nu/other/iceweasel/${pkgname}_${_brandingver}-${_brandingrel}.branding.tar.xz{,.sig} libre.patch libre-searchengines.patch + libre-0001-always-sync-remote-settings-with-local-dump.patch vendor.js.in) source_armv7h=(arm.patch build-arm-libopus.patch) @@ -286,11 +290,18 @@ END export QUILT_PATCHES="${brandingsrcdir}"/patches export QUILT_REFRESH_ARGS='-p ab --no-timestamps --no-index' export QUILT_DIFF_ARGS='--no-timestamps' + export QUILT_PC=$srcdir/.pc quilt push -av ## searchengines ## + # Load custom searchplugins + # FIXME: no longer applicable - replace with (broken) replacement fo libre/mozilla-searchplugins + # https://git.parabola.nu/packages/iceweasel.git/tree/branding-dev-build/mozilla-searchplugins?h=68.0 +# rm -rvf -- browser/components/search/searchplugins/{*.xml,images/} +# cp -av -- /usr/lib/mozilla/searchplugins/* browser/components/search/searchplugins/ + # Patch search-engines config # FIXME: custom searchplugins ID per new format above ("ddg" is standard) # browser/components/search/extensions/<ID>/ @@ -333,6 +344,23 @@ END ## libre patching ## + # Remove remaining non-free bits + # Remove test-related networking dumps, because they contain code from + # some Amazon webpage with no clear licensing, thus nonfree. + # Also they interfere with checking of Remote Settings patching done later, + # because communication with RS server has been captured in them too. + rm python/mozperftest/mozperftest/system/example.zip + rm testing/mozbase/mozproxy/tests/files/mitm5-linux-firefox-amazon.zip + + # Disable/neutralize Remote Settings (as best we can) + echo "applying libre-0001-always-sync-remote-settings-with-local-dump.patch" + patch -Np1 --no-backup-if-mismatch -i ../libre-0001-always-sync-remote-settings-with-local-dump.patch + + # Check Remote Settings patched completely by + # libre-0001-always-sync-remote-settings-with-local-dump.patch + local settings_server='firefox.settings.services.mozilla.com' + ! grep -qr $settings_server || { echo 'Remote Settings patching needs rework'; return 1; } + # Disable various components at the source level sed -i 's/;1/;0/' toolkit/components/telemetry/components.conf sed -Ei 's/((MOZ_SERVICES_HEALTHREPORT|MOZ_NORMANDY).+)True/\1False/' browser/moz.configure @@ -354,7 +382,7 @@ END # Patch and remove anything that's left echo "applying libre.patch" - patch -Np1 -i "${srcdir}"/libre.patch + patch -Np1 --no-backup-if-mismatch -i "${srcdir}"/libre.patch } build() { diff --git a/libre/iceweasel/libre-0001-always-sync-remote-settings-with-local-dump.patch b/libre/iceweasel/libre-0001-always-sync-remote-settings-with-local-dump.patch new file mode 100644 index 000000000..4e3732845 --- /dev/null +++ b/libre/iceweasel/libre-0001-always-sync-remote-settings-with-local-dump.patch @@ -0,0 +1,987 @@ +From c115ebc66ae779c18128e0b815fcf29da268a4f8 Mon Sep 17 00:00:00 2001 +From: grizzlyuser <grizzlyuser@protonmail.com> +Date: Wed, 30 Dec 2020 17:20:39 +0200 +Subject: [PATCH 01/13] Point to local omni.ja files, not remote server + +Basically replace every occurrence of Remote Settings server domain name +with URIs that point to built-in local files within omni.ja. + +Some links to json files may point to non-existing files, but that's OK +because it's better than leave them point to Remote Settings server. +If necessary, missing files can be added later. +--- + .../components/ASRouterAdmin/ASRouterAdmin.jsx | 2 +- + .../newtab/data/content/activity-stream.bundle.js | 2 +- + modules/libpref/init/all.js | 2 +- + services/settings/Utils.jsm | 4 ++-- + .../periodic-updates/scripts/periodic_file_updates.sh | 2 +- + toolkit/components/search/SearchUtils.jsm | 8 ++++---- + toolkit/components/search/docs/DefaultSearchEngines.rst | 2 +- + .../components/search/docs/SearchEngineConfiguration.rst | 2 +- + toolkit/mozapps/defaultagent/RemoteSettings.cpp | 2 +- + 9 files changed, 13 insertions(+), 13 deletions(-) + +diff --git a/browser/components/newtab/content-src/components/ASRouterAdmin/ASRouterAdmin.jsx b/browser/components/newtab/content-src/components/ASRouterAdmin/ASRouterAdmin.jsx +index 8c5e540bd2..329e4e66f9 100644 +--- a/browser/components/newtab/content-src/components/ASRouterAdmin/ASRouterAdmin.jsx ++++ b/browser/components/newtab/content-src/components/ASRouterAdmin/ASRouterAdmin.jsx +@@ -1260,7 +1260,7 @@ export class ASRouterAdminInner extends React.PureComponent { + <a + className="providerUrl" + target="_blank" +- href="https://firefox.settings.services.mozilla.com/v1/buckets/main/collections/nimbus-desktop-experiments/records" ++ href="resource://app/defaults/settings/main/nimbus-desktop-experiments.json" + rel="noopener noreferrer" + > + nimbus-desktop-experiments +diff --git a/browser/components/newtab/data/content/activity-stream.bundle.js b/browser/components/newtab/data/content/activity-stream.bundle.js +index fb6d64d432..d8213e7e20 100644 +--- a/browser/components/newtab/data/content/activity-stream.bundle.js ++++ b/browser/components/newtab/data/content/activity-stream.bundle.js +@@ -1835,7 +1835,7 @@ class ASRouterAdminInner extends react__WEBPACK_IMPORTED_MODULE_4___default.a.Pu + label = react__WEBPACK_IMPORTED_MODULE_4___default.a.createElement("span", null, "remote settings (", react__WEBPACK_IMPORTED_MODULE_4___default.a.createElement("a", { + className: "providerUrl", + target: "_blank", +- href: "https://firefox.settings.services.mozilla.com/v1/buckets/main/collections/nimbus-desktop-experiments/records", ++ href: "resource://app/defaults/settings/main/nimbus-desktop-experiments.json", + rel: "noopener noreferrer" + }, "nimbus-desktop-experiments"), ")"); + } +diff --git a/modules/libpref/init/all.js b/modules/libpref/init/all.js +index 2406affed4..907e8ebcef 100644 +--- a/modules/libpref/init/all.js ++++ b/modules/libpref/init/all.js +@@ -2250,7 +2250,7 @@ pref("security.cert_pinning.hpkp.enabled", false); + // Remote settings preferences + // Note: if you change this, make sure to also review security.onecrl.maximum_staleness_in_seconds + pref("services.settings.poll_interval", 86400); // 24H +-pref("services.settings.server", "https://firefox.settings.services.mozilla.com/v1"); ++pref("services.settings.server", "resource://app/defaults/settings"); + pref("services.settings.default_bucket", "main"); + + // The percentage of clients who will report uptake telemetry as +diff --git a/services/settings/Utils.jsm b/services/settings/Utils.jsm +index 66df850904..0df0fdc677 100644 +--- a/services/settings/Utils.jsm ++++ b/services/settings/Utils.jsm +@@ -60,11 +60,11 @@ var Utils = { + ); + const isXpcshell = env.exists("XPCSHELL_TEST_PROFILE_DIR"); + return AppConstants.RELEASE_OR_BETA && !Cu.isInAutomation && !isXpcshell +- ? "https://firefox.settings.services.mozilla.com/v1" ++ ? "resource://app/defaults/settings" + : gServerURL; + }, + +- CHANGES_PATH: "/buckets/monitor/collections/changes/records", ++ CHANGES_PATH: "/monitor/changes.json", + + /** + * Logger instance. +diff --git a/taskcluster/docker/periodic-updates/scripts/periodic_file_updates.sh b/taskcluster/docker/periodic-updates/scripts/periodic_file_updates.sh +index c2492615e0..2b10ad01f6 100755 +--- a/taskcluster/docker/periodic-updates/scripts/periodic_file_updates.sh ++++ b/taskcluster/docker/periodic-updates/scripts/periodic_file_updates.sh +@@ -279,7 +279,7 @@ function compare_suffix_lists { + } + + function compare_remote_settings_files { +- REMOTE_SETTINGS_SERVER="https://firefox.settings.services.mozilla.com/v1" ++ REMOTE_SETTINGS_SERVER="resource://app/defaults/settings" + + # 1. List remote settings collections from server. + echo "INFO: fetch remote settings list from server" +diff --git a/toolkit/components/search/SearchUtils.jsm b/toolkit/components/search/SearchUtils.jsm +index 278f9f9c08..7bac023c11 100644 +--- a/toolkit/components/search/SearchUtils.jsm ++++ b/toolkit/components/search/SearchUtils.jsm +@@ -139,13 +139,13 @@ var SearchUtils = { + + ENGINES_URLS: { + "prod-main": +- "https://firefox.settings.services.mozilla.com/v1/buckets/main/collections/search-config/records", ++ "resource://app/defaults/settings/main/search-config.json", + "prod-preview": +- "https://firefox.settings.services.mozilla.com/v1/buckets/main-preview/collections/search-config/records", ++ "resource://app/defaults/settings/main/search-config.json", + "stage-main": +- "https://settings.stage.mozaws.net/v1/buckets/main/collections/search-config/records", ++ "resource://app/defaults/settings/main/search-config.json", + "stage-preview": +- "https://settings.stage.mozaws.net/v1/buckets/main-preview/collections/search-config/records", ++ "resource://app/defaults/settings/main/search-config.json", + }, + + // The following constants are left undocumented in nsISearchService.idl +diff --git a/toolkit/components/search/docs/DefaultSearchEngines.rst b/toolkit/components/search/docs/DefaultSearchEngines.rst +index 5668646648..deb7d20185 100644 +--- a/toolkit/components/search/docs/DefaultSearchEngines.rst ++++ b/toolkit/components/search/docs/DefaultSearchEngines.rst +@@ -63,4 +63,4 @@ is updated. + + .. _configuration schema: SearchConfigurationSchema.html + .. _remote settings: /services/common/services/RemoteSettings.html +-.. _search-default-override-allowlist bucket: https://firefox.settings.services.mozilla.com/v1/buckets/main/collections/search-default-override-allowlist/records ++.. _search-default-override-allowlist bucket: resource://app/defaults/settings/main/search-default-override-allowlist.json +diff --git a/toolkit/components/search/docs/SearchEngineConfiguration.rst b/toolkit/components/search/docs/SearchEngineConfiguration.rst +index e9041affb8..7a9466d294 100644 +--- a/toolkit/components/search/docs/SearchEngineConfiguration.rst ++++ b/toolkit/components/search/docs/SearchEngineConfiguration.rst +@@ -68,5 +68,5 @@ related. As a result several situations may occur: + .. _JSON schema: https://json-schema.org/ + .. _stored in mozilla-central: https://searchfox.org/mozilla-central/source/toolkit/components/search/schema/ + .. _Search Configuration Schema: SearchConfigurationSchema.html +-.. _viewed live: https://firefox.settings.services.mozilla.com/v1/buckets/main/collections/search-config/records ++.. _viewed live: resource://app/defaults/settings/main/search-config.json + .. _Normandy: /toolkit/components/normandy/normandy/services.html +diff --git a/toolkit/mozapps/defaultagent/RemoteSettings.cpp b/toolkit/mozapps/defaultagent/RemoteSettings.cpp +index 667d9fc628..b2bf628f29 100644 +--- a/toolkit/mozapps/defaultagent/RemoteSettings.cpp ++++ b/toolkit/mozapps/defaultagent/RemoteSettings.cpp +@@ -23,7 +23,7 @@ extern "C" { + HRESULT IsAgentRemoteDisabledRust(const char* szUrl, DWORD* lpdwDisabled); + } + +-#define PROD_ENDPOINT "https://firefox.settings.services.mozilla.com/v1" ++#define PROD_ENDPOINT "resource://app/defaults/settings" + #define PROD_BID "main" + #define PROD_CID "windows-default-browser-agent" + #define PROD_ID "state" +-- +2.30.0 + + +From f76e1c055def541e358886d0e73ba4710d3e5084 Mon Sep 17 00:00:00 2001 +From: grizzlyuser <grizzlyuser@protonmail.com> +Date: Wed, 30 Dec 2020 17:34:08 +0200 +Subject: [PATCH 02/13] Remove polling triggered by push broadcasts + +When initialized, remote-settings.js adds a listener to push broadcasts, +that let Remote Settings server send push messages to trigger polling +for changes from the client side. This is not needed for local-only +setup. Remove the record from broadcast-listeners.json file stored in +the user profile, so that it doesn't get picked up by push broadcast +service. +--- + dom/push/PushBroadcastService.jsm | 13 +++++++++++++ + services/settings/remote-settings.js | 7 ++----- + 2 files changed, 15 insertions(+), 5 deletions(-) + +diff --git a/dom/push/PushBroadcastService.jsm b/dom/push/PushBroadcastService.jsm +index 27ed31ee9c..71f1316994 100644 +--- a/dom/push/PushBroadcastService.jsm ++++ b/dom/push/PushBroadcastService.jsm +@@ -179,6 +179,19 @@ var BroadcastService = class { + } + } + ++ async deleteListener(broadcastId) { ++ await this.initializePromise; ++ ++ if (this.jsonFile.data.listeners.hasOwnProperty(broadcastId)) { ++ console.info( ++ "deleteListener: deleting listener", ++ broadcastId ++ ); ++ delete this.jsonFile.data.listeners[broadcastId]; ++ this.jsonFile.saveSoon(); ++ } ++ } ++ + /** + * Call the listeners of the specified broadcasts. + * +diff --git a/services/settings/remote-settings.js b/services/settings/remote-settings.js +index 6d0185faf9..aae93fa440 100644 +--- a/services/settings/remote-settings.js ++++ b/services/settings/remote-settings.js +@@ -441,7 +441,7 @@ function remoteSettingsFunction() { + moduleURI: __URI__, + symbolName: "remoteSettingsBroadcastHandler", + }; +- pushBroadcastService.addListener(BROADCAST_ID, currentVersion, moduleInfo); ++ pushBroadcastService.deleteListener(BROADCAST_ID); + }; + + return remoteSettings; +@@ -461,9 +461,6 @@ var remoteSettingsBroadcastHandler = { + `Push notification received (version=${version} phase=${phase})` + ); + +- return RemoteSettings.pollChanges({ +- expectedTimestamp: version, +- trigger: isStartup ? "startup" : "broadcast", +- }); ++ return; + }, + }; +-- +2.30.0 + + +From fcf60ef5835e673e03807d898f90e48907f44e63 Mon Sep 17 00:00:00 2001 +From: grizzlyuser <grizzlyuser@protonmail.com> +Date: Wed, 30 Dec 2020 17:41:54 +0200 +Subject: [PATCH 03/13] Remove timer that triggers polling for changes + +That is not needed for local-only setup. +--- + services/settings/components.conf | 9 +-------- + services/settings/servicesSettings.manifest | 4 ---- + 2 files changed, 1 insertion(+), 12 deletions(-) + +diff --git a/services/settings/components.conf b/services/settings/components.conf +index 9a737802ee..25109415a7 100644 +--- a/services/settings/components.conf ++++ b/services/settings/components.conf +@@ -4,11 +4,4 @@ + # License, v. 2.0. If a copy of the MPL was not distributed with this + # file, You can obtain one at http://mozilla.org/MPL/2.0/. + +-Classes = [ +- { +- 'cid': '{5e756573-234a-49ea-bbe4-59ec7a70657d}', +- 'contract_ids': ['@mozilla.org/services/settings;1'], +- 'jsm': 'resource://services-settings/RemoteSettingsComponents.jsm', +- 'constructor': 'RemoteSettingsTimer', +- }, +-] ++Classes = [] +diff --git a/services/settings/servicesSettings.manifest b/services/settings/servicesSettings.manifest +index 3bfed26ea4..807eb220ec 100644 +--- a/services/settings/servicesSettings.manifest ++++ b/services/settings/servicesSettings.manifest +@@ -1,7 +1,3 @@ + # Register resource aliases + resource services-settings resource://gre/modules/services-settings/ + +-# Schedule polling of remote settings changes +-# (default 24H, max 72H) +-# see syntax https://searchfox.org/mozilla-central/rev/cc280c4be94ff8cf64a27cc9b3d6831ffa49fa45/toolkit/components/timermanager/UpdateTimerManager.jsm#155 +-category update-timer RemoteSettingsComponents @mozilla.org/services/settings;1,getService,services-settings-poll-changes,services.settings.poll_interval,86400,259200 +-- +2.30.0 + + +From 8600d50c402f523973f67dfb03c1ea8614fcf48c Mon Sep 17 00:00:00 2001 +From: grizzlyuser <grizzlyuser@protonmail.com> +Date: Wed, 30 Dec 2020 17:47:41 +0200 +Subject: [PATCH 04/13] Utils: fetch timestamps of each collection locally + +Utils.CHANGES_PATH points to +services/settings/dumps/monitor/changes.json +which will be generated later by JSON processing script. Fetch the +timestamps from that file and mock response headers to not confuse any +code that expects them. +--- + browser/installer/package-manifest.in | 1 + + services/settings/Utils.jsm | 14 ++++++++++++-- + services/settings/dumps/monitor/moz.build | 8 ++++++++ + services/settings/dumps/moz.build | 1 + + 4 files changed, 22 insertions(+), 2 deletions(-) + create mode 100644 services/settings/dumps/monitor/moz.build + +diff --git a/browser/installer/package-manifest.in b/browser/installer/package-manifest.in +index 75c79a7168..ac01689596 100644 +--- a/browser/installer/package-manifest.in ++++ b/browser/installer/package-manifest.in +@@ -295,6 +295,7 @@ + @RESPATH@/browser/defaults/settings/blocklists + @RESPATH@/browser/defaults/settings/pinning + @RESPATH@/browser/defaults/settings/main ++@RESPATH@/browser/defaults/settings/monitor + @RESPATH@/browser/defaults/settings/security-state + + ; Warning: changing the path to channel-prefs.js can cause bugs (Bug 756325) +diff --git a/services/settings/Utils.jsm b/services/settings/Utils.jsm +index 0df0fdc677..0e631ddc0d 100644 +--- a/services/settings/Utils.jsm ++++ b/services/settings/Utils.jsm +@@ -145,7 +145,7 @@ var Utils = { + // "collection":"certificates" + // }]} + +- let url = serverUrl + Utils.CHANGES_PATH; ++ let url = Utils.SERVER_URL + Utils.CHANGES_PATH; + + // Use ETag to obtain a `304 Not modified` when no change occurred, + // and `?_since` parameter to only keep entries that weren't processed yet. +@@ -166,6 +166,9 @@ var Utils = { + .join("&"); + } + const response = await fetch(url, { headers }); ++ const responseDate = new Date().toUTCString() ++ response.headers.set("Date", responseDate); ++ response.headers.set("Last-Modified", responseDate); + + let changes = []; + // If no changes since last time, go on with empty list of changes. +@@ -203,7 +206,14 @@ var Utils = { + ); + } + } else { +- changes = payload.data; ++ const { bucket, collection } = filters; ++ if (!bucket || !collection) { ++ throw new Error('Unable to fetch latest change without bucket or collection'); ++ } ++ const change = payload.data.find( ++ change => change.bucket === bucket && change.collection === collection ++ ) ?? { last_modified: 0, bucket, collection }; ++ changes = [change]; + } + } + // The server should always return ETag. But we've had situations where the CDN +diff --git a/services/settings/dumps/monitor/moz.build b/services/settings/dumps/monitor/moz.build +new file mode 100644 +index 0000000000..d3d017fda5 +--- /dev/null ++++ b/services/settings/dumps/monitor/moz.build +@@ -0,0 +1,8 @@ ++# This Source Code Form is subject to the terms of the Mozilla Public ++# License, v. 2.0. If a copy of the MPL was not distributed with this ++# file, You can obtain one at http://mozilla.org/MPL/2.0/. ++ ++FINAL_TARGET_FILES.defaults.settings.monitor += ["changes.json"] ++ ++if CONFIG["MOZ_BUILD_APP"] == "browser": ++ DIST_SUBDIR = "browser" +diff --git a/services/settings/dumps/moz.build b/services/settings/dumps/moz.build +index 3cc9436f61..3742da5667 100644 +--- a/services/settings/dumps/moz.build ++++ b/services/settings/dumps/moz.build +@@ -5,6 +5,7 @@ + DIRS += [ + "blocklists", + "main", ++ "monitor", + "pinning", + "security-state", + ] +-- +2.30.0 + + +From 99af961e47d810c965c0145f19a92eedec1abafc Mon Sep 17 00:00:00 2001 +From: grizzlyuser <grizzlyuser@protonmail.com> +Date: Wed, 30 Dec 2020 17:52:10 +0200 +Subject: [PATCH 05/13] Utils: disable offline checking + +Since only local data is read now, it should always return false for the +current any any future code that relies on it. +--- + services/settings/Utils.jsm | 9 --------- + 1 file changed, 9 deletions(-) + +diff --git a/services/settings/Utils.jsm b/services/settings/Utils.jsm +index 0e631ddc0d..d034d8ea78 100644 +--- a/services/settings/Utils.jsm ++++ b/services/settings/Utils.jsm +@@ -80,15 +80,6 @@ var Utils = { + * @return {bool} Whether network is down or not. + */ + get isOffline() { +- try { +- return ( +- Services.io.offline || +- CaptivePortalService.state == CaptivePortalService.LOCKED_PORTAL || +- !gNetworkLinkService.isLinkUp +- ); +- } catch (ex) { +- log.warn("Could not determine network status.", ex); +- } + return false; + }, + +-- +2.30.0 + + +From 55830695a1e2457aecca1392d46c98c3e210d58a Mon Sep 17 00:00:00 2001 +From: grizzlyuser <grizzlyuser@protonmail.com> +Date: Wed, 30 Dec 2020 17:56:02 +0200 +Subject: [PATCH 06/13] Refactor hashing logic to a separate function + +It is used instead of internal signature validation mechanism, for +integrity checking of the locally cached data. +--- + services/settings/RemoteSettingsWorker.jsm | 4 ++++ + services/settings/SharedUtils.jsm | 9 +++++++-- + 2 files changed, 11 insertions(+), 2 deletions(-) + +diff --git a/services/settings/RemoteSettingsWorker.jsm b/services/settings/RemoteSettingsWorker.jsm +index 147ebb6b13..c86e218fd3 100644 +--- a/services/settings/RemoteSettingsWorker.jsm ++++ b/services/settings/RemoteSettingsWorker.jsm +@@ -189,6 +189,10 @@ class Worker { + // task on the current thread instead of the worker thread. + return SharedUtils.checkContentHash(buffer, size, hash); + } ++ ++ async getContentHash(bytes) { ++ return SharedUtils.getContentHash(bytes); ++ } + } + + // Now, first add a shutdown blocker. If that fails, we must have +diff --git a/services/settings/SharedUtils.jsm b/services/settings/SharedUtils.jsm +index db5017a742..1a8e83c2e8 100644 +--- a/services/settings/SharedUtils.jsm ++++ b/services/settings/SharedUtils.jsm +@@ -28,11 +28,16 @@ var SharedUtils = { + return false; + } + // Has expected content? ++ const hashStr = await this.getContentHash(bytes); ++ return hashStr == hash; ++ }, ++ ++ async getContentHash(bytes) { + const hashBuffer = await crypto.subtle.digest("SHA-256", bytes); + const hashBytes = new Uint8Array(hashBuffer); + const toHex = b => b.toString(16).padStart(2, "0"); +- const hashStr = Array.from(hashBytes, toHex).join(""); +- return hashStr == hash; ++ ++ return Array.from(hashBytes, toHex).join(""); + }, + + /** +-- +2.30.0 + + +From 9dc19397aebfb6b9f74afa0fb4d653a29e14e964 Mon Sep 17 00:00:00 2001 +From: grizzlyuser <grizzlyuser@protonmail.com> +Date: Wed, 30 Dec 2020 18:05:02 +0200 +Subject: [PATCH 07/13] Client: Fetch and hash records from local dump + +Read the records from local dumps. See [1] for details on how to prepare +custom dumps). Records are cached in the local IndexedDB, and the client +updates cached records each time there's a change. Also it verifies +integrity of the data. Then the list of current / created / updated / +deleted records is generated and emitted to every registered listener. + +Change upstream signature validation mechanism to a simpler one. +Otherwise, it'd be necessary to sign local records, which is redundant, +because the application package should be signed already by the distro. + +Instead of signature property from metadata records, json_dump_metadata +has been introduced. It contains the checksum of the records and size in +bytes. Also added app_build_id property for version checking and updates +of cached data. + +Although it's possible to disable integrity checking via preference, it +seems to be not a good idea, because the logic that detects invalid +local data relies on it. In the context of local-only setup, data that +has been received from real Remote Settings server will not contain the +custom metadata, and thus will be considered invalid and then discarded, +while the client gets a chance to gracefully inform registered listeners +about these changes so that they can discard the data received before +the upgrade to local-only setup. + +[1] https://firefox-source-docs.mozilla.org/services/common/services/RemoteSettings.html#initial-data +--- + services/settings/RemoteSettingsClient.jsm | 62 ++++++++++------------ + 1 file changed, 27 insertions(+), 35 deletions(-) + +diff --git a/services/settings/RemoteSettingsClient.jsm b/services/settings/RemoteSettingsClient.jsm +index 80dd563e11..1025ab33a2 100644 +--- a/services/settings/RemoteSettingsClient.jsm ++++ b/services/settings/RemoteSettingsClient.jsm +@@ -556,11 +556,9 @@ class RemoteSettingsClient extends EventEmitter { + + // If the data is up-to-date but don't have metadata (records loaded from dump), + // we fetch them and validate the signature immediately. +- if (this.verifySignature && ObjectUtils.isEmpty(localMetadata)) { ++ if (this.verifySignature && ObjectUtils.isEmpty(localMetadata?.json_dump_metadata)) { + console.debug(`${this.identifier} pull collection metadata`); +- const metadata = await this.httpClient().getData({ +- query: { _expected: expectedTimestamp }, +- }); ++ const { metadata } = await this._fetchChangeset(expectedTimestamp); + await this.db.importChanges(metadata); + // We don't bother validating the signature if the dump was just loaded. We do + // if the dump was loaded at some other point (eg. from .get()). +@@ -813,32 +811,23 @@ class RemoteSettingsClient extends EventEmitter { + async _validateCollectionSignature(records, timestamp, metadata) { + const start = Cu.now() * 1000; + +- if (!metadata?.signature) { ++ if (!metadata?.json_dump_metadata) { + throw new MissingSignatureError(this.identifier); + } + +- if (!this._verifier) { +- this._verifier = Cc[ +- "@mozilla.org/security/contentsignatureverifier;1" +- ].createInstance(Ci.nsIContentSignatureVerifier); +- } +- +- // This is a content-signature field from an autograph response. + const { +- signature: { x5u, signature }, ++ json_dump_metadata: { hash, size }, + } = metadata; +- const certChain = await (await fetch(x5u)).text(); + // Merge remote records with local ones and serialize as canonical JSON. + const serialized = await RemoteSettingsWorker.canonicalStringify( + records, + timestamp + ); + if ( +- !(await this._verifier.asyncVerifyContentSignature( +- serialized, +- "p384ecdsa=" + signature, +- certChain, +- this.signerName ++ !(await RemoteSettingsWorker.checkContentHash( ++ new TextEncoder().encode(serialized), ++ size, ++ hash + )) + ) { + throw new InvalidSignatureError(this.identifier); +@@ -1030,24 +1019,27 @@ class RemoteSettingsClient extends EventEmitter { + * @param since timestamp of last sync (optional) + */ + async _fetchChangeset(expectedTimestamp, since) { +- const client = this.httpClient(); +- const { +- metadata, +- timestamp: remoteTimestamp, +- changes: remoteRecords, +- } = await client.execute( +- { +- path: `/buckets/${this.bucketName}/collections/${this.collectionName}/changeset`, +- }, +- { +- query: { +- _expected: expectedTimestamp, +- _since: since, +- }, +- } ++ const { data } = await SharedUtils.loadJSONDump( ++ this.bucketName, ++ this.collectionName + ); ++ const remoteRecords = data ?? []; ++ ++ const serialized = await RemoteSettingsWorker.canonicalStringify( ++ remoteRecords, ++ expectedTimestamp ++ ); ++ const bytes = new TextEncoder().encode(serialized); ++ const metadata = { ++ app_build_id: Services.appinfo.appBuildID, ++ json_dump_metadata: { ++ hash: await RemoteSettingsWorker.getContentHash(bytes), ++ size: bytes.length, ++ }, ++ } ++ + return { +- remoteTimestamp, ++ remoteTimestamp: expectedTimestamp, + metadata, + remoteRecords, + }; +-- +2.30.0 + + +From 4dfbb2ccacd011b05839c19127942b497141f2a1 Mon Sep 17 00:00:00 2001 +From: grizzlyuser <grizzlyuser@protonmail.com> +Date: Wed, 30 Dec 2020 18:42:56 +0200 +Subject: [PATCH 08/13] Client: start deferred sync on get() or on() + +The users of the RemoteSettingsClient.jsm can receive records from it in +two ways: by calling get(), and by subscribing to events by calling +on(). + +So hook a deferred sync whenever something calls these methods. Because +multiple of those calls can be made quite early and in very short time, +set up a deferred task that will be armed only when needed and only once +in a second. When the task is running it first checks if the local data +came from the dump of the current app build, and no-ops if true. If +false, it triggers a sync. Then adds a flag if the client has been +correctly synchronized with the dump, so that no metadata checking +occurs during the session. +--- + services/settings/RemoteSettingsClient.jsm | 32 ++++++++++++++++++++-- + 1 file changed, 30 insertions(+), 2 deletions(-) + +diff --git a/services/settings/RemoteSettingsClient.jsm b/services/settings/RemoteSettingsClient.jsm +index 1025ab33a2..253251823a 100644 +--- a/services/settings/RemoteSettingsClient.jsm ++++ b/services/settings/RemoteSettingsClient.jsm +@@ -16,6 +16,7 @@ XPCOMUtils.defineLazyModuleGetters(this, { + ClientEnvironmentBase: + "resource://gre/modules/components-utils/ClientEnvironment.jsm", + Database: "resource://services-settings/Database.jsm", ++ DeferredTask: "resource://gre/modules/DeferredTask.jsm", + Downloader: "resource://services-settings/Attachments.jsm", + IDBHelpers: "resource://services-settings/IDBHelpers.jsm", + KintoHttpClient: "resource://services-common/kinto-http-client.js", +@@ -30,6 +31,7 @@ XPCOMUtils.defineLazyModuleGetters(this, { + XPCOMUtils.defineLazyGlobalGetters(this, ["fetch"]); + + const TELEMETRY_COMPONENT = "remotesettings"; ++const DEFERRED_SYNC_DELAY_MILLISECONDS = 1000 + + XPCOMUtils.defineLazyGetter(this, "console", () => Utils.log); + +@@ -259,6 +261,14 @@ class RemoteSettingsClient extends EventEmitter { + this._lastCheckTimePref = lastCheckTimePref; + this._verifier = null; + this._syncRunning = false; ++ this._deferredSync = new DeferredTask( ++ async () => { ++ if (!this._syncRunning && !(await this._isSynced())) { ++ await this.sync(); ++ } ++ }, ++ DEFERRED_SYNC_DELAY_MILLISECONDS ++ ); + + // This attribute allows signature verification to be disabled, when running tests + // or when pulling data from a dev server. +@@ -290,6 +300,11 @@ class RemoteSettingsClient extends EventEmitter { + ); + } + ++ on(event, callback) { ++ super.on(event, callback); ++ this._deferredSync.arm(); ++ } ++ + get identifier() { + return `${this.bucketName}/${this.collectionName}`; + } +@@ -353,7 +368,11 @@ class RemoteSettingsClient extends EventEmitter { + try { + let hasLocalData = await Utils.hasLocalData(this); + +- if (syncIfEmpty && !hasLocalData) { ++ if (!(await this._isSynced())) { ++ throw new MissingSignatureError(this.identifier); ++ } ++ ++ if ((syncIfEmpty && !hasLocalData)) { + // .get() was called before we had the chance to synchronize the local database. + // We'll try to avoid returning an empty list. + if (!this._importingPromise) { +@@ -414,7 +433,10 @@ class RemoteSettingsClient extends EventEmitter { + // No need to verify signature on JSON dumps. + // If local DB cannot be read, then we don't even try to do anything, + // we return results early. +- return this._filterEntries(data); ++ const filtered = this._filterEntries(data); ++ this._deferredSync.arm(); ++ ++ return filtered; + } + + console.debug( +@@ -452,6 +474,12 @@ class RemoteSettingsClient extends EventEmitter { + return final; + } + ++ async _isSynced() { ++ this._synced ||= ++ Services.appinfo.appBuildID === (await this.db.getMetadata())?.app_build_id; ++ return this._synced; ++ } ++ + /** + * Synchronize the local database with the remote server. + * +-- +2.30.0 + + +From 89dc912d08b7d4eed113439efa17dce57eda106e Mon Sep 17 00:00:00 2001 +From: grizzlyuser <grizzlyuser@protonmail.com> +Date: Wed, 30 Dec 2020 18:53:51 +0200 +Subject: [PATCH 09/13] Client: deep compare records if timestamps match + +When the list of current / updated / deleted records is generated, their +modification timestamps are compared to detect the updates. + +Although in practice this is unlikely to happen, in theory the +timestamp of some older record received from Remote Settings can match +with the modified record in the dump. Although JSON processing script +makes sure to add unique timestamps to each of the modified records, +it's still possible to update dumps manually and simply forget to update +timestamps. So serialize the records and compare them as strings to be +on the safe side. This should happen only once after upgrading to each +new version of the application, so is not likely to introduce any +noticeable performance issues. +--- + services/settings/RemoteSettingsClient.jsm | 6 +++++- + 1 file changed, 5 insertions(+), 1 deletion(-) + +diff --git a/services/settings/RemoteSettingsClient.jsm b/services/settings/RemoteSettingsClient.jsm +index 253251823a..b45a55919c 100644 +--- a/services/settings/RemoteSettingsClient.jsm ++++ b/services/settings/RemoteSettingsClient.jsm +@@ -13,6 +13,7 @@ const { Services } = ChromeUtils.import("resource://gre/modules/Services.jsm"); + + XPCOMUtils.defineLazyModuleGetters(this, { + AppConstants: "resource://gre/modules/AppConstants.jsm", ++ CanonicalJSON: "resource://gre/modules/CanonicalJSON.jsm", + ClientEnvironmentBase: + "resource://gre/modules/components-utils/ClientEnvironment.jsm", + Database: "resource://services-settings/Database.jsm", +@@ -1022,7 +1023,10 @@ class RemoteSettingsClient extends EventEmitter { + const old = oldById.get(r.id); + if (old) { + oldById.delete(r.id); +- if (r.last_modified != old.last_modified) { ++ if ( ++ r.last_modified != old.last_modified || ++ CanonicalJSON.stringify(r) != CanonicalJSON.stringify(old) ++ ) { + syncResult.updated.push({ old, new: r }); + } + } else { +-- +2.30.0 + + +From 5f4207fd155c2dae802522070627c34ced6a192e Mon Sep 17 00:00:00 2001 +From: grizzlyuser <grizzlyuser@protonmail.com> +Date: Wed, 30 Dec 2020 19:01:39 +0200 +Subject: [PATCH 10/13] Client: delete more data on cleanup + +When the client detects the local data is invalid (i.e. it came from +real Remote Settings and can have unwanted records), delete not only +the records, but also the attachments that came with them, because they +too can be problematic. And last check time preference, because it's not +useful anyway when remote-settings.js doesn't do any polling for changes. + +Note that attachments should be deleted before the records, because the +logic gets the data about the attachments from those records. +--- + services/settings/RemoteSettingsClient.jsm | 15 ++++++++++++--- + 1 file changed, 12 insertions(+), 3 deletions(-) + +diff --git a/services/settings/RemoteSettingsClient.jsm b/services/settings/RemoteSettingsClient.jsm +index b45a55919c..3dbd972f87 100644 +--- a/services/settings/RemoteSettingsClient.jsm ++++ b/services/settings/RemoteSettingsClient.jsm +@@ -221,7 +221,10 @@ class AttachmentDownloader extends Downloader { + async deleteAll() { + let allRecords = await this._client.db.list(); + return Promise.all( +- allRecords.filter(r => !!r.attachment).map(r => this.delete(r)) ++ allRecords.filter(r => !!r.attachment).map(r => { ++ this.delete(r); ++ this.deleteCached(r.id); ++ }) + ); + } + } +@@ -982,7 +985,7 @@ class RemoteSettingsClient extends EventEmitter { + // Signature failed, clear local DB because it contains + // bad data (local + remote changes). + console.debug(`${this.identifier} clear local data`); +- await this.db.clear(); ++ await this._clearAll(); + // Local data was tampered, throw and it will retry from empty DB. + console.error(`${this.identifier} local data was corrupted`); + throw new CorruptedDataError(this.identifier); +@@ -1004,7 +1007,7 @@ class RemoteSettingsClient extends EventEmitter { + // _importJSONDump() only clears DB if dump is available, + // therefore do it here! + if (imported < 0) { +- await this.db.clear(); ++ await this._clearAll(); + } + } + } +@@ -1044,6 +1047,12 @@ class RemoteSettingsClient extends EventEmitter { + return syncResult; + } + ++ async _clearAll() { ++ await this.attachments.deleteAll(); ++ await this.db.clear(); ++ Services.prefs.clearUserPref(this.lastCheckTimePref); ++ } ++ + /** + * Fetch information from changeset endpoint. + * +-- +2.30.0 + + +From b3c304e2fe95d0a75c524a5aa96a68a34ce2ae34 Mon Sep 17 00:00:00 2001 +From: grizzlyuser <grizzlyuser@protonmail.com> +Date: Wed, 30 Dec 2020 19:07:56 +0200 +Subject: [PATCH 11/13] Client: remove comparison of collection timestamps + +In case if the cached data that came from real Remote Settings server +(before the upgrade to local-only setup) has collection timestamp, that +is newer than the packaged local dump, then this comparison logic can +lead to early return of old data, skipping the integrity checking and +necessary cleanup. So remove the checks. +--- + services/settings/RemoteSettingsClient.jsm | 5 ----- + 1 file changed, 5 deletions(-) + +diff --git a/services/settings/RemoteSettingsClient.jsm b/services/settings/RemoteSettingsClient.jsm +index 3dbd972f87..d319a77c27 100644 +--- a/services/settings/RemoteSettingsClient.jsm ++++ b/services/settings/RemoteSettingsClient.jsm +@@ -917,14 +917,9 @@ class RemoteSettingsClient extends EventEmitter { + updated: [], + deleted: [], + }; +- // If data wasn't changed, return empty sync result. +- // This can happen when we update the signature but not the data. + console.debug( + `${this.identifier} local timestamp: ${localTimestamp}, remote: ${remoteTimestamp}` + ); +- if (localTimestamp && remoteTimestamp < localTimestamp) { +- return syncResult; +- } + + const start = Cu.now() * 1000; + await this.db.importChanges(metadata, remoteTimestamp, remoteRecords, { +-- +2.30.0 + + +From 804007d2861f829315e575660d47b9a4c430d151 Mon Sep 17 00:00:00 2001 +From: grizzlyuser <grizzlyuser@protonmail.com> +Date: Wed, 30 Dec 2020 19:15:44 +0200 +Subject: [PATCH 12/13] Attachments: load only from dump and drop cached + +--- + services/settings/Attachments.jsm | 35 +++++++------------------------ + 1 file changed, 8 insertions(+), 27 deletions(-) + +diff --git a/services/settings/Attachments.jsm b/services/settings/Attachments.jsm +index 0eeb632799..eaa7db8c81 100644 +--- a/services/settings/Attachments.jsm ++++ b/services/settings/Attachments.jsm +@@ -143,10 +143,11 @@ class Downloader { + checkHash, + attachmentId = record?.id, + useCache = false, +- fallbackToCache = false, + fallbackToDump = false, + } = options || {}; + ++ const fallbackToCache = false; ++ + if (!useCache) { + // For backwards compatibility. + // WARNING: Its return type is different from what's documented. +@@ -206,6 +207,7 @@ class Downloader { + const newBuffer = await this.downloadAsBytes(record, { + retries, + checkHash, ++ dumpInfo, + }); + const blob = new Blob([newBuffer]); + if (useCache) { +@@ -241,7 +243,7 @@ class Downloader { + } + + try { +- return { ...(await cacheInfo.getResult()), _source: "cache_fallback" }; ++ await this.cacheImpl.delete(attachmentId); + } catch (e) { + // Failed to read from cache, e.g. IndexedDB unusable. + Cu.reportError(e); +@@ -278,7 +280,7 @@ class Downloader { + * @returns {String} the absolute file path to the downloaded attachment. + */ + async downloadToDisk(record, options = {}) { +- const { retries = 3 } = options; ++ const retries = 0; + const { + attachment: { filename, size, hash }, + } = record; +@@ -335,31 +337,10 @@ class Downloader { + */ + async downloadAsBytes(record, options = {}) { + const { +- attachment: { location, hash, size }, +- } = record; +- +- const remoteFileUrl = (await this._baseAttachmentsURL()) + location; ++ dumpInfo = new LazyRecordAndBuffer(() => this._readAttachmentDump(attachmentId)) ++ } = options; + +- const { retries = 3, checkHash = true } = options; +- let retried = 0; +- while (true) { +- try { +- const buffer = await this._fetchAttachment(remoteFileUrl); +- if (!checkHash) { +- return buffer; +- } +- if (await RemoteSettingsWorker.checkContentHash(buffer, size, hash)) { +- return buffer; +- } +- // Content is corrupted. +- throw new Downloader.BadContentError(location); +- } catch (e) { +- if (retried >= retries) { +- throw e; +- } +- } +- retried++; +- } ++ return (await dumpInfo.getResult()).buffer; + } + + /** +-- +2.30.0 + + +From 4ee3b715e10a5f9a5ef7830b047ae7f64e55e5f4 Mon Sep 17 00:00:00 2001 +From: grizzlyuser <grizzlyuser@protonmail.com> +Date: Wed, 30 Dec 2020 19:22:20 +0200 +Subject: [PATCH 13/13] Disable CRLite entirely for now + +It's designed to fetch the data from Remote Settings. One of the main +selling points is that new revocations can be pushed to the clients +within minutes. That won't work with local-only setup. Although (some?) +of the JSON dumps for it are in place, obviously the updates won't +happen that fast. + +Right now CRLite doesn't enforce anything, and works just for telemetry +collection (which is hopefully disabled anyway). So disable the +preference right in the source code, so that the patch fails to apply +when the upstream decides to set it to enforcing mode by default. + +The solution with CRLite is up for discussion. If necessary, it's +possible to make clients for blessed collections to communicate to real +Remote Settings server. For example, for collections related to +certificate revocations. +--- + modules/libpref/init/all.js | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/modules/libpref/init/all.js b/modules/libpref/init/all.js +index 907e8ebcef..65db6bd04c 100644 +--- a/modules/libpref/init/all.js ++++ b/modules/libpref/init/all.js +@@ -186,7 +186,7 @@ pref("security.pki.distrust_ca_policy", 2); + #if defined(NIGHTLY_BUILD) + pref("security.pki.crlite_mode", 2); + #else +-pref("security.pki.crlite_mode", 1); ++pref("security.pki.crlite_mode", 0); + #endif + + // Represents the expected certificate transparency log merge delay (including +-- +2.30.0 + |