summaryrefslogtreecommitdiff
path: root/libre/iceweasel
diff options
context:
space:
mode:
Diffstat (limited to 'libre/iceweasel')
-rw-r--r--libre/iceweasel/PKGBUILD30
-rw-r--r--libre/iceweasel/libre-0001-always-sync-remote-settings-with-local-dump.patch987
2 files changed, 1016 insertions, 1 deletions
diff --git a/libre/iceweasel/PKGBUILD b/libre/iceweasel/PKGBUILD
index fe53ac039..fffe5070b 100644
--- a/libre/iceweasel/PKGBUILD
+++ b/libre/iceweasel/PKGBUILD
@@ -27,6 +27,9 @@
# - Modify the addons pages to use GNU IceCat plugins sources, rather
# than addons.mozilla.org, which hosts non-free addons
# - Disable EME, which is implemented via the non-free libWideVine CDM
+# - Disable Normandy that let Mozilla push messages with recommendations
+# of nonfree software
+# - Make Remote Settings work completely offline using local data
# - Rebrand to Iceweasel, per the mozilla trademark policy,
# due to these FSDG changes
#
@@ -74,6 +77,7 @@ source=(https://archive.mozilla.org/pub/firefox/releases/$pkgver/source/firefox-
source+=(https://repo.parabola.nu/other/iceweasel/${pkgname}_${_brandingver}-${_brandingrel}.branding.tar.xz{,.sig}
libre.patch
libre-searchengines.patch
+ libre-0001-always-sync-remote-settings-with-local-dump.patch
vendor.js.in)
source_armv7h=(arm.patch
build-arm-libopus.patch)
@@ -286,11 +290,18 @@ END
export QUILT_PATCHES="${brandingsrcdir}"/patches
export QUILT_REFRESH_ARGS='-p ab --no-timestamps --no-index'
export QUILT_DIFF_ARGS='--no-timestamps'
+ export QUILT_PC=$srcdir/.pc
quilt push -av
## searchengines ##
+ # Load custom searchplugins
+ # FIXME: no longer applicable - replace with (broken) replacement fo libre/mozilla-searchplugins
+ # https://git.parabola.nu/packages/iceweasel.git/tree/branding-dev-build/mozilla-searchplugins?h=68.0
+# rm -rvf -- browser/components/search/searchplugins/{*.xml,images/}
+# cp -av -- /usr/lib/mozilla/searchplugins/* browser/components/search/searchplugins/
+
# Patch search-engines config
# FIXME: custom searchplugins ID per new format above ("ddg" is standard)
# browser/components/search/extensions/<ID>/
@@ -333,6 +344,23 @@ END
## libre patching ##
+ # Remove remaining non-free bits
+ # Remove test-related networking dumps, because they contain code from
+ # some Amazon webpage with no clear licensing, thus nonfree.
+ # Also they interfere with checking of Remote Settings patching done later,
+ # because communication with RS server has been captured in them too.
+ rm python/mozperftest/mozperftest/system/example.zip
+ rm testing/mozbase/mozproxy/tests/files/mitm5-linux-firefox-amazon.zip
+
+ # Disable/neutralize Remote Settings (as best we can)
+ echo "applying libre-0001-always-sync-remote-settings-with-local-dump.patch"
+ patch -Np1 --no-backup-if-mismatch -i ../libre-0001-always-sync-remote-settings-with-local-dump.patch
+
+ # Check Remote Settings patched completely by
+ # libre-0001-always-sync-remote-settings-with-local-dump.patch
+ local settings_server='firefox.settings.services.mozilla.com'
+ ! grep -qr $settings_server || { echo 'Remote Settings patching needs rework'; return 1; }
+
# Disable various components at the source level
sed -i 's/;1/;0/' toolkit/components/telemetry/components.conf
sed -Ei 's/((MOZ_SERVICES_HEALTHREPORT|MOZ_NORMANDY).+)True/\1False/' browser/moz.configure
@@ -354,7 +382,7 @@ END
# Patch and remove anything that's left
echo "applying libre.patch"
- patch -Np1 -i "${srcdir}"/libre.patch
+ patch -Np1 --no-backup-if-mismatch -i "${srcdir}"/libre.patch
}
build() {
diff --git a/libre/iceweasel/libre-0001-always-sync-remote-settings-with-local-dump.patch b/libre/iceweasel/libre-0001-always-sync-remote-settings-with-local-dump.patch
new file mode 100644
index 000000000..4e3732845
--- /dev/null
+++ b/libre/iceweasel/libre-0001-always-sync-remote-settings-with-local-dump.patch
@@ -0,0 +1,987 @@
+From c115ebc66ae779c18128e0b815fcf29da268a4f8 Mon Sep 17 00:00:00 2001
+From: grizzlyuser <grizzlyuser@protonmail.com>
+Date: Wed, 30 Dec 2020 17:20:39 +0200
+Subject: [PATCH 01/13] Point to local omni.ja files, not remote server
+
+Basically replace every occurrence of Remote Settings server domain name
+with URIs that point to built-in local files within omni.ja.
+
+Some links to json files may point to non-existing files, but that's OK
+because it's better than leave them point to Remote Settings server.
+If necessary, missing files can be added later.
+---
+ .../components/ASRouterAdmin/ASRouterAdmin.jsx | 2 +-
+ .../newtab/data/content/activity-stream.bundle.js | 2 +-
+ modules/libpref/init/all.js | 2 +-
+ services/settings/Utils.jsm | 4 ++--
+ .../periodic-updates/scripts/periodic_file_updates.sh | 2 +-
+ toolkit/components/search/SearchUtils.jsm | 8 ++++----
+ toolkit/components/search/docs/DefaultSearchEngines.rst | 2 +-
+ .../components/search/docs/SearchEngineConfiguration.rst | 2 +-
+ toolkit/mozapps/defaultagent/RemoteSettings.cpp | 2 +-
+ 9 files changed, 13 insertions(+), 13 deletions(-)
+
+diff --git a/browser/components/newtab/content-src/components/ASRouterAdmin/ASRouterAdmin.jsx b/browser/components/newtab/content-src/components/ASRouterAdmin/ASRouterAdmin.jsx
+index 8c5e540bd2..329e4e66f9 100644
+--- a/browser/components/newtab/content-src/components/ASRouterAdmin/ASRouterAdmin.jsx
++++ b/browser/components/newtab/content-src/components/ASRouterAdmin/ASRouterAdmin.jsx
+@@ -1260,7 +1260,7 @@ export class ASRouterAdminInner extends React.PureComponent {
+ <a
+ className="providerUrl"
+ target="_blank"
+- href="https://firefox.settings.services.mozilla.com/v1/buckets/main/collections/nimbus-desktop-experiments/records"
++ href="resource://app/defaults/settings/main/nimbus-desktop-experiments.json"
+ rel="noopener noreferrer"
+ >
+ nimbus-desktop-experiments
+diff --git a/browser/components/newtab/data/content/activity-stream.bundle.js b/browser/components/newtab/data/content/activity-stream.bundle.js
+index fb6d64d432..d8213e7e20 100644
+--- a/browser/components/newtab/data/content/activity-stream.bundle.js
++++ b/browser/components/newtab/data/content/activity-stream.bundle.js
+@@ -1835,7 +1835,7 @@ class ASRouterAdminInner extends react__WEBPACK_IMPORTED_MODULE_4___default.a.Pu
+ label = react__WEBPACK_IMPORTED_MODULE_4___default.a.createElement("span", null, "remote settings (", react__WEBPACK_IMPORTED_MODULE_4___default.a.createElement("a", {
+ className: "providerUrl",
+ target: "_blank",
+- href: "https://firefox.settings.services.mozilla.com/v1/buckets/main/collections/nimbus-desktop-experiments/records",
++ href: "resource://app/defaults/settings/main/nimbus-desktop-experiments.json",
+ rel: "noopener noreferrer"
+ }, "nimbus-desktop-experiments"), ")");
+ }
+diff --git a/modules/libpref/init/all.js b/modules/libpref/init/all.js
+index 2406affed4..907e8ebcef 100644
+--- a/modules/libpref/init/all.js
++++ b/modules/libpref/init/all.js
+@@ -2250,7 +2250,7 @@ pref("security.cert_pinning.hpkp.enabled", false);
+ // Remote settings preferences
+ // Note: if you change this, make sure to also review security.onecrl.maximum_staleness_in_seconds
+ pref("services.settings.poll_interval", 86400); // 24H
+-pref("services.settings.server", "https://firefox.settings.services.mozilla.com/v1");
++pref("services.settings.server", "resource://app/defaults/settings");
+ pref("services.settings.default_bucket", "main");
+
+ // The percentage of clients who will report uptake telemetry as
+diff --git a/services/settings/Utils.jsm b/services/settings/Utils.jsm
+index 66df850904..0df0fdc677 100644
+--- a/services/settings/Utils.jsm
++++ b/services/settings/Utils.jsm
+@@ -60,11 +60,11 @@ var Utils = {
+ );
+ const isXpcshell = env.exists("XPCSHELL_TEST_PROFILE_DIR");
+ return AppConstants.RELEASE_OR_BETA && !Cu.isInAutomation && !isXpcshell
+- ? "https://firefox.settings.services.mozilla.com/v1"
++ ? "resource://app/defaults/settings"
+ : gServerURL;
+ },
+
+- CHANGES_PATH: "/buckets/monitor/collections/changes/records",
++ CHANGES_PATH: "/monitor/changes.json",
+
+ /**
+ * Logger instance.
+diff --git a/taskcluster/docker/periodic-updates/scripts/periodic_file_updates.sh b/taskcluster/docker/periodic-updates/scripts/periodic_file_updates.sh
+index c2492615e0..2b10ad01f6 100755
+--- a/taskcluster/docker/periodic-updates/scripts/periodic_file_updates.sh
++++ b/taskcluster/docker/periodic-updates/scripts/periodic_file_updates.sh
+@@ -279,7 +279,7 @@ function compare_suffix_lists {
+ }
+
+ function compare_remote_settings_files {
+- REMOTE_SETTINGS_SERVER="https://firefox.settings.services.mozilla.com/v1"
++ REMOTE_SETTINGS_SERVER="resource://app/defaults/settings"
+
+ # 1. List remote settings collections from server.
+ echo "INFO: fetch remote settings list from server"
+diff --git a/toolkit/components/search/SearchUtils.jsm b/toolkit/components/search/SearchUtils.jsm
+index 278f9f9c08..7bac023c11 100644
+--- a/toolkit/components/search/SearchUtils.jsm
++++ b/toolkit/components/search/SearchUtils.jsm
+@@ -139,13 +139,13 @@ var SearchUtils = {
+
+ ENGINES_URLS: {
+ "prod-main":
+- "https://firefox.settings.services.mozilla.com/v1/buckets/main/collections/search-config/records",
++ "resource://app/defaults/settings/main/search-config.json",
+ "prod-preview":
+- "https://firefox.settings.services.mozilla.com/v1/buckets/main-preview/collections/search-config/records",
++ "resource://app/defaults/settings/main/search-config.json",
+ "stage-main":
+- "https://settings.stage.mozaws.net/v1/buckets/main/collections/search-config/records",
++ "resource://app/defaults/settings/main/search-config.json",
+ "stage-preview":
+- "https://settings.stage.mozaws.net/v1/buckets/main-preview/collections/search-config/records",
++ "resource://app/defaults/settings/main/search-config.json",
+ },
+
+ // The following constants are left undocumented in nsISearchService.idl
+diff --git a/toolkit/components/search/docs/DefaultSearchEngines.rst b/toolkit/components/search/docs/DefaultSearchEngines.rst
+index 5668646648..deb7d20185 100644
+--- a/toolkit/components/search/docs/DefaultSearchEngines.rst
++++ b/toolkit/components/search/docs/DefaultSearchEngines.rst
+@@ -63,4 +63,4 @@ is updated.
+
+ .. _configuration schema: SearchConfigurationSchema.html
+ .. _remote settings: /services/common/services/RemoteSettings.html
+-.. _search-default-override-allowlist bucket: https://firefox.settings.services.mozilla.com/v1/buckets/main/collections/search-default-override-allowlist/records
++.. _search-default-override-allowlist bucket: resource://app/defaults/settings/main/search-default-override-allowlist.json
+diff --git a/toolkit/components/search/docs/SearchEngineConfiguration.rst b/toolkit/components/search/docs/SearchEngineConfiguration.rst
+index e9041affb8..7a9466d294 100644
+--- a/toolkit/components/search/docs/SearchEngineConfiguration.rst
++++ b/toolkit/components/search/docs/SearchEngineConfiguration.rst
+@@ -68,5 +68,5 @@ related. As a result several situations may occur:
+ .. _JSON schema: https://json-schema.org/
+ .. _stored in mozilla-central: https://searchfox.org/mozilla-central/source/toolkit/components/search/schema/
+ .. _Search Configuration Schema: SearchConfigurationSchema.html
+-.. _viewed live: https://firefox.settings.services.mozilla.com/v1/buckets/main/collections/search-config/records
++.. _viewed live: resource://app/defaults/settings/main/search-config.json
+ .. _Normandy: /toolkit/components/normandy/normandy/services.html
+diff --git a/toolkit/mozapps/defaultagent/RemoteSettings.cpp b/toolkit/mozapps/defaultagent/RemoteSettings.cpp
+index 667d9fc628..b2bf628f29 100644
+--- a/toolkit/mozapps/defaultagent/RemoteSettings.cpp
++++ b/toolkit/mozapps/defaultagent/RemoteSettings.cpp
+@@ -23,7 +23,7 @@ extern "C" {
+ HRESULT IsAgentRemoteDisabledRust(const char* szUrl, DWORD* lpdwDisabled);
+ }
+
+-#define PROD_ENDPOINT "https://firefox.settings.services.mozilla.com/v1"
++#define PROD_ENDPOINT "resource://app/defaults/settings"
+ #define PROD_BID "main"
+ #define PROD_CID "windows-default-browser-agent"
+ #define PROD_ID "state"
+--
+2.30.0
+
+
+From f76e1c055def541e358886d0e73ba4710d3e5084 Mon Sep 17 00:00:00 2001
+From: grizzlyuser <grizzlyuser@protonmail.com>
+Date: Wed, 30 Dec 2020 17:34:08 +0200
+Subject: [PATCH 02/13] Remove polling triggered by push broadcasts
+
+When initialized, remote-settings.js adds a listener to push broadcasts,
+that let Remote Settings server send push messages to trigger polling
+for changes from the client side. This is not needed for local-only
+setup. Remove the record from broadcast-listeners.json file stored in
+the user profile, so that it doesn't get picked up by push broadcast
+service.
+---
+ dom/push/PushBroadcastService.jsm | 13 +++++++++++++
+ services/settings/remote-settings.js | 7 ++-----
+ 2 files changed, 15 insertions(+), 5 deletions(-)
+
+diff --git a/dom/push/PushBroadcastService.jsm b/dom/push/PushBroadcastService.jsm
+index 27ed31ee9c..71f1316994 100644
+--- a/dom/push/PushBroadcastService.jsm
++++ b/dom/push/PushBroadcastService.jsm
+@@ -179,6 +179,19 @@ var BroadcastService = class {
+ }
+ }
+
++ async deleteListener(broadcastId) {
++ await this.initializePromise;
++
++ if (this.jsonFile.data.listeners.hasOwnProperty(broadcastId)) {
++ console.info(
++ "deleteListener: deleting listener",
++ broadcastId
++ );
++ delete this.jsonFile.data.listeners[broadcastId];
++ this.jsonFile.saveSoon();
++ }
++ }
++
+ /**
+ * Call the listeners of the specified broadcasts.
+ *
+diff --git a/services/settings/remote-settings.js b/services/settings/remote-settings.js
+index 6d0185faf9..aae93fa440 100644
+--- a/services/settings/remote-settings.js
++++ b/services/settings/remote-settings.js
+@@ -441,7 +441,7 @@ function remoteSettingsFunction() {
+ moduleURI: __URI__,
+ symbolName: "remoteSettingsBroadcastHandler",
+ };
+- pushBroadcastService.addListener(BROADCAST_ID, currentVersion, moduleInfo);
++ pushBroadcastService.deleteListener(BROADCAST_ID);
+ };
+
+ return remoteSettings;
+@@ -461,9 +461,6 @@ var remoteSettingsBroadcastHandler = {
+ `Push notification received (version=${version} phase=${phase})`
+ );
+
+- return RemoteSettings.pollChanges({
+- expectedTimestamp: version,
+- trigger: isStartup ? "startup" : "broadcast",
+- });
++ return;
+ },
+ };
+--
+2.30.0
+
+
+From fcf60ef5835e673e03807d898f90e48907f44e63 Mon Sep 17 00:00:00 2001
+From: grizzlyuser <grizzlyuser@protonmail.com>
+Date: Wed, 30 Dec 2020 17:41:54 +0200
+Subject: [PATCH 03/13] Remove timer that triggers polling for changes
+
+That is not needed for local-only setup.
+---
+ services/settings/components.conf | 9 +--------
+ services/settings/servicesSettings.manifest | 4 ----
+ 2 files changed, 1 insertion(+), 12 deletions(-)
+
+diff --git a/services/settings/components.conf b/services/settings/components.conf
+index 9a737802ee..25109415a7 100644
+--- a/services/settings/components.conf
++++ b/services/settings/components.conf
+@@ -4,11 +4,4 @@
+ # License, v. 2.0. If a copy of the MPL was not distributed with this
+ # file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+-Classes = [
+- {
+- 'cid': '{5e756573-234a-49ea-bbe4-59ec7a70657d}',
+- 'contract_ids': ['@mozilla.org/services/settings;1'],
+- 'jsm': 'resource://services-settings/RemoteSettingsComponents.jsm',
+- 'constructor': 'RemoteSettingsTimer',
+- },
+-]
++Classes = []
+diff --git a/services/settings/servicesSettings.manifest b/services/settings/servicesSettings.manifest
+index 3bfed26ea4..807eb220ec 100644
+--- a/services/settings/servicesSettings.manifest
++++ b/services/settings/servicesSettings.manifest
+@@ -1,7 +1,3 @@
+ # Register resource aliases
+ resource services-settings resource://gre/modules/services-settings/
+
+-# Schedule polling of remote settings changes
+-# (default 24H, max 72H)
+-# see syntax https://searchfox.org/mozilla-central/rev/cc280c4be94ff8cf64a27cc9b3d6831ffa49fa45/toolkit/components/timermanager/UpdateTimerManager.jsm#155
+-category update-timer RemoteSettingsComponents @mozilla.org/services/settings;1,getService,services-settings-poll-changes,services.settings.poll_interval,86400,259200
+--
+2.30.0
+
+
+From 8600d50c402f523973f67dfb03c1ea8614fcf48c Mon Sep 17 00:00:00 2001
+From: grizzlyuser <grizzlyuser@protonmail.com>
+Date: Wed, 30 Dec 2020 17:47:41 +0200
+Subject: [PATCH 04/13] Utils: fetch timestamps of each collection locally
+
+Utils.CHANGES_PATH points to
+services/settings/dumps/monitor/changes.json
+which will be generated later by JSON processing script. Fetch the
+timestamps from that file and mock response headers to not confuse any
+code that expects them.
+---
+ browser/installer/package-manifest.in | 1 +
+ services/settings/Utils.jsm | 14 ++++++++++++--
+ services/settings/dumps/monitor/moz.build | 8 ++++++++
+ services/settings/dumps/moz.build | 1 +
+ 4 files changed, 22 insertions(+), 2 deletions(-)
+ create mode 100644 services/settings/dumps/monitor/moz.build
+
+diff --git a/browser/installer/package-manifest.in b/browser/installer/package-manifest.in
+index 75c79a7168..ac01689596 100644
+--- a/browser/installer/package-manifest.in
++++ b/browser/installer/package-manifest.in
+@@ -295,6 +295,7 @@
+ @RESPATH@/browser/defaults/settings/blocklists
+ @RESPATH@/browser/defaults/settings/pinning
+ @RESPATH@/browser/defaults/settings/main
++@RESPATH@/browser/defaults/settings/monitor
+ @RESPATH@/browser/defaults/settings/security-state
+
+ ; Warning: changing the path to channel-prefs.js can cause bugs (Bug 756325)
+diff --git a/services/settings/Utils.jsm b/services/settings/Utils.jsm
+index 0df0fdc677..0e631ddc0d 100644
+--- a/services/settings/Utils.jsm
++++ b/services/settings/Utils.jsm
+@@ -145,7 +145,7 @@ var Utils = {
+ // "collection":"certificates"
+ // }]}
+
+- let url = serverUrl + Utils.CHANGES_PATH;
++ let url = Utils.SERVER_URL + Utils.CHANGES_PATH;
+
+ // Use ETag to obtain a `304 Not modified` when no change occurred,
+ // and `?_since` parameter to only keep entries that weren't processed yet.
+@@ -166,6 +166,9 @@ var Utils = {
+ .join("&");
+ }
+ const response = await fetch(url, { headers });
++ const responseDate = new Date().toUTCString()
++ response.headers.set("Date", responseDate);
++ response.headers.set("Last-Modified", responseDate);
+
+ let changes = [];
+ // If no changes since last time, go on with empty list of changes.
+@@ -203,7 +206,14 @@ var Utils = {
+ );
+ }
+ } else {
+- changes = payload.data;
++ const { bucket, collection } = filters;
++ if (!bucket || !collection) {
++ throw new Error('Unable to fetch latest change without bucket or collection');
++ }
++ const change = payload.data.find(
++ change => change.bucket === bucket && change.collection === collection
++ ) ?? { last_modified: 0, bucket, collection };
++ changes = [change];
+ }
+ }
+ // The server should always return ETag. But we've had situations where the CDN
+diff --git a/services/settings/dumps/monitor/moz.build b/services/settings/dumps/monitor/moz.build
+new file mode 100644
+index 0000000000..d3d017fda5
+--- /dev/null
++++ b/services/settings/dumps/monitor/moz.build
+@@ -0,0 +1,8 @@
++# This Source Code Form is subject to the terms of the Mozilla Public
++# License, v. 2.0. If a copy of the MPL was not distributed with this
++# file, You can obtain one at http://mozilla.org/MPL/2.0/.
++
++FINAL_TARGET_FILES.defaults.settings.monitor += ["changes.json"]
++
++if CONFIG["MOZ_BUILD_APP"] == "browser":
++ DIST_SUBDIR = "browser"
+diff --git a/services/settings/dumps/moz.build b/services/settings/dumps/moz.build
+index 3cc9436f61..3742da5667 100644
+--- a/services/settings/dumps/moz.build
++++ b/services/settings/dumps/moz.build
+@@ -5,6 +5,7 @@
+ DIRS += [
+ "blocklists",
+ "main",
++ "monitor",
+ "pinning",
+ "security-state",
+ ]
+--
+2.30.0
+
+
+From 99af961e47d810c965c0145f19a92eedec1abafc Mon Sep 17 00:00:00 2001
+From: grizzlyuser <grizzlyuser@protonmail.com>
+Date: Wed, 30 Dec 2020 17:52:10 +0200
+Subject: [PATCH 05/13] Utils: disable offline checking
+
+Since only local data is read now, it should always return false for the
+current any any future code that relies on it.
+---
+ services/settings/Utils.jsm | 9 ---------
+ 1 file changed, 9 deletions(-)
+
+diff --git a/services/settings/Utils.jsm b/services/settings/Utils.jsm
+index 0e631ddc0d..d034d8ea78 100644
+--- a/services/settings/Utils.jsm
++++ b/services/settings/Utils.jsm
+@@ -80,15 +80,6 @@ var Utils = {
+ * @return {bool} Whether network is down or not.
+ */
+ get isOffline() {
+- try {
+- return (
+- Services.io.offline ||
+- CaptivePortalService.state == CaptivePortalService.LOCKED_PORTAL ||
+- !gNetworkLinkService.isLinkUp
+- );
+- } catch (ex) {
+- log.warn("Could not determine network status.", ex);
+- }
+ return false;
+ },
+
+--
+2.30.0
+
+
+From 55830695a1e2457aecca1392d46c98c3e210d58a Mon Sep 17 00:00:00 2001
+From: grizzlyuser <grizzlyuser@protonmail.com>
+Date: Wed, 30 Dec 2020 17:56:02 +0200
+Subject: [PATCH 06/13] Refactor hashing logic to a separate function
+
+It is used instead of internal signature validation mechanism, for
+integrity checking of the locally cached data.
+---
+ services/settings/RemoteSettingsWorker.jsm | 4 ++++
+ services/settings/SharedUtils.jsm | 9 +++++++--
+ 2 files changed, 11 insertions(+), 2 deletions(-)
+
+diff --git a/services/settings/RemoteSettingsWorker.jsm b/services/settings/RemoteSettingsWorker.jsm
+index 147ebb6b13..c86e218fd3 100644
+--- a/services/settings/RemoteSettingsWorker.jsm
++++ b/services/settings/RemoteSettingsWorker.jsm
+@@ -189,6 +189,10 @@ class Worker {
+ // task on the current thread instead of the worker thread.
+ return SharedUtils.checkContentHash(buffer, size, hash);
+ }
++
++ async getContentHash(bytes) {
++ return SharedUtils.getContentHash(bytes);
++ }
+ }
+
+ // Now, first add a shutdown blocker. If that fails, we must have
+diff --git a/services/settings/SharedUtils.jsm b/services/settings/SharedUtils.jsm
+index db5017a742..1a8e83c2e8 100644
+--- a/services/settings/SharedUtils.jsm
++++ b/services/settings/SharedUtils.jsm
+@@ -28,11 +28,16 @@ var SharedUtils = {
+ return false;
+ }
+ // Has expected content?
++ const hashStr = await this.getContentHash(bytes);
++ return hashStr == hash;
++ },
++
++ async getContentHash(bytes) {
+ const hashBuffer = await crypto.subtle.digest("SHA-256", bytes);
+ const hashBytes = new Uint8Array(hashBuffer);
+ const toHex = b => b.toString(16).padStart(2, "0");
+- const hashStr = Array.from(hashBytes, toHex).join("");
+- return hashStr == hash;
++
++ return Array.from(hashBytes, toHex).join("");
+ },
+
+ /**
+--
+2.30.0
+
+
+From 9dc19397aebfb6b9f74afa0fb4d653a29e14e964 Mon Sep 17 00:00:00 2001
+From: grizzlyuser <grizzlyuser@protonmail.com>
+Date: Wed, 30 Dec 2020 18:05:02 +0200
+Subject: [PATCH 07/13] Client: Fetch and hash records from local dump
+
+Read the records from local dumps. See [1] for details on how to prepare
+custom dumps). Records are cached in the local IndexedDB, and the client
+updates cached records each time there's a change. Also it verifies
+integrity of the data. Then the list of current / created / updated /
+deleted records is generated and emitted to every registered listener.
+
+Change upstream signature validation mechanism to a simpler one.
+Otherwise, it'd be necessary to sign local records, which is redundant,
+because the application package should be signed already by the distro.
+
+Instead of signature property from metadata records, json_dump_metadata
+has been introduced. It contains the checksum of the records and size in
+bytes. Also added app_build_id property for version checking and updates
+of cached data.
+
+Although it's possible to disable integrity checking via preference, it
+seems to be not a good idea, because the logic that detects invalid
+local data relies on it. In the context of local-only setup, data that
+has been received from real Remote Settings server will not contain the
+custom metadata, and thus will be considered invalid and then discarded,
+while the client gets a chance to gracefully inform registered listeners
+about these changes so that they can discard the data received before
+the upgrade to local-only setup.
+
+[1] https://firefox-source-docs.mozilla.org/services/common/services/RemoteSettings.html#initial-data
+---
+ services/settings/RemoteSettingsClient.jsm | 62 ++++++++++------------
+ 1 file changed, 27 insertions(+), 35 deletions(-)
+
+diff --git a/services/settings/RemoteSettingsClient.jsm b/services/settings/RemoteSettingsClient.jsm
+index 80dd563e11..1025ab33a2 100644
+--- a/services/settings/RemoteSettingsClient.jsm
++++ b/services/settings/RemoteSettingsClient.jsm
+@@ -556,11 +556,9 @@ class RemoteSettingsClient extends EventEmitter {
+
+ // If the data is up-to-date but don't have metadata (records loaded from dump),
+ // we fetch them and validate the signature immediately.
+- if (this.verifySignature && ObjectUtils.isEmpty(localMetadata)) {
++ if (this.verifySignature && ObjectUtils.isEmpty(localMetadata?.json_dump_metadata)) {
+ console.debug(`${this.identifier} pull collection metadata`);
+- const metadata = await this.httpClient().getData({
+- query: { _expected: expectedTimestamp },
+- });
++ const { metadata } = await this._fetchChangeset(expectedTimestamp);
+ await this.db.importChanges(metadata);
+ // We don't bother validating the signature if the dump was just loaded. We do
+ // if the dump was loaded at some other point (eg. from .get()).
+@@ -813,32 +811,23 @@ class RemoteSettingsClient extends EventEmitter {
+ async _validateCollectionSignature(records, timestamp, metadata) {
+ const start = Cu.now() * 1000;
+
+- if (!metadata?.signature) {
++ if (!metadata?.json_dump_metadata) {
+ throw new MissingSignatureError(this.identifier);
+ }
+
+- if (!this._verifier) {
+- this._verifier = Cc[
+- "@mozilla.org/security/contentsignatureverifier;1"
+- ].createInstance(Ci.nsIContentSignatureVerifier);
+- }
+-
+- // This is a content-signature field from an autograph response.
+ const {
+- signature: { x5u, signature },
++ json_dump_metadata: { hash, size },
+ } = metadata;
+- const certChain = await (await fetch(x5u)).text();
+ // Merge remote records with local ones and serialize as canonical JSON.
+ const serialized = await RemoteSettingsWorker.canonicalStringify(
+ records,
+ timestamp
+ );
+ if (
+- !(await this._verifier.asyncVerifyContentSignature(
+- serialized,
+- "p384ecdsa=" + signature,
+- certChain,
+- this.signerName
++ !(await RemoteSettingsWorker.checkContentHash(
++ new TextEncoder().encode(serialized),
++ size,
++ hash
+ ))
+ ) {
+ throw new InvalidSignatureError(this.identifier);
+@@ -1030,24 +1019,27 @@ class RemoteSettingsClient extends EventEmitter {
+ * @param since timestamp of last sync (optional)
+ */
+ async _fetchChangeset(expectedTimestamp, since) {
+- const client = this.httpClient();
+- const {
+- metadata,
+- timestamp: remoteTimestamp,
+- changes: remoteRecords,
+- } = await client.execute(
+- {
+- path: `/buckets/${this.bucketName}/collections/${this.collectionName}/changeset`,
+- },
+- {
+- query: {
+- _expected: expectedTimestamp,
+- _since: since,
+- },
+- }
++ const { data } = await SharedUtils.loadJSONDump(
++ this.bucketName,
++ this.collectionName
+ );
++ const remoteRecords = data ?? [];
++
++ const serialized = await RemoteSettingsWorker.canonicalStringify(
++ remoteRecords,
++ expectedTimestamp
++ );
++ const bytes = new TextEncoder().encode(serialized);
++ const metadata = {
++ app_build_id: Services.appinfo.appBuildID,
++ json_dump_metadata: {
++ hash: await RemoteSettingsWorker.getContentHash(bytes),
++ size: bytes.length,
++ },
++ }
++
+ return {
+- remoteTimestamp,
++ remoteTimestamp: expectedTimestamp,
+ metadata,
+ remoteRecords,
+ };
+--
+2.30.0
+
+
+From 4dfbb2ccacd011b05839c19127942b497141f2a1 Mon Sep 17 00:00:00 2001
+From: grizzlyuser <grizzlyuser@protonmail.com>
+Date: Wed, 30 Dec 2020 18:42:56 +0200
+Subject: [PATCH 08/13] Client: start deferred sync on get() or on()
+
+The users of the RemoteSettingsClient.jsm can receive records from it in
+two ways: by calling get(), and by subscribing to events by calling
+on().
+
+So hook a deferred sync whenever something calls these methods. Because
+multiple of those calls can be made quite early and in very short time,
+set up a deferred task that will be armed only when needed and only once
+in a second. When the task is running it first checks if the local data
+came from the dump of the current app build, and no-ops if true. If
+false, it triggers a sync. Then adds a flag if the client has been
+correctly synchronized with the dump, so that no metadata checking
+occurs during the session.
+---
+ services/settings/RemoteSettingsClient.jsm | 32 ++++++++++++++++++++--
+ 1 file changed, 30 insertions(+), 2 deletions(-)
+
+diff --git a/services/settings/RemoteSettingsClient.jsm b/services/settings/RemoteSettingsClient.jsm
+index 1025ab33a2..253251823a 100644
+--- a/services/settings/RemoteSettingsClient.jsm
++++ b/services/settings/RemoteSettingsClient.jsm
+@@ -16,6 +16,7 @@ XPCOMUtils.defineLazyModuleGetters(this, {
+ ClientEnvironmentBase:
+ "resource://gre/modules/components-utils/ClientEnvironment.jsm",
+ Database: "resource://services-settings/Database.jsm",
++ DeferredTask: "resource://gre/modules/DeferredTask.jsm",
+ Downloader: "resource://services-settings/Attachments.jsm",
+ IDBHelpers: "resource://services-settings/IDBHelpers.jsm",
+ KintoHttpClient: "resource://services-common/kinto-http-client.js",
+@@ -30,6 +31,7 @@ XPCOMUtils.defineLazyModuleGetters(this, {
+ XPCOMUtils.defineLazyGlobalGetters(this, ["fetch"]);
+
+ const TELEMETRY_COMPONENT = "remotesettings";
++const DEFERRED_SYNC_DELAY_MILLISECONDS = 1000
+
+ XPCOMUtils.defineLazyGetter(this, "console", () => Utils.log);
+
+@@ -259,6 +261,14 @@ class RemoteSettingsClient extends EventEmitter {
+ this._lastCheckTimePref = lastCheckTimePref;
+ this._verifier = null;
+ this._syncRunning = false;
++ this._deferredSync = new DeferredTask(
++ async () => {
++ if (!this._syncRunning && !(await this._isSynced())) {
++ await this.sync();
++ }
++ },
++ DEFERRED_SYNC_DELAY_MILLISECONDS
++ );
+
+ // This attribute allows signature verification to be disabled, when running tests
+ // or when pulling data from a dev server.
+@@ -290,6 +300,11 @@ class RemoteSettingsClient extends EventEmitter {
+ );
+ }
+
++ on(event, callback) {
++ super.on(event, callback);
++ this._deferredSync.arm();
++ }
++
+ get identifier() {
+ return `${this.bucketName}/${this.collectionName}`;
+ }
+@@ -353,7 +368,11 @@ class RemoteSettingsClient extends EventEmitter {
+ try {
+ let hasLocalData = await Utils.hasLocalData(this);
+
+- if (syncIfEmpty && !hasLocalData) {
++ if (!(await this._isSynced())) {
++ throw new MissingSignatureError(this.identifier);
++ }
++
++ if ((syncIfEmpty && !hasLocalData)) {
+ // .get() was called before we had the chance to synchronize the local database.
+ // We'll try to avoid returning an empty list.
+ if (!this._importingPromise) {
+@@ -414,7 +433,10 @@ class RemoteSettingsClient extends EventEmitter {
+ // No need to verify signature on JSON dumps.
+ // If local DB cannot be read, then we don't even try to do anything,
+ // we return results early.
+- return this._filterEntries(data);
++ const filtered = this._filterEntries(data);
++ this._deferredSync.arm();
++
++ return filtered;
+ }
+
+ console.debug(
+@@ -452,6 +474,12 @@ class RemoteSettingsClient extends EventEmitter {
+ return final;
+ }
+
++ async _isSynced() {
++ this._synced ||=
++ Services.appinfo.appBuildID === (await this.db.getMetadata())?.app_build_id;
++ return this._synced;
++ }
++
+ /**
+ * Synchronize the local database with the remote server.
+ *
+--
+2.30.0
+
+
+From 89dc912d08b7d4eed113439efa17dce57eda106e Mon Sep 17 00:00:00 2001
+From: grizzlyuser <grizzlyuser@protonmail.com>
+Date: Wed, 30 Dec 2020 18:53:51 +0200
+Subject: [PATCH 09/13] Client: deep compare records if timestamps match
+
+When the list of current / updated / deleted records is generated, their
+modification timestamps are compared to detect the updates.
+
+Although in practice this is unlikely to happen, in theory the
+timestamp of some older record received from Remote Settings can match
+with the modified record in the dump. Although JSON processing script
+makes sure to add unique timestamps to each of the modified records,
+it's still possible to update dumps manually and simply forget to update
+timestamps. So serialize the records and compare them as strings to be
+on the safe side. This should happen only once after upgrading to each
+new version of the application, so is not likely to introduce any
+noticeable performance issues.
+---
+ services/settings/RemoteSettingsClient.jsm | 6 +++++-
+ 1 file changed, 5 insertions(+), 1 deletion(-)
+
+diff --git a/services/settings/RemoteSettingsClient.jsm b/services/settings/RemoteSettingsClient.jsm
+index 253251823a..b45a55919c 100644
+--- a/services/settings/RemoteSettingsClient.jsm
++++ b/services/settings/RemoteSettingsClient.jsm
+@@ -13,6 +13,7 @@ const { Services } = ChromeUtils.import("resource://gre/modules/Services.jsm");
+
+ XPCOMUtils.defineLazyModuleGetters(this, {
+ AppConstants: "resource://gre/modules/AppConstants.jsm",
++ CanonicalJSON: "resource://gre/modules/CanonicalJSON.jsm",
+ ClientEnvironmentBase:
+ "resource://gre/modules/components-utils/ClientEnvironment.jsm",
+ Database: "resource://services-settings/Database.jsm",
+@@ -1022,7 +1023,10 @@ class RemoteSettingsClient extends EventEmitter {
+ const old = oldById.get(r.id);
+ if (old) {
+ oldById.delete(r.id);
+- if (r.last_modified != old.last_modified) {
++ if (
++ r.last_modified != old.last_modified ||
++ CanonicalJSON.stringify(r) != CanonicalJSON.stringify(old)
++ ) {
+ syncResult.updated.push({ old, new: r });
+ }
+ } else {
+--
+2.30.0
+
+
+From 5f4207fd155c2dae802522070627c34ced6a192e Mon Sep 17 00:00:00 2001
+From: grizzlyuser <grizzlyuser@protonmail.com>
+Date: Wed, 30 Dec 2020 19:01:39 +0200
+Subject: [PATCH 10/13] Client: delete more data on cleanup
+
+When the client detects the local data is invalid (i.e. it came from
+real Remote Settings and can have unwanted records), delete not only
+the records, but also the attachments that came with them, because they
+too can be problematic. And last check time preference, because it's not
+useful anyway when remote-settings.js doesn't do any polling for changes.
+
+Note that attachments should be deleted before the records, because the
+logic gets the data about the attachments from those records.
+---
+ services/settings/RemoteSettingsClient.jsm | 15 ++++++++++++---
+ 1 file changed, 12 insertions(+), 3 deletions(-)
+
+diff --git a/services/settings/RemoteSettingsClient.jsm b/services/settings/RemoteSettingsClient.jsm
+index b45a55919c..3dbd972f87 100644
+--- a/services/settings/RemoteSettingsClient.jsm
++++ b/services/settings/RemoteSettingsClient.jsm
+@@ -221,7 +221,10 @@ class AttachmentDownloader extends Downloader {
+ async deleteAll() {
+ let allRecords = await this._client.db.list();
+ return Promise.all(
+- allRecords.filter(r => !!r.attachment).map(r => this.delete(r))
++ allRecords.filter(r => !!r.attachment).map(r => {
++ this.delete(r);
++ this.deleteCached(r.id);
++ })
+ );
+ }
+ }
+@@ -982,7 +985,7 @@ class RemoteSettingsClient extends EventEmitter {
+ // Signature failed, clear local DB because it contains
+ // bad data (local + remote changes).
+ console.debug(`${this.identifier} clear local data`);
+- await this.db.clear();
++ await this._clearAll();
+ // Local data was tampered, throw and it will retry from empty DB.
+ console.error(`${this.identifier} local data was corrupted`);
+ throw new CorruptedDataError(this.identifier);
+@@ -1004,7 +1007,7 @@ class RemoteSettingsClient extends EventEmitter {
+ // _importJSONDump() only clears DB if dump is available,
+ // therefore do it here!
+ if (imported < 0) {
+- await this.db.clear();
++ await this._clearAll();
+ }
+ }
+ }
+@@ -1044,6 +1047,12 @@ class RemoteSettingsClient extends EventEmitter {
+ return syncResult;
+ }
+
++ async _clearAll() {
++ await this.attachments.deleteAll();
++ await this.db.clear();
++ Services.prefs.clearUserPref(this.lastCheckTimePref);
++ }
++
+ /**
+ * Fetch information from changeset endpoint.
+ *
+--
+2.30.0
+
+
+From b3c304e2fe95d0a75c524a5aa96a68a34ce2ae34 Mon Sep 17 00:00:00 2001
+From: grizzlyuser <grizzlyuser@protonmail.com>
+Date: Wed, 30 Dec 2020 19:07:56 +0200
+Subject: [PATCH 11/13] Client: remove comparison of collection timestamps
+
+In case if the cached data that came from real Remote Settings server
+(before the upgrade to local-only setup) has collection timestamp, that
+is newer than the packaged local dump, then this comparison logic can
+lead to early return of old data, skipping the integrity checking and
+necessary cleanup. So remove the checks.
+---
+ services/settings/RemoteSettingsClient.jsm | 5 -----
+ 1 file changed, 5 deletions(-)
+
+diff --git a/services/settings/RemoteSettingsClient.jsm b/services/settings/RemoteSettingsClient.jsm
+index 3dbd972f87..d319a77c27 100644
+--- a/services/settings/RemoteSettingsClient.jsm
++++ b/services/settings/RemoteSettingsClient.jsm
+@@ -917,14 +917,9 @@ class RemoteSettingsClient extends EventEmitter {
+ updated: [],
+ deleted: [],
+ };
+- // If data wasn't changed, return empty sync result.
+- // This can happen when we update the signature but not the data.
+ console.debug(
+ `${this.identifier} local timestamp: ${localTimestamp}, remote: ${remoteTimestamp}`
+ );
+- if (localTimestamp && remoteTimestamp < localTimestamp) {
+- return syncResult;
+- }
+
+ const start = Cu.now() * 1000;
+ await this.db.importChanges(metadata, remoteTimestamp, remoteRecords, {
+--
+2.30.0
+
+
+From 804007d2861f829315e575660d47b9a4c430d151 Mon Sep 17 00:00:00 2001
+From: grizzlyuser <grizzlyuser@protonmail.com>
+Date: Wed, 30 Dec 2020 19:15:44 +0200
+Subject: [PATCH 12/13] Attachments: load only from dump and drop cached
+
+---
+ services/settings/Attachments.jsm | 35 +++++++------------------------
+ 1 file changed, 8 insertions(+), 27 deletions(-)
+
+diff --git a/services/settings/Attachments.jsm b/services/settings/Attachments.jsm
+index 0eeb632799..eaa7db8c81 100644
+--- a/services/settings/Attachments.jsm
++++ b/services/settings/Attachments.jsm
+@@ -143,10 +143,11 @@ class Downloader {
+ checkHash,
+ attachmentId = record?.id,
+ useCache = false,
+- fallbackToCache = false,
+ fallbackToDump = false,
+ } = options || {};
+
++ const fallbackToCache = false;
++
+ if (!useCache) {
+ // For backwards compatibility.
+ // WARNING: Its return type is different from what's documented.
+@@ -206,6 +207,7 @@ class Downloader {
+ const newBuffer = await this.downloadAsBytes(record, {
+ retries,
+ checkHash,
++ dumpInfo,
+ });
+ const blob = new Blob([newBuffer]);
+ if (useCache) {
+@@ -241,7 +243,7 @@ class Downloader {
+ }
+
+ try {
+- return { ...(await cacheInfo.getResult()), _source: "cache_fallback" };
++ await this.cacheImpl.delete(attachmentId);
+ } catch (e) {
+ // Failed to read from cache, e.g. IndexedDB unusable.
+ Cu.reportError(e);
+@@ -278,7 +280,7 @@ class Downloader {
+ * @returns {String} the absolute file path to the downloaded attachment.
+ */
+ async downloadToDisk(record, options = {}) {
+- const { retries = 3 } = options;
++ const retries = 0;
+ const {
+ attachment: { filename, size, hash },
+ } = record;
+@@ -335,31 +337,10 @@ class Downloader {
+ */
+ async downloadAsBytes(record, options = {}) {
+ const {
+- attachment: { location, hash, size },
+- } = record;
+-
+- const remoteFileUrl = (await this._baseAttachmentsURL()) + location;
++ dumpInfo = new LazyRecordAndBuffer(() => this._readAttachmentDump(attachmentId))
++ } = options;
+
+- const { retries = 3, checkHash = true } = options;
+- let retried = 0;
+- while (true) {
+- try {
+- const buffer = await this._fetchAttachment(remoteFileUrl);
+- if (!checkHash) {
+- return buffer;
+- }
+- if (await RemoteSettingsWorker.checkContentHash(buffer, size, hash)) {
+- return buffer;
+- }
+- // Content is corrupted.
+- throw new Downloader.BadContentError(location);
+- } catch (e) {
+- if (retried >= retries) {
+- throw e;
+- }
+- }
+- retried++;
+- }
++ return (await dumpInfo.getResult()).buffer;
+ }
+
+ /**
+--
+2.30.0
+
+
+From 4ee3b715e10a5f9a5ef7830b047ae7f64e55e5f4 Mon Sep 17 00:00:00 2001
+From: grizzlyuser <grizzlyuser@protonmail.com>
+Date: Wed, 30 Dec 2020 19:22:20 +0200
+Subject: [PATCH 13/13] Disable CRLite entirely for now
+
+It's designed to fetch the data from Remote Settings. One of the main
+selling points is that new revocations can be pushed to the clients
+within minutes. That won't work with local-only setup. Although (some?)
+of the JSON dumps for it are in place, obviously the updates won't
+happen that fast.
+
+Right now CRLite doesn't enforce anything, and works just for telemetry
+collection (which is hopefully disabled anyway). So disable the
+preference right in the source code, so that the patch fails to apply
+when the upstream decides to set it to enforcing mode by default.
+
+The solution with CRLite is up for discussion. If necessary, it's
+possible to make clients for blessed collections to communicate to real
+Remote Settings server. For example, for collections related to
+certificate revocations.
+---
+ modules/libpref/init/all.js | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/modules/libpref/init/all.js b/modules/libpref/init/all.js
+index 907e8ebcef..65db6bd04c 100644
+--- a/modules/libpref/init/all.js
++++ b/modules/libpref/init/all.js
+@@ -186,7 +186,7 @@ pref("security.pki.distrust_ca_policy", 2);
+ #if defined(NIGHTLY_BUILD)
+ pref("security.pki.crlite_mode", 2);
+ #else
+-pref("security.pki.crlite_mode", 1);
++pref("security.pki.crlite_mode", 0);
+ #endif
+
+ // Represents the expected certificate transparency log merge delay (including
+--
+2.30.0
+