summaryrefslogtreecommitdiff
path: root/libre/iceweasel/9001-FSDG-always-sync-remote-settings-with-local-dump.patch
diff options
context:
space:
mode:
Diffstat (limited to 'libre/iceweasel/9001-FSDG-always-sync-remote-settings-with-local-dump.patch')
-rw-r--r--libre/iceweasel/9001-FSDG-always-sync-remote-settings-with-local-dump.patch998
1 files changed, 998 insertions, 0 deletions
diff --git a/libre/iceweasel/9001-FSDG-always-sync-remote-settings-with-local-dump.patch b/libre/iceweasel/9001-FSDG-always-sync-remote-settings-with-local-dump.patch
new file mode 100644
index 000000000..de515a452
--- /dev/null
+++ b/libre/iceweasel/9001-FSDG-always-sync-remote-settings-with-local-dump.patch
@@ -0,0 +1,998 @@
+From 15cacd9bfdb9c08def24e780d83bb8dd672c711f Mon Sep 17 00:00:00 2001
+From: grizzlyuser <grizzlyuser@protonmail.com>
+Date: Wed, 30 Dec 2020 17:20:39 +0200
+Subject: [PATCH 01/13] Point to local omni.ja files, not remote server
+
+This patch series tries to remove any network communication with Remote
+Settings [1], which can be used by Mozilla to silently push data to client
+browsers. This data can include references to nonfree software, for example,
+to search engines or other websites that contain nonfree JavaScript code.
+Without this patching, it would be hard to make sure the browser does not
+violate paragraph 4 of [2]: "Programs in the system should not suggest
+installing nonfree plugins, documentation, and so on."
+
+Changes in the current patch:
+First of all, replace every occurrence of Remote Settings server domain name
+with URIs that point to built-in local files within omni.ja.
+
+Some links to json files may point to non-existing files, but that's OK
+because it's better than leave them point to Remote Settings server.
+If necessary, missing files can be added later.
+
+[1] https://remote-settings.readthedocs.io/en/latest/introduction.html
+[2] https://www.gnu.org/distros/free-system-distribution-guidelines.en.html#license-rules
+---
+ .../components/ASRouterAdmin/ASRouterAdmin.jsx | 2 +-
+ .../newtab/data/content/activity-stream.bundle.js | 2 +-
+ modules/libpref/init/all.js | 2 +-
+ services/settings/Utils.jsm | 4 ++--
+ .../periodic-updates/scripts/periodic_file_updates.sh | 2 +-
+ toolkit/components/search/SearchUtils.jsm | 8 ++++----
+ toolkit/components/search/docs/DefaultSearchEngines.rst | 2 +-
+ .../components/search/docs/SearchEngineConfiguration.rst | 2 +-
+ toolkit/mozapps/defaultagent/RemoteSettings.cpp | 2 +-
+ 9 files changed, 13 insertions(+), 13 deletions(-)
+
+diff --git a/browser/components/newtab/content-src/components/ASRouterAdmin/ASRouterAdmin.jsx b/browser/components/newtab/content-src/components/ASRouterAdmin/ASRouterAdmin.jsx
+index e0d8e45cc0..4cb5fca1c6 100644
+--- a/browser/components/newtab/content-src/components/ASRouterAdmin/ASRouterAdmin.jsx
++++ b/browser/components/newtab/content-src/components/ASRouterAdmin/ASRouterAdmin.jsx
+@@ -1230,7 +1230,7 @@ export class ASRouterAdminInner extends React.PureComponent {
+ <a
+ className="providerUrl"
+ target="_blank"
+- href="https://firefox.settings.services.mozilla.com/v1/buckets/main/collections/nimbus-desktop-experiments/records"
++ href="resource://app/defaults/settings/main/nimbus-desktop-experiments.json"
+ rel="noopener noreferrer"
+ >
+ nimbus-desktop-experiments
+diff --git a/browser/components/newtab/data/content/activity-stream.bundle.js b/browser/components/newtab/data/content/activity-stream.bundle.js
+index 5227e5af81..427759dcfe 100644
+--- a/browser/components/newtab/data/content/activity-stream.bundle.js
++++ b/browser/components/newtab/data/content/activity-stream.bundle.js
+@@ -1841,7 +1841,7 @@ class ASRouterAdminInner extends react__WEBPACK_IMPORTED_MODULE_3___default.a.Pu
+ label = /*#__PURE__*/react__WEBPACK_IMPORTED_MODULE_3___default.a.createElement("span", null, "remote settings (", /*#__PURE__*/react__WEBPACK_IMPORTED_MODULE_3___default.a.createElement("a", {
+ className: "providerUrl",
+ target: "_blank",
+- href: "https://firefox.settings.services.mozilla.com/v1/buckets/main/collections/nimbus-desktop-experiments/records",
++ href: "resource://app/defaults/settings/main/nimbus-desktop-experiments.json",
+ rel: "noopener noreferrer"
+ }, "nimbus-desktop-experiments"), ")");
+ }
+diff --git a/modules/libpref/init/all.js b/modules/libpref/init/all.js
+index 1ed81b6f53..edca292681 100644
+--- a/modules/libpref/init/all.js
++++ b/modules/libpref/init/all.js
+@@ -2210,7 +2210,7 @@ pref("security.cert_pinning.hpkp.enabled", false);
+ // Remote settings preferences
+ // Note: if you change this, make sure to also review security.onecrl.maximum_staleness_in_seconds
+ pref("services.settings.poll_interval", 86400); // 24H
+-pref("services.settings.server", "https://firefox.settings.services.mozilla.com/v1");
++pref("services.settings.server", "resource://app/defaults/settings");
+ pref("services.settings.default_bucket", "main");
+
+ // The percentage of clients who will report uptake telemetry as
+diff --git a/services/settings/Utils.jsm b/services/settings/Utils.jsm
+index ee23591a6a..ef91781ac6 100644
+--- a/services/settings/Utils.jsm
++++ b/services/settings/Utils.jsm
+@@ -60,11 +60,11 @@ var Utils = {
+ !Cu.isInAutomation &&
+ !isXpcshell &&
+ isNotThunderbird
+- ? "https://firefox.settings.services.mozilla.com/v1"
++ ? "resource://app/defaults/settings"
+ : gServerURL;
+ },
+
+- CHANGES_PATH: "/buckets/monitor/collections/changes/changeset",
++ CHANGES_PATH: "/monitor/changes.json",
+
+ /**
+ * Logger instance.
+diff --git a/taskcluster/docker/periodic-updates/scripts/periodic_file_updates.sh b/taskcluster/docker/periodic-updates/scripts/periodic_file_updates.sh
+index 7764777c1a..3c8db49743 100755
+--- a/taskcluster/docker/periodic-updates/scripts/periodic_file_updates.sh
++++ b/taskcluster/docker/periodic-updates/scripts/periodic_file_updates.sh
+@@ -279,7 +279,7 @@ function compare_suffix_lists {
+ }
+
+ function compare_remote_settings_files {
+- REMOTE_SETTINGS_SERVER="https://firefox.settings.services.mozilla.com/v1"
++ REMOTE_SETTINGS_SERVER="resource://app/defaults/settings"
+
+ # 1. List remote settings collections from server.
+ echo "INFO: fetch remote settings list from server"
+diff --git a/toolkit/components/search/SearchUtils.jsm b/toolkit/components/search/SearchUtils.jsm
+index 8a3c6acb84..b0a9c4b86f 100644
+--- a/toolkit/components/search/SearchUtils.jsm
++++ b/toolkit/components/search/SearchUtils.jsm
+@@ -159,13 +159,13 @@ var SearchUtils = {
+
+ ENGINES_URLS: {
+ "prod-main":
+- "https://firefox.settings.services.mozilla.com/v1/buckets/main/collections/search-config/records",
++ "resource://app/defaults/settings/main/search-config.json",
+ "prod-preview":
+- "https://firefox.settings.services.mozilla.com/v1/buckets/main-preview/collections/search-config/records",
++ "resource://app/defaults/settings/main/search-config.json",
+ "stage-main":
+- "https://settings.stage.mozaws.net/v1/buckets/main/collections/search-config/records",
++ "resource://app/defaults/settings/main/search-config.json",
+ "stage-preview":
+- "https://settings.stage.mozaws.net/v1/buckets/main-preview/collections/search-config/records",
++ "resource://app/defaults/settings/main/search-config.json",
+ },
+
+ // The following constants are left undocumented in nsISearchService.idl
+diff --git a/toolkit/components/search/docs/DefaultSearchEngines.rst b/toolkit/components/search/docs/DefaultSearchEngines.rst
+index 0648471396..37948dca31 100644
+--- a/toolkit/components/search/docs/DefaultSearchEngines.rst
++++ b/toolkit/components/search/docs/DefaultSearchEngines.rst
+@@ -86,4 +86,4 @@ is updated.
+
+ .. _configuration schema: SearchConfigurationSchema.html
+ .. _remote settings: /services/common/services/RemoteSettings.html
+-.. _search-default-override-allowlist bucket: https://firefox.settings.services.mozilla.com/v1/buckets/main/collections/search-default-override-allowlist/records
++.. _search-default-override-allowlist bucket: resource://app/defaults/settings/main/search-default-override-allowlist.json
+diff --git a/toolkit/components/search/docs/SearchEngineConfiguration.rst b/toolkit/components/search/docs/SearchEngineConfiguration.rst
+index e9041affb8..7a9466d294 100644
+--- a/toolkit/components/search/docs/SearchEngineConfiguration.rst
++++ b/toolkit/components/search/docs/SearchEngineConfiguration.rst
+@@ -68,5 +68,5 @@ related. As a result several situations may occur:
+ .. _JSON schema: https://json-schema.org/
+ .. _stored in mozilla-central: https://searchfox.org/mozilla-central/source/toolkit/components/search/schema/
+ .. _Search Configuration Schema: SearchConfigurationSchema.html
+-.. _viewed live: https://firefox.settings.services.mozilla.com/v1/buckets/main/collections/search-config/records
++.. _viewed live: resource://app/defaults/settings/main/search-config.json
+ .. _Normandy: /toolkit/components/normandy/normandy/services.html
+diff --git a/toolkit/mozapps/defaultagent/RemoteSettings.cpp b/toolkit/mozapps/defaultagent/RemoteSettings.cpp
+index 667d9fc628..b2bf628f29 100644
+--- a/toolkit/mozapps/defaultagent/RemoteSettings.cpp
++++ b/toolkit/mozapps/defaultagent/RemoteSettings.cpp
+@@ -23,7 +23,7 @@ extern "C" {
+ HRESULT IsAgentRemoteDisabledRust(const char* szUrl, DWORD* lpdwDisabled);
+ }
+
+-#define PROD_ENDPOINT "https://firefox.settings.services.mozilla.com/v1"
++#define PROD_ENDPOINT "resource://app/defaults/settings"
+ #define PROD_BID "main"
+ #define PROD_CID "windows-default-browser-agent"
+ #define PROD_ID "state"
+--
+2.31.1
+
+
+From f1e92b5fb7844a57ad63d8a52b4867db9817fc14 Mon Sep 17 00:00:00 2001
+From: grizzlyuser <grizzlyuser@protonmail.com>
+Date: Wed, 30 Dec 2020 17:34:08 +0200
+Subject: [PATCH 02/13] Remove polling triggered by push broadcasts
+
+When initialized, remote-settings.js adds a listener to push broadcasts,
+that let Remote Settings server send push messages to trigger polling
+for changes from the client side. This is not needed for local-only
+setup. Remove the record from broadcast-listeners.json file stored in
+the user profile, so that it doesn't get picked up by push broadcast
+service.
+---
+ dom/push/PushBroadcastService.jsm | 13 +++++++++++++
+ services/settings/remote-settings.js | 7 ++-----
+ 2 files changed, 15 insertions(+), 5 deletions(-)
+
+diff --git a/dom/push/PushBroadcastService.jsm b/dom/push/PushBroadcastService.jsm
+index aa1504211d..d635a2c3aa 100644
+--- a/dom/push/PushBroadcastService.jsm
++++ b/dom/push/PushBroadcastService.jsm
+@@ -178,6 +178,19 @@ var BroadcastService = class {
+ }
+ }
+
++ async deleteListener(broadcastId) {
++ await this.initializePromise;
++
++ if (this.jsonFile.data.listeners.hasOwnProperty(broadcastId)) {
++ console.info(
++ "deleteListener: deleting listener",
++ broadcastId
++ );
++ delete this.jsonFile.data.listeners[broadcastId];
++ this.jsonFile.saveSoon();
++ }
++ }
++
+ /**
+ * Call the listeners of the specified broadcasts.
+ *
+diff --git a/services/settings/remote-settings.js b/services/settings/remote-settings.js
+index 6d0185faf9..aae93fa440 100644
+--- a/services/settings/remote-settings.js
++++ b/services/settings/remote-settings.js
+@@ -441,7 +441,7 @@ function remoteSettingsFunction() {
+ moduleURI: __URI__,
+ symbolName: "remoteSettingsBroadcastHandler",
+ };
+- pushBroadcastService.addListener(BROADCAST_ID, currentVersion, moduleInfo);
++ pushBroadcastService.deleteListener(BROADCAST_ID);
+ };
+
+ return remoteSettings;
+@@ -461,9 +461,6 @@ var remoteSettingsBroadcastHandler = {
+ `Push notification received (version=${version} phase=${phase})`
+ );
+
+- return RemoteSettings.pollChanges({
+- expectedTimestamp: version,
+- trigger: isStartup ? "startup" : "broadcast",
+- });
++ return;
+ },
+ };
+--
+2.31.1
+
+
+From 3054d3efe22802ab5503dd812e0a0283bbd791f1 Mon Sep 17 00:00:00 2001
+From: grizzlyuser <grizzlyuser@protonmail.com>
+Date: Wed, 30 Dec 2020 17:41:54 +0200
+Subject: [PATCH 03/13] Remove timer that triggers polling for changes
+
+That is not needed for local-only setup.
+---
+ services/settings/components.conf | 9 +--------
+ services/settings/servicesSettings.manifest | 4 ----
+ 2 files changed, 1 insertion(+), 12 deletions(-)
+
+diff --git a/services/settings/components.conf b/services/settings/components.conf
+index 9a737802ee..25109415a7 100644
+--- a/services/settings/components.conf
++++ b/services/settings/components.conf
+@@ -4,11 +4,4 @@
+ # License, v. 2.0. If a copy of the MPL was not distributed with this
+ # file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+-Classes = [
+- {
+- 'cid': '{5e756573-234a-49ea-bbe4-59ec7a70657d}',
+- 'contract_ids': ['@mozilla.org/services/settings;1'],
+- 'jsm': 'resource://services-settings/RemoteSettingsComponents.jsm',
+- 'constructor': 'RemoteSettingsTimer',
+- },
+-]
++Classes = []
+diff --git a/services/settings/servicesSettings.manifest b/services/settings/servicesSettings.manifest
+index 3bfed26ea4..807eb220ec 100644
+--- a/services/settings/servicesSettings.manifest
++++ b/services/settings/servicesSettings.manifest
+@@ -1,7 +1,3 @@
+ # Register resource aliases
+ resource services-settings resource://gre/modules/services-settings/
+
+-# Schedule polling of remote settings changes
+-# (default 24H, max 72H)
+-# see syntax https://searchfox.org/mozilla-central/rev/cc280c4be94ff8cf64a27cc9b3d6831ffa49fa45/toolkit/components/timermanager/UpdateTimerManager.jsm#155
+-category update-timer RemoteSettingsComponents @mozilla.org/services/settings;1,getService,services-settings-poll-changes,services.settings.poll_interval,86400,259200
+--
+2.31.1
+
+
+From c856641861ca70da2d5aa720e402f2e505ebe5ac Mon Sep 17 00:00:00 2001
+From: grizzlyuser <grizzlyuser@protonmail.com>
+Date: Wed, 30 Dec 2020 17:47:41 +0200
+Subject: [PATCH 04/13] Utils: fetch timestamps of each collection locally
+
+Utils.CHANGES_PATH points to
+services/settings/dumps/monitor/changes.json
+which will be generated later by JSON processing script. Fetch the
+timestamps from that file and mock response headers to not confuse any
+code that expects them.
+---
+ browser/installer/package-manifest.in | 1 +
+ services/settings/Utils.jsm | 13 ++++++++++++-
+ services/settings/dumps/monitor/moz.build | 8 ++++++++
+ services/settings/dumps/moz.build | 1 +
+ 4 files changed, 22 insertions(+), 1 deletion(-)
+ create mode 100644 services/settings/dumps/monitor/moz.build
+
+diff --git a/browser/installer/package-manifest.in b/browser/installer/package-manifest.in
+index ec20499166..c618c02a8f 100644
+--- a/browser/installer/package-manifest.in
++++ b/browser/installer/package-manifest.in
+@@ -298,6 +298,7 @@
+ @RESPATH@/browser/defaults/settings/blocklists
+ @RESPATH@/browser/defaults/settings/pinning
+ @RESPATH@/browser/defaults/settings/main
++@RESPATH@/browser/defaults/settings/monitor
+ @RESPATH@/browser/defaults/settings/security-state
+
+ ; Warning: changing the path to channel-prefs.js can cause bugs (Bug 756325)
+diff --git a/services/settings/Utils.jsm b/services/settings/Utils.jsm
+index ef91781ac6..8736951968 100644
+--- a/services/settings/Utils.jsm
++++ b/services/settings/Utils.jsm
+@@ -150,7 +150,7 @@ var Utils = {
+ async fetchLatestChanges(serverUrl, options = {}) {
+ const { expectedTimestamp, lastEtag = "", filters = {} } = options;
+
+- let url = serverUrl + Utils.CHANGES_PATH;
++ let url = Utils.SERVER_URL + Utils.CHANGES_PATH;
+ const params = {
+ ...filters,
+ _expected: expectedTimestamp ?? 0,
+@@ -166,6 +166,9 @@ var Utils = {
+ .join("&");
+ }
+ const response = await fetch(url);
++ const responseDate = new Date().toUTCString()
++ response.headers.set("Date", responseDate);
++ response.headers.set("Last-Modified", responseDate);
+
+ if (response.status >= 500) {
+ throw new Error(`Server error ${response.status} ${response.statusText}`);
+@@ -200,7 +194,15 @@ var Utils = {
+ }
+ }
+
+- const { changes = [], timestamp } = payload;
++ const { timestamp } = payload;
++ const { bucket, collection } = filters;
++ if (!bucket || !collection) {
++ throw new Error('Unable to fetch latest change without bucket or collection');
++ }
++ const change = payload.changes.find(
++ change => change.bucket === bucket && change.collection === collection
++ ) ?? { last_modified: 0, bucket, collection };
++ const changes = [change];
+
+ let serverTimeMillis = Date.parse(response.headers.get("Date"));
+ // Since the response is served via a CDN, the Date header value could have been cached.
+diff --git a/services/settings/dumps/monitor/moz.build b/services/settings/dumps/monitor/moz.build
+new file mode 100644
+index 0000000000..d3d017fda5
+--- /dev/null
++++ b/services/settings/dumps/monitor/moz.build
+@@ -0,0 +1,8 @@
++# This Source Code Form is subject to the terms of the Mozilla Public
++# License, v. 2.0. If a copy of the MPL was not distributed with this
++# file, You can obtain one at http://mozilla.org/MPL/2.0/.
++
++FINAL_TARGET_FILES.defaults.settings.monitor += ["changes.json"]
++
++if CONFIG["MOZ_BUILD_APP"] == "browser":
++ DIST_SUBDIR = "browser"
+diff --git a/services/settings/dumps/moz.build b/services/settings/dumps/moz.build
+index 3cc9436f61..3742da5667 100644
+--- a/services/settings/dumps/moz.build
++++ b/services/settings/dumps/moz.build
+@@ -5,6 +5,7 @@
+ DIRS += [
+ "blocklists",
+ "main",
++ "monitor",
+ "pinning",
+ "security-state",
+ ]
+--
+2.31.1
+
+
+From a0f311bed359d484fbf85e696b5b7e3a288292f8 Mon Sep 17 00:00:00 2001
+From: grizzlyuser <grizzlyuser@protonmail.com>
+Date: Wed, 30 Dec 2020 17:52:10 +0200
+Subject: [PATCH 05/13] Utils: disable offline checking
+
+Since only local data is read now, it should always return false for the
+current and any future code that relies on it.
+---
+ services/settings/Utils.jsm | 9 ---------
+ 1 file changed, 9 deletions(-)
+
+diff --git a/services/settings/Utils.jsm b/services/settings/Utils.jsm
+index 8736951968..8ac085feea 100644
+--- a/services/settings/Utils.jsm
++++ b/services/settings/Utils.jsm
+@@ -80,15 +80,6 @@ var Utils = {
+ * @return {bool} Whether network is down or not.
+ */
+ get isOffline() {
+- try {
+- return (
+- Services.io.offline ||
+- CaptivePortalService.state == CaptivePortalService.LOCKED_PORTAL ||
+- !gNetworkLinkService.isLinkUp
+- );
+- } catch (ex) {
+- log.warn("Could not determine network status.", ex);
+- }
+ return false;
+ },
+
+--
+2.31.1
+
+
+From 95e4979573c79c987160a71e913c7564de22127f Mon Sep 17 00:00:00 2001
+From: grizzlyuser <grizzlyuser@protonmail.com>
+Date: Wed, 30 Dec 2020 17:56:02 +0200
+Subject: [PATCH 06/13] Refactor hashing logic to a separate function
+
+It is used instead of internal signature validation mechanism, for
+integrity checking of the locally cached data.
+---
+ services/settings/RemoteSettingsWorker.jsm | 4 ++++
+ services/settings/SharedUtils.jsm | 9 +++++++--
+ 2 files changed, 11 insertions(+), 2 deletions(-)
+
+diff --git a/services/settings/RemoteSettingsWorker.jsm b/services/settings/RemoteSettingsWorker.jsm
+index 147ebb6b13..c86e218fd3 100644
+--- a/services/settings/RemoteSettingsWorker.jsm
++++ b/services/settings/RemoteSettingsWorker.jsm
+@@ -189,6 +189,10 @@ class Worker {
+ // task on the current thread instead of the worker thread.
+ return SharedUtils.checkContentHash(buffer, size, hash);
+ }
++
++ async getContentHash(bytes) {
++ return SharedUtils.getContentHash(bytes);
++ }
+ }
+
+ // Now, first add a shutdown blocker. If that fails, we must have
+diff --git a/services/settings/SharedUtils.jsm b/services/settings/SharedUtils.jsm
+index db5017a742..1a8e83c2e8 100644
+--- a/services/settings/SharedUtils.jsm
++++ b/services/settings/SharedUtils.jsm
+@@ -28,11 +28,16 @@ var SharedUtils = {
+ return false;
+ }
+ // Has expected content?
++ const hashStr = await this.getContentHash(bytes);
++ return hashStr == hash;
++ },
++
++ async getContentHash(bytes) {
+ const hashBuffer = await crypto.subtle.digest("SHA-256", bytes);
+ const hashBytes = new Uint8Array(hashBuffer);
+ const toHex = b => b.toString(16).padStart(2, "0");
+- const hashStr = Array.from(hashBytes, toHex).join("");
+- return hashStr == hash;
++
++ return Array.from(hashBytes, toHex).join("");
+ },
+
+ /**
+--
+2.31.1
+
+
+From daac032f8a4e12451cda9ec6b1eca29f5a35521f Mon Sep 17 00:00:00 2001
+From: grizzlyuser <grizzlyuser@protonmail.com>
+Date: Wed, 30 Dec 2020 18:05:02 +0200
+Subject: [PATCH 07/13] Client: Fetch and hash records from local dump
+
+Read the records from local dumps. See [1] for details on how to prepare
+custom dumps). Records are cached in the local IndexedDB, and the client
+updates cached records each time there's a change. Also it verifies
+integrity of the data. Then the list of current / created / updated /
+deleted records is generated and emitted to every registered listener.
+
+Change upstream signature validation mechanism to a simpler one.
+Otherwise, it'd be necessary to sign local records, which is redundant,
+because the application package should be signed already by the distro.
+
+Instead of signature property from metadata records, json_dump_metadata
+has been introduced. It contains the checksum of the records and size in
+bytes. Also added app_build_id property for version checking and updates
+of cached data.
+
+Although it's possible to disable integrity checking via preference, it
+seems to be not a good idea, because the logic that detects invalid
+local data relies on it. In the context of local-only setup, data that
+has been received from real Remote Settings server will not contain the
+custom metadata, and thus will be considered invalid and then discarded,
+while the client gets a chance to gracefully inform registered listeners
+about these changes so that they can discard the data received before
+the upgrade to local-only setup.
+
+[1] https://firefox-source-docs.mozilla.org/services/common/services/RemoteSettings.html#initial-data
+---
+ services/settings/RemoteSettingsClient.jsm | 62 ++++++++++------------
+ 1 file changed, 27 insertions(+), 35 deletions(-)
+
+diff --git a/services/settings/RemoteSettingsClient.jsm b/services/settings/RemoteSettingsClient.jsm
+index 80dd563e11..1025ab33a2 100644
+--- a/services/settings/RemoteSettingsClient.jsm
++++ b/services/settings/RemoteSettingsClient.jsm
+@@ -556,11 +556,9 @@ class RemoteSettingsClient extends EventEmitter {
+
+ // If the data is up-to-date but don't have metadata (records loaded from dump),
+ // we fetch them and validate the signature immediately.
+- if (this.verifySignature && ObjectUtils.isEmpty(localMetadata)) {
++ if (this.verifySignature && ObjectUtils.isEmpty(localMetadata?.json_dump_metadata)) {
+ console.debug(`${this.identifier} pull collection metadata`);
+- const metadata = await this.httpClient().getData({
+- query: { _expected: expectedTimestamp },
+- });
++ const { metadata } = await this._fetchChangeset(expectedTimestamp);
+ await this.db.importChanges(metadata);
+ // We don't bother validating the signature if the dump was just loaded. We do
+ // if the dump was loaded at some other point (eg. from .get()).
+@@ -813,32 +811,23 @@ class RemoteSettingsClient extends EventEmitter {
+ async _validateCollectionSignature(records, timestamp, metadata) {
+ const start = Cu.now() * 1000;
+
+- if (!metadata?.signature) {
++ if (!metadata?.json_dump_metadata) {
+ throw new MissingSignatureError(this.identifier);
+ }
+
+- if (!this._verifier) {
+- this._verifier = Cc[
+- "@mozilla.org/security/contentsignatureverifier;1"
+- ].createInstance(Ci.nsIContentSignatureVerifier);
+- }
+-
+- // This is a content-signature field from an autograph response.
+ const {
+- signature: { x5u, signature },
++ json_dump_metadata: { hash, size },
+ } = metadata;
+- const certChain = await (await fetch(x5u)).text();
+ // Merge remote records with local ones and serialize as canonical JSON.
+ const serialized = await RemoteSettingsWorker.canonicalStringify(
+ records,
+ timestamp
+ );
+ if (
+- !(await this._verifier.asyncVerifyContentSignature(
+- serialized,
+- "p384ecdsa=" + signature,
+- certChain,
+- this.signerName
++ !(await RemoteSettingsWorker.checkContentHash(
++ new TextEncoder().encode(serialized),
++ size,
++ hash
+ ))
+ ) {
+ throw new InvalidSignatureError(this.identifier);
+@@ -1030,24 +1019,27 @@ class RemoteSettingsClient extends EventEmitter {
+ * @param since timestamp of last sync (optional)
+ */
+ async _fetchChangeset(expectedTimestamp, since) {
+- const client = this.httpClient();
+- const {
+- metadata,
+- timestamp: remoteTimestamp,
+- changes: remoteRecords,
+- } = await client.execute(
+- {
+- path: `/buckets/${this.bucketName}/collections/${this.collectionName}/changeset`,
+- },
+- {
+- query: {
+- _expected: expectedTimestamp,
+- _since: since,
+- },
+- }
++ const { data } = await SharedUtils.loadJSONDump(
++ this.bucketName,
++ this.collectionName
+ );
++ const remoteRecords = data ?? [];
++
++ const serialized = await RemoteSettingsWorker.canonicalStringify(
++ remoteRecords,
++ expectedTimestamp
++ );
++ const bytes = new TextEncoder().encode(serialized);
++ const metadata = {
++ app_build_id: Services.appinfo.appBuildID,
++ json_dump_metadata: {
++ hash: await RemoteSettingsWorker.getContentHash(bytes),
++ size: bytes.length,
++ },
++ }
++
+ return {
+- remoteTimestamp,
++ remoteTimestamp: expectedTimestamp,
+ metadata,
+ remoteRecords,
+ };
+--
+2.31.1
+
+
+From 4d36e599e6b24e10960ebb978c038c66c5ade06d Mon Sep 17 00:00:00 2001
+From: grizzlyuser <grizzlyuser@protonmail.com>
+Date: Wed, 30 Dec 2020 18:42:56 +0200
+Subject: [PATCH 08/13] Client: start deferred sync on get() or on()
+
+The users of the RemoteSettingsClient.jsm can receive records from it in
+two ways: by calling get(), and by subscribing to events by calling
+on().
+
+So hook a deferred sync whenever something calls these methods. Because
+multiple of those calls can be made quite early and in very short time,
+set up a deferred task that will be armed only when needed and only once
+in a second. When the task is running it first checks if the local data
+came from the dump of the current app build, and no-ops if true. If
+false, it triggers a sync. Then adds a flag if the client has been
+correctly synchronized with the dump, so that no metadata checking
+occurs during the session.
+---
+ services/settings/RemoteSettingsClient.jsm | 30 +++++++++++++++++++++-
+ 1 file changed, 29 insertions(+), 1 deletion(-)
+
+diff --git a/services/settings/RemoteSettingsClient.jsm b/services/settings/RemoteSettingsClient.jsm
+index 1025ab33a2..1cebf2bc29 100644
+--- a/services/settings/RemoteSettingsClient.jsm
++++ b/services/settings/RemoteSettingsClient.jsm
+@@ -16,6 +16,7 @@ XPCOMUtils.defineLazyModuleGetters(this, {
+ ClientEnvironmentBase:
+ "resource://gre/modules/components-utils/ClientEnvironment.jsm",
+ Database: "resource://services-settings/Database.jsm",
++ DeferredTask: "resource://gre/modules/DeferredTask.jsm",
+ Downloader: "resource://services-settings/Attachments.jsm",
+ IDBHelpers: "resource://services-settings/IDBHelpers.jsm",
+ KintoHttpClient: "resource://services-common/kinto-http-client.js",
+@@ -30,6 +31,7 @@ XPCOMUtils.defineLazyModuleGetters(this, {
+ XPCOMUtils.defineLazyGlobalGetters(this, ["fetch"]);
+
+ const TELEMETRY_COMPONENT = "remotesettings";
++const DEFERRED_SYNC_DELAY_MILLISECONDS = 1000;
+
+ XPCOMUtils.defineLazyGetter(this, "console", () => Utils.log);
+
+@@ -259,6 +261,14 @@ class RemoteSettingsClient extends EventEmitter {
+ this._lastCheckTimePref = lastCheckTimePref;
+ this._verifier = null;
+ this._syncRunning = false;
++ this._deferredSync = new DeferredTask(
++ async () => {
++ if (!this._syncRunning && !(await this._isSynced())) {
++ await this.sync();
++ }
++ },
++ DEFERRED_SYNC_DELAY_MILLISECONDS
++ );
+
+ // This attribute allows signature verification to be disabled, when running tests
+ // or when pulling data from a dev server.
+@@ -290,6 +300,11 @@ class RemoteSettingsClient extends EventEmitter {
+ );
+ }
+
++ on(event, callback) {
++ super.on(event, callback);
++ this._deferredSync.arm();
++ }
++
+ get identifier() {
+ return `${this.bucketName}/${this.collectionName}`;
+ }
+@@ -353,6 +368,10 @@ class RemoteSettingsClient extends EventEmitter {
+ let lastModified = await this.db.getLastModified();
+ let hasLocalData = lastModified !== null;
+
++ if (!(await this._isSynced())) {
++ throw new MissingSignatureError(this.identifier);
++ }
++
+ if (syncIfEmpty && !hasLocalData) {
+ // .get() was called before we had the chance to synchronize the local database.
+ // We'll try to avoid returning an empty list.
+@@ -414,7 +433,10 @@ class RemoteSettingsClient extends EventEmitter {
+ // No need to verify signature on JSON dumps.
+ // If local DB cannot be read, then we don't even try to do anything,
+ // we return results early.
+- return this._filterEntries(data);
++ const filtered = this._filterEntries(data);
++ this._deferredSync.arm();
++
++ return filtered;
+ }
+
+ console.debug(
+@@ -452,6 +474,12 @@ class RemoteSettingsClient extends EventEmitter {
+ return final;
+ }
+
++ async _isSynced() {
++ this._synced ||=
++ Services.appinfo.appBuildID === (await this.db.getMetadata())?.app_build_id;
++ return this._synced;
++ }
++
+ /**
+ * Synchronize the local database with the remote server.
+ *
+--
+2.31.1
+
+
+From defc4080596f5407a98f0c9f1a456f685226054f Mon Sep 17 00:00:00 2001
+From: grizzlyuser <grizzlyuser@protonmail.com>
+Date: Wed, 30 Dec 2020 18:53:51 +0200
+Subject: [PATCH 09/13] Client: deep compare records if timestamps match
+
+When the list of current / updated / deleted records is generated, their
+modification timestamps are compared to detect the updates.
+
+Although in practice this is unlikely to happen, in theory the
+timestamp of some older record received from Remote Settings can match
+with the modified record in the dump. Although JSON processing script
+makes sure to add unique timestamps to each of the modified records,
+it's still possible to update dumps manually and simply forget to update
+timestamps. So serialize the records and compare them as strings to be
+on the safe side. This should happen only once after upgrading to each
+new version of the application, so is not likely to introduce any
+noticeable performance issues.
+---
+ services/settings/RemoteSettingsClient.jsm | 6 +++++-
+ 1 file changed, 5 insertions(+), 1 deletion(-)
+
+diff --git a/services/settings/RemoteSettingsClient.jsm b/services/settings/RemoteSettingsClient.jsm
+index 1cebf2bc29..2c18c5cfb6 100644
+--- a/services/settings/RemoteSettingsClient.jsm
++++ b/services/settings/RemoteSettingsClient.jsm
+@@ -13,6 +13,7 @@ const { Services } = ChromeUtils.import("resource://gre/modules/Services.jsm");
+
+ XPCOMUtils.defineLazyModuleGetters(this, {
+ AppConstants: "resource://gre/modules/AppConstants.jsm",
++ CanonicalJSON: "resource://gre/modules/CanonicalJSON.jsm",
+ ClientEnvironmentBase:
+ "resource://gre/modules/components-utils/ClientEnvironment.jsm",
+ Database: "resource://services-settings/Database.jsm",
+@@ -1022,7 +1023,10 @@ class RemoteSettingsClient extends EventEmitter {
+ const old = oldById.get(r.id);
+ if (old) {
+ oldById.delete(r.id);
+- if (r.last_modified != old.last_modified) {
++ if (
++ r.last_modified != old.last_modified ||
++ CanonicalJSON.stringify(r) != CanonicalJSON.stringify(old)
++ ) {
+ syncResult.updated.push({ old, new: r });
+ }
+ } else {
+--
+2.31.1
+
+
+From c2ee19f01bc37e15c7742af8a502ffaa10745a52 Mon Sep 17 00:00:00 2001
+From: grizzlyuser <grizzlyuser@protonmail.com>
+Date: Wed, 30 Dec 2020 19:01:39 +0200
+Subject: [PATCH 10/13] Client: delete more data on cleanup
+
+When the client detects the local data is invalid (i.e. it came from
+real Remote Settings and can have unwanted records), delete not only
+the records, but also the attachments that came with them, because they
+too can be problematic. And last check time preference, because it's not
+useful anyway when remote-settings.js doesn't do any polling for changes.
+
+Note that attachments should be deleted before the records, because the
+logic gets the data about the attachments from those records.
+---
+ services/settings/RemoteSettingsClient.jsm | 15 ++++++++++++---
+ 1 file changed, 12 insertions(+), 3 deletions(-)
+
+diff --git a/services/settings/RemoteSettingsClient.jsm b/services/settings/RemoteSettingsClient.jsm
+index 2c18c5cfb6..8b65dc0cba 100644
+--- a/services/settings/RemoteSettingsClient.jsm
++++ b/services/settings/RemoteSettingsClient.jsm
+@@ -221,7 +221,10 @@ class AttachmentDownloader extends Downloader {
+ async deleteAll() {
+ let allRecords = await this._client.db.list();
+ return Promise.all(
+- allRecords.filter(r => !!r.attachment).map(r => this.delete(r))
++ allRecords.filter(r => !!r.attachment).map(r => {
++ this.delete(r);
++ this.deleteCached(r.id);
++ })
+ );
+ }
+ }
+@@ -982,7 +985,7 @@ class RemoteSettingsClient extends EventEmitter {
+ // Signature failed, clear local DB because it contains
+ // bad data (local + remote changes).
+ console.debug(`${this.identifier} clear local data`);
+- await this.db.clear();
++ await this._clearAll();
+ // Local data was tampered, throw and it will retry from empty DB.
+ console.error(`${this.identifier} local data was corrupted`);
+ throw new CorruptedDataError(this.identifier);
+@@ -1004,7 +1007,7 @@ class RemoteSettingsClient extends EventEmitter {
+ // _importJSONDump() only clears DB if dump is available,
+ // therefore do it here!
+ if (imported < 0) {
+- await this.db.clear();
++ await this._clearAll();
+ }
+ }
+ }
+@@ -1044,6 +1047,12 @@ class RemoteSettingsClient extends EventEmitter {
+ return syncResult;
+ }
+
++ async _clearAll() {
++ await this.attachments.deleteAll();
++ await this.db.clear();
++ Services.prefs.clearUserPref(this.lastCheckTimePref);
++ }
++
+ /**
+ * Fetch information from changeset endpoint.
+ *
+--
+2.31.1
+
+
+From 56d2af487f7077753ea4df6bd0b1e6c91ed7ab9f Mon Sep 17 00:00:00 2001
+From: grizzlyuser <grizzlyuser@protonmail.com>
+Date: Wed, 30 Dec 2020 19:07:56 +0200
+Subject: [PATCH 11/13] Client: remove comparison of collection timestamps
+
+In case if the cached data that came from real Remote Settings server
+(before the upgrade to local-only setup) has collection timestamp, that
+is newer than the packaged local dump, then this comparison logic can
+lead to early return of old data, skipping the integrity checking and
+necessary cleanup. So remove the checks.
+---
+ services/settings/RemoteSettingsClient.jsm | 5 -----
+ 1 file changed, 5 deletions(-)
+
+diff --git a/services/settings/RemoteSettingsClient.jsm b/services/settings/RemoteSettingsClient.jsm
+index 8b65dc0cba..6274596591 100644
+--- a/services/settings/RemoteSettingsClient.jsm
++++ b/services/settings/RemoteSettingsClient.jsm
+@@ -917,14 +917,9 @@ class RemoteSettingsClient extends EventEmitter {
+ updated: [],
+ deleted: [],
+ };
+- // If data wasn't changed, return empty sync result.
+- // This can happen when we update the signature but not the data.
+ console.debug(
+ `${this.identifier} local timestamp: ${localTimestamp}, remote: ${remoteTimestamp}`
+ );
+- if (localTimestamp && remoteTimestamp < localTimestamp) {
+- return syncResult;
+- }
+
+ const start = Cu.now() * 1000;
+ await this.db.importChanges(metadata, remoteTimestamp, remoteRecords, {
+--
+2.31.1
+
+
+From c009c1a9ba2477c9335921b706256f115ecfd498 Mon Sep 17 00:00:00 2001
+From: grizzlyuser <grizzlyuser@protonmail.com>
+Date: Wed, 30 Dec 2020 19:15:44 +0200
+Subject: [PATCH 12/13] Attachments: load only from dump and drop cached
+
+---
+ services/settings/Attachments.jsm | 35 +++++++------------------------
+ 1 file changed, 8 insertions(+), 27 deletions(-)
+
+diff --git a/services/settings/Attachments.jsm b/services/settings/Attachments.jsm
+index 0eeb632799..eaa7db8c81 100644
+--- a/services/settings/Attachments.jsm
++++ b/services/settings/Attachments.jsm
+@@ -143,10 +143,11 @@ class Downloader {
+ checkHash,
+ attachmentId = record?.id,
+ useCache = false,
+- fallbackToCache = false,
+ fallbackToDump = false,
+ } = options || {};
+
++ const fallbackToCache = false;
++
+ if (!useCache) {
+ // For backwards compatibility.
+ // WARNING: Its return type is different from what's documented.
+@@ -206,6 +207,7 @@ class Downloader {
+ const newBuffer = await this.downloadAsBytes(record, {
+ retries,
+ checkHash,
++ dumpInfo,
+ });
+ const blob = new Blob([newBuffer]);
+ if (useCache) {
+@@ -241,7 +243,7 @@ class Downloader {
+ }
+
+ try {
+- return { ...(await cacheInfo.getResult()), _source: "cache_fallback" };
++ await this.cacheImpl.delete(attachmentId);
+ } catch (e) {
+ // Failed to read from cache, e.g. IndexedDB unusable.
+ Cu.reportError(e);
+@@ -278,7 +280,7 @@ class Downloader {
+ * @returns {String} the absolute file path to the downloaded attachment.
+ */
+ async downloadToDisk(record, options = {}) {
+- const { retries = 3 } = options;
++ const retries = 0;
+ const {
+ attachment: { filename, size, hash },
+ } = record;
+@@ -335,31 +337,10 @@ class Downloader {
+ */
+ async downloadAsBytes(record, options = {}) {
+ const {
+- attachment: { location, hash, size },
+- } = record;
+-
+- const remoteFileUrl = (await this._baseAttachmentsURL()) + location;
++ dumpInfo = new LazyRecordAndBuffer(() => this._readAttachmentDump(attachmentId))
++ } = options;
+
+- const { retries = 3, checkHash = true } = options;
+- let retried = 0;
+- while (true) {
+- try {
+- const buffer = await this._fetchAttachment(remoteFileUrl);
+- if (!checkHash) {
+- return buffer;
+- }
+- if (await RemoteSettingsWorker.checkContentHash(buffer, size, hash)) {
+- return buffer;
+- }
+- // Content is corrupted.
+- throw new Downloader.BadContentError(location);
+- } catch (e) {
+- if (retried >= retries) {
+- throw e;
+- }
+- }
+- retried++;
+- }
++ return (await dumpInfo.getResult()).buffer;
+ }
+
+ /**
+--
+2.31.1
+
+
+From 2035bd7a6ce1816417b619d3f1ce994a8b44ce9d Mon Sep 17 00:00:00 2001
+From: grizzlyuser <grizzlyuser@protonmail.com>
+Date: Wed, 30 Dec 2020 19:22:20 +0200
+Subject: [PATCH 13/13] Disable CRLite entirely for now
+
+It's designed to fetch the data from Remote Settings. One of the main
+selling points is that new revocations can be pushed to the clients
+within minutes. That won't work with local-only setup. Although (some?)
+of the JSON dumps for it are in place, obviously the updates won't
+happen that fast.
+
+Right now CRLite doesn't enforce anything, and works just for telemetry
+collection (which is hopefully disabled anyway). So disable the
+preference right in the source code, so that the patch fails to apply
+when the upstream decides to set it to enforcing mode by default.
+
+The solution with CRLite is up for discussion. If necessary, it's
+possible to make clients for blessed collections to communicate to real
+Remote Settings server. For example, for collections related to
+certificate revocations.
+---
+ modules/libpref/init/all.js | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/modules/libpref/init/all.js b/modules/libpref/init/all.js
+index edca292681..da7e23d674 100644
+--- a/modules/libpref/init/all.js
++++ b/modules/libpref/init/all.js
+@@ -172,7 +172,7 @@ pref("security.cert_pinning.max_max_age_seconds", 5184000);
+ // 0: Disable CRLite entirely
+ // 1: Enable and check revocations via CRLite, but only collect telemetry
+ // 2: Enable and enforce revocations via CRLite
+-pref("security.pki.crlite_mode", 1);
++pref("security.pki.crlite_mode", 0);
+
+ // Represents the expected certificate transparency log merge delay (including
+ // the time to generate a CRLite filter). Currently 28 hours in seconds.
+--
+2.31.1
+