diff options
Diffstat (limited to 'libre/iceweasel/9001-always-sync-remote-settings-with-local-dump.patch')
| -rw-r--r-- | libre/iceweasel/9001-always-sync-remote-settings-with-local-dump.patch | 998 |
1 files changed, 0 insertions, 998 deletions
diff --git a/libre/iceweasel/9001-always-sync-remote-settings-with-local-dump.patch b/libre/iceweasel/9001-always-sync-remote-settings-with-local-dump.patch deleted file mode 100644 index 30b64089e..000000000 --- a/libre/iceweasel/9001-always-sync-remote-settings-with-local-dump.patch +++ /dev/null @@ -1,998 +0,0 @@ -From 15cacd9bfdb9c08def24e780d83bb8dd672c711f Mon Sep 17 00:00:00 2001 -From: grizzlyuser <grizzlyuser@protonmail.com> -Date: Wed, 30 Dec 2020 17:20:39 +0200 -Subject: [PATCH 01/13] Point to local omni.ja files, not remote server - -This patch series tries to remove any network communication with Remote -Settings [1], which can be used by Mozilla to silently push data to client -browsers. This data can include references to nonfree software, for example, -to search engines or other websites that contain nonfree JavaScript code. -Without this patching, it would be hard to make sure the browser does not -violate paragraph 4 of [2]: "Programs in the system should not suggest -installing nonfree plugins, documentation, and so on." - -Changes in the current patch: -First of all, replace every occurrence of Remote Settings server domain name -with URIs that point to built-in local files within omni.ja. - -Some links to json files may point to non-existing files, but that's OK -because it's better than leave them point to Remote Settings server. -If necessary, missing files can be added later. - -[1] https://remote-settings.readthedocs.io/en/latest/introduction.html -[2] https://www.gnu.org/distros/free-system-distribution-guidelines.en.html#license-rules ---- - .../components/ASRouterAdmin/ASRouterAdmin.jsx | 2 +- - .../newtab/data/content/activity-stream.bundle.js | 2 +- - modules/libpref/init/all.js | 2 +- - services/settings/Utils.jsm | 4 ++-- - .../periodic-updates/scripts/periodic_file_updates.sh | 2 +- - toolkit/components/search/SearchUtils.jsm | 8 ++++---- - toolkit/components/search/docs/DefaultSearchEngines.rst | 2 +- - .../components/search/docs/SearchEngineConfiguration.rst | 2 +- - toolkit/mozapps/defaultagent/RemoteSettings.cpp | 2 +- - 9 files changed, 13 insertions(+), 13 deletions(-) - -diff --git a/browser/components/newtab/content-src/components/ASRouterAdmin/ASRouterAdmin.jsx b/browser/components/newtab/content-src/components/ASRouterAdmin/ASRouterAdmin.jsx -index e0d8e45cc0..4cb5fca1c6 100644 ---- a/browser/components/newtab/content-src/components/ASRouterAdmin/ASRouterAdmin.jsx -+++ b/browser/components/newtab/content-src/components/ASRouterAdmin/ASRouterAdmin.jsx -@@ -1230,7 +1230,7 @@ export class ASRouterAdminInner extends React.PureComponent { - <a - className="providerUrl" - target="_blank" -- href="https://firefox.settings.services.mozilla.com/v1/buckets/main/collections/nimbus-desktop-experiments/records" -+ href="resource://app/defaults/settings/main/nimbus-desktop-experiments.json" - rel="noopener noreferrer" - > - nimbus-desktop-experiments -diff --git a/browser/components/newtab/data/content/activity-stream.bundle.js b/browser/components/newtab/data/content/activity-stream.bundle.js -index 5227e5af81..427759dcfe 100644 ---- a/browser/components/newtab/data/content/activity-stream.bundle.js -+++ b/browser/components/newtab/data/content/activity-stream.bundle.js -@@ -1841,7 +1841,7 @@ class ASRouterAdminInner extends react__WEBPACK_IMPORTED_MODULE_3___default.a.Pu - label = /*#__PURE__*/react__WEBPACK_IMPORTED_MODULE_3___default.a.createElement("span", null, "remote settings (", /*#__PURE__*/react__WEBPACK_IMPORTED_MODULE_3___default.a.createElement("a", { - className: "providerUrl", - target: "_blank", -- href: "https://firefox.settings.services.mozilla.com/v1/buckets/main/collections/nimbus-desktop-experiments/records", -+ href: "resource://app/defaults/settings/main/nimbus-desktop-experiments.json", - rel: "noopener noreferrer" - }, "nimbus-desktop-experiments"), ")"); - } -diff --git a/modules/libpref/init/all.js b/modules/libpref/init/all.js -index 1ed81b6f53..edca292681 100644 ---- a/modules/libpref/init/all.js -+++ b/modules/libpref/init/all.js -@@ -2210,7 +2210,7 @@ pref("security.cert_pinning.hpkp.enabled", false); - // Remote settings preferences - // Note: if you change this, make sure to also review security.onecrl.maximum_staleness_in_seconds - pref("services.settings.poll_interval", 86400); // 24H --pref("services.settings.server", "https://firefox.settings.services.mozilla.com/v1"); -+pref("services.settings.server", "resource://app/defaults/settings"); - pref("services.settings.default_bucket", "main"); - - // The percentage of clients who will report uptake telemetry as -diff --git a/services/settings/Utils.jsm b/services/settings/Utils.jsm -index ee23591a6a..ef91781ac6 100644 ---- a/services/settings/Utils.jsm -+++ b/services/settings/Utils.jsm -@@ -60,11 +60,11 @@ var Utils = { - ); - const isXpcshell = env.exists("XPCSHELL_TEST_PROFILE_DIR"); - return AppConstants.RELEASE_OR_BETA && !Cu.isInAutomation && !isXpcshell -- ? "https://firefox.settings.services.mozilla.com/v1" -+ ? "resource://app/defaults/settings" - : gServerURL; - }, - -- CHANGES_PATH: "/buckets/monitor/collections/changes/changeset", -+ CHANGES_PATH: "/monitor/changes.json", - - /** - * Logger instance. -diff --git a/taskcluster/docker/periodic-updates/scripts/periodic_file_updates.sh b/taskcluster/docker/periodic-updates/scripts/periodic_file_updates.sh -index 7764777c1a..3c8db49743 100755 ---- a/taskcluster/docker/periodic-updates/scripts/periodic_file_updates.sh -+++ b/taskcluster/docker/periodic-updates/scripts/periodic_file_updates.sh -@@ -279,7 +279,7 @@ function compare_suffix_lists { - } - - function compare_remote_settings_files { -- REMOTE_SETTINGS_SERVER="https://firefox.settings.services.mozilla.com/v1" -+ REMOTE_SETTINGS_SERVER="resource://app/defaults/settings" - - # 1. List remote settings collections from server. - echo "INFO: fetch remote settings list from server" -diff --git a/toolkit/components/search/SearchUtils.jsm b/toolkit/components/search/SearchUtils.jsm -index 8a3c6acb84..b0a9c4b86f 100644 ---- a/toolkit/components/search/SearchUtils.jsm -+++ b/toolkit/components/search/SearchUtils.jsm -@@ -159,13 +159,13 @@ var SearchUtils = { - - ENGINES_URLS: { - "prod-main": -- "https://firefox.settings.services.mozilla.com/v1/buckets/main/collections/search-config/records", -+ "resource://app/defaults/settings/main/search-config.json", - "prod-preview": -- "https://firefox.settings.services.mozilla.com/v1/buckets/main-preview/collections/search-config/records", -+ "resource://app/defaults/settings/main/search-config.json", - "stage-main": -- "https://settings.stage.mozaws.net/v1/buckets/main/collections/search-config/records", -+ "resource://app/defaults/settings/main/search-config.json", - "stage-preview": -- "https://settings.stage.mozaws.net/v1/buckets/main-preview/collections/search-config/records", -+ "resource://app/defaults/settings/main/search-config.json", - }, - - // The following constants are left undocumented in nsISearchService.idl -diff --git a/toolkit/components/search/docs/DefaultSearchEngines.rst b/toolkit/components/search/docs/DefaultSearchEngines.rst -index 0648471396..37948dca31 100644 ---- a/toolkit/components/search/docs/DefaultSearchEngines.rst -+++ b/toolkit/components/search/docs/DefaultSearchEngines.rst -@@ -86,4 +86,4 @@ is updated. - - .. _configuration schema: SearchConfigurationSchema.html - .. _remote settings: /services/common/services/RemoteSettings.html --.. _search-default-override-allowlist bucket: https://firefox.settings.services.mozilla.com/v1/buckets/main/collections/search-default-override-allowlist/records -+.. _search-default-override-allowlist bucket: resource://app/defaults/settings/main/search-default-override-allowlist.json -diff --git a/toolkit/components/search/docs/SearchEngineConfiguration.rst b/toolkit/components/search/docs/SearchEngineConfiguration.rst -index e9041affb8..7a9466d294 100644 ---- a/toolkit/components/search/docs/SearchEngineConfiguration.rst -+++ b/toolkit/components/search/docs/SearchEngineConfiguration.rst -@@ -68,5 +68,5 @@ related. As a result several situations may occur: - .. _JSON schema: https://json-schema.org/ - .. _stored in mozilla-central: https://searchfox.org/mozilla-central/source/toolkit/components/search/schema/ - .. _Search Configuration Schema: SearchConfigurationSchema.html --.. _viewed live: https://firefox.settings.services.mozilla.com/v1/buckets/main/collections/search-config/records -+.. _viewed live: resource://app/defaults/settings/main/search-config.json - .. _Normandy: /toolkit/components/normandy/normandy/services.html -diff --git a/toolkit/mozapps/defaultagent/RemoteSettings.cpp b/toolkit/mozapps/defaultagent/RemoteSettings.cpp -index 667d9fc628..b2bf628f29 100644 ---- a/toolkit/mozapps/defaultagent/RemoteSettings.cpp -+++ b/toolkit/mozapps/defaultagent/RemoteSettings.cpp -@@ -23,7 +23,7 @@ extern "C" { - HRESULT IsAgentRemoteDisabledRust(const char* szUrl, DWORD* lpdwDisabled); - } - --#define PROD_ENDPOINT "https://firefox.settings.services.mozilla.com/v1" -+#define PROD_ENDPOINT "resource://app/defaults/settings" - #define PROD_BID "main" - #define PROD_CID "windows-default-browser-agent" - #define PROD_ID "state" --- -2.31.1 - - -From f1e92b5fb7844a57ad63d8a52b4867db9817fc14 Mon Sep 17 00:00:00 2001 -From: grizzlyuser <grizzlyuser@protonmail.com> -Date: Wed, 30 Dec 2020 17:34:08 +0200 -Subject: [PATCH 02/13] Remove polling triggered by push broadcasts - -When initialized, remote-settings.js adds a listener to push broadcasts, -that let Remote Settings server send push messages to trigger polling -for changes from the client side. This is not needed for local-only -setup. Remove the record from broadcast-listeners.json file stored in -the user profile, so that it doesn't get picked up by push broadcast -service. ---- - dom/push/PushBroadcastService.jsm | 13 +++++++++++++ - services/settings/remote-settings.js | 7 ++----- - 2 files changed, 15 insertions(+), 5 deletions(-) - -diff --git a/dom/push/PushBroadcastService.jsm b/dom/push/PushBroadcastService.jsm -index aa1504211d..d635a2c3aa 100644 ---- a/dom/push/PushBroadcastService.jsm -+++ b/dom/push/PushBroadcastService.jsm -@@ -178,6 +178,19 @@ var BroadcastService = class { - } - } - -+ async deleteListener(broadcastId) { -+ await this.initializePromise; -+ -+ if (this.jsonFile.data.listeners.hasOwnProperty(broadcastId)) { -+ console.info( -+ "deleteListener: deleting listener", -+ broadcastId -+ ); -+ delete this.jsonFile.data.listeners[broadcastId]; -+ this.jsonFile.saveSoon(); -+ } -+ } -+ - /** - * Call the listeners of the specified broadcasts. - * -diff --git a/services/settings/remote-settings.js b/services/settings/remote-settings.js -index 6d0185faf9..aae93fa440 100644 ---- a/services/settings/remote-settings.js -+++ b/services/settings/remote-settings.js -@@ -441,7 +441,7 @@ function remoteSettingsFunction() { - moduleURI: __URI__, - symbolName: "remoteSettingsBroadcastHandler", - }; -- pushBroadcastService.addListener(BROADCAST_ID, currentVersion, moduleInfo); -+ pushBroadcastService.deleteListener(BROADCAST_ID); - }; - - return remoteSettings; -@@ -461,9 +461,6 @@ var remoteSettingsBroadcastHandler = { - `Push notification received (version=${version} phase=${phase})` - ); - -- return RemoteSettings.pollChanges({ -- expectedTimestamp: version, -- trigger: isStartup ? "startup" : "broadcast", -- }); -+ return; - }, - }; --- -2.31.1 - - -From 3054d3efe22802ab5503dd812e0a0283bbd791f1 Mon Sep 17 00:00:00 2001 -From: grizzlyuser <grizzlyuser@protonmail.com> -Date: Wed, 30 Dec 2020 17:41:54 +0200 -Subject: [PATCH 03/13] Remove timer that triggers polling for changes - -That is not needed for local-only setup. ---- - services/settings/components.conf | 9 +-------- - services/settings/servicesSettings.manifest | 4 ---- - 2 files changed, 1 insertion(+), 12 deletions(-) - -diff --git a/services/settings/components.conf b/services/settings/components.conf -index 9a737802ee..25109415a7 100644 ---- a/services/settings/components.conf -+++ b/services/settings/components.conf -@@ -4,11 +4,4 @@ - # License, v. 2.0. If a copy of the MPL was not distributed with this - # file, You can obtain one at http://mozilla.org/MPL/2.0/. - --Classes = [ -- { -- 'cid': '{5e756573-234a-49ea-bbe4-59ec7a70657d}', -- 'contract_ids': ['@mozilla.org/services/settings;1'], -- 'jsm': 'resource://services-settings/RemoteSettingsComponents.jsm', -- 'constructor': 'RemoteSettingsTimer', -- }, --] -+Classes = [] -diff --git a/services/settings/servicesSettings.manifest b/services/settings/servicesSettings.manifest -index 3bfed26ea4..807eb220ec 100644 ---- a/services/settings/servicesSettings.manifest -+++ b/services/settings/servicesSettings.manifest -@@ -1,7 +1,3 @@ - # Register resource aliases - resource services-settings resource://gre/modules/services-settings/ - --# Schedule polling of remote settings changes --# (default 24H, max 72H) --# see syntax https://searchfox.org/mozilla-central/rev/cc280c4be94ff8cf64a27cc9b3d6831ffa49fa45/toolkit/components/timermanager/UpdateTimerManager.jsm#155 --category update-timer RemoteSettingsComponents @mozilla.org/services/settings;1,getService,services-settings-poll-changes,services.settings.poll_interval,86400,259200 --- -2.31.1 - - -From c856641861ca70da2d5aa720e402f2e505ebe5ac Mon Sep 17 00:00:00 2001 -From: grizzlyuser <grizzlyuser@protonmail.com> -Date: Wed, 30 Dec 2020 17:47:41 +0200 -Subject: [PATCH 04/13] Utils: fetch timestamps of each collection locally - -Utils.CHANGES_PATH points to -services/settings/dumps/monitor/changes.json -which will be generated later by JSON processing script. Fetch the -timestamps from that file and mock response headers to not confuse any -code that expects them. ---- - browser/installer/package-manifest.in | 1 + - services/settings/Utils.jsm | 13 ++++++++++++- - services/settings/dumps/monitor/moz.build | 8 ++++++++ - services/settings/dumps/moz.build | 1 + - 4 files changed, 22 insertions(+), 1 deletion(-) - create mode 100644 services/settings/dumps/monitor/moz.build - -diff --git a/browser/installer/package-manifest.in b/browser/installer/package-manifest.in -index ec20499166..c618c02a8f 100644 ---- a/browser/installer/package-manifest.in -+++ b/browser/installer/package-manifest.in -@@ -298,6 +298,7 @@ - @RESPATH@/browser/defaults/settings/blocklists - @RESPATH@/browser/defaults/settings/pinning - @RESPATH@/browser/defaults/settings/main -+@RESPATH@/browser/defaults/settings/monitor - @RESPATH@/browser/defaults/settings/security-state - - ; Warning: changing the path to channel-prefs.js can cause bugs (Bug 756325) -diff --git a/services/settings/Utils.jsm b/services/settings/Utils.jsm -index ef91781ac6..8736951968 100644 ---- a/services/settings/Utils.jsm -+++ b/services/settings/Utils.jsm -@@ -150,7 +150,7 @@ var Utils = { - async fetchLatestChanges(serverUrl, options = {}) { - const { expectedTimestamp, lastEtag = "", filters = {} } = options; - -- let url = serverUrl + Utils.CHANGES_PATH; -+ let url = Utils.SERVER_URL + Utils.CHANGES_PATH; - const params = { - ...filters, - _expected: expectedTimestamp ?? 0, -@@ -166,6 +166,9 @@ var Utils = { - .join("&"); - } - const response = await fetch(url); -+ const responseDate = new Date().toUTCString() -+ response.headers.set("Date", responseDate); -+ response.headers.set("Last-Modified", responseDate); - - if (response.status >= 500) { - throw new Error(`Server error ${response.status} ${response.statusText}`); -@@ -200,7 +194,15 @@ var Utils = { - } - } - -- const { changes = [], timestamp } = payload; -+ const { timestamp } = payload; -+ const { bucket, collection } = filters; -+ if (!bucket || !collection) { -+ throw new Error('Unable to fetch latest change without bucket or collection'); -+ } -+ const change = payload.changes.find( -+ change => change.bucket === bucket && change.collection === collection -+ ) ?? { last_modified: 0, bucket, collection }; -+ const changes = [change]; - - let serverTimeMillis = Date.parse(response.headers.get("Date")); - // Since the response is served via a CDN, the Date header value could have been cached. -diff --git a/services/settings/dumps/monitor/moz.build b/services/settings/dumps/monitor/moz.build -new file mode 100644 -index 0000000000..d3d017fda5 ---- /dev/null -+++ b/services/settings/dumps/monitor/moz.build -@@ -0,0 +1,8 @@ -+# This Source Code Form is subject to the terms of the Mozilla Public -+# License, v. 2.0. If a copy of the MPL was not distributed with this -+# file, You can obtain one at http://mozilla.org/MPL/2.0/. -+ -+FINAL_TARGET_FILES.defaults.settings.monitor += ["changes.json"] -+ -+if CONFIG["MOZ_BUILD_APP"] == "browser": -+ DIST_SUBDIR = "browser" -diff --git a/services/settings/dumps/moz.build b/services/settings/dumps/moz.build -index 3cc9436f61..3742da5667 100644 ---- a/services/settings/dumps/moz.build -+++ b/services/settings/dumps/moz.build -@@ -5,6 +5,7 @@ - DIRS += [ - "blocklists", - "main", -+ "monitor", - "pinning", - "security-state", - ] --- -2.31.1 - - -From a0f311bed359d484fbf85e696b5b7e3a288292f8 Mon Sep 17 00:00:00 2001 -From: grizzlyuser <grizzlyuser@protonmail.com> -Date: Wed, 30 Dec 2020 17:52:10 +0200 -Subject: [PATCH 05/13] Utils: disable offline checking - -Since only local data is read now, it should always return false for the -current and any future code that relies on it. ---- - services/settings/Utils.jsm | 9 --------- - 1 file changed, 9 deletions(-) - -diff --git a/services/settings/Utils.jsm b/services/settings/Utils.jsm -index 8736951968..8ac085feea 100644 ---- a/services/settings/Utils.jsm -+++ b/services/settings/Utils.jsm -@@ -80,15 +80,6 @@ var Utils = { - * @return {bool} Whether network is down or not. - */ - get isOffline() { -- try { -- return ( -- Services.io.offline || -- CaptivePortalService.state == CaptivePortalService.LOCKED_PORTAL || -- !gNetworkLinkService.isLinkUp -- ); -- } catch (ex) { -- log.warn("Could not determine network status.", ex); -- } - return false; - }, - --- -2.31.1 - - -From 95e4979573c79c987160a71e913c7564de22127f Mon Sep 17 00:00:00 2001 -From: grizzlyuser <grizzlyuser@protonmail.com> -Date: Wed, 30 Dec 2020 17:56:02 +0200 -Subject: [PATCH 06/13] Refactor hashing logic to a separate function - -It is used instead of internal signature validation mechanism, for -integrity checking of the locally cached data. ---- - services/settings/RemoteSettingsWorker.jsm | 4 ++++ - services/settings/SharedUtils.jsm | 9 +++++++-- - 2 files changed, 11 insertions(+), 2 deletions(-) - -diff --git a/services/settings/RemoteSettingsWorker.jsm b/services/settings/RemoteSettingsWorker.jsm -index 147ebb6b13..c86e218fd3 100644 ---- a/services/settings/RemoteSettingsWorker.jsm -+++ b/services/settings/RemoteSettingsWorker.jsm -@@ -189,6 +189,10 @@ class Worker { - // task on the current thread instead of the worker thread. - return SharedUtils.checkContentHash(buffer, size, hash); - } -+ -+ async getContentHash(bytes) { -+ return SharedUtils.getContentHash(bytes); -+ } - } - - // Now, first add a shutdown blocker. If that fails, we must have -diff --git a/services/settings/SharedUtils.jsm b/services/settings/SharedUtils.jsm -index db5017a742..1a8e83c2e8 100644 ---- a/services/settings/SharedUtils.jsm -+++ b/services/settings/SharedUtils.jsm -@@ -28,11 +28,16 @@ var SharedUtils = { - return false; - } - // Has expected content? -+ const hashStr = await this.getContentHash(bytes); -+ return hashStr == hash; -+ }, -+ -+ async getContentHash(bytes) { - const hashBuffer = await crypto.subtle.digest("SHA-256", bytes); - const hashBytes = new Uint8Array(hashBuffer); - const toHex = b => b.toString(16).padStart(2, "0"); -- const hashStr = Array.from(hashBytes, toHex).join(""); -- return hashStr == hash; -+ -+ return Array.from(hashBytes, toHex).join(""); - }, - - /** --- -2.31.1 - - -From daac032f8a4e12451cda9ec6b1eca29f5a35521f Mon Sep 17 00:00:00 2001 -From: grizzlyuser <grizzlyuser@protonmail.com> -Date: Wed, 30 Dec 2020 18:05:02 +0200 -Subject: [PATCH 07/13] Client: Fetch and hash records from local dump - -Read the records from local dumps. See [1] for details on how to prepare -custom dumps). Records are cached in the local IndexedDB, and the client -updates cached records each time there's a change. Also it verifies -integrity of the data. Then the list of current / created / updated / -deleted records is generated and emitted to every registered listener. - -Change upstream signature validation mechanism to a simpler one. -Otherwise, it'd be necessary to sign local records, which is redundant, -because the application package should be signed already by the distro. - -Instead of signature property from metadata records, json_dump_metadata -has been introduced. It contains the checksum of the records and size in -bytes. Also added app_build_id property for version checking and updates -of cached data. - -Although it's possible to disable integrity checking via preference, it -seems to be not a good idea, because the logic that detects invalid -local data relies on it. In the context of local-only setup, data that -has been received from real Remote Settings server will not contain the -custom metadata, and thus will be considered invalid and then discarded, -while the client gets a chance to gracefully inform registered listeners -about these changes so that they can discard the data received before -the upgrade to local-only setup. - -[1] https://firefox-source-docs.mozilla.org/services/common/services/RemoteSettings.html#initial-data ---- - services/settings/RemoteSettingsClient.jsm | 62 ++++++++++------------ - 1 file changed, 27 insertions(+), 35 deletions(-) - -diff --git a/services/settings/RemoteSettingsClient.jsm b/services/settings/RemoteSettingsClient.jsm -index 80dd563e11..1025ab33a2 100644 ---- a/services/settings/RemoteSettingsClient.jsm -+++ b/services/settings/RemoteSettingsClient.jsm -@@ -556,11 +556,9 @@ class RemoteSettingsClient extends EventEmitter { - - // If the data is up-to-date but don't have metadata (records loaded from dump), - // we fetch them and validate the signature immediately. -- if (this.verifySignature && ObjectUtils.isEmpty(localMetadata)) { -+ if (this.verifySignature && ObjectUtils.isEmpty(localMetadata?.json_dump_metadata)) { - console.debug(`${this.identifier} pull collection metadata`); -- const metadata = await this.httpClient().getData({ -- query: { _expected: expectedTimestamp }, -- }); -+ const { metadata } = await this._fetchChangeset(expectedTimestamp); - await this.db.importChanges(metadata); - // We don't bother validating the signature if the dump was just loaded. We do - // if the dump was loaded at some other point (eg. from .get()). -@@ -813,32 +811,23 @@ class RemoteSettingsClient extends EventEmitter { - async _validateCollectionSignature(records, timestamp, metadata) { - const start = Cu.now() * 1000; - -- if (!metadata?.signature) { -+ if (!metadata?.json_dump_metadata) { - throw new MissingSignatureError(this.identifier); - } - -- if (!this._verifier) { -- this._verifier = Cc[ -- "@mozilla.org/security/contentsignatureverifier;1" -- ].createInstance(Ci.nsIContentSignatureVerifier); -- } -- -- // This is a content-signature field from an autograph response. - const { -- signature: { x5u, signature }, -+ json_dump_metadata: { hash, size }, - } = metadata; -- const certChain = await (await fetch(x5u)).text(); - // Merge remote records with local ones and serialize as canonical JSON. - const serialized = await RemoteSettingsWorker.canonicalStringify( - records, - timestamp - ); - if ( -- !(await this._verifier.asyncVerifyContentSignature( -- serialized, -- "p384ecdsa=" + signature, -- certChain, -- this.signerName -+ !(await RemoteSettingsWorker.checkContentHash( -+ new TextEncoder().encode(serialized), -+ size, -+ hash - )) - ) { - throw new InvalidSignatureError(this.identifier); -@@ -1030,24 +1019,27 @@ class RemoteSettingsClient extends EventEmitter { - * @param since timestamp of last sync (optional) - */ - async _fetchChangeset(expectedTimestamp, since) { -- const client = this.httpClient(); -- const { -- metadata, -- timestamp: remoteTimestamp, -- changes: remoteRecords, -- } = await client.execute( -- { -- path: `/buckets/${this.bucketName}/collections/${this.collectionName}/changeset`, -- }, -- { -- query: { -- _expected: expectedTimestamp, -- _since: since, -- }, -- } -+ const { data } = await SharedUtils.loadJSONDump( -+ this.bucketName, -+ this.collectionName - ); -+ const remoteRecords = data ?? []; -+ -+ const serialized = await RemoteSettingsWorker.canonicalStringify( -+ remoteRecords, -+ expectedTimestamp -+ ); -+ const bytes = new TextEncoder().encode(serialized); -+ const metadata = { -+ app_build_id: Services.appinfo.appBuildID, -+ json_dump_metadata: { -+ hash: await RemoteSettingsWorker.getContentHash(bytes), -+ size: bytes.length, -+ }, -+ } -+ - return { -- remoteTimestamp, -+ remoteTimestamp: expectedTimestamp, - metadata, - remoteRecords, - }; --- -2.31.1 - - -From 4d36e599e6b24e10960ebb978c038c66c5ade06d Mon Sep 17 00:00:00 2001 -From: grizzlyuser <grizzlyuser@protonmail.com> -Date: Wed, 30 Dec 2020 18:42:56 +0200 -Subject: [PATCH 08/13] Client: start deferred sync on get() or on() - -The users of the RemoteSettingsClient.jsm can receive records from it in -two ways: by calling get(), and by subscribing to events by calling -on(). - -So hook a deferred sync whenever something calls these methods. Because -multiple of those calls can be made quite early and in very short time, -set up a deferred task that will be armed only when needed and only once -in a second. When the task is running it first checks if the local data -came from the dump of the current app build, and no-ops if true. If -false, it triggers a sync. Then adds a flag if the client has been -correctly synchronized with the dump, so that no metadata checking -occurs during the session. ---- - services/settings/RemoteSettingsClient.jsm | 30 +++++++++++++++++++++- - 1 file changed, 29 insertions(+), 1 deletion(-) - -diff --git a/services/settings/RemoteSettingsClient.jsm b/services/settings/RemoteSettingsClient.jsm -index 1025ab33a2..1cebf2bc29 100644 ---- a/services/settings/RemoteSettingsClient.jsm -+++ b/services/settings/RemoteSettingsClient.jsm -@@ -16,6 +16,7 @@ XPCOMUtils.defineLazyModuleGetters(this, { - ClientEnvironmentBase: - "resource://gre/modules/components-utils/ClientEnvironment.jsm", - Database: "resource://services-settings/Database.jsm", -+ DeferredTask: "resource://gre/modules/DeferredTask.jsm", - Downloader: "resource://services-settings/Attachments.jsm", - IDBHelpers: "resource://services-settings/IDBHelpers.jsm", - KintoHttpClient: "resource://services-common/kinto-http-client.js", -@@ -30,6 +31,7 @@ XPCOMUtils.defineLazyModuleGetters(this, { - XPCOMUtils.defineLazyGlobalGetters(this, ["fetch"]); - - const TELEMETRY_COMPONENT = "remotesettings"; -+const DEFERRED_SYNC_DELAY_MILLISECONDS = 1000; - - XPCOMUtils.defineLazyGetter(this, "console", () => Utils.log); - -@@ -259,6 +261,14 @@ class RemoteSettingsClient extends EventEmitter { - this._lastCheckTimePref = lastCheckTimePref; - this._verifier = null; - this._syncRunning = false; -+ this._deferredSync = new DeferredTask( -+ async () => { -+ if (!this._syncRunning && !(await this._isSynced())) { -+ await this.sync(); -+ } -+ }, -+ DEFERRED_SYNC_DELAY_MILLISECONDS -+ ); - - // This attribute allows signature verification to be disabled, when running tests - // or when pulling data from a dev server. -@@ -290,6 +300,11 @@ class RemoteSettingsClient extends EventEmitter { - ); - } - -+ on(event, callback) { -+ super.on(event, callback); -+ this._deferredSync.arm(); -+ } -+ - get identifier() { - return `${this.bucketName}/${this.collectionName}`; - } -@@ -353,6 +368,10 @@ class RemoteSettingsClient extends EventEmitter { - try { - let hasLocalData = await Utils.hasLocalData(this); - -+ if (!(await this._isSynced())) { -+ throw new MissingSignatureError(this.identifier); -+ } -+ - if (syncIfEmpty && !hasLocalData) { - // .get() was called before we had the chance to synchronize the local database. - // We'll try to avoid returning an empty list. -@@ -414,7 +433,10 @@ class RemoteSettingsClient extends EventEmitter { - // No need to verify signature on JSON dumps. - // If local DB cannot be read, then we don't even try to do anything, - // we return results early. -- return this._filterEntries(data); -+ const filtered = this._filterEntries(data); -+ this._deferredSync.arm(); -+ -+ return filtered; - } - - console.debug( -@@ -452,6 +474,12 @@ class RemoteSettingsClient extends EventEmitter { - return final; - } - -+ async _isSynced() { -+ this._synced ||= -+ Services.appinfo.appBuildID === (await this.db.getMetadata())?.app_build_id; -+ return this._synced; -+ } -+ - /** - * Synchronize the local database with the remote server. - * --- -2.31.1 - - -From defc4080596f5407a98f0c9f1a456f685226054f Mon Sep 17 00:00:00 2001 -From: grizzlyuser <grizzlyuser@protonmail.com> -Date: Wed, 30 Dec 2020 18:53:51 +0200 -Subject: [PATCH 09/13] Client: deep compare records if timestamps match - -When the list of current / updated / deleted records is generated, their -modification timestamps are compared to detect the updates. - -Although in practice this is unlikely to happen, in theory the -timestamp of some older record received from Remote Settings can match -with the modified record in the dump. Although JSON processing script -makes sure to add unique timestamps to each of the modified records, -it's still possible to update dumps manually and simply forget to update -timestamps. So serialize the records and compare them as strings to be -on the safe side. This should happen only once after upgrading to each -new version of the application, so is not likely to introduce any -noticeable performance issues. ---- - services/settings/RemoteSettingsClient.jsm | 6 +++++- - 1 file changed, 5 insertions(+), 1 deletion(-) - -diff --git a/services/settings/RemoteSettingsClient.jsm b/services/settings/RemoteSettingsClient.jsm -index 1cebf2bc29..2c18c5cfb6 100644 ---- a/services/settings/RemoteSettingsClient.jsm -+++ b/services/settings/RemoteSettingsClient.jsm -@@ -13,6 +13,7 @@ const { Services } = ChromeUtils.import("resource://gre/modules/Services.jsm"); - - XPCOMUtils.defineLazyModuleGetters(this, { - AppConstants: "resource://gre/modules/AppConstants.jsm", -+ CanonicalJSON: "resource://gre/modules/CanonicalJSON.jsm", - ClientEnvironmentBase: - "resource://gre/modules/components-utils/ClientEnvironment.jsm", - Database: "resource://services-settings/Database.jsm", -@@ -1022,7 +1023,10 @@ class RemoteSettingsClient extends EventEmitter { - const old = oldById.get(r.id); - if (old) { - oldById.delete(r.id); -- if (r.last_modified != old.last_modified) { -+ if ( -+ r.last_modified != old.last_modified || -+ CanonicalJSON.stringify(r) != CanonicalJSON.stringify(old) -+ ) { - syncResult.updated.push({ old, new: r }); - } - } else { --- -2.31.1 - - -From c2ee19f01bc37e15c7742af8a502ffaa10745a52 Mon Sep 17 00:00:00 2001 -From: grizzlyuser <grizzlyuser@protonmail.com> -Date: Wed, 30 Dec 2020 19:01:39 +0200 -Subject: [PATCH 10/13] Client: delete more data on cleanup - -When the client detects the local data is invalid (i.e. it came from -real Remote Settings and can have unwanted records), delete not only -the records, but also the attachments that came with them, because they -too can be problematic. And last check time preference, because it's not -useful anyway when remote-settings.js doesn't do any polling for changes. - -Note that attachments should be deleted before the records, because the -logic gets the data about the attachments from those records. ---- - services/settings/RemoteSettingsClient.jsm | 15 ++++++++++++--- - 1 file changed, 12 insertions(+), 3 deletions(-) - -diff --git a/services/settings/RemoteSettingsClient.jsm b/services/settings/RemoteSettingsClient.jsm -index 2c18c5cfb6..8b65dc0cba 100644 ---- a/services/settings/RemoteSettingsClient.jsm -+++ b/services/settings/RemoteSettingsClient.jsm -@@ -221,7 +221,10 @@ class AttachmentDownloader extends Downloader { - async deleteAll() { - let allRecords = await this._client.db.list(); - return Promise.all( -- allRecords.filter(r => !!r.attachment).map(r => this.delete(r)) -+ allRecords.filter(r => !!r.attachment).map(r => { -+ this.delete(r); -+ this.deleteCached(r.id); -+ }) - ); - } - } -@@ -982,7 +985,7 @@ class RemoteSettingsClient extends EventEmitter { - // Signature failed, clear local DB because it contains - // bad data (local + remote changes). - console.debug(`${this.identifier} clear local data`); -- await this.db.clear(); -+ await this._clearAll(); - // Local data was tampered, throw and it will retry from empty DB. - console.error(`${this.identifier} local data was corrupted`); - throw new CorruptedDataError(this.identifier); -@@ -1004,7 +1007,7 @@ class RemoteSettingsClient extends EventEmitter { - // _importJSONDump() only clears DB if dump is available, - // therefore do it here! - if (imported < 0) { -- await this.db.clear(); -+ await this._clearAll(); - } - } - } -@@ -1044,6 +1047,12 @@ class RemoteSettingsClient extends EventEmitter { - return syncResult; - } - -+ async _clearAll() { -+ await this.attachments.deleteAll(); -+ await this.db.clear(); -+ Services.prefs.clearUserPref(this.lastCheckTimePref); -+ } -+ - /** - * Fetch information from changeset endpoint. - * --- -2.31.1 - - -From 56d2af487f7077753ea4df6bd0b1e6c91ed7ab9f Mon Sep 17 00:00:00 2001 -From: grizzlyuser <grizzlyuser@protonmail.com> -Date: Wed, 30 Dec 2020 19:07:56 +0200 -Subject: [PATCH 11/13] Client: remove comparison of collection timestamps - -In case if the cached data that came from real Remote Settings server -(before the upgrade to local-only setup) has collection timestamp, that -is newer than the packaged local dump, then this comparison logic can -lead to early return of old data, skipping the integrity checking and -necessary cleanup. So remove the checks. ---- - services/settings/RemoteSettingsClient.jsm | 5 ----- - 1 file changed, 5 deletions(-) - -diff --git a/services/settings/RemoteSettingsClient.jsm b/services/settings/RemoteSettingsClient.jsm -index 8b65dc0cba..6274596591 100644 ---- a/services/settings/RemoteSettingsClient.jsm -+++ b/services/settings/RemoteSettingsClient.jsm -@@ -917,14 +917,9 @@ class RemoteSettingsClient extends EventEmitter { - updated: [], - deleted: [], - }; -- // If data wasn't changed, return empty sync result. -- // This can happen when we update the signature but not the data. - console.debug( - `${this.identifier} local timestamp: ${localTimestamp}, remote: ${remoteTimestamp}` - ); -- if (localTimestamp && remoteTimestamp < localTimestamp) { -- return syncResult; -- } - - const start = Cu.now() * 1000; - await this.db.importChanges(metadata, remoteTimestamp, remoteRecords, { --- -2.31.1 - - -From c009c1a9ba2477c9335921b706256f115ecfd498 Mon Sep 17 00:00:00 2001 -From: grizzlyuser <grizzlyuser@protonmail.com> -Date: Wed, 30 Dec 2020 19:15:44 +0200 -Subject: [PATCH 12/13] Attachments: load only from dump and drop cached - ---- - services/settings/Attachments.jsm | 35 +++++++------------------------ - 1 file changed, 8 insertions(+), 27 deletions(-) - -diff --git a/services/settings/Attachments.jsm b/services/settings/Attachments.jsm -index 0eeb632799..eaa7db8c81 100644 ---- a/services/settings/Attachments.jsm -+++ b/services/settings/Attachments.jsm -@@ -143,10 +143,11 @@ class Downloader { - checkHash, - attachmentId = record?.id, - useCache = false, -- fallbackToCache = false, - fallbackToDump = false, - } = options || {}; - -+ const fallbackToCache = false; -+ - if (!useCache) { - // For backwards compatibility. - // WARNING: Its return type is different from what's documented. -@@ -206,6 +207,7 @@ class Downloader { - const newBuffer = await this.downloadAsBytes(record, { - retries, - checkHash, -+ dumpInfo, - }); - const blob = new Blob([newBuffer]); - if (useCache) { -@@ -241,7 +243,7 @@ class Downloader { - } - - try { -- return { ...(await cacheInfo.getResult()), _source: "cache_fallback" }; -+ await this.cacheImpl.delete(attachmentId); - } catch (e) { - // Failed to read from cache, e.g. IndexedDB unusable. - Cu.reportError(e); -@@ -278,7 +280,7 @@ class Downloader { - * @returns {String} the absolute file path to the downloaded attachment. - */ - async downloadToDisk(record, options = {}) { -- const { retries = 3 } = options; -+ const retries = 0; - const { - attachment: { filename, size, hash }, - } = record; -@@ -335,31 +337,10 @@ class Downloader { - */ - async downloadAsBytes(record, options = {}) { - const { -- attachment: { location, hash, size }, -- } = record; -- -- const remoteFileUrl = (await this._baseAttachmentsURL()) + location; -+ dumpInfo = new LazyRecordAndBuffer(() => this._readAttachmentDump(attachmentId)) -+ } = options; - -- const { retries = 3, checkHash = true } = options; -- let retried = 0; -- while (true) { -- try { -- const buffer = await this._fetchAttachment(remoteFileUrl); -- if (!checkHash) { -- return buffer; -- } -- if (await RemoteSettingsWorker.checkContentHash(buffer, size, hash)) { -- return buffer; -- } -- // Content is corrupted. -- throw new Downloader.BadContentError(location); -- } catch (e) { -- if (retried >= retries) { -- throw e; -- } -- } -- retried++; -- } -+ return (await dumpInfo.getResult()).buffer; - } - - /** --- -2.31.1 - - -From 2035bd7a6ce1816417b619d3f1ce994a8b44ce9d Mon Sep 17 00:00:00 2001 -From: grizzlyuser <grizzlyuser@protonmail.com> -Date: Wed, 30 Dec 2020 19:22:20 +0200 -Subject: [PATCH 13/13] Disable CRLite entirely for now - -It's designed to fetch the data from Remote Settings. One of the main -selling points is that new revocations can be pushed to the clients -within minutes. That won't work with local-only setup. Although (some?) -of the JSON dumps for it are in place, obviously the updates won't -happen that fast. - -Right now CRLite doesn't enforce anything, and works just for telemetry -collection (which is hopefully disabled anyway). So disable the -preference right in the source code, so that the patch fails to apply -when the upstream decides to set it to enforcing mode by default. - -The solution with CRLite is up for discussion. If necessary, it's -possible to make clients for blessed collections to communicate to real -Remote Settings server. For example, for collections related to -certificate revocations. ---- - modules/libpref/init/all.js | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/modules/libpref/init/all.js b/modules/libpref/init/all.js -index edca292681..da7e23d674 100644 ---- a/modules/libpref/init/all.js -+++ b/modules/libpref/init/all.js -@@ -172,7 +172,7 @@ pref("security.cert_pinning.max_max_age_seconds", 5184000); - // 0: Disable CRLite entirely - // 1: Enable and check revocations via CRLite, but only collect telemetry - // 2: Enable and enforce revocations via CRLite --pref("security.pki.crlite_mode", 1); -+pref("security.pki.crlite_mode", 0); - - // Represents the expected certificate transparency log merge delay (including - // the time to generate a CRLite filter). Currently 28 hours in seconds. --- -2.31.1 - |