diff options
Diffstat (limited to 'libre/iceweasel/9001-FSDG-sync-remote-settings-with-local-dump.patch')
| -rw-r--r-- | libre/iceweasel/9001-FSDG-sync-remote-settings-with-local-dump.patch | 719 |
1 files changed, 419 insertions, 300 deletions
diff --git a/libre/iceweasel/9001-FSDG-sync-remote-settings-with-local-dump.patch b/libre/iceweasel/9001-FSDG-sync-remote-settings-with-local-dump.patch index c773a8b08..e81f2d598 100644 --- a/libre/iceweasel/9001-FSDG-sync-remote-settings-with-local-dump.patch +++ b/libre/iceweasel/9001-FSDG-sync-remote-settings-with-local-dump.patch @@ -1,4 +1,4 @@ -From bec8749011d7dba665eace92557f9d36b36c3060 Mon Sep 17 00:00:00 2001 +From 924f9c99de6f3223a46e472339c59726e60e5c79 Mon Sep 17 00:00:00 2001 From: grizzlyuser <grizzlyuser@protonmail.com> Date: Wed, 30 Dec 2020 17:20:39 +0200 Subject: [PATCH 01/13] Point to local omni.ja files, not remote server @@ -23,21 +23,26 @@ If necessary, missing files can be added later. [2] https://www.gnu.org/distros/free-system-distribution-guidelines.en.html#license-rules --- .../components/ASRouterAdmin/ASRouterAdmin.jsx | 2 +- - .../newtab/data/content/activity-stream.bundle.js | 2 +- - modules/libpref/init/all.js | 2 +- - services/settings/Utils.jsm | 4 ++-- + .../components/asrouter/content/asrouter-admin.bundle.js | 2 +- + browser/components/newtab/bin/import-rollouts.js | 4 ++-- + services/settings/Utils.sys.mjs | 2 +- + services/settings/docs/index.rst | 4 ++-- .../periodic-updates/scripts/periodic_file_updates.sh | 2 +- - toolkit/components/search/SearchUtils.jsm | 8 ++++---- + third_party/rust/remote_settings/.cargo-checksum.json | 2 +- + third_party/rust/remote_settings/src/client.rs | 4 ++-- + .../components/antitracking/docs/query-stripping/index.md | 2 +- + .../backgroundtasks/BackgroundTask_message.sys.mjs | 4 ++-- + toolkit/components/search/SearchUtils.sys.mjs | 8 ++++---- toolkit/components/search/docs/DefaultSearchEngines.rst | 2 +- .../components/search/docs/SearchEngineConfiguration.rst | 2 +- - toolkit/mozapps/defaultagent/RemoteSettings.cpp | 2 +- - 9 files changed, 13 insertions(+), 13 deletions(-) - -diff --git a/browser/components/newtab/content-src/components/ASRouterAdmin/ASRouterAdmin.jsx b/browser/components/newtab/content-src/components/ASRouterAdmin/ASRouterAdmin.jsx -index 0a9da8d804..fdc387aa63 100644 ---- a/browser/components/newtab/content-src/components/ASRouterAdmin/ASRouterAdmin.jsx -+++ b/browser/components/newtab/content-src/components/ASRouterAdmin/ASRouterAdmin.jsx -@@ -1226,7 +1226,7 @@ export class ASRouterAdminInner extends React.PureComponent { + toolkit/modules/AppConstants.sys.mjs | 4 ++-- + 14 files changed, 22 insertions(+), 22 deletions(-) + +diff --git a/browser/components/asrouter/content-src/components/ASRouterAdmin/ASRouterAdmin.jsx b/browser/components/asrouter/content-src/components/ASRouterAdmin/ASRouterAdmin.jsx +index 7d4484dc83..9346d518e6 100644 +--- a/browser/components/asrouter/content-src/components/ASRouterAdmin/ASRouterAdmin.jsx ++++ b/browser/components/asrouter/content-src/components/ASRouterAdmin/ASRouterAdmin.jsx +@@ -874,7 +874,7 @@ export class ASRouterAdminInner extends React.PureComponent { <a className="providerUrl" target="_blank" @@ -46,12 +51,12 @@ index 0a9da8d804..fdc387aa63 100644 rel="noopener noreferrer" > nimbus-desktop-experiments -diff --git a/browser/components/newtab/data/content/activity-stream.bundle.js b/browser/components/newtab/data/content/activity-stream.bundle.js -index 6e7cc5e33d..f2b4f3fcc9 100644 ---- a/browser/components/newtab/data/content/activity-stream.bundle.js -+++ b/browser/components/newtab/data/content/activity-stream.bundle.js -@@ -1819,7 +1819,7 @@ class ASRouterAdminInner extends react__WEBPACK_IMPORTED_MODULE_3___default.a.Pu - label = /*#__PURE__*/react__WEBPACK_IMPORTED_MODULE_3___default.a.createElement("span", null, "remote settings (", /*#__PURE__*/react__WEBPACK_IMPORTED_MODULE_3___default.a.createElement("a", { +diff --git a/browser/components/asrouter/content/asrouter-admin.bundle.js b/browser/components/asrouter/content/asrouter-admin.bundle.js +index 614e8a62a1..3ce250c6ed 100644 +--- a/browser/components/asrouter/content/asrouter-admin.bundle.js ++++ b/browser/components/asrouter/content/asrouter-admin.bundle.js +@@ -1641,7 +1641,7 @@ class ASRouterAdminInner extends (react__WEBPACK_IMPORTED_MODULE_1___default().P + label = /*#__PURE__*/react__WEBPACK_IMPORTED_MODULE_1___default().createElement("span", null, "remote settings (", /*#__PURE__*/react__WEBPACK_IMPORTED_MODULE_1___default().createElement("a", { className: "providerUrl", target: "_blank", - href: "https://firefox.settings.services.mozilla.com/v1/buckets/main/collections/nimbus-desktop-experiments/records", @@ -59,41 +64,64 @@ index 6e7cc5e33d..f2b4f3fcc9 100644 rel: "noopener noreferrer" }, "nimbus-desktop-experiments"), ")"); } -diff --git a/modules/libpref/init/all.js b/modules/libpref/init/all.js -index 651d38c0e2..cb72224e1b 100644 ---- a/modules/libpref/init/all.js -+++ b/modules/libpref/init/all.js -@@ -2198,7 +2198,7 @@ pref("security.insecure_field_warning.ignore_local_ip_address", true); - // Remote settings preferences - // Note: if you change this, make sure to also review security.onecrl.maximum_staleness_in_seconds - pref("services.settings.poll_interval", 86400); // 24H --pref("services.settings.server", "https://firefox.settings.services.mozilla.com/v1"); -+pref("services.settings.server", "resource://app/defaults/settings"); - pref("services.settings.default_bucket", "main"); - - // The percentage of clients who will report uptake telemetry as -diff --git a/services/settings/Utils.jsm b/services/settings/Utils.jsm -index f91e5dcb67..c09c58e693 100644 ---- a/services/settings/Utils.jsm -+++ b/services/settings/Utils.jsm -@@ -90,10 +90,10 @@ var Utils = { - get SERVER_URL() { - return allowServerURLOverride - ? gServerURL -- : "https://firefox.settings.services.mozilla.com/v1"; -+ : "resource://app/defaults/settings"; +diff --git a/browser/components/newtab/bin/import-rollouts.js b/browser/components/newtab/bin/import-rollouts.js +index ce22d4a114..b2dc35e9d9 100644 +--- a/browser/components/newtab/bin/import-rollouts.js ++++ b/browser/components/newtab/bin/import-rollouts.js +@@ -36,7 +36,7 @@ const jsonschema = require("../../../../third_party/js/cfworker/json-schema.js") + + const DEFAULT_COLLECTION_ID = "nimbus-desktop-experiments"; + const BASE_URL = +- "https://firefox.settings.services.mozilla.com/v1/buckets/main/collections/"; ++ "resource://app/defaults/settings/main/"; + const EXPERIMENTER_URL = "https://experimenter.services.mozilla.com/nimbus/"; + const OUTPUT_PATH = "./test/NimbusRolloutMessageProvider.sys.mjs"; + const LICENSE_STRING = `/* This Source Code Form is subject to the terms of the Mozilla Public +@@ -237,7 +237,7 @@ async function main() { + } + ); + +- const RECORDS_URL = `${BASE_URL}${cli.flags.collection}/records`; ++ const RECORDS_URL = `${BASE_URL}${cli.flags.collection}.json`; + + console.log(`Fetching records from ${chalk.underline.yellow(RECORDS_URL)}`); + +diff --git a/services/settings/Utils.sys.mjs b/services/settings/Utils.sys.mjs +index 73c83e526b..653cabe7e1 100644 +--- a/services/settings/Utils.sys.mjs ++++ b/services/settings/Utils.sys.mjs +@@ -100,7 +100,7 @@ export var Utils = { + : AppConstants.REMOTE_SETTINGS_SERVER_URL; }, - CHANGES_PATH: "/buckets/monitor/collections/changes/changeset", -+ CHANGES_PATH: "/monitor/changes.json", ++ CHANGES_PATH: "/monitor/changes", /** * Logger instance. +diff --git a/services/settings/docs/index.rst b/services/settings/docs/index.rst +index 3e042f67e4..dc8584c318 100644 +--- a/services/settings/docs/index.rst ++++ b/services/settings/docs/index.rst +@@ -177,11 +177,11 @@ It is possible to package a dump of the server records that will be loaded into + + The JSON dump will serve as the default dataset for ``.get()``, instead of doing a round-trip to pull the latest data. It will also reduce the amount of data to be downloaded on the first synchronization. + +-#. Place the JSON dump of the server records in the ``services/settings/dumps/main/`` folder. Using this command: ++#. Place the JSON dump of the server records in the ``services/settings/dumps/main/`` folder. Using this command (replace 'https://settings.invalid' with your Remote Settings address): + :: + + CID="your-collection" +- curl "https://firefox.settings.services.mozilla.com/v1/buckets/main/collections/${CID}/changeset?_expected=0" | jq '{"data": .changes, "timestamp": .timestamp}' > services/settings/dumps/main/${CID}.json`` ++ curl "https://settings.invalid/v1/buckets/main/collections/${CID}/changeset?_expected=0" | jq '{"data": .changes, "timestamp": .timestamp}' > services/settings/dumps/main/${CID}.json`` + + #. Add the filename to the ``FINAL_TARGET_FILES`` list in ``services/settings/dumps/main/moz.build`` + #. Add the filename to the ``[browser]`` section of ``mobile/android/installer/package-manifest.in`` IF the file should be bundled with Android. diff --git a/taskcluster/docker/periodic-updates/scripts/periodic_file_updates.sh b/taskcluster/docker/periodic-updates/scripts/periodic_file_updates.sh -index 191d3d563b..49fb961be2 100755 +index 160b7fbc35..2e9dcde464 100755 --- a/taskcluster/docker/periodic-updates/scripts/periodic_file_updates.sh +++ b/taskcluster/docker/periodic-updates/scripts/periodic_file_updates.sh -@@ -285,7 +285,7 @@ function compare_suffix_lists { +@@ -286,7 +286,7 @@ function compare_suffix_lists { } function compare_remote_settings_files { @@ -102,11 +130,69 @@ index 191d3d563b..49fb961be2 100755 # 1. List remote settings collections from server. echo "INFO: fetch remote settings list from server" -diff --git a/toolkit/components/search/SearchUtils.jsm b/toolkit/components/search/SearchUtils.jsm -index 41d2a23e55..e123f3f392 100644 ---- a/toolkit/components/search/SearchUtils.jsm -+++ b/toolkit/components/search/SearchUtils.jsm -@@ -159,13 +159,13 @@ var SearchUtils = { +diff --git a/third_party/rust/remote_settings/.cargo-checksum.json b/third_party/rust/remote_settings/.cargo-checksum.json +index 28ac46a2c2..e8a54e1bac 100644 +--- a/third_party/rust/remote_settings/.cargo-checksum.json ++++ b/third_party/rust/remote_settings/.cargo-checksum.json +@@ -1 +1 @@ +-{"files":{"Cargo.toml":"cf0a08d6b0d6285a459b78115aa24818a04b5987652655e64b80ffd8c8ae0813","build.rs":"4326f03729cf8f1673e4228e6dc111de1ea4d8bcc06351f7ae563efb2613f866","src/client.rs":"3d87162e6913a81cc6f5178a7ca791e262d0d029e7dedf3df4fe2f66e5501185","src/config.rs":"7bb678addfae3b4ed5f2892d32263e5b33cc05e5a12a250f664150e78211f94a","src/error.rs":"192ca42af7c6b882f3129378c23b45dab8a0d2b179e23a8813a335ffd56b21dc","src/lib.rs":"416e99894e152f6cea7418ad2fabfd94bc3d907efd9f33fbd2a83fb99452b2df","src/remote_settings.udl":"e38758592ca75adbebb8fe688b10520d9931a5f3292d94f229cba05310756a43","uniffi.toml":"f8ec8dc593e0d501c2e9e40368ec93ec33b1edd8608e29495e0a54b63144e880"},"package":null} +\ No newline at end of file ++{"files":{"Cargo.toml":"cf0a08d6b0d6285a459b78115aa24818a04b5987652655e64b80ffd8c8ae0813","build.rs":"4326f03729cf8f1673e4228e6dc111de1ea4d8bcc06351f7ae563efb2613f866","src/client.rs":"ff1ab5961fe695989d1ff343c5042ce0f907dc323da2ba87005fc9437919b448","src/config.rs":"7bb678addfae3b4ed5f2892d32263e5b33cc05e5a12a250f664150e78211f94a","src/error.rs":"192ca42af7c6b882f3129378c23b45dab8a0d2b179e23a8813a335ffd56b21dc","src/lib.rs":"416e99894e152f6cea7418ad2fabfd94bc3d907efd9f33fbd2a83fb99452b2df","src/remote_settings.udl":"e38758592ca75adbebb8fe688b10520d9931a5f3292d94f229cba05310756a43","uniffi.toml":"f8ec8dc593e0d501c2e9e40368ec93ec33b1edd8608e29495e0a54b63144e880"},"package":null} +diff --git a/third_party/rust/remote_settings/src/client.rs b/third_party/rust/remote_settings/src/client.rs +index 9585a7cc7c..1c97208b82 100644 +--- a/third_party/rust/remote_settings/src/client.rs ++++ b/third_party/rust/remote_settings/src/client.rs +@@ -33,7 +33,7 @@ impl Client { + pub fn new(config: RemoteSettingsConfig) -> Result<Self> { + let server_url = config + .server_url +- .unwrap_or_else(|| String::from("https://firefox.settings.services.mozilla.com")); ++ .unwrap_or_else(|| String::from("resource://app/defaults/settings")); + let bucket_name = config.bucket_name.unwrap_or_else(|| String::from("main")); + let base_url = Url::parse(&server_url)?; + +@@ -518,7 +518,7 @@ mod test { + }; + let client = Client::new(config).unwrap(); + assert_eq!( +- Url::parse("https://firefox.settings.services.mozilla.com").unwrap(), ++ Url::parse("resource://app/defaults/settings").unwrap(), + client.base_url + ); + assert_eq!(String::from("main"), client.bucket_name); +diff --git a/toolkit/components/antitracking/docs/query-stripping/index.md b/toolkit/components/antitracking/docs/query-stripping/index.md +index e49d8513ba..c1c6ae5deb 100644 +--- a/toolkit/components/antitracking/docs/query-stripping/index.md ++++ b/toolkit/components/antitracking/docs/query-stripping/index.md +@@ -63,7 +63,7 @@ of query params: + It is enabled in Nightly by default in all modes with an extended + strip-list. You can find the current list of parameters that are + stripped +-[here](https://firefox.settings.services.mozilla.com/v1/buckets/main/collections/query-stripping/records). ++[here](resource://app/defaults/settings/main/collections/query-stripping/records). + Note that some records have a *filter\_expression* that limits where + they apply. + +diff --git a/toolkit/components/backgroundtasks/BackgroundTask_message.sys.mjs b/toolkit/components/backgroundtasks/BackgroundTask_message.sys.mjs +index bb9d71b5c0..ec0c5f11d7 100644 +--- a/toolkit/components/backgroundtasks/BackgroundTask_message.sys.mjs ++++ b/toolkit/components/backgroundtasks/BackgroundTask_message.sys.mjs +@@ -19,8 +19,8 @@ + // environment variables still apply. + // + // --stage: use stage Remote Settings +-// (`https://firefox.settings.services.allizom.org/v1`) rather than production +-// (`https://firefox.settings.services.mozilla.com/v1`) ++// (`resource://app/defaults/settings/`) rather than production ++// (`resource://app/defaults/settings/`) + // + // --preview: enable Remote Settings and Experiment previews. + // +diff --git a/toolkit/components/search/SearchUtils.sys.mjs b/toolkit/components/search/SearchUtils.sys.mjs +index 8e474b58a2..a8cd3cccac 100644 +--- a/toolkit/components/search/SearchUtils.sys.mjs ++++ b/toolkit/components/search/SearchUtils.sys.mjs +@@ -193,13 +193,13 @@ export var SearchUtils = { ENGINES_URLS: { "prod-main": @@ -116,26 +202,26 @@ index 41d2a23e55..e123f3f392 100644 - "https://firefox.settings.services.mozilla.com/v1/buckets/main-preview/collections/search-config/records", + "resource://app/defaults/settings/main/search-config.json", "stage-main": -- "https://settings.stage.mozaws.net/v1/buckets/main/collections/search-config/records", +- "https://firefox.settings.services.allizom.org/v1/buckets/main/collections/search-config/records", + "resource://app/defaults/settings/main/search-config.json", "stage-preview": -- "https://settings.stage.mozaws.net/v1/buckets/main-preview/collections/search-config/records", +- "https://firefox.settings.services.allizom.org/v1/buckets/main-preview/collections/search-config/records", + "resource://app/defaults/settings/main/search-config.json", }, // The following constants are left undocumented in nsISearchService.idl diff --git a/toolkit/components/search/docs/DefaultSearchEngines.rst b/toolkit/components/search/docs/DefaultSearchEngines.rst -index 1506af2913..5f43ff26f3 100644 +index 3dfe68abb1..26d5f18a7b 100644 --- a/toolkit/components/search/docs/DefaultSearchEngines.rst +++ b/toolkit/components/search/docs/DefaultSearchEngines.rst @@ -99,4 +99,4 @@ is updated. .. _configuration schema: SearchConfigurationSchema.html - .. _remote settings: /services/common/services/RemoteSettings.html + .. _remote settings: /services/settings/index.html -.. _search-default-override-allowlist bucket: https://firefox.settings.services.mozilla.com/v1/buckets/main/collections/search-default-override-allowlist/records +.. _search-default-override-allowlist bucket: resource://app/defaults/settings/main/search-default-override-allowlist.json diff --git a/toolkit/components/search/docs/SearchEngineConfiguration.rst b/toolkit/components/search/docs/SearchEngineConfiguration.rst -index e9041affb8..7a9466d294 100644 +index c782f9f7c3..4d773d27c5 100644 --- a/toolkit/components/search/docs/SearchEngineConfiguration.rst +++ b/toolkit/components/search/docs/SearchEngineConfiguration.rst @@ -68,5 +68,5 @@ related. As a result several situations may occur: @@ -145,44 +231,47 @@ index e9041affb8..7a9466d294 100644 -.. _viewed live: https://firefox.settings.services.mozilla.com/v1/buckets/main/collections/search-config/records +.. _viewed live: resource://app/defaults/settings/main/search-config.json .. _Normandy: /toolkit/components/normandy/normandy/services.html -diff --git a/toolkit/mozapps/defaultagent/RemoteSettings.cpp b/toolkit/mozapps/defaultagent/RemoteSettings.cpp -index 667d9fc628..b2bf628f29 100644 ---- a/toolkit/mozapps/defaultagent/RemoteSettings.cpp -+++ b/toolkit/mozapps/defaultagent/RemoteSettings.cpp -@@ -23,7 +23,7 @@ extern "C" { - HRESULT IsAgentRemoteDisabledRust(const char* szUrl, DWORD* lpdwDisabled); - } +diff --git a/toolkit/modules/AppConstants.sys.mjs b/toolkit/modules/AppConstants.sys.mjs +index 36b26056ec..ba54202045 100644 +--- a/toolkit/modules/AppConstants.sys.mjs ++++ b/toolkit/modules/AppConstants.sys.mjs +@@ -416,9 +416,9 @@ export var AppConstants = Object.freeze({ --#define PROD_ENDPOINT "https://firefox.settings.services.mozilla.com/v1" -+#define PROD_ENDPOINT "resource://app/defaults/settings" - #define PROD_BID "main" - #define PROD_CID "windows-default-browser-agent" - #define PROD_ID "state" + REMOTE_SETTINGS_SERVER_URL: + #ifdef MOZ_THUNDERBIRD +- "https://thunderbird-settings.thunderbird.net/v1", ++ "resource://app/defaults/settings", + #else +- "https://firefox.settings.services.mozilla.com/v1", ++ "resource://app/defaults/settings", + #endif + + REMOTE_SETTINGS_VERIFY_SIGNATURE: -- -2.35.1 +2.43.0 -From 99e7bd7760ebcd68575d19761d35a6db20ac8b48 Mon Sep 17 00:00:00 2001 +From 3536b16aa46bda48ac8692ca72e978837fcef796 Mon Sep 17 00:00:00 2001 From: grizzlyuser <grizzlyuser@protonmail.com> Date: Wed, 30 Dec 2020 17:34:08 +0200 Subject: [PATCH 02/13] Remove polling triggered by push broadcasts -When initialized, remote-settings.js adds a listener to push broadcasts, +When initialized, remote-settings.sys.mjs adds a listener to push broadcasts, that let Remote Settings server send push messages to trigger polling for changes from the client side. This is not needed for local-only setup. Remove the record from broadcast-listeners.json file stored in the user profile, so that it doesn't get picked up by push broadcast service. --- - dom/push/PushBroadcastService.jsm | 13 +++++++++++++ - services/settings/remote-settings.js | 7 ++----- - 2 files changed, 15 insertions(+), 5 deletions(-) - -diff --git a/dom/push/PushBroadcastService.jsm b/dom/push/PushBroadcastService.jsm -index aa1504211d..d635a2c3aa 100644 ---- a/dom/push/PushBroadcastService.jsm -+++ b/dom/push/PushBroadcastService.jsm -@@ -178,6 +178,19 @@ var BroadcastService = class { + dom/push/PushBroadcastService.sys.mjs | 13 +++++++++++++ + services/settings/remote-settings.sys.mjs | 11 ++--------- + 2 files changed, 15 insertions(+), 9 deletions(-) + +diff --git a/dom/push/PushBroadcastService.sys.mjs b/dom/push/PushBroadcastService.sys.mjs +index eea31ef192..529ecd6917 100644 +--- a/dom/push/PushBroadcastService.sys.mjs ++++ b/dom/push/PushBroadcastService.sys.mjs +@@ -168,6 +168,19 @@ export var BroadcastService = class { } } @@ -202,20 +291,24 @@ index aa1504211d..d635a2c3aa 100644 /** * Call the listeners of the specified broadcasts. * -diff --git a/services/settings/remote-settings.js b/services/settings/remote-settings.js -index 4307597351..64cad55929 100644 ---- a/services/settings/remote-settings.js -+++ b/services/settings/remote-settings.js -@@ -446,7 +446,7 @@ function remoteSettingsFunction() { - moduleURI: __URI__, +diff --git a/services/settings/remote-settings.sys.mjs b/services/settings/remote-settings.sys.mjs +index b93da0f0e4..8d59bc6b05 100644 +--- a/services/settings/remote-settings.sys.mjs ++++ b/services/settings/remote-settings.sys.mjs +@@ -575,11 +575,7 @@ function remoteSettingsFunction() { + moduleURI: import.meta.url, symbolName: "remoteSettingsBroadcastHandler", }; -- pushBroadcastService.addListener(BROADCAST_ID, currentVersion, moduleInfo); -+ pushBroadcastService.deleteListener(BROADCAST_ID); +- lazy.pushBroadcastService.addListener( +- BROADCAST_ID, +- currentVersion, +- moduleInfo +- ); ++ lazy.pushBroadcastService.deleteListener(BROADCAST_ID); }; return remoteSettings; -@@ -466,9 +466,6 @@ var remoteSettingsBroadcastHandler = { +@@ -599,9 +595,6 @@ export var remoteSettingsBroadcastHandler = { `Push notification received (version=${version} phase=${phase})` ); @@ -227,10 +320,10 @@ index 4307597351..64cad55929 100644 }, }; -- -2.35.1 +2.43.0 -From 52e9a179bc94e96fd8959b5fea9884ef9aa5fe77 Mon Sep 17 00:00:00 2001 +From 528c5d6d9437c003b731b3531273dbeb10422a0c Mon Sep 17 00:00:00 2001 From: grizzlyuser <grizzlyuser@protonmail.com> Date: Wed, 30 Dec 2020 17:41:54 +0200 Subject: [PATCH 03/13] Remove timer that triggers polling for changes @@ -242,7 +335,7 @@ That is not needed for local-only setup. 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/services/settings/components.conf b/services/settings/components.conf -index 9a737802ee..25109415a7 100644 +index fbc4df9bfc..25109415a7 100644 --- a/services/settings/components.conf +++ b/services/settings/components.conf @@ -4,11 +4,4 @@ @@ -253,7 +346,7 @@ index 9a737802ee..25109415a7 100644 - { - 'cid': '{5e756573-234a-49ea-bbe4-59ec7a70657d}', - 'contract_ids': ['@mozilla.org/services/settings;1'], -- 'jsm': 'resource://services-settings/RemoteSettingsComponents.jsm', +- 'esModule': 'resource://services-settings/RemoteSettingsComponents.sys.mjs', - 'constructor': 'RemoteSettingsTimer', - }, -] @@ -271,32 +364,32 @@ index 3bfed26ea4..807eb220ec 100644 -# see syntax https://searchfox.org/mozilla-central/rev/cc280c4be94ff8cf64a27cc9b3d6831ffa49fa45/toolkit/components/timermanager/UpdateTimerManager.jsm#155 -category update-timer RemoteSettingsComponents @mozilla.org/services/settings;1,getService,services-settings-poll-changes,services.settings.poll_interval,86400,259200 -- -2.35.1 +2.43.0 -From 04b7e9a6af031487d55eef867e581eb68c4d07c9 Mon Sep 17 00:00:00 2001 +From d13036fd21af770da7d305d058ecedd560afba2f Mon Sep 17 00:00:00 2001 From: grizzlyuser <grizzlyuser@protonmail.com> Date: Wed, 30 Dec 2020 17:47:41 +0200 Subject: [PATCH 04/13] Utils: fetch timestamps of each collection locally Utils.CHANGES_PATH points to -services/settings/dumps/monitor/changes.json +services/settings/dumps/monitor/changes which will be generated later by JSON processing script. Fetch the timestamps from that file and mock response headers to not confuse any code that expects them. --- browser/installer/package-manifest.in | 1 + - services/settings/Utils.jsm | 17 ++++++++++++++--- - services/settings/dumps/monitor/moz.build | 8 ++++++++ + services/settings/Utils.sys.mjs | 28 ++++++++++++++++++++--- + services/settings/dumps/monitor/moz.build | 8 +++++++ services/settings/dumps/moz.build | 1 + - 4 files changed, 24 insertions(+), 3 deletions(-) + 4 files changed, 35 insertions(+), 3 deletions(-) create mode 100644 services/settings/dumps/monitor/moz.build diff --git a/browser/installer/package-manifest.in b/browser/installer/package-manifest.in -index 73a41dc25b..04c2a3fb6a 100644 +index b3213b8c44..0f953c3425 100644 --- a/browser/installer/package-manifest.in +++ b/browser/installer/package-manifest.in -@@ -275,6 +275,7 @@ +@@ -258,6 +258,7 @@ @RESPATH@/browser/defaults/settings/last_modified.json @RESPATH@/browser/defaults/settings/blocklists @RESPATH@/browser/defaults/settings/main @@ -304,11 +397,11 @@ index 73a41dc25b..04c2a3fb6a 100644 @RESPATH@/browser/defaults/settings/security-state ; Warning: changing the path to channel-prefs.js can cause bugs (Bug 756325) -diff --git a/services/settings/Utils.jsm b/services/settings/Utils.jsm -index c09c58e693..f5605b1b16 100644 ---- a/services/settings/Utils.jsm -+++ b/services/settings/Utils.jsm -@@ -300,7 +300,7 @@ var Utils = { +diff --git a/services/settings/Utils.sys.mjs b/services/settings/Utils.sys.mjs +index 653cabe7e1..e4d4247253 100644 +--- a/services/settings/Utils.sys.mjs ++++ b/services/settings/Utils.sys.mjs +@@ -376,7 +376,7 @@ export var Utils = { async fetchLatestChanges(serverUrl, options = {}) { const { expectedTimestamp, lastEtag = "", filters = {} } = options; @@ -317,19 +410,30 @@ index c09c58e693..f5605b1b16 100644 const params = { ...filters, _expected: expectedTimestamp ?? 0, -@@ -315,7 +315,10 @@ var Utils = { +@@ -391,7 +391,21 @@ export var Utils = { .map(([k, v]) => `${k}=${encodeURIComponent(v)}`) .join("&"); } - const response = await Utils.fetch(url); -+ const response = await fetch(url); -+ const responseDate = new Date().toUTCString() -+ response.headers.set("Date", responseDate); -+ response.headers.set("Last-Modified", responseDate); ++ const rawResponse = await fetch(url); ++ const responseDate = new Date().toUTCString(); ++ ++ const headers = new Headers(); ++ headers.set("Date", responseDate); ++ headers.set("Last-Modified", responseDate); ++ headers.set("Content-Type", "application/json"); ++ ++ const responseAttributes = { ++ status: rawResponse.status, ++ statusText: rawResponse.statusText, ++ headers, ++ }; ++ ++ const response = new Response(rawResponse.body, responseAttributes); if (response.status >= 500) { throw new Error(`Server error ${response.status} ${response.statusText}`); -@@ -350,7 +353,15 @@ var Utils = { +@@ -426,7 +440,15 @@ export var Utils = { } } @@ -348,7 +452,7 @@ index c09c58e693..f5605b1b16 100644 // Since the response is served via a CDN, the Date header value could have been cached. diff --git a/services/settings/dumps/monitor/moz.build b/services/settings/dumps/monitor/moz.build new file mode 100644 -index 0000000000..d3d017fda5 +index 0000000000..25c53a2eeb --- /dev/null +++ b/services/settings/dumps/monitor/moz.build @@ -0,0 +1,8 @@ @@ -356,7 +460,7 @@ index 0000000000..d3d017fda5 +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + -+FINAL_TARGET_FILES.defaults.settings.monitor += ["changes.json"] ++FINAL_TARGET_FILES.defaults.settings.monitor += ["changes"] + +if CONFIG["MOZ_BUILD_APP"] == "browser": + DIST_SUBDIR = "browser" @@ -373,10 +477,10 @@ index f407580bfa..53e9d8b45e 100644 ] -- -2.35.1 +2.43.0 -From d2b5ce3db6af3e000f9ab7a10b2608e77cde0067 Mon Sep 17 00:00:00 2001 +From b2a061b8b79988eeef189af214fa03e4d7c76153 Mon Sep 17 00:00:00 2001 From: grizzlyuser <grizzlyuser@protonmail.com> Date: Wed, 30 Dec 2020 17:52:10 +0200 Subject: [PATCH 05/13] Utils: disable offline checking @@ -384,22 +488,23 @@ Subject: [PATCH 05/13] Utils: disable offline checking Since only local data is read now, it should always return false for the current and any future code that relies on it. --- - services/settings/Utils.jsm | 9 --------- - 1 file changed, 9 deletions(-) - -diff --git a/services/settings/Utils.jsm b/services/settings/Utils.jsm -index f5605b1b16..cd189f1500 100644 ---- a/services/settings/Utils.jsm -+++ b/services/settings/Utils.jsm -@@ -109,15 +109,6 @@ var Utils = { + services/settings/Utils.sys.mjs | 10 ---------- + 1 file changed, 10 deletions(-) + +diff --git a/services/settings/Utils.sys.mjs b/services/settings/Utils.sys.mjs +index e4d4247253..296001536c 100644 +--- a/services/settings/Utils.sys.mjs ++++ b/services/settings/Utils.sys.mjs +@@ -190,16 +190,6 @@ export var Utils = { * @return {bool} Whether network is down or not. */ get isOffline() { - try { - return ( - Services.io.offline || -- CaptivePortalService.state == CaptivePortalService.LOCKED_PORTAL || -- !gNetworkLinkService.isLinkUp +- lazy.CaptivePortalService.state == +- lazy.CaptivePortalService.LOCKED_PORTAL || +- !lazy.gNetworkLinkService.isLinkUp - ); - } catch (ex) { - log.warn("Could not determine network status.", ex); @@ -408,10 +513,10 @@ index f5605b1b16..cd189f1500 100644 }, -- -2.35.1 +2.43.0 -From 588cd28c3cd000501be66af573ac136e55d7e1ea Mon Sep 17 00:00:00 2001 +From 96d145c3ca83e835c4ec6711e75c01627e509ea0 Mon Sep 17 00:00:00 2001 From: grizzlyuser <grizzlyuser@protonmail.com> Date: Wed, 30 Dec 2020 17:56:02 +0200 Subject: [PATCH 06/13] Refactor hashing logic to a separate function @@ -419,30 +524,30 @@ Subject: [PATCH 06/13] Refactor hashing logic to a separate function It is used instead of internal signature validation mechanism, for integrity checking of the locally cached data. --- - services/settings/RemoteSettingsWorker.jsm | 4 ++++ - services/settings/SharedUtils.jsm | 9 +++++++-- + services/settings/RemoteSettingsWorker.sys.mjs | 4 ++++ + services/settings/SharedUtils.sys.mjs | 9 +++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) -diff --git a/services/settings/RemoteSettingsWorker.jsm b/services/settings/RemoteSettingsWorker.jsm -index 147ebb6b13..c86e218fd3 100644 ---- a/services/settings/RemoteSettingsWorker.jsm -+++ b/services/settings/RemoteSettingsWorker.jsm -@@ -189,6 +189,10 @@ class Worker { +diff --git a/services/settings/RemoteSettingsWorker.sys.mjs b/services/settings/RemoteSettingsWorker.sys.mjs +index 57bfb3f3d5..52a0b7961a 100644 +--- a/services/settings/RemoteSettingsWorker.sys.mjs ++++ b/services/settings/RemoteSettingsWorker.sys.mjs +@@ -177,6 +177,10 @@ class Worker { // task on the current thread instead of the worker thread. - return SharedUtils.checkContentHash(buffer, size, hash); + return lazy.SharedUtils.checkContentHash(buffer, size, hash); } + + async getContentHash(bytes) { -+ return SharedUtils.getContentHash(bytes); ++ return lazy.SharedUtils.getContentHash(bytes); + } } // Now, first add a shutdown blocker. If that fails, we must have -diff --git a/services/settings/SharedUtils.jsm b/services/settings/SharedUtils.jsm -index db5017a742..1a8e83c2e8 100644 ---- a/services/settings/SharedUtils.jsm -+++ b/services/settings/SharedUtils.jsm -@@ -28,11 +28,16 @@ var SharedUtils = { +diff --git a/services/settings/SharedUtils.sys.mjs b/services/settings/SharedUtils.sys.mjs +index 1eeaf0bed9..d998a2d3b1 100644 +--- a/services/settings/SharedUtils.sys.mjs ++++ b/services/settings/SharedUtils.sys.mjs +@@ -21,11 +21,16 @@ export var SharedUtils = { return false; } // Has expected content? @@ -462,10 +567,10 @@ index db5017a742..1a8e83c2e8 100644 /** -- -2.35.1 +2.43.0 -From bce803bdb3e4f7203042454c65c386db6f385439 Mon Sep 17 00:00:00 2001 +From b612fe1de7381746ac51694a4110268d0769ca6c Mon Sep 17 00:00:00 2001 From: grizzlyuser <grizzlyuser@protonmail.com> Date: Wed, 30 Dec 2020 18:05:02 +0200 Subject: [PATCH 07/13] Client: Fetch and hash records from local dump @@ -496,20 +601,20 @@ the upgrade to local-only setup. [1] https://firefox-source-docs.mozilla.org/services/common/services/RemoteSettings.html#initial-data --- - services/settings/RemoteSettingsClient.jsm | 62 ++++++++++------------ - 1 file changed, 27 insertions(+), 35 deletions(-) + .../settings/RemoteSettingsClient.sys.mjs | 66 ++++++++----------- + 1 file changed, 28 insertions(+), 38 deletions(-) -diff --git a/services/settings/RemoteSettingsClient.jsm b/services/settings/RemoteSettingsClient.jsm -index bd3fddbcf2..1c9dd2378f 100644 ---- a/services/settings/RemoteSettingsClient.jsm -+++ b/services/settings/RemoteSettingsClient.jsm -@@ -587,11 +587,9 @@ class RemoteSettingsClient extends EventEmitter { +diff --git a/services/settings/RemoteSettingsClient.sys.mjs b/services/settings/RemoteSettingsClient.sys.mjs +index 7e95b9baab..989a3180b8 100644 +--- a/services/settings/RemoteSettingsClient.sys.mjs ++++ b/services/settings/RemoteSettingsClient.sys.mjs +@@ -706,11 +706,9 @@ export class RemoteSettingsClient extends EventEmitter { // If the data is up-to-date but don't have metadata (records loaded from dump), // we fetch them and validate the signature immediately. -- if (this.verifySignature && ObjectUtils.isEmpty(localMetadata)) { -+ if (this.verifySignature && ObjectUtils.isEmpty(localMetadata?.json_dump_metadata)) { - console.debug(`${this.identifier} pull collection metadata`); +- if (this.verifySignature && lazy.ObjectUtils.isEmpty(localMetadata)) { ++ if (this.verifySignature && lazy.ObjectUtils.isEmpty(localMetadata?.json_dump_metadata)) { + lazy.console.debug(`${this.identifier} pull collection metadata`); - const metadata = await this.httpClient().getData({ - query: { _expected: expectedTimestamp }, - }); @@ -517,10 +622,10 @@ index bd3fddbcf2..1c9dd2378f 100644 await this.db.importChanges(metadata); // We don't bother validating the signature if the dump was just loaded. We do // if the dump was loaded at some other point (eg. from .get()). -@@ -844,32 +842,23 @@ class RemoteSettingsClient extends EventEmitter { +@@ -987,38 +985,28 @@ export class RemoteSettingsClient extends EventEmitter { + * @returns {Promise} + */ async _validateCollectionSignature(records, timestamp, metadata) { - const start = Cu.now() * 1000; - - if (!metadata?.signature) { + if (!metadata?.json_dump_metadata) { throw new MissingSignatureError(this.identifier); @@ -537,26 +642,34 @@ index bd3fddbcf2..1c9dd2378f 100644 - signature: { x5u, signature }, + json_dump_metadata: { hash, size }, } = metadata; -- const certChain = await (await Utils.fetch(x5u)).text(); +- const certChain = await (await lazy.Utils.fetch(x5u)).text(); // Merge remote records with local ones and serialize as canonical JSON. - const serialized = await RemoteSettingsWorker.canonicalStringify( + const serialized = await lazy.RemoteSettingsWorker.canonicalStringify( records, timestamp ); + +- lazy.console.debug(`${this.identifier} verify signature using ${x5u}`); ++ lazy.console.debug(`${this.identifier} verify signature using size ${size} and hash ${hash}`); if ( - !(await this._verifier.asyncVerifyContentSignature( - serialized, - "p384ecdsa=" + signature, - certChain, -- this.signerName -+ !(await RemoteSettingsWorker.checkContentHash( +- this.signerName, +- lazy.Utils.CERT_CHAIN_ROOT_IDENTIFIER ++ !(await lazy.RemoteSettingsWorker.checkContentHash( + new TextEncoder().encode(serialized), + size, + hash )) ) { - throw new InvalidSignatureError(this.identifier); -@@ -1061,24 +1050,27 @@ class RemoteSettingsClient extends EventEmitter { +- throw new InvalidSignatureError(this.identifier, x5u); ++ throw new InvalidSignatureError(this.identifier); + } + } + +@@ -1185,24 +1173,26 @@ export class RemoteSettingsClient extends EventEmitter { * @param since timestamp of last sync (optional) */ async _fetchChangeset(expectedTimestamp, since) { @@ -575,13 +688,13 @@ index bd3fddbcf2..1c9dd2378f 100644 - _since: since, - }, - } -+ const { data } = await SharedUtils.loadJSONDump( ++ const { data } = await lazy.SharedUtils.loadJSONDump( + this.bucketName, + this.collectionName ); + const remoteRecords = data ?? []; + -+ const serialized = await RemoteSettingsWorker.canonicalStringify( ++ const serialized = await lazy.RemoteSettingsWorker.canonicalStringify( + remoteRecords, + expectedTimestamp + ); @@ -589,11 +702,10 @@ index bd3fddbcf2..1c9dd2378f 100644 + const metadata = { + app_build_id: Services.appinfo.appBuildID, + json_dump_metadata: { -+ hash: await RemoteSettingsWorker.getContentHash(bytes), ++ hash: await lazy.RemoteSettingsWorker.getContentHash(bytes), + size: bytes.length, + }, + } -+ return { - remoteTimestamp, + remoteTimestamp: expectedTimestamp, @@ -601,15 +713,15 @@ index bd3fddbcf2..1c9dd2378f 100644 remoteRecords, }; -- -2.35.1 +2.43.0 -From dd4897ba2ece2d0deb23953c044c44323d9934d0 Mon Sep 17 00:00:00 2001 +From 9eaa21162fc1e2abeb24381edd4bea01fd103065 Mon Sep 17 00:00:00 2001 From: grizzlyuser <grizzlyuser@protonmail.com> Date: Wed, 30 Dec 2020 18:42:56 +0200 Subject: [PATCH 08/13] Client: start deferred sync on get() or on() -The users of the RemoteSettingsClient.jsm can receive records from it in +The users of the RemoteSettingsClient.sys.mjs can receive records from it in two ways: by calling get(), and by subscribing to events by calling on(). @@ -622,46 +734,43 @@ false, it triggers a sync. Then adds a flag if the client has been correctly synchronized with the dump, so that no metadata checking occurs during the session. --- - services/settings/RemoteSettingsClient.jsm | 30 +++++++++++++++++++++- - 1 file changed, 29 insertions(+), 1 deletion(-) - -diff --git a/services/settings/RemoteSettingsClient.jsm b/services/settings/RemoteSettingsClient.jsm -index 1c9dd2378f..f7a0b3bdaa 100644 ---- a/services/settings/RemoteSettingsClient.jsm -+++ b/services/settings/RemoteSettingsClient.jsm -@@ -16,6 +16,7 @@ XPCOMUtils.defineLazyModuleGetters(this, { + .../settings/RemoteSettingsClient.sys.mjs | 27 ++++++++++++++++++- + 1 file changed, 26 insertions(+), 1 deletion(-) + +diff --git a/services/settings/RemoteSettingsClient.sys.mjs b/services/settings/RemoteSettingsClient.sys.mjs +index 989a3180b8..c2c2a5734b 100644 +--- a/services/settings/RemoteSettingsClient.sys.mjs ++++ b/services/settings/RemoteSettingsClient.sys.mjs +@@ -12,6 +12,7 @@ ChromeUtils.defineESModuleGetters(lazy, { ClientEnvironmentBase: - "resource://gre/modules/components-utils/ClientEnvironment.jsm", - Database: "resource://services-settings/Database.jsm", -+ DeferredTask: "resource://gre/modules/DeferredTask.jsm", - Downloader: "resource://services-settings/Attachments.jsm", - IDBHelpers: "resource://services-settings/IDBHelpers.jsm", - KintoHttpClient: "resource://services-common/kinto-http-client.js", -@@ -28,6 +29,7 @@ XPCOMUtils.defineLazyModuleGetters(this, { + "resource://gre/modules/components-utils/ClientEnvironment.sys.mjs", + Database: "resource://services-settings/Database.sys.mjs", ++ DeferredTask: "resource://gre/modules/DeferredTask.sys.mjs", + IDBHelpers: "resource://services-settings/IDBHelpers.sys.mjs", + KintoHttpClient: "resource://services-common/kinto-http-client.sys.mjs", + ObjectUtils: "resource://gre/modules/ObjectUtils.sys.mjs", +@@ -23,6 +24,7 @@ ChromeUtils.defineESModuleGetters(lazy, { }); const TELEMETRY_COMPONENT = "remotesettings"; +const DEFERRED_SYNC_DELAY_MILLISECONDS = 1000; - XPCOMUtils.defineLazyGetter(this, "console", () => Utils.log); + ChromeUtils.defineLazyGetter(lazy, "console", () => lazy.Utils.log); -@@ -259,6 +261,14 @@ class RemoteSettingsClient extends EventEmitter { +@@ -334,6 +336,11 @@ export class RemoteSettingsClient extends EventEmitter { this._lastCheckTimePref = lastCheckTimePref; this._verifier = null; this._syncRunning = false; -+ this._deferredSync = new DeferredTask( -+ async () => { -+ if (!this._syncRunning && !(await this._isSynced())) { -+ await this.sync(); -+ } -+ }, -+ DEFERRED_SYNC_DELAY_MILLISECONDS -+ ); ++ this._deferredSync = new lazy.DeferredTask(async () => { ++ if (!this._syncRunning && !(await this._isSynced())) { ++ await this.sync(); ++ } ++ }, DEFERRED_SYNC_DELAY_MILLISECONDS); // This attribute allows signature verification to be disabled, when running tests // or when pulling data from a dev server. -@@ -293,6 +303,11 @@ class RemoteSettingsClient extends EventEmitter { - ); +@@ -363,6 +370,11 @@ export class RemoteSettingsClient extends EventEmitter { + this.db.identifier = this.identifier; } + on(event, callback) { @@ -672,18 +781,18 @@ index 1c9dd2378f..f7a0b3bdaa 100644 get identifier() { return `${this.bucketName}/${this.collectionName}`; } -@@ -361,6 +376,10 @@ class RemoteSettingsClient extends EventEmitter { - let lastModified = await this.db.getLastModified(); +@@ -436,6 +448,10 @@ export class RemoteSettingsClient extends EventEmitter { + let lastModified = forceSync ? null : await this.db.getLastModified(); let hasLocalData = lastModified !== null; + if (!(await this._isSynced())) { + throw new MissingSignatureError(this.identifier); + } + - if (syncIfEmpty && !hasLocalData) { - // .get() was called before we had the chance to synchronize the local database. - // We'll try to avoid returning an empty list. -@@ -445,7 +464,10 @@ class RemoteSettingsClient extends EventEmitter { + if (forceSync) { + if (!this._importingPromise) { + this._importingPromise = (async () => { +@@ -551,7 +567,10 @@ export class RemoteSettingsClient extends EventEmitter { // No need to verify signature on JSON dumps. // If local DB cannot be read, then we don't even try to do anything, // we return results early. @@ -694,14 +803,14 @@ index 1c9dd2378f..f7a0b3bdaa 100644 + return filtered; } - console.debug( -@@ -483,6 +505,12 @@ class RemoteSettingsClient extends EventEmitter { + lazy.console.debug( +@@ -591,6 +610,12 @@ export class RemoteSettingsClient extends EventEmitter { return final; } + async _isSynced() { + this._synced ||= -+ Services.appinfo.appBuildID === (await this.db.getMetadata())?.app_build_id; ++ Services.appinfo.appBuildID === (await this.db?.getMetadata())?.app_build_id; + return this._synced; + } + @@ -709,10 +818,10 @@ index 1c9dd2378f..f7a0b3bdaa 100644 * Synchronize the local database with the remote server. * -- -2.35.1 +2.43.0 -From 192096cd8caa2cbdfdc294a71056e6ee1b7e513d Mon Sep 17 00:00:00 2001 +From 4598808209c3a9a8cd3902053b250c88861bced2 Mon Sep 17 00:00:00 2001 From: grizzlyuser <grizzlyuser@protonmail.com> Date: Wed, 30 Dec 2020 18:53:51 +0200 Subject: [PATCH 09/13] Client: deep compare records if timestamps match @@ -730,38 +839,38 @@ on the safe side. This should happen only once after upgrading to each new version of the application, so is not likely to introduce any noticeable performance issues. --- - services/settings/RemoteSettingsClient.jsm | 6 +++++- + services/settings/RemoteSettingsClient.sys.mjs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) -diff --git a/services/settings/RemoteSettingsClient.jsm b/services/settings/RemoteSettingsClient.jsm -index f7a0b3bdaa..64ac27d149 100644 ---- a/services/settings/RemoteSettingsClient.jsm -+++ b/services/settings/RemoteSettingsClient.jsm -@@ -13,6 +13,7 @@ const { Services } = ChromeUtils.import("resource://gre/modules/Services.jsm"); +diff --git a/services/settings/RemoteSettingsClient.sys.mjs b/services/settings/RemoteSettingsClient.sys.mjs +index c2c2a5734b..ee612a049c 100644 +--- a/services/settings/RemoteSettingsClient.sys.mjs ++++ b/services/settings/RemoteSettingsClient.sys.mjs +@@ -9,6 +9,7 @@ import { Downloader } from "resource://services-settings/Attachments.sys.mjs"; + const lazy = {}; - XPCOMUtils.defineLazyModuleGetters(this, { - AppConstants: "resource://gre/modules/AppConstants.jsm", -+ CanonicalJSON: "resource://gre/modules/CanonicalJSON.jsm", + ChromeUtils.defineESModuleGetters(lazy, { ++ CanonicalJSON: "resource://gre/modules/CanonicalJSON.sys.mjs", ClientEnvironmentBase: - "resource://gre/modules/components-utils/ClientEnvironment.jsm", - Database: "resource://services-settings/Database.jsm", -@@ -1053,7 +1054,10 @@ class RemoteSettingsClient extends EventEmitter { + "resource://gre/modules/components-utils/ClientEnvironment.sys.mjs", + Database: "resource://services-settings/Database.sys.mjs", +@@ -1173,7 +1174,10 @@ export class RemoteSettingsClient extends EventEmitter { const old = oldById.get(r.id); if (old) { oldById.delete(r.id); - if (r.last_modified != old.last_modified) { + if ( + r.last_modified != old.last_modified || -+ CanonicalJSON.stringify(r) != CanonicalJSON.stringify(old) ++ lazy.CanonicalJSON.stringify(r) != lazy.CanonicalJSON.stringify(old) + ) { syncResult.updated.push({ old, new: r }); } } else { -- -2.35.1 +2.43.0 -From c834cd3beeb989f30c88aabea480d08989c02abc Mon Sep 17 00:00:00 2001 +From 79445efeb9c18ad9c2e2c4070cdc72aa6b736d1e Mon Sep 17 00:00:00 2001 From: grizzlyuser <grizzlyuser@protonmail.com> Date: Wed, 30 Dec 2020 19:01:39 +0200 Subject: [PATCH 10/13] Client: delete more data on cleanup @@ -770,40 +879,37 @@ When the client detects the local data is invalid (i.e. it came from real Remote Settings and can have unwanted records), delete not only the records, but also the attachments that came with them, because they too can be problematic. And last check time preference, because it's not -useful anyway when remote-settings.js doesn't do any polling for changes. +useful anyway when remote-settings.sys.mjs doesn't do any polling for changes. Note that attachments should be deleted before the records, because the logic gets the data about the attachments from those records. --- - services/settings/RemoteSettingsClient.jsm | 15 ++++++++++++--- - 1 file changed, 12 insertions(+), 3 deletions(-) - -diff --git a/services/settings/RemoteSettingsClient.jsm b/services/settings/RemoteSettingsClient.jsm -index 64ac27d149..ac6259ca2c 100644 ---- a/services/settings/RemoteSettingsClient.jsm -+++ b/services/settings/RemoteSettingsClient.jsm -@@ -219,7 +219,10 @@ class AttachmentDownloader extends Downloader { - async deleteAll() { - let allRecords = await this._client.db.list(); - return Promise.all( -- allRecords.filter(r => !!r.attachment).map(r => this.delete(r)) -+ allRecords.filter(r => !!r.attachment).map(r => { -+ this.delete(r); -+ this.deleteCached(r.id); -+ }) + services/settings/RemoteSettingsClient.sys.mjs | 12 +++++++++--- + 1 file changed, 9 insertions(+), 3 deletions(-) + +diff --git a/services/settings/RemoteSettingsClient.sys.mjs b/services/settings/RemoteSettingsClient.sys.mjs +index ee612a049c..b3c6caebba 100644 +--- a/services/settings/RemoteSettingsClient.sys.mjs ++++ b/services/settings/RemoteSettingsClient.sys.mjs +@@ -255,7 +255,7 @@ class AttachmentDownloader extends Downloader { + allRecords + .filter(r => !!r.attachment) + .map(r => +- Promise.all([this.deleteDownloaded(r), this.deleteFromDisk(r)]) ++ Promise.all([this.deleteDownloaded(r), this.deleteFromDisk(r), this.deleteCached(r.id)]) + ) ); } - } -@@ -1013,7 +1016,7 @@ class RemoteSettingsClient extends EventEmitter { +@@ -1133,7 +1133,7 @@ export class RemoteSettingsClient extends EventEmitter { // Signature failed, clear local DB because it contains // bad data (local + remote changes). - console.debug(`${this.identifier} clear local data`); + lazy.console.debug(`${this.identifier} clear local data`); - await this.db.clear(); + await this._clearAll(); // Local data was tampered, throw and it will retry from empty DB. - console.error(`${this.identifier} local data was corrupted`); + lazy.console.error(`${this.identifier} local data was corrupted`); throw new CorruptedDataError(this.identifier); -@@ -1035,7 +1038,7 @@ class RemoteSettingsClient extends EventEmitter { +@@ -1155,7 +1155,7 @@ export class RemoteSettingsClient extends EventEmitter { // _importJSONDump() only clears DB if dump is available, // therefore do it here! if (imported < 0) { @@ -812,7 +918,7 @@ index 64ac27d149..ac6259ca2c 100644 } } } -@@ -1075,6 +1078,12 @@ class RemoteSettingsClient extends EventEmitter { +@@ -1195,6 +1195,12 @@ export class RemoteSettingsClient extends EventEmitter { return syncResult; } @@ -826,10 +932,10 @@ index 64ac27d149..ac6259ca2c 100644 * Fetch information from changeset endpoint. * -- -2.35.1 +2.43.0 -From 68ee3019155c605ea3fefd9370209d548d1049c2 Mon Sep 17 00:00:00 2001 +From 24ba3c8ca82381eba887a318fb09f6d24e07064c Mon Sep 17 00:00:00 2001 From: grizzlyuser <grizzlyuser@protonmail.com> Date: Wed, 30 Dec 2020 19:07:56 +0200 Subject: [PATCH 11/13] Client: remove comparison of collection timestamps @@ -840,67 +946,66 @@ is newer than the packaged local dump, then this comparison logic can lead to early return of old data, skipping the integrity checking and necessary cleanup. So remove the checks. --- - services/settings/RemoteSettingsClient.jsm | 5 ----- + services/settings/RemoteSettingsClient.sys.mjs | 5 ----- 1 file changed, 5 deletions(-) -diff --git a/services/settings/RemoteSettingsClient.jsm b/services/settings/RemoteSettingsClient.jsm -index ac6259ca2c..e59845a337 100644 ---- a/services/settings/RemoteSettingsClient.jsm -+++ b/services/settings/RemoteSettingsClient.jsm -@@ -948,14 +948,9 @@ class RemoteSettingsClient extends EventEmitter { +diff --git a/services/settings/RemoteSettingsClient.sys.mjs b/services/settings/RemoteSettingsClient.sys.mjs +index b3c6caebba..264a6cc831 100644 +--- a/services/settings/RemoteSettingsClient.sys.mjs ++++ b/services/settings/RemoteSettingsClient.sys.mjs +@@ -1075,14 +1075,9 @@ export class RemoteSettingsClient extends EventEmitter { updated: [], deleted: [], }; - // If data wasn't changed, return empty sync result. - // This can happen when we update the signature but not the data. - console.debug( + lazy.console.debug( `${this.identifier} local timestamp: ${localTimestamp}, remote: ${remoteTimestamp}` ); - if (localTimestamp && remoteTimestamp < localTimestamp) { - return syncResult; - } - const start = Cu.now() * 1000; await this.db.importChanges(metadata, remoteTimestamp, remoteRecords, { + clear: retry, -- -2.35.1 +2.43.0 -From aa15f1fee8030b3a315312fba2f1b19e6694aad4 Mon Sep 17 00:00:00 2001 +From 633d8d40e00786b388e62898e32414d1305fa91f Mon Sep 17 00:00:00 2001 From: grizzlyuser <grizzlyuser@protonmail.com> Date: Wed, 30 Dec 2020 19:15:44 +0200 Subject: [PATCH 12/13] Attachments: load only from dump and drop cached --- - services/settings/Attachments.jsm | 35 +++++++------------------------ - 1 file changed, 8 insertions(+), 27 deletions(-) - -diff --git a/services/settings/Attachments.jsm b/services/settings/Attachments.jsm -index 70c02627ac..d24d8907fd 100644 ---- a/services/settings/Attachments.jsm -+++ b/services/settings/Attachments.jsm -@@ -141,10 +141,11 @@ class Downloader { + services/settings/Attachments.sys.mjs | 37 ++++++--------------------- + 1 file changed, 8 insertions(+), 29 deletions(-) + +diff --git a/services/settings/Attachments.sys.mjs b/services/settings/Attachments.sys.mjs +index 5ddc6bb046..f7888cb769 100644 +--- a/services/settings/Attachments.sys.mjs ++++ b/services/settings/Attachments.sys.mjs +@@ -147,9 +147,10 @@ export class Downloader { + retries, checkHash, attachmentId = record?.id, - useCache = false, - fallbackToCache = false, fallbackToDump = false, } = options || {}; - + const fallbackToCache = false; + - if (!useCache) { - // For backwards compatibility. - // WARNING: Its return type is different from what's documented. -@@ -204,6 +205,7 @@ class Downloader { + if (!attachmentId) { + // Check for pre-condition. This should not happen, but it is explicitly + // checked to avoid mixing up attachments, which could be dangerous. +@@ -200,6 +201,7 @@ export class Downloader { const newBuffer = await this.downloadAsBytes(record, { retries, checkHash, + dumpInfo, }); const blob = new Blob([newBuffer]); - if (useCache) { -@@ -239,7 +241,7 @@ class Downloader { + // Store in cache but don't wait for it before returning. +@@ -233,7 +235,7 @@ export class Downloader { } try { @@ -908,8 +1013,8 @@ index 70c02627ac..d24d8907fd 100644 + await this.cacheImpl.delete(attachmentId); } catch (e) { // Failed to read from cache, e.g. IndexedDB unusable. - Cu.reportError(e); -@@ -276,7 +278,7 @@ class Downloader { + console.error(e); +@@ -318,7 +320,7 @@ export class Downloader { * @returns {String} the absolute file path to the downloaded attachment. */ async downloadToDisk(record, options = {}) { @@ -918,7 +1023,7 @@ index 70c02627ac..d24d8907fd 100644 const { attachment: { filename, size, hash }, } = record; -@@ -333,31 +335,10 @@ class Downloader { +@@ -373,33 +375,10 @@ export class Downloader { */ async downloadAsBytes(record, options = {}) { const { @@ -937,7 +1042,9 @@ index 70c02627ac..d24d8907fd 100644 - if (!checkHash) { - return buffer; - } -- if (await RemoteSettingsWorker.checkContentHash(buffer, size, hash)) { +- if ( +- await lazy.RemoteSettingsWorker.checkContentHash(buffer, size, hash) +- ) { - return buffer; - } - // Content is corrupted. @@ -954,10 +1061,10 @@ index 70c02627ac..d24d8907fd 100644 /** -- -2.35.1 +2.43.0 -From 271e88d8a18a6ae30ccc616c57fe7a36290ba520 Mon Sep 17 00:00:00 2001 +From c0b96623a42ed3c74218fcc2574f78196fd901ff Mon Sep 17 00:00:00 2001 From: grizzlyuser <grizzlyuser@protonmail.com> Date: Wed, 30 Dec 2020 19:22:20 +0200 Subject: [PATCH 13/13] Disable CRLite entirely for now @@ -968,32 +1075,44 @@ within minutes. That won't work with local-only setup. Although (some?) of the JSON dumps for it are in place, obviously the updates won't happen that fast. -Right now CRLite doesn't enforce anything, and works just for telemetry -collection (which is hopefully disabled anyway). So disable the -preference right in the source code, so that the patch fails to apply -when the upstream decides to set it to enforcing mode by default. +Disable the preference right in the source code, so that the patch fails +to apply when the upstream decides to set it to fully enforcing mode +by default. The solution with CRLite is up for discussion. If necessary, it's possible to make clients for blessed collections to communicate to real Remote Settings server. For example, for collections related to certificate revocations. --- - modules/libpref/init/all.js | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/modules/libpref/init/all.js b/modules/libpref/init/all.js -index cb72224e1b..79940b598a 100644 ---- a/modules/libpref/init/all.js -+++ b/modules/libpref/init/all.js -@@ -160,7 +160,7 @@ pref("security.xfocsp.errorReporting.automatic", false); - // 1: Consult CRLite but only collect telemetry. - // 2: Consult CRLite and enforce both "Revoked" and "Not Revoked" results. - // 3: Consult CRLite and enforce "Not Revoked" results, but defer to OCSP for "Revoked". --pref("security.pki.crlite_mode", 1); -+pref("security.pki.crlite_mode", 0); - - // Issuer we use to detect MitM proxies. Set to the issuer of the cert of the - // Firefox update service. The string format is whatever NSS uses to print a DN. + modules/libpref/init/StaticPrefList.yaml | 2 +- + security/manager/ssl/nsNSSComponent.cpp | 1 + + 2 files changed, 2 insertions(+), 1 deletion(-) + +diff --git a/modules/libpref/init/StaticPrefList.yaml b/modules/libpref/init/StaticPrefList.yaml +index 592ddd3229..9688ee64fd 100644 +--- a/modules/libpref/init/StaticPrefList.yaml ++++ b/modules/libpref/init/StaticPrefList.yaml +@@ -14167,7 +14167,7 @@ + # 3: Consult CRLite and enforce "Not Revoked" results, but defer to OCSP for "Revoked". + - name: security.pki.crlite_mode + type: RelaxedAtomicUint32 +- value: 3 ++ value: 0 + mirror: always + + - name: security.tls.version.min +diff --git a/security/manager/ssl/nsNSSComponent.cpp b/security/manager/ssl/nsNSSComponent.cpp +index dd14b5c24c..90350be691 100644 +--- a/security/manager/ssl/nsNSSComponent.cpp ++++ b/security/manager/ssl/nsNSSComponent.cpp +@@ -1129,6 +1129,7 @@ void nsNSSComponent::setValidationOptions( + CRLiteMode defaultCRLiteMode = CRLiteMode::Disabled; + CRLiteMode crliteMode = + static_cast<CRLiteMode>(StaticPrefs::security_pki_crlite_mode()); ++ // Adding a comment just in case so that the patch breaks as soon as any surrounding lines get changed + switch (crliteMode) { + case CRLiteMode::Disabled: + case CRLiteMode::TelemetryOnly: -- -2.35.1 +2.43.0 |