Support fetching assets from CDNs when auto-updating

gorhill · gorhill · commit 4687c60bf9ce · 2020-04-08T09:57:55.000-04:00
This commit add the ability to fetch from CDN servers
when an asset is fetched as a result of auto-update.

If an asset has a `cdnURLs` entry in `assets.json`,
the asset will be auto-updated using one of those
CDN URLs. When many CDN URLs are specified, those
URLs will be shuffled in order to spread the bandwidth
across all specified CDN servers. If all specified CDN
servers fail to respond, uBO will fall back to usual
`contentURLs` entry.

The `cdnURLs` are used only when an asset is
auto-updated, this ensures a user will get the more
recent available version of an asset when manually
updating.

The motivation of this new feature is to relieve
GitHub from acting as a CDN (which it is not) for
uBO -- an increasing concern with the growing adoption
of uBO along with the growing size of key uBO assets.
diff --git a/src/js/assets.js b/src/js/assets.js
@@ -33,6 +33,10 @@ const errorCantConnectTo = vAPI.i18n('errorCantConnectTo');
 
 const api = {};
 
+// A hint for various pieces of code to take measures if possible to save
+// bandwidth of remote servers.
+let remoteServerFriendly = false;
+
 /******************************************************************************/
 
 const observers = [];
@@ -157,14 +161,15 @@ api.fetchText = async function(url) {
     // https://github.com/gorhill/uBlock/issues/2592
     //   Force browser cache to be bypassed, but only for resources which have
     //   been fetched more than one hour ago.
-    //
     // https://github.com/uBlockOrigin/uBlock-issues/issues/682#issuecomment-515197130
     //   Provide filter list authors a way to completely bypass
     //   the browser cache.
     // https://github.com/gorhill/uBlock/commit/048bfd251c9b#r37972005
     //   Use modulo prime numbers to avoid generating the same token at the
     //   same time across different days.
-    if ( isExternal ) {
+    // Do not bypass browser cache if we are asked to be gentle on remote
+    // servers.
+    if ( isExternal && remoteServerFriendly !== true ) {
         const cacheBypassToken =
             µBlock.hiddenSettings.updateAssetBypassBrowserCache
                 ? Math.floor(Date.now() /    1000) % 86413
@@ -743,6 +748,19 @@ const getRemote = async function(assetKey) {
         contentURLs = assetDetails.contentURL.slice(0);
     }
 
+    // If asked to be gentle on remote servers, favour using dedicated CDN
+    // servers. If more than one CDN server is present, randomly shuffle the
+    // set of servers so as to spread the bandwidth burden.
+    if ( remoteServerFriendly && Array.isArray(assetDetails.cdnURLs) ) {
+        const cdnURLs = assetDetails.cdnURLs.slice();
+        for ( let i = 0, n = cdnURLs.length; i < n; i++ ) {
+            const j = Math.floor(Math.random() * n);
+            if ( j === i ) { continue; }
+            [ cdnURLs[j], cdnURLs[i] ] = [ cdnURLs[i], cdnURLs[j] ];
+        }
+        contentURLs.unshift(...cdnURLs);
+    }
+
     for ( const contentURL of contentURLs ) {
         if ( reIsExternalPath.test(contentURL) === false ) { continue; }
 
@@ -756,18 +774,17 @@ const getRemote = async function(assetKey) {
             if ( result.statusCode === 0 ) {
                 error = 'network error';
             }
-            registerAssetSource(
-                assetKey,
-                { error: { time: Date.now(), error } }
-            );
+            registerAssetSource(assetKey, {
+                error: { time: Date.now(), error }
+            });
             continue;
         }
 
         // Success
-        assetCacheWrite(
-            assetKey,
-            { content: result.content, url: contentURL }
-        );
+        assetCacheWrite(assetKey, {
+            content: result.content,
+            url: contentURL
+        });
         registerAssetSource(assetKey, { error: undefined });
         return reportBack(result.content);
     }
@@ -835,9 +852,10 @@ const updaterAssetDelayDefault = 120000;
 const updaterUpdated = [];
 const updaterFetched = new Set();
 
-let updaterStatus,
-    updaterTimer,
-    updaterAssetDelay = updaterAssetDelayDefault;
+let updaterStatus;
+let updaterTimer;
+let updaterAssetDelay = updaterAssetDelayDefault;
+let updaterAuto = false;
 
 const updateFirst = function() {
     updaterStatus = 'updating';
@@ -861,25 +879,22 @@ const updateNext = async function() {
         if ( updaterFetched.has(assetKey) ) { continue; }
         const cacheEntry = cacheDict[assetKey];
         if (
-            cacheEntry &&
+            (cacheEntry instanceof Object) &&
             (cacheEntry.writeTime + assetEntry.updateAfter * 86400000) > now
         ) {
             continue;
         }
         if (
-            fireNotification(
-                'before-asset-updated',
-                { assetKey: assetKey,  type: assetEntry.content }
-            ) === true
+            fireNotification('before-asset-updated', {
+                assetKey,
+                type: assetEntry.content
+            }) === true
         ) {
             assetKeyToUpdate = assetKey;
             break;
         }
         // This will remove a cached asset when it's no longer in use.
-        if (
-            cacheEntry &&
-            cacheEntry.readTime < assetCacheRegistryStartTime
-        ) {
+        if ( cacheEntry && cacheEntry.readTime < assetCacheRegistryStartTime ) {
             assetCacheRemove(assetKey);
         }
     }
@@ -888,7 +903,13 @@ const updateNext = async function() {
     }
     updaterFetched.add(assetKeyToUpdate);
 
+    // In auto-update context, be gentle on remote servers.
+    remoteServerFriendly = updaterAuto;
+
     const result = await getRemote(assetKeyToUpdate);
+
+    remoteServerFriendly = false;
+
     if ( result.content !== '' ) {
         updaterUpdated.push(result.assetKey);
         if ( result.assetKey === 'assets.json' ) {
@@ -912,10 +933,11 @@ const updateDone = function() {
 
 api.updateStart = function(details) {
     const oldUpdateDelay = updaterAssetDelay;
-    const newUpdateDelay = typeof details.delay === 'number' ?
-        details.delay :
-        updaterAssetDelayDefault;
+    const newUpdateDelay = typeof details.delay === 'number'
+        ? details.delay
+        : updaterAssetDelayDefault;
     updaterAssetDelay = Math.min(oldUpdateDelay, newUpdateDelay);
+    updaterAuto = details.auto === true;
     if ( updaterStatus !== undefined ) {
         if ( newUpdateDelay < oldUpdateDelay ) {
             clearTimeout(updaterTimer);
diff --git a/src/js/storage.js b/src/js/storage.js
@@ -1144,6 +1144,8 @@ self.addEventListener('hiddenSettingsChanged', ( ) => {
         const json = await vAPI.adminStorage.getItem('adminSettings');
         if ( typeof json === 'string' && json !== '' ) {
             data = JSON.parse(json);
+        } else if ( json instanceof Object ) {
+            data = json;
         }
     } catch (ex) {
         console.error(ex);
@@ -1247,7 +1249,7 @@ self.addEventListener('hiddenSettingsChanged', ( ) => {
 
 /******************************************************************************/
 
-µBlock.scheduleAssetUpdater = (function() {
+µBlock.scheduleAssetUpdater = (( ) => {
     let timer, next = 0;
 
     return function(updateDelay) {
@@ -1271,7 +1273,8 @@ self.addEventListener('hiddenSettingsChanged', ( ) => {
             next = 0;
             this.assets.updateStart({
                 delay: this.hiddenSettings.autoUpdateAssetFetchPeriod * 1000 ||
-                       120000
+                       120000,
+                auto: true,
             });
         }, updateDelay);
     };
