Fine tune hostname uncloaking through CNAME-lookup

gorhill · gorhill · commit a16e4161de5b · 2019-11-23T13:07:23.000-05:00
Related issue: - uBlockOrigin/uBlock-issues#780 Related commit: - 3a564c199260 This adds two new advanced settings: - cnameIgnoreRootDocument - Default to `true` - Tells uBO to skip CNAME-lookup for root document. - cnameReplayFullURL - Default to `false` - Tells uBO whether to replay the whole URL or just the origin part of it. Replaying only the origin part is meant to lower undue breakage and improve performance by avoiding repeating the pattern-matching of the whole URL -- which pattern-matching was most likely already accomplished with the original request. This commit is meant to explore enabling CNAME-lookup by default for the next stable release while: - Eliminating a development burden by removing the need to create a new filtering syntax to deal with undesirable CNAME-cloaked hostnames - Eliminating a filter list maintainer burden by removing the need to find/deal with all base domains which engage in undesirable CNAME-cloaked hostnames The hope is that the approach implemented in this commit should require at most a few unbreak rules with no further need for special filtering syntax or filter list maintance efforts.
diff --git a/platform/chromium/vapi-background.js b/platform/chromium/vapi-background.js
@@ -1259,18 +1259,29 @@ vAPI.Net = class {
             console.info('No requests found to benchmark');
             return;
         }
+        const mappedTypes = new Map([
+            [ 'document', 'main_frame' ],
+            [ 'subdocument', 'sub_frame' ],
+        ]);
         console.info('vAPI.net.onBeforeSuspendableRequest()...');
         const t0 = self.performance.now();
         const promises = [];
+        const details = {
+            documentUrl: '',
+            tabId: -1,
+            parentFrameId: -1,
+            frameId: 0,
+            type: '',
+            url: '',
+        };
         for ( const request of requests ) {
-            const details = {
-                documentUrl: request.frameUrl,
-                tabId: Number.MAX_SAFE_INTEGER,
-                parentFrameId: -1,
-                frameId: 0,
-                type: request.cpt,
-                url: request.url,
-            };
+            details.documentUrl = request.frameUrl;
+            details.tabId = -1;
+            details.parentFrameId = -1;
+            details.frameId = 0;
+            details.type = mappedTypes.get(request.cpt) || request.cpt;
+            details.url = request.url;
+            if ( details.type === 'main_frame' ) { continue; }
             promises.push(this.onBeforeSuspendableRequest(details));
         }
         return Promise.all(promises).then(results => {
diff --git a/platform/firefox/vapi-webrequest.js b/platform/firefox/vapi-webrequest.js
@@ -63,16 +63,20 @@
             this.cnames = new Map([ [ '', '' ] ]);
             this.cnameAliasList = null;
             this.cnameIgnoreList = null;
-            this.url = new URL(vAPI.getURL('/'));
+            this.cnameIgnore1stParty = true;
+            this.cnameIgnoreRootDocument = true;
             this.cnameMaxTTL = 60;
+            this.cnameReplayFullURL = false;
             this.cnameTimer = undefined;
         }
         setOptions(options) {
             super.setOptions(options);
             this.cnameAliasList = this.regexFromStrList(options.cnameAliasList);
             this.cnameIgnoreList = this.regexFromStrList(options.cnameIgnoreList);
-            this.cnameIgnore1stParty = options.cnameIgnore1stParty === true;
+            this.cnameIgnore1stParty = options.cnameIgnore1stParty !== false;
+            this.cnameIgnoreRootDocument = options.cnameIgnoreRootDocument !== false;
             this.cnameMaxTTL = options.cnameMaxTTL || 120;
+            this.cnameReplayFullURL = options.cnameReplayFullURL === true;
             this.cnames.clear(); this.cnames.set('', '');
         }
         normalizeDetails(details) {
@@ -123,11 +127,22 @@
             }
             return Array.from(out);
         }
-        processCanonicalName(cname, details) {
-            this.url.href = details.url;
-            details.cnameOf = this.url.hostname;
-            this.url.hostname = cname;
-            details.url = this.url.href;
+        processCanonicalName(hn, cn, details) {
+            const hnBeg = details.url.indexOf(hn);
+            if ( hnBeg === -1 ) { return; }
+            const oldURL = details.url;
+            let newURL = oldURL.slice(0, hnBeg) + cn;
+            const hnEnd = hnBeg + hn.length;
+            if ( this.cnameReplayFullURL ) {
+                newURL += oldURL.slice(hnEnd);
+            } else {
+                const pathBeg = oldURL.indexOf('/', hnEnd);
+                if ( pathBeg !== -1 ) {
+                    newURL += oldURL.slice(hnEnd, pathBeg + 1);
+                }
+            }
+            details.url = newURL;
+            details.aliasURL = oldURL;
             return super.onBeforeSuspendableRequest(details);
         }
         recordCanonicalName(hn, record) {
@@ -187,11 +202,14 @@
             let r = super.onBeforeSuspendableRequest(details);
             if ( r !== undefined ) { return r; }
             if ( this.cnameAliasList === null ) { return; }
+            if ( details.type === 'main_frame' && this.cnameIgnoreRootDocument ) {
+                return;
+            }
             const hn = vAPI.hostnameFromNetworkURL(details.url);
             let cname = this.cnames.get(hn);
             if ( cname === '' ) { return; }
             if ( cname !== undefined ) {
-                return this.processCanonicalName(cname, details);
+                return this.processCanonicalName(hn, cname, details);
             }
             if ( this.cnameAliasList.test(hn) === false ) {
                 this.cnames.set(hn, '');
@@ -201,7 +219,7 @@
                 rec => {
                     const cname = this.recordCanonicalName(hn, rec);
                     if ( cname === '' ) { return; }
-                    return this.processCanonicalName(cname, details);
+                    return this.processCanonicalName(hn, cname, details);
                 },
                 ( ) => {
                     this.cnames.set(hn, '');
diff --git a/src/css/logger-ui.css b/src/css/logger-ui.css
@@ -268,7 +268,7 @@ body.colorBlind #vwRenderer .logEntry > div.cosmeticRealm,
 body.colorBlind #vwRenderer .logEntry > div.redirect {
     background-color: rgba(0, 19, 110, 0.1);
     }
-#vwRenderer .logEntry > div[data-cnameof] {
+#vwRenderer .logEntry > div[data-aliasid] {
     color: mediumblue;
     }
 #vwRenderer .logEntry > div[data-type="tabLoad"] {
diff --git a/src/js/background.js b/src/js/background.js
@@ -49,7 +49,9 @@ const µBlock = (( ) => { // jshint ignore:line
         cnameAliasList: 'unset',
         cnameIgnoreList: 'unset',
         cnameIgnore1stParty: true,
+        cnameIgnoreRootDocument: true,
         cnameMaxTTL: 120,
+        cnameReplayFullURL: false,
         consoleLogLevel: 'unset',
         debugScriptlets: false,
         debugScriptletInjector: false,
diff --git a/src/js/filtering-context.js b/src/js/filtering-context.js
@@ -29,9 +29,10 @@
     }
     this.tstamp = 0;
     this.realm = '';
+    this.id = undefined;
     this.type = undefined;
-    this.cnameOf = undefined;
     this.url = undefined;
+    this.aliasURL = undefined;
     this.hostname = undefined;
     this.domain = undefined;
     this.docId = undefined;
@@ -64,9 +65,10 @@
         }
         this.fromTabId(tabId);
         this.realm = '';
+        this.id = details.requestId;
         this.type = details.type;
         this.setURL(details.url);
-        this.cnameOf = details.cnameOf || undefined;
+        this.aliasURL = details.aliasURL || undefined;
         this.docId = details.type !== 'sub_frame'
             ? details.frameId
             : details.parentFrameId;
diff --git a/src/js/logger-ui.js b/src/js/logger-ui.js
@@ -157,9 +157,12 @@ const regexFromURLFilteringResult = function(result) {
 
 const nodeFromURL = function(parent, url, re) {
     const fragment = document.createDocumentFragment();
-    if ( re instanceof RegExp === false ) {
+    if ( re === undefined ) {
         fragment.textContent = url;
     } else {
+        if ( typeof re === 'string' ) {
+            re = new RegExp(re.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'), 'g');
+        }
         const matches = re.exec(url);
         if ( matches === null || matches[0].length === 0 ) {
             fragment.textContent = url;
@@ -211,6 +214,9 @@ const LogEntry = function(details) {
             this[prop] = details[prop];
         }
     }
+    if ( details.aliasURL !== undefined ) {
+        this.aliased = true;
+    }
     if ( this.tabDomain === '' ) {
         this.tabDomain = this.tabHostname || '';
     }
@@ -222,12 +228,13 @@ const LogEntry = function(details) {
     }
 };
 LogEntry.prototype = {
-    cnameOf: '',
+    aliased: false,
     dead: false,
     docDomain: '',
     docHostname: '',
     domain: '',
     filter: undefined,
+    id: '',
     realm: '',
     tabDomain: '',
     tabHostname: '',
@@ -294,7 +301,7 @@ const processLoggerEntries = function(response) {
             if ( autoDeleteVoidedRows ) { continue; }
             parsed.voided = true;
         }
-        if ( parsed.type === 'main_frame' && parsed.cnameOf === '' ) {
+        if ( parsed.type === 'main_frame' && parsed.aliased === false ) {
             const separator = createLogSeparator(parsed, unboxed.url);
             loggerEntries.unshift(separator);
             if ( rowFilterer.filterOne(separator) ) {
@@ -304,7 +311,7 @@ const processLoggerEntries = function(response) {
                 }
             }
         }
-        if ( cnameOfEnabled === false && parsed.cnameOf !== '' ) {
+        if ( cnameOfEnabled === false && parsed.aliased ) {
             uDom.nodeFromId('filterExprCnameOf').style.display = '';
             cnameOfEnabled = true;
         }
@@ -405,8 +412,10 @@ const parseLogEntry = function(details) {
     textContent.push(normalizeToStr(details.url));
 
     // Hidden cells -- useful for row-filtering purpose
-    if ( entry.cnameOf !== '' ) {
-        textContent.push(`cnameOf=${entry.cnameOf}`);
+
+    // Cell 7
+    if ( entry.aliased ) {
+        textContent.push(`aliasURL=${details.aliasURL}`);
     }
 
     entry.textContent = textContent.join('\t');
@@ -723,17 +732,20 @@ const viewPort = (( ) => {
         span.textContent = cells[5];
 
         // URL
-        let re = null;
+        let re;
         if ( filteringType === 'static' ) {
             re = new RegExp(filter.regex, 'gi');
         } else if ( filteringType === 'dynamicUrl' ) {
             re = regexFromURLFilteringResult(filter.rule.join(' '));
         }
         nodeFromURL(div.children[6], cells[6], re);
 
-        // Cname
-        if ( details.cnameOf !== '' ) {
-            div.setAttribute('data-cnameof', details.cnameOf);
+        // Alias URL (CNAME, etc.)
+        if ( cells.length > 7 ) {
+            const pos = details.textContent.lastIndexOf('\taliasURL=');
+            if ( pos !== -1 ) {
+                div.setAttribute('data-aliasid', details.id);
+            }
         }
 
         return div;
@@ -1452,6 +1464,16 @@ const reloadTab = function(ev) {
         return targetRow.children[1].textContent;
     };
 
+    const aliasURLFromID = function(id) {
+        if ( id === '' ) { return ''; }
+        for ( const entry of loggerEntries ) {
+            if ( entry.id !== id || entry.aliased ) { continue; }
+            const fields = entry.textContent.split('\t');
+            return fields[6] || '';
+        }
+        return '';
+    };
+
     const toSummaryPaneFilterNode = async function(receiver, filter) {
         receiver.children[1].textContent = filter;
         if ( filterAuthorMode !== true ) { return; }
@@ -1613,8 +1635,8 @@ const reloadTab = function(ev) {
             rows[6].style.display = 'none';
         }
         // URL
-        text = trch[6].textContent;
-        if ( text !== '' ) {
+        const canonicalURL = trch[6].textContent;
+        if ( canonicalURL !== '' ) {
             const attr = tr.getAttribute('data-status') || '';
             if ( attr !== '' ) {
                 rows[7].setAttribute('data-status', attr);
@@ -1623,12 +1645,17 @@ const reloadTab = function(ev) {
         } else {
             rows[7].style.display = 'none';
         }
-        // CNAME of
-        text = tr.getAttribute('data-cnameof') || '';
-        if ( text !== '' ) {
-            rows[8].children[1].textContent = text;
+        // Alias URL
+        text = tr.getAttribute('data-aliasid');
+        const aliasURL = text ? aliasURLFromID(text) : '';
+        if ( aliasURL !== '' ) {
+            rows[8].children[1].textContent =
+                vAPI.hostnameFromURI(aliasURL) + ' \u21d2\n\u2003' +
+                vAPI.hostnameFromURI(canonicalURL);
+            rows[9].children[1].textContent = aliasURL;
         } else {
             rows[8].style.display = 'none';
+            rows[9].style.display = 'none';
         }
     };
 
diff --git a/src/js/storage.js b/src/js/storage.js
@@ -139,7 +139,9 @@ self.addEventListener('hiddenSettingsChanged', ( ) => {
         cnameAliasList: µBlock.hiddenSettings.cnameAliasList,
         cnameIgnoreList: µBlock.hiddenSettings.cnameIgnoreList,
         cnameIgnore1stParty: µBlock.hiddenSettings.cnameIgnore1stParty,
+        cnameIgnoreRootDocument: µBlock.hiddenSettings.cnameIgnoreRootDocument,
         cnameMaxTTL: µBlock.hiddenSettings.cnameMaxTTL,
+        cnameReplayFullURL: µBlock.hiddenSettings.cnameReplayFullURL,
     });
 });
 
diff --git a/src/js/traffic.js b/src/js/traffic.js
@@ -99,7 +99,7 @@ const onBeforeRequest = function(details) {
         if (
             details.parentFrameId !== -1 &&
             details.type === 'sub_frame' &&
-            details.cnameOf === undefined
+            details.aliasURL === undefined
         ) {
             pageStore.setFrame(details.frameId, details.url);
         }
diff --git a/src/logger-ui.html b/src/logger-ui.html
@@ -65,7 +65,7 @@
                                 </span>
                             </div>
                             <div><span data-filtex="!" data-i18n="loggerRowFiltererBuiltinNot"></span><span data-filtex="\t(?:0,)?1\t" data-i18n="loggerRowFiltererBuiltin1p"></span><span data-filtex="\t(?:3(?:,\d)?|0,3)\t" data-i18n="loggerRowFiltererBuiltin3p"></span></div>
-                            <div id="filterExprCnameOf" style="display:none"><span data-filtex="!" data-i18n="loggerRowFiltererBuiltinNot"></span><span data-filtex="\tcnameOf=.">CNAME</span></div>
+                            <div id="filterExprCnameOf" style="display:none"><span data-filtex="!" data-i18n="loggerRowFiltererBuiltinNot"></span><span data-filtex="\taliasURL=.">CNAME</span></div>
                         </div>
                     </span>
                 </span>
@@ -121,7 +121,8 @@
                 <div><span data-i18n="loggerEntryDetailsPartyness"></span><span class="prose"></span></div>
                 <div><span data-i18n="loggerEntryDetailsType"></span><span></span></div>
                 <div><span data-i18n="loggerEntryDetailsURL"></span><span></span></div>
-                <div><span >CNAME of</span><span></span></div>
+                <div><span>CNAME</span><span></span></div>
+                <div><span>Original URL</span><span></span></div>
             </div>
             <div class="pane dynamic hide" data-pane="dynamic">
                 <div class="toolbar row">

Original file line number	Diff line number	Diff line change
`@@ -268,7 +268,7 @@ body.colorBlind #vwRenderer .logEntry > div.cosmeticRealm,`
`268`	`268`	`body.colorBlind #vwRenderer .logEntry > div.redirect {`
`269`	`269`	`background-color: rgba(0, 19, 110, 0.1);`
`270`	`270`	`}`
`271`		`-#vwRenderer .logEntry > div[data-cnameof] {`
	`271`	`+#vwRenderer .logEntry > div[data-aliasid] {`
`272`	`272`	`color: mediumblue;`
`273`	`273`	`}`
`274`	`274`	`#vwRenderer .logEntry > div[data-type="tabLoad"] {`
Original file line number	Diff line number	Diff line change
`@@ -99,7 +99,7 @@ const onBeforeRequest = function(details) {`
`99`	`99`	`if (`
`100`	`100`	`details.parentFrameId !== -1 &&`
`101`	`101`	`details.type === 'sub_frame' &&`
`102`		`- details.cnameOf === undefined`
	`102`	`+ details.aliasURL === undefined`
`103`	`103`	`) {`
`104`	`104`	`pageStore.setFrame(details.frameId, details.url);`
`105`	`105`	`}`