Better file: protocol support.

9 files changed, 146 insertions, 81 deletions

diff --git a/src/bg/ChildPolicies.js b/src/bg/ChildPolicies.js
index 91263fd..4fceb0f 100644
--- a/src/bg/ChildPolicies.js
+++ b/src/bg/ChildPolicies.js
@@ -2,7 +2,7 @@
 {
   let marker = JSON.stringify(uuid());
   let allUrls = ["<all_urls>"];
-  
+
   let Scripts = {
     references: new Set(),
     opts: {
@@ -17,7 +17,7 @@
       opts.matches = allUrls;
       delete opts.excludedMatches;
       this._stubScript = await browser.contentScripts.register(opts);
-      
+
       this.init = this.forget;
     },
     forget() {
@@ -29,7 +29,7 @@
     debug: false,
     trace(code) {
       return this.debug
-        ? `console.debug("Executing child policy", ${JSON.stringify(code)});${code}`
+        ? `console.debug("Executing child policy on %s", document.URL, ${JSON.stringify(code)});${code}`
         : code
         ;
     },
@@ -38,7 +38,7 @@
       if (!matches.length) return;
       try {
         let opts = Object.assign({}, this.opts);
-        opts.js[0].code = this.trace(code); 
+        opts.js[0].code = this.trace(code);
         opts.matches = matches;
         if (excludeMatches && excludeMatches.length) {
           opts.excludeMatches = excludeMatches;
@@ -48,38 +48,51 @@
         error(e);
       }
     },
-    
+
     buildPerms(perms, finalizeSetup = false) {
       if (typeof perms !== "string") {
         perms = JSON.stringify(perms);
       }
       return finalizeSetup
-        ? `ns.setup(${perms}, ${marker});` 
+        ? `ns.setup(${perms}, ${marker});`
         : `ns.config.CURRENT = ${perms};`
         ;
     }
   };
-  
+
   let flatten = arr => arr.reduce((a, b) => a.concat(Array.isArray(b) ? flatten(b) : b), []);
-  
-  let protocolRx = /^(https?):/i;
-  let pathRx = /[^:/]\//;
+
+  let protocolRx = /^(\w+):/i;
+  let pathRx = /(?:[^:/]\/|:\/{3})$/;
   let portRx = /:\d+(?=\/|$)/;
-  let validMatchPatternRx = /^(?:https?|\*):\/\/(?:\*\.)?(?:[\w\u0100-\uf000][\w\u0100-\uf000.-]*)?[\w\u0100-\uf000]\/(\*|[^*]*)$/;
-  
+  let validMatchPatternRx = /^(?:\*|(?:http|ws|ftp)s?|file):\/\/(?:\*\.)?(?:[\w\u0100-\uf000][\w\u0100-\uf000.-]*)?\/(\*|[^*]*)$/;
+
   let siteKey2MatchPattern = site => {
     let hasProtocol = site.match(protocolRx);
-    let protocol = hasProtocol ? ''
-      : Sites.isSecureDomainKey(site) ? "https://" : "*://";
-    let hostname = Sites.toggleSecureDomainKey(site, false)
-      .replace(portRx, '');
-    if (!hasProtocol) hostname = `*.${hostname}`;
-    let path = pathRx.test(hostname) ? "" : "/*";
-    let mp = `${protocol}${hostname}${path}`;
-    return  validMatchPatternRx.test(mp) && (path ? mp : [mp, `${mp}?*`, `${mp}#*`]);
+    let mp = site;
+    if (hasProtocol) {
+      try {
+        let url = new URL(site);
+        url.port = "";
+        url.search = "";
+        url.hash = "";
+        mp = url.href;
+      } catch (e) {
+        return false;
+      }
+    } else {
+      let protocol = Sites.isSecureDomainKey(site) ? "https://" : "*://";
+      let hostname = Sites.toggleSecureDomainKey(site, false)
+          .replace(portRx, '');
+      mp = `${protocol}*.${hostname}`;
+      if (!hostname.includes("/")) mp += "/";
+    }
+
+    return validMatchPatternRx.test(mp) && (
+      mp.endsWith("/") ? `${mp}*` : [mp, `${mp}?*`, `${mp}#*`]);
   };
-  
-  let siteKeys2MatchPatterns = keys => keys && flatten(keys.map(siteKey2MatchPattern)).filter(p => !!p) || [];  
+
+  let siteKeys2MatchPatterns = keys => keys && flatten(keys.map(siteKey2MatchPattern)).filter(p => !!p) || [];
 
   var ChildPolicies = {
     async storeTabInfo(tabId, info) {
@@ -90,21 +103,21 @@
           allFrames: false,
           matchAboutBlank: true,
           runAt: "document_start",
-        });    
+        });
       } catch (e) {
         error(e);
       }
     },
     async update(policy, debug) {
       if (debug !== "undefined") Scripts.debug = debug;
-      
+
       await Scripts.init();
-      
+
       if (!policy.enforced) {
         await Scripts.register(`ns.setup(null, ${marker});`, allUrls);
         return;
       }
-      
+
       let serialized = policy.dry ? policy.dry(true) : policy;
       let permsMap = new Map();
       let trusted = JSON.stringify(serialized.TRUSTED);
@@ -120,7 +133,7 @@
         if (!(newKeys && newKeys.length)) continue;
         let keys = permsMap.get(perms);
         if (keys) {
-          newKeys = keys.concat(newKeys); 
+          newKeys = keys.concat(newKeys);
         }
         permsMap.set(perms, newKeys);
       }
@@ -134,11 +147,11 @@
           permsMap.set(permsKey, [key]);
         }
       }
-      
+
       // compute exclusions
       let permsMapEntries = [...permsMap];
       let excludeMap = new Map();
-      
+
       for (let [perms, keys] of permsMapEntries) {
         excludeMap.set(perms, siteKeys2MatchPatterns(flatten(
           permsMapEntries.filter(([other]) => other !== perms)
@@ -146,14 +159,14 @@
               .filter(k => k && k.includes("/") && keys.some(by => Sites.isImplied(k, by)))
           ));
       }
-      
+
       // register new content scripts
       for (let [perms, keys] of [...permsMap]) {
         await Scripts.register(Scripts.buildPerms(perms), siteKeys2MatchPatterns(keys), excludeMap.get(perms));
       }
       await Scripts.register(Scripts.buildPerms(serialized.DEFAULT, true), allUrls);
     },
-    
+
     getForDocument(policy, url, context = null) {
       return {
         CURRENT: policy.get(url, context).perms.dry(),
@@ -161,7 +174,7 @@
         MARKER: marker
       };
     },
-    
+
     async updateFrame(tabId, frameId, perms, defaultPreset) {
       let code = Scripts.buildPerms(perms) + Scripts.buildPerms(defaultPreset, true);
       await browser.tabs.executeScript(tabId, {
@@ -169,7 +182,7 @@
         frameId,
         matchAboutBlank: true,
         runAt: "document_start"
-      });   
+      });
     }
   };
 }
diff --git a/src/bg/RequestGuard.js b/src/bg/RequestGuard.js
index 5dea994..b585935 100644
--- a/src/bg/RequestGuard.js
+++ b/src/bg/RequestGuard.js
@@ -177,14 +177,12 @@ var RequestGuard = (() => {
       let {siteKey} = Sites.parse(url);
       let options;
       if (siteKey === origin) {
-        TAG += `@${siteKey}`;
-      } else {
-        options = [
-          {label: _("allowLocal", siteKey), checked: true},
-          {label: _("allowLocal", origin)}
-        ];
+        origin = new URL(url).protocol;
       }
-      // let parsedDoc = Sites.parse(documentUrl);
+      options = [
+        {label: _("allowLocal", siteKey), checked: true},
+        {label: _("allowLocal", origin)}
+      ];
       let t = u => `${TAG}@${u}`;
       let ret = await Prompts.prompt({
         title: _("BlockedObjects"),
diff --git a/src/bg/main.js b/src/bg/main.js
index 58e7aef..9976f8f 100644
--- a/src/bg/main.js
+++ b/src/bg/main.js
@@ -141,8 +141,9 @@
       return await Settings.import(data);
     },
     
-    async fetchChildPolicy({url, contextUrl}) {
-      return ChildPolicies.getForDocument(ns.policy, url, contextUrl);
+    async fetchChildPolicy({url, contextUrl}, sender) {
+      return ChildPolicies.getForDocument(ns.policy, 
+        url || sender.url, contextUrl || sender.tab.url);
     },
     
     async openStandalonePopup() {
diff --git a/src/common/Policy.js b/src/common/Policy.js
index 9afc92e..f4479db 100644
--- a/src/common/Policy.js
+++ b/src/common/Policy.js
@@ -5,9 +5,10 @@ var {Permissions, Policy, Sites} = (() => {
   const SECURE_DOMAIN_RX = new RegExp(`^${SECURE_DOMAIN_PREFIX}`);
   const DOMAIN_RX = new RegExp(`(?:^\\w+://|${SECURE_DOMAIN_PREFIX})?([^/]*)`, "i");
   const SKIP_RX = /^(?:(?:about|chrome|resource|moz-.*):|\[System)/;
-  
+  const VALID_SITE_RX = /^(?:(?:(?:(?:http|ftp|ws)s?|file):)(?:(?:\/\/)[\w\u0100-\uf000][\w\u0100-\uf000.-]*[\w\u0100-\uf000](?:$|\/))?|[\w\u0100-\uf000][\w\u0100-\uf000.-]*[\w\u0100-\uf000]$)/;
+
   let rxQuote = s => s.replace(/[.?*+^$[\]\\(){}|-]/g, "\\$&");
-  
+
   class Sites extends Map {
     static secureDomainKey(domain) {
       return domain.includes(":") ? domain : `${SECURE_DOMAIN_PREFIX}${domain}`;
@@ -20,14 +21,14 @@ var {Permissions, Policy, Sites} = (() => {
     }
 
     static isValid(site) {
-      return /^(?:https?:(?:\/\/)?)?([\w\u0100-\uf000][\w\u0100-\uf000.-]*)?[\w\u0100-\uf000](?::\d+)?$/.test(site);
+      return VALID_SITE_RX.test(site);
     }
-    
-    
+
+
     static originImplies(originKey, site) {
       return originKey === site || site.startsWith(`${originKey}/`);
     }
-    
+
     static domainImplies(domainKey, site, protocol ="https?") {
       if (Sites.isSecureDomainKey(domainKey)) {
         protocol = "https";
@@ -42,13 +43,13 @@ var {Permissions, Policy, Sites} = (() => {
         return false;
       }
     }
-    
+
     static isImplied(site, byKey) {
-      return byKey.includes("://") 
+      return byKey.includes("://")
         ? Sites.originImplies(byKey, site)
         : Sites.domainImplies(byKey, site);
     }
-    
+
     static parse(site) {
       let url, siteKey = "";
       if (site instanceof URL) {
@@ -63,7 +64,11 @@ var {Permissions, Policy, Sites} = (() => {
       if (url) {
         let path = url.pathname;
         siteKey = url.origin;
-        if (path !== '/') siteKey += path;
+        if (siteKey === "null") {
+          siteKey = site;
+        } else if (path !== '/') {
+          siteKey += path;
+        }
       }
       return {url, siteKey};
     }
@@ -71,14 +76,16 @@ var {Permissions, Policy, Sites} = (() => {
     static optimalKey(site) {
       let {url, siteKey} = Sites.parse(site);
       if (url && url.protocol === "https:") return Sites.secureDomainKey(tld.getDomain(url.hostname));
-      return url && url.origin || siteKey;
+      return Sites.origin(url) || siteKey;
     }
 
     static origin(site) {
       try {
-        return new URL(site).origin;
+        let objUrl = site.href ? site : new URL(site);
+        let origin = objUrl.origin;
+        return origin === "null" ? objUrl.href : origin;
       } catch (e) {};
-      return site;
+      return site.origin || site;
     }
 
     static toExternal(url) { // domains are stored in punycode internally
@@ -101,6 +108,7 @@ var {Permissions, Policy, Sites} = (() => {
 
     match(site) {
       if (site && this.size) {
+        if (site instanceof URL) site = site.href;
         if (this.has(site)) return site;
 
         let {url, siteKey} = Sites.parse(site);
diff --git a/src/content/PlaceHolder.js b/src/content/PlaceHolder.js
index f32c812..ac4dc14 100644
--- a/src/content/PlaceHolder.js
+++ b/src/content/PlaceHolder.js
@@ -1,6 +1,6 @@
 var PlaceHolder = (() => {
   const HANDLERS = new Map();
-  
+
   let checkStyle = async () => {
     checkStyle = () => {};
     if (!ns.embeddingDocument) return;
@@ -11,7 +11,7 @@ var PlaceHolder = (() => {
         (await fetch(browser.extension.getURL("/content/content.css"))).text();
     }
   }
-  
+
   class Handler {
     constructor(type, selector) {
       this.type = type;
@@ -20,10 +20,16 @@ var PlaceHolder = (() => {
       HANDLERS.set(type, this);
     }
     filter(element, request) {
-      if (request.embeddingDocument) return true;
+      if (request.embeddingDocument) {
+        return document.URL === request.url;
+      }
       let url = request.initialUrl || request.url;
       return "data" in element ? element.data === url : element.src === url;
     }
+    selectFor(request) {
+      return [...document.querySelectorAll(this.selector)]
+        .filter(element => this.filter(element, request))
+    }
   }
 
   new Handler("frame", "iframe");
@@ -59,6 +65,9 @@ var PlaceHolder = (() => {
     static canReplace(policyType) {
       return HANDLERS.has(policyType);
     }
+    static handlerFor(policyType) {
+      return HANDLERS.get(policyType);
+    }
 
     static listen() {
       PlaceHolder.listen = () => {};
@@ -83,7 +92,7 @@ var PlaceHolder = (() => {
       this.policyType = policyType;
       this.request = request;
       this.replacements = new Set();
-      this.handler = HANDLERS.get(policyType);
+      this.handler = PlaceHolder.handlerFor(policyType);
       if (this.handler) {
         [...document.querySelectorAll(this.handler.selector)]
         .filter(element => this.handler.filter(element, request))
@@ -100,7 +109,11 @@ var PlaceHolder = (() => {
       let {
         url
       } = this.request;
-      this.origin = new URL(url).origin;
+      let objUrl = new URL(url)
+      this.origin = objUrl.origin;
+      if (this.origin === "null") {
+        this.origin = objUrl.protocol;
+      }
       let TYPE = `<${this.policyType.toUpperCase()}>`;
 
       let replacement = createHTMLElement("a");
@@ -129,7 +142,7 @@ var PlaceHolder = (() => {
 
       replacement._placeHolderObj = this;
       replacement._placeHolderElement = element;
-      
+
 
       element.parentNode.replaceChild(replacement, element);
       this.replacements.add(replacement);
diff --git a/src/content/content.js b/src/content/content.js
index e73feb6..fb68ae4 100644
--- a/src/content/content.js
+++ b/src/content/content.js
@@ -1,6 +1,5 @@
 'use strict';
 // debug = () => {}; // REL_ONLY
-
 var _ = browser.i18n.getMessage;
 
 function createHTMLElement(name) {
@@ -50,7 +49,7 @@ var notifyPage = async () => {
   if (document.readyState === "complete") {
     try {
       if (!("canScript" in ns)) {
-        let childPolicy = await Messages.send("fetchChildPolicy", {url: document.URL, contextUrl: top.location.href});
+        let childPolicy = await Messages.send("fetchChildPolicy", {url: document.URL});
         ns.config.CURRENT = childPolicy.CURRENT;
         ns.setup(childPolicy.DEFAULT, childPolicy.MARKER);
         return;
@@ -82,11 +81,22 @@ ns.on("capabilities", () => {
       },
       allowed: ns.canScript
     });
-  
-  if (!ns.canScript) {
 
+  if (!ns.canScript) {
+    addEventListener("beforescriptexecute", e => e.preventDefault());
+    let mo = new MutationObserver(mutations => {
+      for (let m of mutations) {
+        console.log(`Mutation `, m);
+        if (m.type !== "attribute") continue;
+        if (/^on\w+/i.test(m.attributeName)) {
+          m.target.removeAttribute(m.attributeName);
+        } else if (/^\s*(javascript|data):/i.test(m.target.attributes[m.attributeName])) {
+          m.target.setAttribute(m.attributeName, "#");
+        }
+      }
+    });
+    // mo.observe(document.documentElement, {attributes: true, subtree: true});
     if ("serviceWorker" in navigator && navigator.serviceWorker.controller) {
-      addEventListener("beforescriptexecute", e => e.preventDefault());
       (async () => {
         for (let r of await navigator.serviceWorker.getRegistrations()) {
           await r.unregister();
@@ -97,6 +107,6 @@ ns.on("capabilities", () => {
     if (document.readyState !== "loading") onScriptDisabled();
     window.addEventListener("DOMContentLoaded", onScriptDisabled);
   }
-  
+
   notifyPage();
 });
diff --git a/src/content/embeddingDocument.js b/src/content/embeddingDocument.js
index 75b0db0..eed04b1 100644
--- a/src/content/embeddingDocument.js
+++ b/src/content/embeddingDocument.js
@@ -1,18 +1,25 @@
 if (ns.embeddingDocument) {
   ns.on("capabilities", () => {
     for (let policyType of ["object", "media"]) {
-      if (!ns.allows(policyType)) {
-        let request = {
-          id: `noscript-${policyType}-doc`,
-          type: policyType,
-          url: document.URL,
-          documentUrl: document.URL,
-          embeddingDocument: true,
-        };
+      let request = {
+        id: `noscript-${policyType}-doc`,
+        type: policyType,
+        url: document.URL,
+        documentUrl: document.URL,
+        embeddingDocument: true,
+      };
+      
+      if (ns.allows(policyType)) {
+        let handler = PlaceHolder.handlerFor(policyType);
+        if (handler && handler.selectFor(request).length > 0) {
+          seen.record({policyType, request, allowed: true});
+        }
+      } else {
         let ph = PlaceHolder.create(policyType, request);
         if (ph.replacements.size > 0) {
           debug(`Created placeholder for ${policyType} at ${document.URL}`);
           seen.record({policyType, request, allowed: false});
+          break;
         }
       }
     }
diff --git a/src/content/onScriptDisabled.js b/src/content/onScriptDisabled.js
index 3606ede..79912c9 100644
--- a/src/content/onScriptDisabled.js
+++ b/src/content/onScriptDisabled.js
@@ -1,4 +1,20 @@
 function onScriptDisabled() {
+  if (document.URL.startsWith("file:")) {
+    // file: documents are loaded synchronously and may not be affected by
+    // CSP. We already intercept onbeforeexecutescript event, let's cope with
+    // event and URL attributes.
+    for (let e of document.all) {
+      for (let a of e.attributes) {
+        if (/^on\w+/i.test(a.name)) {
+          debug(`Removed %s.%sevent`, e.tagName, a.name);
+          a.value = "";
+        } else if (/^\s*(?:data|javascript):/i.test(unescape(a.value))) {
+          debug(`Neutralized %s.%s="%s" attribute`, e.tagName, a.name, a.value);
+          a.value = "data:";
+        }
+      }
+    }
+  }
   for (let noscript of document.querySelectorAll("noscript")) {
     // force show NOSCRIPT elements content
     let replacement = createHTMLElement("span");
diff --git a/src/ui/popup.js b/src/ui/popup.js
index 8c34beb..c876ed7 100644
--- a/src/ui/popup.js
+++ b/src/ui/popup.js
@@ -177,15 +177,13 @@ addEventListener("unload", e => {
       let domains = new Map();
 
       function urlToLabel(url) {
-        let {
-          origin
-        } = url;
+        let origin = Sites.origin(url);
         let match = policySites.match(url);
         if (match) return match;
         if (domains.has(origin)) {
           if (justDomains) return domains.get(origin);
         } else {
-          let domain = tld.getDomain(url.hostname);
+          let domain = tld.getDomain(url.hostname) || origin;
           domain = url.protocol === "https:" ? Sites.secureDomainKey(domain) : domain;
           domains.set(origin, domain);
           if (justDomains) return domain;
@@ -196,7 +194,8 @@ addEventListener("unload", e => {
       let parsedSeen = seen.map(thing => Object.assign({
           type: thing.policyType
         }, Sites.parse(thing.request.url)))
-        .filter(parsed => parsed.url && parsed.url.origin !== "null");
+        .filter(parsed => parsed.url && (
+            parsed.url.origin !== "null" || parsed.url.protocol === "file:"));
 
       let sitesSet = new Set(
         parsedSeen.map(parsed => parsed.label = urlToLabel(parsed.url))
@@ -206,7 +205,7 @@ addEventListener("unload", e => {
       }
       let sites = [...sitesSet];
       for (let parsed of parsedSeen) {
-        sites.filter(s => parsed.label === s || domains.get(parsed.url.origin) === s).forEach(m => {
+        sites.filter(s => parsed.label === s || domains.get(Sites.origin(parsed.url)) === s).forEach(m => {
           let siteTypes = typesMap.get(m);
           if (!siteTypes) typesMap.set(m, siteTypes = new Set());
           siteTypes.add(parsed.type);