refactor assets management; working webextension version

.jshintrc

@@ -5,9 +5,9 @@
 	"strict": "global",
 	"globals": {
 		"self": false,
-		"vAPI": false,
 		"chrome": false,
-		"safari": false,
+		"vAPI": false,
+		"µMatrix": false,
 		"Components": false // global variable in Firefox
 	}
 }

assets/assets.json

@@ -0,0 +1,75 @@
+{
+	"assets.json": {
+		"content": "internal",
+		"updateAfter": 13,
+		"contentURL": [
+			"https://raw.githubusercontent.com/gorhill/uMatrix/master/assets/assets.json",
+			"assets/assets.json"
+		]
+	},
+	"public_suffix_list.dat": {
+		"content": "internal",
+		"updateAfter": 19,
+		"contentURL": [
+			"https://publicsuffix.org/list/public_suffix_list.dat",
+			"assets/thirdparties/publicsuffix.org/list/effective_tld_names.dat"
+		]
+	},
+	"malware-0": {
+		"content": "filters",
+		"group": "malware",
+		"title": "Malware Domain List",
+		"contentURL": [
+			"https://www.malwaredomainlist.com/hostslist/hosts.txt",
+			"assets/thirdparties/www.malwaredomainlist.com/hostslist/hosts.txt"
+		]
+	},
+	"malware-1": {
+		"content": "filters",
+		"group": "malware",
+		"title": "Malware domains",
+		"contentURL": [
+			"https://mirror.cedia.org.ec/malwaredomains/justdomains",
+			"https://mirror1.malwaredomains.com/files/justdomains",
+			"assets/thirdparties/mirror1.malwaredomains.com/files/justdomains",
+			"assets/thirdparties/mirror1.malwaredomains.com/files/justdomains.txt"
+		],
+		"supportURL": "http://www.malwaredomains.com/"
+	},
+	"dpollock-0": {
+		"content": "filters",
+		"group": "multipurpose",
+		"updateAfter": 11,
+		"title": "Dan Pollock’s hosts file",
+		"contentURL": "http://someonewhocares.org/hosts/hosts",
+		"supportURL": "http://someonewhocares.org/hosts/"
+	},
+	"hphosts": {
+		"content": "filters",
+		"group": "multipurpose",
+		"updateAfter": 11,
+		"title": "hpHosts’ Ad and tracking servers",
+		"contentURL": "https://hosts-file.net/.%5Cad_servers.txt",
+		"supportURL": "https://hosts-file.net/"
+	},
+	"mvps-0": {
+		"content": "filters",
+		"group": "multipurpose",
+		"updateAfter": 11,
+		"title": "MVPS HOSTS",
+		"contentURL": "http://winhelp2002.mvps.org/hosts.txt",
+		"supportURL": "http://winhelp2002.mvps.org/"
+	},
+	"plowe-0": {
+		"content": "filters",
+		"group": "multipurpose",
+		"updateAfter": 13,
+		"title": "Peter Lowe’s Ad and tracking server list",
+		"contentURL": [
+			"https://pgl.yoyo.org/adservers/serverlist.php?hostformat=hosts&showintro=1&mimetype=plaintext",
+			"assets/thirdparties/pgl.yoyo.org/as/serverlist",
+			"assets/thirdparties/pgl.yoyo.org/as/serverlist.txt"
+		],
+		"supportURL": "https://pgl.yoyo.org/adservers/"
+	}
+}

platform/chromium/manifest.json

@@ -78,7 +78,6 @@
         "webNavigation",
         "webRequest",
         "webRequestBlocking",
-        "http://*/*",
+        "<all_urls>"
-        "https://*/*"
     ]
 }

platform/chromium/polyfill.js

@@ -0,0 +1,234 @@
+/*******************************************************************************
+
+    uMatrix - a browser extension to block requests.
+    Copyright (C) 2017 Raymond Hill
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see {http://www.gnu.org/licenses/}.
+
+    Home: https://github.com/gorhill/uMatrix
+
+    This file has been originally imported from:
+    https://github.com/gorhill/uBlock/tree/master/platform/chromium
+
+*/
+
+// For background page or non-background pages
+
+/* exported objectAssign */
+
+'use strict';
+
+/******************************************************************************/
+/******************************************************************************/
+
+// https://github.com/gorhill/uBlock/issues/1067
+// https://developer.mozilla.org/en/docs/Web/JavaScript/Reference/Global_Objects/String/startsWith
+// Firefox 17/Chromium 41 supports `startsWith`.
+
+if ( String.prototype.startsWith instanceof Function === false ) {
+    String.prototype.startsWith = function(needle, pos) {
+        if ( typeof pos !== 'number' ) {
+            pos = 0;
+        }
+        return this.lastIndexOf(needle, pos) === pos;
+    };
+}
+
+/******************************************************************************/
+
+// https://github.com/gorhill/uBlock/issues/1067
+// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/endsWith
+// Firefox 17/Chromium 41 supports `endsWith`.
+
+if ( String.prototype.endsWith instanceof Function === false ) {
+    String.prototype.endsWith = function(needle, pos) {
+        if ( typeof pos !== 'number' ) {
+            pos = this.length;
+        }
+        pos -= needle.length;
+        return this.indexOf(needle, pos) === pos;
+    };
+}
+
+/******************************************************************************/
+
+// As per MDN, Object.assign appeared first in Chromium 45.
+// https://developer.mozilla.org/en/docs/Web/JavaScript/Reference/Global_Objects/Object/assign#Browser_compatibility
+
+var objectAssign = Object.assign || function(target, source) {
+    var keys = Object.keys(source);
+    for ( var i = 0, n = keys.length, key; i < n; i++ ) {
+        key = keys[i];
+        target[key] = source[key];
+    }
+    return target;
+};
+
+/******************************************************************************/
+
+// https://github.com/gorhill/uBlock/issues/1070
+// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Set#Browser_compatibility
+// This polyfill is designed to fulfill *only* what uBlock Origin needs -- this
+// is not an accurate API of the real Set() type.
+
+if ( self.Set instanceof Function === false ) {
+    self.Set = function(iter) {
+        this.clear();
+        if ( Array.isArray(iter) ) {
+            for ( var i = 0, n = iter.length; i < n; i++ ) {
+                this.add(iter[i]);
+            }
+            return;
+        }
+    };
+
+    self.Set.polyfill = true;
+
+    self.Set.prototype.clear = function() {
+        this._set = Object.create(null);
+        this.size = 0;
+        // Iterator stuff
+        this._values = undefined;
+        this._i = undefined;
+        this.value = undefined;
+        this.done = true;
+    };
+
+    self.Set.prototype.add = function(k) {
+        if ( this._set[k] === undefined ) {
+            this._set[k] = true;
+            this.size += 1;
+        }
+        return this;
+    };
+
+    self.Set.prototype.delete = function(k) {
+        if ( this._set[k] !== undefined ) {
+            delete this._set[k];
+            this.size -= 1;
+            return true;
+        }
+        return false;
+    };
+
+    self.Set.prototype.has = function(k) {
+        return this._set[k] !== undefined;
+    };
+
+    self.Set.prototype.next = function() {
+        if ( this._i < this.size ) {
+            this.value = this._values[this._i++];
+        } else {
+            this._values = undefined;
+            this.value = undefined;
+            this.done = true;
+        }
+        return this;
+    };
+
+    self.Set.prototype.values = function() {
+        this._values = Object.keys(this._set);
+        this._i = 0;
+        this.value = undefined;
+        this.done = false;
+        return this;
+    };
+}
+
+/******************************************************************************/
+
+// https://github.com/gorhill/uBlock/issues/1070
+// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Set#Browser_compatibility
+// This polyfill is designed to fulfill *only* what uBlock Origin needs -- this
+// is not an accurate API of the real Map() type.
+
+if ( self.Map instanceof Function === false ) {
+    self.Map = function(iter) {
+        this.clear();
+        if ( Array.isArray(iter) ) {
+            for ( var i = 0, n = iter.length, entry; i < n; i++ ) {
+                entry = iter[i];
+                this.set(entry[0], entry[1]);
+            }
+            return;
+        }
+    };
+
+    self.Map.polyfill = true;
+
+    self.Map.prototype.clear = function() {
+        this._map = Object.create(null);
+        this.size = 0;
+        // Iterator stuff
+        this._keys = undefined;
+        this._i = undefined;
+        this.value = undefined;
+        this.done = true;
+    };
+
+    self.Map.prototype.delete = function(k) {
+        if ( this._map[k] !== undefined ) {
+            delete this._map[k];
+            this.size -= 1;
+            return true;
+        }
+        return false;
+    };
+
+    self.Map.prototype.entries = function() {
+        this._keys = Object.keys(this._map);
+        this._i = 0;
+        this.value = [ undefined, undefined ];
+        this.done = false;
+        return this;
+    };
+
+    self.Map.prototype.get = function(k) {
+        return this._map[k];
+    };
+
+    self.Map.prototype.has = function(k) {
+        return this._map[k] !== undefined;
+    };
+
+    self.Map.prototype.next = function() {
+        if ( this._i < this.size ) {
+            var key = this._keys[this._i++];
+            this.value[0] = key;
+            this.value[1] = this._map[key];
+        } else {
+            this._keys = undefined;
+            this.value = undefined;
+            this.done = true;
+        }
+        return this;
+    };
+
+    self.Map.prototype.set = function(k, v) {
+        if ( v !== undefined ) {
+            if ( this._map[k] === undefined ) {
+                this.size += 1;
+            }
+            this._map[k] = v;
+        } else {
+            if ( this._map[k] !== undefined ) {
+                this.size -= 1;
+            }
+            delete this._map[k];
+        }
+        return this;
+    };
+}
+
+/******************************************************************************/

platform/chromium/vapi-background.js

@@ -1,7 +1,8 @@
 /*******************************************************************************
 
     uMatrix - a browser extension to block requests.
-    Copyright (C) 2014-2016 The uBlock authors
+    Copyright (C) 2014-2017 The uBlock Origin authors
+    Copyright (C)      2017 Raymond Hill
 
     This program is free software: you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
@@ -52,14 +53,16 @@ chrome.privacy.network.networkPredictionEnabled.set({
 // Tell Chromium to allow all javascript: µMatrix will control whether
 // javascript execute through `Content-Policy-Directive` and webRequest.
 //   https://github.com/gorhill/httpswitchboard/issues/74
-chrome.contentSettings.javascript.set({
+if ( chrome.contentSettings instanceof Object ) {
-    primaryPattern: 'https://*/*',
+    chrome.contentSettings.javascript.set({
-    setting: 'allow'
+        primaryPattern: 'https://*/*',
-});
+        setting: 'allow'
-chrome.contentSettings.javascript.set({
+    });
-    primaryPattern: 'http://*/*',
+    chrome.contentSettings.javascript.set({
-    setting: 'allow'
+        primaryPattern: 'http://*/*',
-});
+        setting: 'allow'
+    });
+}
 
 /******************************************************************************/
 
@@ -107,6 +110,7 @@ vAPI.app.restart = function() {
 // chrome.storage.local.get(null, function(bin){ console.debug('%o', bin); });
 
 vAPI.storage = chrome.storage.local;
+vAPI.cacheStorage = chrome.storage.local;
 
 /******************************************************************************/
 
@@ -583,8 +587,9 @@ vAPI.net = {};
 /******************************************************************************/
 
 vAPI.net.registerListeners = function() {
-    var µm = µMatrix;
+    var µm = µMatrix,
-    var httpRequestHeadersJunkyard = [];
+        reNetworkURL = /^(?:https?|wss?):\/\//,
+        httpRequestHeadersJunkyard = [];
 
     // Abstraction layer to deal with request headers
     // >>>>>>>>
@@ -657,11 +662,24 @@ vAPI.net.registerListeners = function() {
 
     // Normalizing request types
     // >>>>>>>>
+    var extToTypeMap = new Map([
+        ['eot','font'],['otf','font'],['svg','font'],['ttf','font'],['woff','font'],['woff2','font'],
+        ['mp3','media'],['mp4','media'],['webm','media'],
+        ['gif','image'],['ico','image'],['jpeg','image'],['jpg','image'],['png','image'],['webp','image']
+    ]);
+
     var normalizeRequestDetails = function(details) {
         details.tabId = details.tabId.toString();
 
+        var type = details.type;
+
+        if ( type === 'imageset' ) {
+            details.type = 'image';
+            return;
+        }
+
         // The rest of the function code is to normalize request type
-        if ( details.type !== 'other' ) {
+        if ( type !== 'other' ) {
             return;
         }
 
@@ -672,26 +690,11 @@ vAPI.net.registerListeners = function() {
             }
         }
 
+        // Try to map known "extension" part of URL to request type.
         var path = µm.URI.pathFromURI(details.url),
             pos = path.indexOf('.', path.length - 6);
- 
+        if ( pos !== -1 && (type = extToTypeMap.get(path.slice(pos + 1))) ) {
-        // https://github.com/chrisaljoudi/uBlock/issues/862
+            details.type = type;
-        // If no transposition possible, transpose to `object` as per
-        // Chromium bug 410382 (see below)
-        if ( pos === -1 ) {
-            return;
-        }
-
-        var needle = path.slice(pos) + '.';
-        if ( '.eot.ttf.otf.svg.woff.woff2.'.indexOf(needle) !== -1 ) {
-            details.type = 'font';
-            return;
-        }
-        // Still need this because often behind-the-scene requests are wrongly
-        // categorized as 'other'
-        if ( '.ico.png.gif.jpg.jpeg.mp3.mp4.webm.webp.'.indexOf(needle) !== -1 ) {
-            details.type = 'image';
-            return;
         }
     };
     // <<<<<<<<
@@ -700,12 +703,13 @@ vAPI.net.registerListeners = function() {
     // Network event handlers
     // >>>>>>>>
     var onBeforeRequestClient = this.onBeforeRequest.callback;
-    var onBeforeRequest = function(details) {
-        normalizeRequestDetails(details);
-        return onBeforeRequestClient(details);
-    };
     chrome.webRequest.onBeforeRequest.addListener(
-        onBeforeRequest,
+        function(details) {
+            if ( reNetworkURL.test(details.url) ) {
+                normalizeRequestDetails(details);
+                return onBeforeRequestClient(details);
+            }
+        },
         //function(details) {
         //    quickProfiler.start('onBeforeRequest');
         //    var r = onBeforeRequest(details);
@@ -713,8 +717,8 @@ vAPI.net.registerListeners = function() {
         //    return r;
         //},
         {
-            'urls': this.onBeforeRequest.urls || ['<all_urls>'],
+            'urls': this.onBeforeRequest.urls || [ '<all_urls>' ],
-            'types': this.onBeforeRequest.types || []
+            'types': this.onBeforeRequest.types || undefined
         },
         this.onBeforeRequest.extra
     );
@@ -735,8 +739,8 @@ vAPI.net.registerListeners = function() {
     chrome.webRequest.onBeforeSendHeaders.addListener(
         onBeforeSendHeaders,
         {
-            'urls': this.onBeforeSendHeaders.urls || ['<all_urls>'],
+            'urls': this.onBeforeSendHeaders.urls || [ '<all_urls>' ],
-            'types': this.onBeforeSendHeaders.types || []
+            'types': this.onBeforeSendHeaders.types || undefined
         },
         this.onBeforeSendHeaders.extra
     );
@@ -749,8 +753,8 @@ vAPI.net.registerListeners = function() {
     chrome.webRequest.onHeadersReceived.addListener(
         onHeadersReceived,
         {
-            'urls': this.onHeadersReceived.urls || ['<all_urls>'],
+            'urls': this.onHeadersReceived.urls || [ '<all_urls>' ],
-            'types': this.onHeadersReceived.types || []
+            'types': this.onHeadersReceived.types || undefined
         },
         this.onHeadersReceived.extra
     );

platform/chromium/vapi-common.js

@@ -88,7 +88,7 @@ vAPI.closePopup = function() {
 // This storage is optional, but it is nice to have, for a more polished user
 // experience.
 
-vAPI.localStorage = window.localStorage;
+vAPI.localStorage = self.localStorage;
 
 /******************************************************************************/
 

platform/firefox/polyfill.js

@@ -0,0 +1,95 @@
+/*******************************************************************************
+
+    uMatrix - a browser extension to block requests.
+    Copyright (C) 2017 Raymond Hill
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see {http://www.gnu.org/licenses/}.
+
+    Home: https://github.com/gorhill/uMatrix
+
+    This file has been originally imported from:
+    https://github.com/gorhill/uBlock/tree/master/platform/chromium
+
+*/
+
+// For background page or non-background pages
+
+/* exported objectAssign */
+
+'use strict';
+
+/******************************************************************************/
+/******************************************************************************/
+
+// As per MDN, Object.assign appeared first in Firefox 34.
+// https://developer.mozilla.org/en/docs/Web/JavaScript/Reference/Global_Objects/Object/assign#Browser_compatibility
+
+var objectAssign = Object.assign || function(target, source) {
+    var keys = Object.keys(source);
+    for ( var i = 0, n = keys.length, key; i < n; i++ ) {
+        key = keys[i];
+        target[key] = source[key];
+    }
+    return target;
+};
+
+/******************************************************************************/
+
+// Patching for Pale Moon which does not implement ES6 Set/Map.
+// Test for non-ES6 Set/Map: check if property `iterator` is present.
+// The code is strictly to satisfy uBO's core, not to be an accurate
+// implementation of ES6.
+
+if ( self.Set.prototype.iterator instanceof Function ) {
+    //console.log('Patching non-ES6 Set() to be more ES6-like.');
+    self.Set.prototype._values = self.Set.prototype.values;
+    self.Set.prototype.values = function() {
+        this._valueIter = this._values();
+        this.value = undefined;
+        this.done = false;
+        return this;
+    };
+    self.Set.prototype.next = function() {
+        try {
+            this.value = this._valueIter.next();
+        } catch (ex) {
+            this._valueIter = undefined;
+            this.value = undefined;
+            this.done = true;
+        }
+        return this;
+    };
+}
+
+if ( self.Map.prototype.iterator instanceof Function ) {
+    //console.log('Patching non-ES6 Map() to be more ES6-like.');
+    self.Map.prototype._entries = self.Map.prototype.entries;
+    self.Map.prototype.entries = function() {
+        this._entryIter = this._entries();
+        this.value = undefined;
+        this.done = false;
+        return this;
+    };
+    self.Map.prototype.next = function() {
+        try {
+            this.value = this._entryIter.next();
+        } catch (ex) {
+            this._entryIter = undefined;
+            this.value = undefined;
+            this.done = true;
+        }
+        return this;
+    };
+}
+

platform/firefox/vapi-background.js

@@ -580,6 +580,8 @@ vAPI.storage = (function() {
     return api;
 })();
 
+vAPI.cacheStorage = vAPI.storage;
+
 /******************************************************************************/
 
 // This must be executed/setup early.

platform/webext/manifest.json

@@ -0,0 +1,61 @@
+{
+    "applications": {
+        "gecko": {
+            "id": "uMatrix@raymondhill.net",
+            "strict_min_version": "53.0a1"
+        }
+    },
+    "author": "Raymond Hill",
+    "background": {
+        "page": "background.html"
+    },
+    "browser_action": {
+        "browser_style": false,
+        "default_icon": {
+            "19": "img/browsericons/icon19.png",
+            "38": "img/browsericons/icon38.png"
+        },
+        "default_title": "uMatrix",
+        "default_popup": "popup.html"
+    },
+    "content_scripts": [
+        {
+            "matches": ["http://*/*", "https://*/*"],
+            "js": ["js/vapi-client.js", "js/contentscript-start.js"],
+            "run_at": "document_start",
+            "all_frames": true
+        },
+        {
+            "matches": ["http://*/*", "https://*/*"],
+            "js": ["js/contentscript-end.js"],
+            "run_at": "document_end",
+            "all_frames": true
+        }
+    ],
+    "default_locale": "en",
+    "description": "__MSG_extShortDesc__",
+    "icons": {
+        "16": "img/icon_16.png",
+        "128": "img/icon_128.png"
+    },
+    "manifest_version": 2,
+    "minimum_chrome_version": "26.0",
+    "name": "uMatrix",
+    "options_ui": {
+        "page": "options_ui.html"
+    },
+    "permissions": [
+        "browsingData",
+        "contentSettings",
+        "cookies",
+        "privacy",
+        "storage",
+        "tabs",
+        "webNavigation",
+        "webRequest",
+        "webRequestBlocking",
+        "<all_urls>"
+    ],
+    "short_name": "uMatrix",
+    "version": "0.9.9"
+}

platform/webext/polyfill.js

@@ -0,0 +1,30 @@
+/*******************************************************************************
+
+    uBlock Origin - a browser extension to block requests.
+    Copyright (C) 2016 The uBlock Origin authors
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see {http://www.gnu.org/licenses/}.
+
+    Home: https://github.com/gorhill/uBlock
+*/
+
+// For background page or non-background pages
+
+'use strict';
+
+/******************************************************************************/
+
+var objectAssign = Object.assign;
+
+/******************************************************************************/

src/_locales/en/messages.json

@@ -50,11 +50,11 @@
         "description": "HAS TO FIT IN MATRIX HEADER!"
     },
     "imagePrettyName": {
-        "message": "media",
+        "message": "image",
         "description": "HAS TO FIT IN MATRIX HEADER!"
     },
     "pluginPrettyName": {
-        "message": "plugin",
+        "message": "media",
         "description": "HAS TO FIT IN MATRIX HEADER!"
     },
     "scriptPrettyName": {

src/background.html

@@ -5,9 +5,9 @@
 <title>uMatrix</title>
 </head>
 <body>
+<script src="js/polyfill.js"></script>
 <script src="lib/punycode.js"></script>
 <script src="lib/publicsuffixlist.js"></script>
-<script src="lib/yamd5.js"></script>
 <script src="js/vapi-common.js"></script>
 <script src="js/vapi-background.js"></script>
 <script src="js/background.js"></script>

src/css/common.css

@@ -80,3 +80,40 @@ body[dir="ltr"] .tip-anchor-right[data-i18n-tip]:hover:after,
 body[dir="rtl"] .tip-anchor-left[data-i18n-tip]:hover:after {
     right: -3vw;
     }
+button.custom {
+    padding: 0.6em 1em;
+    border: 1px solid transparent;
+    border-color: #ccc #ccc #bbb #bbb;
+    border-radius: 3px;
+    background-color: hsl(216, 0%, 75%);
+    background-image: linear-gradient(#f2f2f2, #dddddd);
+    background-repeat: repeat-x;
+    color: #000;
+    opacity: 0.8;
+    }
+button.custom:hover {
+    opacity: 1.0;
+    }
+button.custom.important {
+    padding: 0.6em 1em;
+    border: 1px solid transparent;
+    border-color: #ffcc7f #ffcc7f hsl(36, 100%, 73%);
+    border-radius: 3px;
+    background-color: hsl(36, 100%, 75%);
+    background-image: linear-gradient(#ffdca8, #ffcc7f);
+    background-repeat: repeat-x;
+    color: #222;
+    opacity: 0.8;
+    }
+button.custom.important:hover {
+    opacity: 1.0;
+    }
+button.custom.disabled,
+button.custom[disabled] {
+    border-color: #ddd #ddd hsl(36, 0%, 85%);
+    background-color: hsl(36, 0%, 72%);
+    background-image: linear-gradient(#f2f2f2, #dddddd);
+    color: #666;
+    opacity: 0.6;
+    pointer-events: none;
+    }

src/css/hosts-files.css

@@ -1,3 +1,13 @@
+@keyframes spin {
+    0% { transform: rotate(0deg); -webkit-transform: rotate(0deg); }
+    12.5% { transform: rotate(45deg); -webkit-transform: rotate(45deg); }
+    25% { transform: rotate(90deg); -webkit-transform: rotate(90deg); }
+    37.5% { transform: rotate(135deg); -webkit-transform: rotate(135deg); }
+    50% { transform: rotate(180deg); -webkit-transform: rotate(180deg); }
+    67.5% { transform: rotate(225deg); -webkit-transform: rotate(225deg); }
+    75% { transform: rotate(270deg); -webkit-transform: rotate(270deg); }
+    87.5% { transform: rotate(315deg); -webkit-transform: rotate(315deg); }
+    }
 ul {
     padding: 0;
     list-style-type: none;
@@ -25,85 +35,96 @@ body[dir="rtl"] li.listEntry {
     margin-right: 1em;
     }
 li.listEntry > * {
+    margin-right: 0.5em;
+    text-indent: 0;
     unicode-bidi: embed;
     }
 li.listEntry > a:nth-of-type(2) {
     font-size: 13px;
     opacity: 0.5;
     }
-.dim {
+li.listEntry.toRemove > input[type="checkbox"] {
-    opacity: 0.5;
+    visibility: hidden;
     }
-/* I designed the button with: http://charliepark.org/bootstrap_buttons/ */
+li.listEntry.toRemove > a.content {
-button.custom {
+    text-decoration: line-through;
-    padding: 5px;
-    border: 1px solid transparent;
-    border-color: #80b3ff #80b3ff hsl(216, 100%, 75%);
-    border-radius: 3px;
-    background-color: hsl(216, 100%, 75%);
-    background-image: linear-gradient(#a8cbff, #80b3ff);
-    background-repeat: repeat-x;
-    color: #222;
-    cursor: pointer;
-    opacity: 0.8;
     }
-button.custom.disabled {
+li.listEntry > .fa {
-    border-color: #dddddd #dddddd hsl(36, 0%, 85%);
+    color: inherit;
-    background-color: hsl(36, 0%, 72%);
+    display: none;
-    background-image: linear-gradient(#f2f2f2, #dddddd);
+    font-size: 110%;
-    color: #aaa;
+    opacity: 0.5;
-    pointer-events: none;
+    vertical-align: baseline;
     }
-button.custom:hover {
+li.listEntry > a.fa:hover {
-    opacity: 1.0;
+    opacity: 1;
     }
-button.custom.reloadAll:not(.disabled) {
+li.listEntry.support > a.support {
-    border-color: #ffcc7f #ffcc7f hsl(36, 100%, 73%);
+    display: inline-block;
-    background-color: hsl(36, 100%, 75%);
-    background-image: linear-gradient(#ffdca8, #ffcc7f);
     }
-#buttonApply {
+li.listEntry > a.remove,
-    display: initial;
+li.listEntry > a.remove:visited {
-    padding: 1em 1em;
+    color: darkred;
-    position: fixed;
-    top: 1em;
     }
-body[dir="ltr"] #buttonApply {
+li.listEntry.external > a.remove {
-    right: 1em;
+    display: inline-block;
     }
-body[dir="rtl"] #buttonApply {
+li.listEntry.mustread > a.mustread {
-    left: 1em;
+    display: inline-block;
     }
-#buttonApply.disabled {
+li.listEntry.mustread > a.mustread:hover {
-    display: none;
+    color: mediumblue;
     }
-span.status {
+li.listEntry > .counts {
-    border: 1px solid transparent;
+    display: none;
-    color: #444;
-    display: inline-block;
     font-size: smaller;
-    line-height: 1;
-    margin: 0 0 0 0.5em;
-    opacity: 0.8;
-    padding: 1px 2px;
 }
-span.unsecure {
+li.listEntry > input[type="checkbox"]:checked ~ .counts {
-    background-color: hsl(0, 100%, 88%);
+    display: inline;
-    border-color: hsl(0, 100%, 83%);
+}
+li.listEntry span.status {
+    color: #444;
+    cursor: default;
+    display: none;
+}
+li.listEntry span.status:hover {
+    opacity: 1;
+    }
+li.listEntry span.unsecure {
+    color: darkred;
+    }
+li.listEntry.unsecure > input[type="checkbox"]:checked ~ span.unsecure {
+    display: inline-block;
     }
-span.purge {
+li.listEntry span.failed {
-    border-color: #ddd;
+    color: darkred;
-    background-color: #eee;
+    }
+li.listEntry.failed span.failed {
+    display: inline-block;
+    }
+li.listEntry span.cache {
     cursor: pointer;
     }
-span.purge:hover {
+li.listEntry.cached:not(.obsolete) > input[type="checkbox"]:checked ~ span.cache {
-    opacity: 1;
+    display: inline-block;
     }
-span.obsolete {
+li.listEntry span.obsolete {
-    border-color: hsl(36, 100%, 73%);
+    color: hsl(36, 100%, 40%);
-    background-color: hsl(36, 100%, 75%);
     }
-#externalListsDiv {
+body:not(.updating) li.listEntry.obsolete > input[type="checkbox"]:checked ~ span.obsolete {
+    display: inline-block;
+    }
+li.listEntry span.updating {
+    transform-origin: 50% 46%;
+    }
+body.updating li.listEntry.obsolete > input[type="checkbox"]:checked ~ span.updating {
+    animation: spin 1s step-start infinite;
+    display: inline-block;
+    }
+.dim {
+    opacity: 0.5;
+    }
+#externalLists {
     margin: 2em auto 0 auto;
     }
 body[dir="ltr"] #externalListsDiv {
@@ -119,64 +140,3 @@ body[dir="rtl"] #externalListsDiv {
     height: 12em;
     white-space: pre;
     }
-body #busyOverlay {
-    background-color: transparent;
-    bottom: 0;
-    cursor: wait;
-    display: none;
-    left: 0;
-    position: fixed;
-    right: 0;
-    top: 0;
-    z-index: 1000;
-    }
-body.busy #busyOverlay {
-    display: block;
-    }
-#busyOverlay > div:nth-of-type(1) {
-    background-color: white;
-    bottom: 0;
-    left: 0;
-    opacity: 0.75;
-    position: absolute;
-    right: 0;
-    top: 0;
-    }
-#busyOverlay > div:nth-of-type(2) {
-    background-color: #eee;
-    border: 1px solid transparent;
-    border-color: #80b3ff #80b3ff hsl(216, 100%, 75%);
-    border-radius: 3px;
-    box-sizing: border-box;
-    height: 3em;
-    left: 10%;
-    position: absolute;
-    bottom: 75%;
-    width: 80%;
-    }
-#busyOverlay > div:nth-of-type(2) > div:nth-of-type(1) {
-    background-color: hsl(216, 100%, 75%);
-    background-image: linear-gradient(#a8cbff, #80b3ff);
-    background-repeat: repeat-x;
-    border: 0;
-    box-sizing: border-box;
-    color: #222;
-    height: 100%;
-    left: 0;
-    padding: 0;
-    position: absolute;
-    width: 25%;
-    }
-#busyOverlay > div:nth-of-type(2) > div:nth-of-type(2) {
-    background-color: transparent;
-    border: 0;
-    box-sizing: border-box;
-    height: 100%;
-    left: 0;
-    line-height: 3em;
-    overflow: hidden;
-    position: absolute;
-    text-align: center;
-    top: 0;
-    width: 100%;
-    }

src/css/popup.css

@@ -74,7 +74,9 @@ body[dir="rtl"] #gotoDashboard > span:last-of-type {
     padding: 02px 0;
     text-align: center;
     }
-
+#toolbarContainer {
+    position: relative;
+}
 .toolbar {
     border: 0;
     display: inline-block;

src/hosts-files.html

@@ -11,43 +11,38 @@
 <body>
 
 <p data-i18n="hostsFilesPrompt"></p>
+<p>
+    <button id="buttonApply" class="custom important reloadAll disabled" data-i18n="hostsFilesApplyChanges"></button>
+    <button id="buttonUpdate" class="custom important reloadAll disabled" data-i18n="hostsFilesUpdateNow"></button>
+    <button id="buttonPurgeAll" class="custom disabled" data-i18n="hostsFilesPurgeAll"></button>
+</p>
 <ul id="options">
     <li><input type="checkbox" id="autoUpdate"><label data-i18n="hostsFilesAutoUpdatePrompt" for="autoUpdate"></label>
-        <button class="custom reloadAll disabled" id="buttonUpdate" data-i18n="hostsFilesUpdateNow"></button>
+    <li><span id="listsOfBlockedHostsPrompt"></span>
-        <button id="buttonPurgeAll" class="custom disabled" data-i18n="hostsFilesPurgeAll"></button>
-    <li><p id="listsOfBlockedHostsPrompt"></p>
 </ul>
-<button id="buttonApply" class="custom reloadAll disabled" data-i18n="hostsFilesApplyChanges"></button>
 <ul id="lists">
 </ul>
 
-<div id="externalListsDiv">
+<div id="externalLists">
 <p data-i18n="hostsFilesExternalListsHint" style="margin: 0 0 0.25em 0; font-size: 13px;"></p>
 <textarea id="externalHostsFiles" dir="ltr" spellcheck="false"></textarea>
-<p style="margin: 0.25em 0 0 0"><button id="externalListsParse" disabled="true" data-i18n="hostsFilesExternalListsParse"></button></p>
-</div>
-
-<div id="busyOverlay">
-    <div></div>
-    <!-- progress bar widget -->
-    <div><div></div><div></div></div>
 </div>
 
 <div id="templates" style="display: none;">
     <ul>
-        <li class="groupEntry"><span class="geName"></span> <span class="geCount dim"></span>
-            <ul></ul>
-        </li>
         <li class="listEntry">
-            <input type="checkbox">
+            <input type="checkbox"><!--
-            <a type="text/plain" target="_blank" href=""></a>
+         --><a class="content" type="text/plain" target="_blank" href=""></a>&#8203;<!--
-            <a href="" style="display: none;" target="_blank"></a>: <!--
+         --><a class="fa support" href="" target="_blank">&#xf015;</a>&#8203;<!--
-         --><span class="dim"></span><!--
+         --><a class="fa remove" href="">&#xf00d;</a>&#8203;<!--
-         --><span class="status unsecure" style="display: none;">http</span><!--
+         --><a class="fa mustread" href="" target="_blank">&#xf05a;</a>&#8203;<!--
-         --><span class="status new" style="display: none;" data-i18n="hostsFilesExternalListNew"></span><!--
+         --><span class="fa status unsecure" title="http">&#xf13e;</span>&#8203;<!--
-         --><span class="status obsolete" style="display: none;" data-i18n="hostsFilesExternalListObsolete"></span><!--
+         --><span class="counts dim"></span>&#8203;<!--
-         --><span class="status purge" style="display: none;" data-i18n="hostsFilesExternalListPurge"></span>
+         --><span class="fa status obsolete" title="hostsFilesExternalListObsolete">&#xf071;</span>&#8203;<!--
-        </li>
+         --><span class="fa status cache">&#xf017;</span>&#8203;<!--
+         --><span class="fa status updating">&#xf110;</span>&#8203;<!--
+         --><span class="fa status failed">&#xf06a;</span>
+            </li>
     </ul>
 </div>
 

src/js/asset-viewer.js

@@ -1,7 +1,7 @@
 /*******************************************************************************
 
-    µMatrix - a Chromium browser extension to block requests.
+    uMatrix - a Chromium browser extension to block requests.
-    Copyright (C) 2014 Raymond Hill
+    Copyright (C) 2014-2017 Raymond Hill
 
     This program is free software: you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
@@ -19,14 +19,14 @@
     Home: https://github.com/gorhill/uMatrix
 */
 
-/* global vAPI, uDom */
+/* global uDom */
+
+'use strict';
 
 /******************************************************************************/
 
 (function() {
 
-'use strict';
-
 /******************************************************************************/
 
 var messager = vAPI.messaging.channel('asset-viewer.js');

src/js/background.js

@@ -1,7 +1,7 @@
 /*******************************************************************************
 
     µMatrix - a Chromium browser extension to black/white list requests.
-    Copyright (C) 2014  Raymond Hill
+    Copyright (C) 2014-2017 Raymond Hill
 
     This program is free software: you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
@@ -19,11 +19,11 @@
     Home: https://github.com/gorhill/uMatrix
 */
 
-/* global chrome */
+'use strict';
 
 /******************************************************************************/
 
-var µMatrix = (function() {
+var µMatrix = (function() { // jshint ignore:line
 
 /******************************************************************************/
 
@@ -127,12 +127,8 @@ return {
     updateAssetsEvery: 11 * oneDay + 1 * oneHour + 1 * oneMinute + 1 * oneSecond,
     firstUpdateAfter: 11 * oneMinute,
     nextUpdateAfter: 11 * oneHour,
-    projectServerRoot: 'https://raw.githubusercontent.com/gorhill/umatrix/master/',
+    assetsBootstrapLocation: 'assets/assets.json',
-    pslPath: 'assets/thirdparties/publicsuffix.org/list/effective_tld_names.dat',
+    pslAssetKey: 'public_suffix_list.dat',
-
-    // permanent hosts files
-    permanentHostsFiles: {
-    },
 
     // list of live hosts files
     liveHostsFiles: {

src/js/hosts-files.js

@@ -1,7 +1,7 @@
 /*******************************************************************************
 
-    µMatrix - a Chromium browser extension to black/white list requests.
+    uMatrix - a Chromium browser extension to black/white list requests.
-    Copyright (C) 2014-2015 Raymond Hill
+    Copyright (C) 2014-2017 Raymond Hill
 
     This program is free software: you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
@@ -19,37 +19,34 @@
     Home: https://github.com/gorhill/uMatrix
 */
 
-/* global vAPI, uDom */
+/* global uDom */
+
+'use strict';
 
 /******************************************************************************/
 
 (function() {
 
-'use strict';
-
 /******************************************************************************/
 
-var listDetails = {};
+var listDetails = {},
-var externalHostsFiles = '';
+    lastUpdateTemplateString = vAPI.i18n('hostsFilesLastUpdate'),
-var cacheWasPurged = false;
+    hostsFilesSettingsHash,
-var needUpdate = false;
+    reValidExternalList = /[a-z-]+:\/\/\S*\/\S+/;
-var hasCachedContent = false;
 
 /******************************************************************************/
 
 var onMessage = function(msg) {
     switch ( msg.what ) {
+    case 'assetUpdated':
+        updateAssetStatus(msg);
+        break;
+    case 'assetsUpdated':
+        document.body.classList.remove('updating');
+        break;
     case 'loadHostsFilesCompleted':
         renderHostsFiles();
         break;
-
-    case 'forceUpdateAssetsProgress':
-        renderBusyOverlay(true, msg.progress);
-        if ( msg.done ) {
-            messager.send({ what: 'reloadHostsFiles' });
-        }
-        break;
-
     default:
         break;
     }
@@ -65,115 +62,120 @@ var renderNumber = function(value) {
 
 /******************************************************************************/
 
-// TODO: get rid of background page dependencies
+var renderHostsFiles = function(soft) {
-
+    var listEntryTemplate = uDom('#templates .listEntry'),
-var renderHostsFiles = function() {
+        listStatsTemplate = vAPI.i18n('hostsFilesPerFileStats'),
-    var listEntryTemplate = uDom('#templates .listEntry');
+        renderElapsedTimeToString = vAPI.i18n.renderElapsedTimeToString,
-    var listStatsTemplate = vAPI.i18n('hostsFilesPerFileStats');
+        reExternalHostFile = /^https?:/;
-    var lastUpdateString = vAPI.i18n('hostsFilesLastUpdate');
-    var renderElapsedTimeToString = vAPI.i18n.renderElapsedTimeToString;
-    var reExternalHostFile = /^https?:/;
 
-    // Assemble a pretty blacklist name if possible
+    // Assemble a pretty list name if possible
     var listNameFromListKey = function(listKey) {
         var list = listDetails.current[listKey] || listDetails.available[listKey];
         var listTitle = list ? list.title : '';
-        if ( listTitle === '' ) {
+        if ( listTitle === '' ) { return listKey; }
-            return listKey;
-        }
         return listTitle;
     };
 
-    var liFromListEntry = function(listKey) {
+    var liFromListEntry = function(listKey, li) {
-        var elem, text;
+        var entry = listDetails.available[listKey],
-        var entry = listDetails.available[listKey];
+            elem;
-        var li = listEntryTemplate.clone();
+        if ( !li ) {
-
+            li = listEntryTemplate.clone().nodeAt(0);
-        if ( entry.off !== true ) {
-            li.descendants('input').attr('checked', '');
         }
-
+        if ( li.getAttribute('data-listkey') !== listKey ) {
-        elem = li.descendants('a:nth-of-type(1)');
+            li.setAttribute('data-listkey', listKey);
-        elem.attr('href', encodeURI(listKey));
+            elem = li.querySelector('input[type="checkbox"]');
-        elem.text(listNameFromListKey(listKey) + '\u200E');
+            elem.checked = entry.off !== true;
-
+            elem = li.querySelector('a:nth-of-type(1)');
-        elem = li.descendants('a:nth-of-type(2)');
+            elem.setAttribute('href', 'asset-viewer.html?url=' + encodeURI(listKey));
-        if ( entry.homeDomain ) {
+            elem.setAttribute('type', 'text/html');
-            elem.attr('href', 'http://' + encodeURI(entry.homeHostname));
+            elem.textContent = listNameFromListKey(listKey);
-            elem.text('(' + entry.homeDomain + ')');
+            li.classList.remove('toRemove');
-            elem.css('display', '');
+            if ( entry.supportName ) {
+                li.classList.add('support');
+                elem = li.querySelector('a.support');
+                elem.setAttribute('href', entry.supportURL);
+                elem.setAttribute('title', entry.supportName);
+            } else {
+                li.classList.remove('support');
+            }
+            if ( entry.external ) {
+                li.classList.add('external');
+            } else {
+                li.classList.remove('external');
+            }
+            if ( entry.instructionURL ) {
+                li.classList.add('mustread');
+                elem = li.querySelector('a.mustread');
+                elem.setAttribute('href', entry.instructionURL);
+            } else {
+                li.classList.remove('mustread');
+            }
         }
-
+        // https://github.com/gorhill/uBlock/issues/1429
-        elem = li.descendants('span:nth-of-type(1)');
+        if ( !soft ) {
-        text = listStatsTemplate
+            elem = li.querySelector('input[type="checkbox"]');
-            .replace('{{used}}', renderNumber(!entry.off && !isNaN(+entry.entryUsedCount) ? entry.entryUsedCount : 0))
+            elem.checked = entry.off !== true;
-            .replace('{{total}}', !isNaN(+entry.entryCount) ? renderNumber(entry.entryCount) : '?');
-        elem.text(text);
-
-        // https://github.com/gorhill/uBlock/issues/78
-        // Badge for non-secure connection
-        var remoteURL = listKey;
-        if ( remoteURL.lastIndexOf('http:', 0) !== 0 ) {
-            remoteURL = entry.homeURL || '';
         }
-        if ( remoteURL.lastIndexOf('http:', 0) === 0 ) {
+        elem = li.querySelector('span.counts');
-            li.descendants('span.status.unsecure').css('display', '');
+        var text = '';
+        if ( !isNaN(+entry.entryUsedCount) && !isNaN(+entry.entryCount) ) {
+            text = listStatsTemplate
+                .replace('{{used}}', renderNumber(entry.off ? 0 : entry.entryUsedCount))
+                .replace('{{total}}', renderNumber(entry.entryCount));
         }
-
+        elem.textContent = text;
         // https://github.com/chrisaljoudi/uBlock/issues/104
         var asset = listDetails.cache[listKey] || {};
-
+        var remoteURL = asset.remoteURL;
-        // Badge for update status
+        li.classList.toggle(
-        if ( entry.off !== true ) {
+            'unsecure',
-            if ( asset.repoObsolete ) {
+            typeof remoteURL === 'string' && remoteURL.lastIndexOf('http:', 0) === 0
-                li.descendants('span.status.new').css('display', '');
+        );
-                needUpdate = true;
+        li.classList.toggle('failed', asset.error !== undefined);
-            } else if ( asset.cacheObsolete ) {
+        li.classList.toggle('obsolete', asset.obsolete === true);
-                li.descendants('span.status.obsolete').css('display', '');
+        li.classList.toggle('cached', asset.cached === true && asset.writeTime > 0);
-                needUpdate = true;
-            } else if ( entry.external && !asset.cached ) {
-                li.descendants('span.status.obsolete').css('display', '');
-                needUpdate = true;
-            }
-        }
-
-        // In cache
         if ( asset.cached ) {
-            elem = li.descendants('span.status.purge');
+            li.querySelector('.status.cache').setAttribute(
-            elem.css('display', '');
+                'title',
-            elem.attr('title', lastUpdateString.replace('{{ago}}', renderElapsedTimeToString(asset.lastModified)));
+                lastUpdateTemplateString.replace('{{ago}}', renderElapsedTimeToString(asset.writeTime))
-            hasCachedContent = true;
+            );
         }
+        li.classList.remove('discard');
         return li;
     };
 
     var onListsReceived = function(details) {
         // Before all, set context vars
         listDetails = details;
-        needUpdate = false;
-        hasCachedContent = false;
 
-        var availableLists = details.available;
+        // Incremental rendering: this will allow us to easily discard unused
-        var listKeys = Object.keys(details.available);
+        // DOM list entries.
+        uDom('#lists .listEntry').addClass('discard');
+
+        var availableLists = details.available,
+            listKeys = Object.keys(details.available);
 
         // Sort works this way:
         // - Send /^https?:/ items at the end (custom hosts file URL)
         listKeys.sort(function(a, b) {
-            var ta = availableLists[a].title || a;
+            var ta = availableLists[a].title || a,
-            var tb = availableLists[b].title || b;
+                tb = availableLists[b].title || b;
             if ( reExternalHostFile.test(ta) === reExternalHostFile.test(tb) ) {
                 return ta.localeCompare(tb);
             }
-            if ( reExternalHostFile.test(tb) ) {
+            return reExternalHostFile.test(tb) ? -1 : 1;
-                return -1;
-            }
-            return 1;
         });
-        var ulList = uDom('#lists').empty();
+
+        var ulList = document.querySelector('#lists');
         for ( var i = 0; i < listKeys.length; i++ ) {
-            ulList.append(liFromListEntry(listKeys[i]));
+            var liEntry = liFromListEntry(listKeys[i], ulList.children[i]);
+            if ( liEntry.parentElement === null ) {
+                ulList.appendChild(liEntry);
+            }
         }
 
+        uDom('#lists .listEntry.discard').remove();
         uDom('#listsOfBlockedHostsPrompt').text(
             vAPI.i18n('hostsFilesStats').replace(
                 '{{blockedHostnameCount}}',
@@ -182,8 +184,10 @@ var renderHostsFiles = function() {
         );
         uDom('#autoUpdate').prop('checked', listDetails.autoUpdate === true);
 
+        if ( !soft ) {
+            hostsFilesSettingsHash = hashFromCurrentFromSettings();
+        }
         renderWidgets();
-        renderBusyOverlay(details.manualUpdate, details.manualUpdateProgress);
     };
 
     messager.send({ what: 'getLists' }, onListsReceived);
@@ -191,198 +195,163 @@ var renderHostsFiles = function() {
 
 /******************************************************************************/
 
-// Progress must be normalized to [0, 1], or can be undefined.
-
-var renderBusyOverlay = function(state, progress) {
-    progress = progress || {};
-    var showProgress = typeof progress.value === 'number';
-    if ( showProgress ) {
-        uDom('#busyOverlay > div:nth-of-type(2) > div:first-child').css(
-            'width',
-            (progress.value * 100).toFixed(1) + '%'
-        );
-        var text = progress.text || '';
-        if ( text !== '' ) {
-            uDom('#busyOverlay > div:nth-of-type(2) > div:last-child').text(text);
-        }
-    }
-    uDom('#busyOverlay > div:nth-of-type(2)').css('display', showProgress ? '' : 'none');
-    uDom('body').toggleClass('busy', !!state);
-};
-
-/******************************************************************************/
-
-// This is to give a visual hint that the selection of blacklists has changed.
-
 var renderWidgets = function() {
-    uDom('#buttonApply').toggleClass('disabled', !listsSelectionChanged());
+    uDom('#buttonUpdate').toggleClass('disabled', document.querySelector('body:not(.updating) #lists .listEntry.obsolete > input[type="checkbox"]:checked') === null);
-    uDom('#buttonUpdate').toggleClass('disabled', !listsContentChanged());
+    uDom('#buttonPurgeAll').toggleClass('disabled', document.querySelector('#lists .listEntry.cached') === null);
-    uDom('#buttonPurgeAll').toggleClass('disabled', !hasCachedContent);
+    uDom('#buttonApply').toggleClass('disabled', hostsFilesSettingsHash === hashFromCurrentFromSettings());
 };
 
 /******************************************************************************/
 
-// Return whether selection of lists changed.
+var updateAssetStatus = function(details) {
-
+    var li = document.querySelector('#lists .listEntry[data-listkey="' + details.key + '"]');
-var listsSelectionChanged = function() {
+    if ( li === null ) { return; }
-    if ( cacheWasPurged ) {
+    li.classList.toggle('failed', !!details.failed);
-        return true;
+    li.classList.toggle('obsolete', !details.cached);
-    }
+    li.classList.toggle('cached', !!details.cached);
-    var availableLists = listDetails.available;
+    if ( details.cached ) {
-    var currentLists = listDetails.current;
+        li.querySelector('.status.cache').setAttribute(
-    var location, availableOff, currentOff;
+            'title',
-    // This check existing entries
+            lastUpdateTemplateString.replace(
-    for ( location in availableLists ) {
+                '{{ago}}',
-        if ( availableLists.hasOwnProperty(location) === false ) {
+                vAPI.i18n.renderElapsedTimeToString(Date.now())
-            continue;
+            )
-        }
+        );
-        availableOff = availableLists[location].off === true;
-        currentOff = currentLists[location] === undefined || currentLists[location].off === true;
-        if ( availableOff !== currentOff ) {
-            return true;
-        }
-    }
-    // This check removed entries
-    for ( location in currentLists ) {
-        if ( currentLists.hasOwnProperty(location) === false ) {
-            continue;
-        }
-        currentOff = currentLists[location].off === true;
-        availableOff = availableLists[location] === undefined || availableLists[location].off === true;
-        if ( availableOff !== currentOff ) {
-            return true;
-        }
     }
-    return false;
+    renderWidgets();
 };
 
-/******************************************************************************/
+/*******************************************************************************
-
-// Return whether content need update.
 
-var listsContentChanged = function() {
+    Compute a hash from all the settings affecting how filter lists are loaded
-    return needUpdate;
+    in memory.
-};
 
-/******************************************************************************/
+**/
 
-// This is to give a visual hint that the selection of blacklists has changed.
+var hashFromCurrentFromSettings = function() {
-
+    var hash = [],
-var updateWidgets = function() {
+        listHash = [],
-    uDom('#buttonApply').toggleClass('disabled', !listsSelectionChanged());
+        listEntries = document.querySelectorAll('#lists .listEntry[data-listkey]:not(.toRemove)'),
-    uDom('#buttonUpdate').toggleClass('disabled', !listsContentChanged());
+        liEntry,
-    uDom('#buttonPurgeAll').toggleClass('disabled', !hasCachedContent);
+        i = listEntries.length;
-    uDom('body').toggleClass('busy', false);
+    while ( i-- ) {
+        liEntry = listEntries[i];
+        if ( liEntry.querySelector('input[type="checkbox"]:checked') !== null ) {
+            listHash.push(liEntry.getAttribute('data-listkey'));
+        }
+    }
+    hash.push(
+        listHash.sort().join(),
+        reValidExternalList.test(document.getElementById('externalHostsFiles').value),
+        document.querySelector('#lists .listEntry.toRemove') !== null
+    );
+    return hash.join();
 };
 
 /******************************************************************************/
 
-var onListCheckboxChanged = function() {
+var onHostsFilesSettingsChanged = function() {
-    var href = uDom(this).parent().descendants('a').first().attr('href');
+    renderWidgets();
-    if ( typeof href !== 'string' ) {
-        return;
-    }
-    if ( listDetails.available[href] === undefined ) {
-        return;
-    }
-    listDetails.available[href].off = !this.checked;
-    updateWidgets();
 };
 
 /******************************************************************************/
 
-var onListLinkClicked = function(ev) {
+var onRemoveExternalHostsFile = function(ev) {
-    messager.send({
+    var liEntry = uDom(this).ancestors('[data-listkey]'),
-        what: 'gotoExtensionURL',
+        listKey = liEntry.attr('data-listkey');
-        url: 'asset-viewer.html?url=' + uDom(this).attr('href')
+    if ( listKey ) {
-    });
+        liEntry.toggleClass('toRemove');
+        renderWidgets();
+    }
     ev.preventDefault();
 };
 
 /******************************************************************************/
 
 var onPurgeClicked = function() {
-    var button = uDom(this);
+    var button = uDom(this),
-    var li = button.parent();
+        liEntry = button.ancestors('[data-listkey]'),
-    var href = li.descendants('a').first().attr('href');
+        listKey = liEntry.attr('data-listkey');
-    if ( !href ) {
+    if ( !listKey ) { return; }
-        return;
+
-    }
+    messager.send({ what: 'purgeCache', assetKey: listKey });
-    messager.send({ what: 'purgeCache', path: href });
+    liEntry.addClass('obsolete');
-    button.remove();
+    liEntry.removeClass('cached');
-    if ( li.descendants('input').first().prop('checked') ) {
+
-        cacheWasPurged = true;
+    if ( liEntry.descendants('input').first().prop('checked') ) {
-        updateWidgets();
+        renderWidgets();
     }
 };
 
 /******************************************************************************/
 
 var selectHostsFiles = function(callback) {
-    var switches = [];
+    // Hosts files to select
-    var lis = uDom('#lists .listEntry'), li;
+    var toSelect = [],
-    var i = lis.length;
+        liEntries = document.querySelectorAll('#lists .listEntry[data-listkey]:not(.toRemove)'),
+        i = liEntries.length,
+        liEntry;
     while ( i-- ) {
-        li = lis.at(i);
+        liEntry = liEntries[i];
-        switches.push({
+        if ( liEntry.querySelector('input[type="checkbox"]:checked') !== null ) {
-            location: li.descendants('a').attr('href'),
+            toSelect.push(liEntry.getAttribute('data-listkey'));
-            off: li.descendants('input').prop('checked') === false
+        }
-        });
     }
 
+    // External hosts files to remove
+    var toRemove = [];
+    liEntries = document.querySelectorAll('#lists .listEntry.toRemove[data-listkey]');
+    i = liEntries.length;
+    while ( i-- ) {
+        toRemove.push(liEntries[i].getAttribute('data-listkey'));
+    }
+
+    // External hosts files to import
+    var externalListsElem = document.getElementById('externalHostsFiles'),
+        toImport = externalListsElem.value.trim();
+    externalListsElem.value = '';
+
     messager.send({
-        what: 'selectHostsFiles',
+            what: 'selectHostsFiles',
-        switches: switches
+            toSelect: toSelect,
-    }, callback);
+            toImport: toImport,
+            toRemove: toRemove
+        },
+        callback
+    );
+
+    hostsFilesSettingsHash = hashFromCurrentFromSettings();
 };
 
 /******************************************************************************/
 
 var buttonApplyHandler = function() {
     uDom('#buttonApply').removeClass('enabled');
-
+    selectHostsFiles(function() {
-    renderBusyOverlay(true);
-
-    var onSelectionDone = function() {
         messager.send({ what: 'reloadHostsFiles' });
-    };
+    });
-
+    renderWidgets();
-    selectHostsFiles(onSelectionDone);
-
-    cacheWasPurged = false;
 };
 
 /******************************************************************************/
 
 var buttonUpdateHandler = function() {
     uDom('#buttonUpdate').removeClass('enabled');
-
+    selectHostsFiles(function() {
-    if ( needUpdate ) {
+        document.body.classList.add('updating');
-        renderBusyOverlay(true);
+        messager.send({ what: 'forceUpdateAssets' });
-
+        renderWidgets();
-        var onSelectionDone = function() {
+    });
-            messager.send({ what: 'forceUpdateAssets' });
+    renderWidgets();
-        };
-
-        selectHostsFiles(onSelectionDone);
-
-        cacheWasPurged = false;
-    }
 };
 
 /******************************************************************************/
 
 var buttonPurgeAllHandler = function() {
     uDom('#buttonPurgeAll').removeClass('enabled');
-
+    messager.send({ what: 'purgeAllCaches' }, function() {
-    renderBusyOverlay(true);
+        renderHostsFiles(true);
-
+    });
-    var onCompleted = function() {
-        cacheWasPurged = true;
-        renderHostsFiles();
-    };
-
-    messager.send({ what: 'purgeAllCaches' }, onCompleted);
 };
 
 /******************************************************************************/
@@ -397,52 +366,16 @@ var autoUpdateCheckboxChanged = function() {
 
 /******************************************************************************/
 
-var renderExternalLists = function() {
+uDom('#autoUpdate').on('change', autoUpdateCheckboxChanged);
-    var onReceived = function(details) {
+uDom('#buttonApply').on('click', buttonApplyHandler);
-        uDom('#externalHostsFiles').val(details);
+uDom('#buttonUpdate').on('click', buttonUpdateHandler);
-        externalHostsFiles = details;
+uDom('#buttonPurgeAll').on('click', buttonPurgeAllHandler);
-    };
+uDom('#lists').on('change', '.listEntry > input', onHostsFilesSettingsChanged);
-    messager.send({ what: 'userSettings', name: 'externalHostsFiles' }, onReceived);
+uDom('#lists').on('click', '.listEntry > a.remove', onRemoveExternalHostsFile);
-};
+uDom('#lists').on('click', 'span.cache', onPurgeClicked);
-
+uDom('#externalHostsFiles').on('input', onHostsFilesSettingsChanged);
-/******************************************************************************/
-
-var externalListsChangeHandler = function() {
-    uDom('#externalListsParse').prop(
-        'disabled',
-        this.value.trim() === externalHostsFiles
-    );
-};
-
-/******************************************************************************/
-
-var externalListsApplyHandler = function() {
-    externalHostsFiles = uDom('#externalHostsFiles').val();
-    messager.send({
-        what: 'userSettings',
-        name: 'externalHostsFiles',
-        value: externalHostsFiles
-    });
-    renderHostsFiles();
-    uDom('#externalListsParse').prop('disabled', true);
-};
-
-/******************************************************************************/
 
-uDom.onLoad(function() {
+renderHostsFiles();
-    uDom('#autoUpdate').on('change', autoUpdateCheckboxChanged);
-    uDom('#buttonApply').on('click', buttonApplyHandler);
-    uDom('#buttonUpdate').on('click', buttonUpdateHandler);
-    uDom('#buttonPurgeAll').on('click', buttonPurgeAllHandler);
-    uDom('#lists').on('change', '.listEntry > input', onListCheckboxChanged);
-    uDom('#lists').on('click', '.listEntry > a:nth-of-type(1)', onListLinkClicked);
-    uDom('#lists').on('click', 'span.purge', onPurgeClicked);
-    uDom('#externalHostsFiles').on('input', externalListsChangeHandler);
-    uDom('#externalListsParse').on('click', externalListsApplyHandler);
-
-    renderHostsFiles();
-    renderExternalLists();
-});
 
 /******************************************************************************/
 

src/js/messaging.js

@@ -19,7 +19,7 @@
     Home: https://github.com/gorhill/uMatrix
 */
 
-/* global µMatrix, vAPI */
+'use strict';
 
 /******************************************************************************/
 /******************************************************************************/
@@ -28,8 +28,6 @@
 
 (function() {
 
-'use strict';
-
 var µm = µMatrix;
 
 /******************************************************************************/
@@ -40,7 +38,12 @@ function onMessage(request, sender, callback) {
     // Async
     switch ( request.what ) {
     case 'getAssetContent':
-        return µm.assets.getLocal(request.url, callback);
+        µm.assets.get(request.url, { dontCache: true }, callback);
+        return;
+
+    case 'selectHostsFiles':
+        µm.selectHostsFiles(request, callback);
+        return;
 
     default:
         break;
@@ -55,7 +58,8 @@ function onMessage(request, sender, callback) {
         break;
 
     case 'forceUpdateAssets':
-        µm.assetUpdater.force();
+        µm.scheduleAssetUpdater(0);
+        µm.assets.updateStart({ delay: 2000 });
         break;
 
     case 'getUserSettings':
@@ -82,10 +86,6 @@ function onMessage(request, sender, callback) {
         µm.reloadHostsFiles();
         break;
 
-    case 'selectHostsFiles':
-        µm.selectHostsFiles(request.switches);
-        break;
-
     case 'userSettings':
         if ( request.hasOwnProperty('value') === false ) {
             request.value = undefined;
@@ -571,8 +571,6 @@ vAPI.messaging.listen('contentscript-end.js', onMessage);
 
 (function() {
 
-'use strict';
-
 /******************************************************************************/
 
 var µm = µMatrix;
@@ -800,8 +798,6 @@ var getLists = function(callback) {
     var onMetadataReady = function(entries) {
         r.cache = entries;
         prepEntries(r.cache);
-        r.manualUpdate = µm.assetUpdater.manualUpdate;
-        r.manualUpdateProgress = µm.assetUpdater.manualUpdateProgress;
         callback(r);
     };
     var onAvailableHostsFilesReady = function(lists) {
@@ -822,9 +818,6 @@ var onMessage = function(request, sender, callback) {
     case 'getLists':
         return getLists(callback);
 
-    case 'purgeAllCaches':
-        return µm.assets.purgeAll(callback);
-
     default:
         break;
     }
@@ -834,7 +827,16 @@ var onMessage = function(request, sender, callback) {
 
     switch ( request.what ) {
     case 'purgeCache':
-        µm.assets.purge(request.path);
+        µm.assets.purge(request.assetKey);
+        µm.assets.remove('compiled/' + request.assetKey);
+        break;
+
+    case 'purgeAllCaches':
+        if ( request.hard ) {
+            µm.assets.remove(/./);
+        } else {
+            µm.assets.purge(/./, 'public_suffix_list.dat');
+        }
         break;
 
     default:
@@ -944,8 +946,6 @@ vAPI.messaging.listen('about.js', onMessage);
 
 (function() {
 
-'use strict';
-
 /******************************************************************************/
 
 var µm = µMatrix;

src/js/popup.js

@@ -19,7 +19,7 @@
     Home: https://github.com/gorhill/uMatrix
 */
 
-/* global punycode, vAPI, uDom */
+/* global punycode, uDom */
 /* jshint esnext: true, bitwise: false */
 
 'use strict';

src/js/start.js

@@ -1,7 +1,7 @@
 /*******************************************************************************
 
     µMatrix - a Chromium browser extension to black/white list requests.
-    Copyright (C) 2014 Raymond Hill
+    Copyright (C) 2014-2017 Raymond Hill
 
     This program is free software: you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
@@ -89,22 +89,11 @@ var rwLocalUserSettings = {
 
 /******************************************************************************/
 
-// Important: raise barrier to remote fetching: we do not want resources to
-// be pulled from remote server at start up time.
-
-µm.assets.remoteFetchBarrier += 1;
-
-/******************************************************************************/
-
 var onAllDone = function() {
     µm.webRequest.start();
 
-    // https://github.com/chrisaljoudi/uBlock/issues/184
+    µm.assets.addObserver(µm.assetObserver.bind(µm));
-    // Check for updates not too far in the future.
+    µm.scheduleAssetUpdater(µm.userSettings.autoUpdate ? 7 * 60 * 1000 : 0);
-    µm.assetUpdater.onStart.addListener(µm.updateStartHandler.bind(µm));
-    µm.assetUpdater.onCompleted.addListener(µm.updateCompleteHandler.bind(µm));
-    µm.assetUpdater.onAssetUpdated.addListener(µm.assetUpdatedHandler.bind(µm));
-    µm.assets.onAssetCacheRemoved.addListener(µm.assetCacheRemovedHandler.bind(µm));
 
     for ( var key in defaultLocalUserSettings ) {
         if ( defaultLocalUserSettings.hasOwnProperty(key) === false ) {
@@ -118,9 +107,7 @@ var onAllDone = function() {
         }
     }
 
-    vAPI.cloud.start([
+    vAPI.cloud.start([ 'myRulesPane' ]);
-        'myRulesPane'
-    ]);
 };
 
 var onTabsReady = function(tabs) {
@@ -135,12 +122,6 @@ var onTabsReady = function(tabs) {
     onAllDone();
 };
 
-var onHostsFilesLoaded = function() {
-    // Important: remove barrier to remote fetching, this was useful only
-    // for launch time.
-    µm.assets.remoteFetchBarrier -= 1;
-};
-
 var onUserSettingsLoaded = function() {
     // Version 0.9.0.0
     // Remove obsolete user settings which may have been loaded.
@@ -154,7 +135,7 @@ var onUserSettingsLoaded = function() {
     delete µm.userSettings.subframeColor;
     delete µm.userSettings.subframeOpacity;
 
-    µm.loadHostsFiles(onHostsFilesLoaded);
+    µm.loadHostsFiles();
 };
 
 var onPSLReady = function() {

src/js/storage.js

@@ -19,7 +19,9 @@
     Home: https://github.com/gorhill/uMatrix
 */
 
-/* global chrome, µMatrix, punycode, publicSuffixList */
+/* global objectAssign, punycode, publicSuffixList */
+
+'use strict';
 
 /******************************************************************************/
 
@@ -28,7 +30,12 @@
     var getBytesInUseHandler = function(bytesInUse) {
         µm.storageUsed = bytesInUse;
     };
-    vAPI.storage.getBytesInUse(null, getBytesInUseHandler);
+    // Not all WebExtension implementations support getBytesInUse().
+    if ( typeof vAPI.storage.getBytesInUse === 'function' ) {
+        vAPI.storage.getBytesInUse(null, getBytesInUseHandler);
+    } else {
+        µm.storageUsed = undefined;
+    }
 };
 
 /******************************************************************************/
@@ -89,65 +96,84 @@
 
 /******************************************************************************/
 
+µMatrix.listKeysFromCustomHostsFiles = function(raw) {
+    var out = new Set(),
+        reIgnore = /^[!#]/,
+        reValid = /^[a-z-]+:\/\/\S+/,
+        lineIter = new this.utils.LineIterator(raw),
+        location;
+    while ( lineIter.eot() === false ) {
+        location = lineIter.next().trim();
+        if ( reIgnore.test(location) || !reValid.test(location) ) { continue; }
+        out.add(location);
+    }
+    return this.utils.setToArray(out);
+};
+
+/******************************************************************************/
+
 µMatrix.getAvailableHostsFiles = function(callback) {
-    var availableHostsFiles = {};
+    var µm = this,
-    var redirections = {};
+        availableHostsFiles = {};
-    var µm = this;
 
-    var fixLocation = function(location) {
+    // Custom filter lists.
-        // https://github.com/chrisaljoudi/uBlock/issues/418
+    var importedListKeys = this.listKeysFromCustomHostsFiles(µm.userSettings.externalHostsFiles),
-        // We now support built-in external filter lists
+        i = importedListKeys.length,
-        if ( /^https?:/.test(location) === false ) {
+        listKey, entry;
-            location = 'assets/thirdparties/' + location;
+    while ( i-- ) {
-        }
+        listKey = importedListKeys[i];
-        return location;
+        entry = {
-    };
+            content: 'filters',
+            contentURL: listKey,
+            external: true,
+            submitter: 'user',
+            title: listKey
+        };
+        availableHostsFiles[listKey] = entry;
+        this.assets.registerAssetSource(listKey, entry);
+    }
 
     // selected lists
-    var onSelectedHostsFilesLoaded = function(store) {
+    var onSelectedHostsFilesLoaded = function(bin) {
-        var lists = store.liveHostsFiles;
+        // Now get user's selection of lists
-        var locations = Object.keys(lists);
+        for ( var assetKey in bin.liveHostsFiles ) {
-        var oldLocation, newLocation;
+            var availableEntry = availableHostsFiles[assetKey];
-        var availableEntry, storedEntry;
+            if ( availableEntry === undefined ) { continue; }
-
+            var liveEntry = bin.liveHostsFiles[assetKey];
-        while ( (oldLocation = locations.pop()) ) {
+            availableEntry.off = liveEntry.off || false;
-            newLocation = redirections[oldLocation] || oldLocation;
+            if ( liveEntry.entryCount !== undefined ) {
-            availableEntry = availableHostsFiles[newLocation];
+                availableEntry.entryCount = liveEntry.entryCount;
-            if ( availableEntry === undefined ) {
-                continue;
-            }
-            storedEntry = lists[oldLocation] || {};
-            availableEntry.off = storedEntry.off || false;
-            µm.assets.setHomeURL(newLocation, availableEntry.homeURL || '');
-            if ( storedEntry.entryCount !== undefined ) {
-                availableEntry.entryCount = storedEntry.entryCount;
             }
-            if ( storedEntry.entryUsedCount !== undefined ) {
+            if ( liveEntry.entryUsedCount !== undefined ) {
-                availableEntry.entryUsedCount = storedEntry.entryUsedCount;
+                availableEntry.entryUsedCount = liveEntry.entryUsedCount;
             }
             // This may happen if the list name was pulled from the list content
-            if ( availableEntry.title === '' && storedEntry.title !== undefined ) {
+            if ( availableEntry.title === '' && liveEntry.title !== undefined ) {
-                availableEntry.title = storedEntry.title;
+                availableEntry.title = liveEntry.title;
             }
         }
+
+        // Remove unreferenced imported filter lists.
+        var dict = new Set(importedListKeys);
+        for ( assetKey in availableHostsFiles ) {
+            var entry = availableHostsFiles[assetKey];
+            if ( entry.submitter !== 'user' ) { continue; }
+            if ( dict.has(assetKey) ) { continue; }
+            delete availableHostsFiles[assetKey];
+            µm.assets.unregisterAssetSource(assetKey);
+            µm.assets.remove(assetKey);
+        }
+
         callback(availableHostsFiles);
     };
 
     // built-in lists
-    var onBuiltinHostsFilesLoaded = function(details) {
+    var onBuiltinHostsFilesLoaded = function(entries) {
-        var location, locations;
+        for ( var assetKey in entries ) {
-        try {
+            if ( entries.hasOwnProperty(assetKey) === false ) { continue; }
-            locations = JSON.parse(details.content);
+            entry = entries[assetKey];
-        } catch (e) {
+            if ( entry.content !== 'filters' ) { continue; }
-            locations = {};
+            availableHostsFiles[assetKey] = objectAssign({}, entry);
-        }
-        var hostsFileEntry;
-        for ( location in locations ) {
-            if ( locations.hasOwnProperty(location) === false ) {
-                continue;
-            }
-            hostsFileEntry = locations[location];
-            availableHostsFiles[fixLocation(location)] = hostsFileEntry;
         }
 
         // Now get user's selection of lists
@@ -157,37 +183,7 @@
         );
     };
 
-    // permanent hosts files
+    this.assets.metadata(onBuiltinHostsFilesLoaded);
-    var location;
-    var lists = this.permanentHostsFiles;
-    for ( location in lists ) {
-        if ( lists.hasOwnProperty(location) === false ) {
-            continue;
-        }
-        availableHostsFiles[location] = lists[location];
-    }
-
-    // custom lists
-    var c;
-    var locations = this.userSettings.externalHostsFiles.split('\n');
-    for ( var i = 0; i < locations.length; i++ ) {
-        location = locations[i].trim();
-        c = location.charAt(0);
-        if ( location === '' || c === '!' || c === '#' ) {
-            continue;
-        }
-        // Coarse validation
-        if ( /[^0-9A-Za-z!*'();:@&=+$,\/?%#\[\]_.~-]/.test(location) ) {
-            continue;
-        }
-        availableHostsFiles[location] = {
-            title: location,
-            external: true
-        };
-    }
-
-    // get built-in block lists.
-    this.assets.get('assets/umatrix/hosts-files.json', onBuiltinHostsFilesLoaded);
 };
 
 /******************************************************************************/
@@ -245,8 +241,6 @@
 /******************************************************************************/
 
 µMatrix.mergeHostsFile = function(details) {
-    //console.log('storage.js > mergeHostsFile from "%s": "%s..."', details.path, details.content.slice(0, 40));
-
     var usedCount = this.ubiquitousBlacklist.count;
     var duplicateCount = this.ubiquitousBlacklist.duplicateCount;
 
@@ -255,7 +249,7 @@
     usedCount = this.ubiquitousBlacklist.count - usedCount;
     duplicateCount = this.ubiquitousBlacklist.duplicateCount - duplicateCount;
 
-    var hostsFilesMeta = this.liveHostsFiles[details.path];
+    var hostsFilesMeta = this.liveHostsFiles[details.assetKey];
     hostsFilesMeta.entryCount = usedCount + duplicateCount;
     hostsFilesMeta.entryUsedCount = usedCount;
 };
@@ -263,8 +257,6 @@
 /******************************************************************************/
 
 µMatrix.mergeHostsFileContent = function(rawText) {
-    //console.log('storage.js > mergeHostsFileContent from "%s": "%s..."', details.path, details.content.slice(0, 40));
-
     var rawEnd = rawText.length;
     var ubiquitousBlacklist = this.ubiquitousBlacklist;
     var reLocalhost = /(^|\s)(localhost\.localdomain|localhost|local|broadcasthost|0\.0\.0\.0|127\.0\.0\.1|::1|fe80::1%lo0)(?=\s|$)/g;
@@ -307,7 +299,6 @@
         // For example, when a filter contains whitespace characters, or
         // whatever else outside the range of printable ascii characters.
         if ( matches[0] !== line ) {
-            //console.error('"%s": "%s" !== "%s"', details.path, matches[0], line);
             continue;
         }
 
@@ -324,30 +315,97 @@
 
 // `switches` contains the filter lists for which the switch must be revisited.
 
-µMatrix.selectHostsFiles = function(switches) {
+µMatrix.selectHostsFiles = function(details, callback) {
-    switches = switches || {};
+    var µm = this,
+        externalHostsFiles = this.userSettings.externalHostsFiles,
+        i, n, assetKey;
 
-    // Only the lists referenced by the switches are touched.
+    // Hosts file to select
-    var liveHostsFiles = this.liveHostsFiles;
+    if ( Array.isArray(details.toSelect) ) {
-    var entry, state, location;
+        for ( assetKey in this.liveHostsFiles ) {
-    var i = switches.length;
+            if ( this.liveHostsFiles.hasOwnProperty(assetKey) === false ) {
-    while ( i-- ) {
+                continue;
-        entry = switches[i];
+            }
-        state = entry.off === true;
+            if ( details.toSelect.indexOf(assetKey) !== -1 ) {
-        location = entry.location;
+                this.liveHostsFiles[assetKey].off = false;
-        if ( liveHostsFiles.hasOwnProperty(location) === false ) {
+            } else if ( details.merge !== true ) {
-            if ( state !== true ) {
+                this.liveHostsFiles[assetKey].off = true;
-                liveHostsFiles[location] = { off: state };
             }
-            continue;
         }
-        if ( liveHostsFiles[location].off === state ) {
+    }
-            continue;
+
+    // Imported hosts files to remove
+    if ( Array.isArray(details.toRemove) ) {
+        var removeURLFromHaystack = function(haystack, needle) {
+            return haystack.replace(
+                new RegExp(
+                    '(^|\\n)' +
+                    needle.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') +
+                    '(\\n|$)', 'g'),
+                '\n'
+            ).trim();
+        };
+        for ( i = 0, n = details.toRemove.length; i < n; i++ ) {
+            assetKey = details.toRemove[i];
+            delete this.liveHostsFiles[assetKey];
+            externalHostsFiles = removeURLFromHaystack(externalHostsFiles, assetKey);
+            this.assets.remove(assetKey);
         }
-        liveHostsFiles[location].off = state;
     }
 
-    vAPI.storage.set({ 'liveHostsFiles': liveHostsFiles });
+    // Hosts file to import
+    if ( typeof details.toImport === 'string' ) {
+        // https://github.com/gorhill/uBlock/issues/1181
+        //   Try mapping the URL of an imported filter list to the assetKey of an
+        //   existing stock list.
+        var assetKeyFromURL = function(url) {
+            var needle = url.replace(/^https?:/, '');
+            var assets = µm.liveHostsFiles, asset;
+            for ( var assetKey in assets ) {
+                asset = assets[assetKey];
+                if ( asset.content !== 'filters' ) { continue; }
+                if ( typeof asset.contentURL === 'string' ) {
+                    if ( asset.contentURL.endsWith(needle) ) { return assetKey; }
+                    continue;
+                }
+                if ( Array.isArray(asset.contentURL) === false ) { continue; }
+                for ( i = 0, n = asset.contentURL.length; i < n; i++ ) {
+                    if ( asset.contentURL[i].endsWith(needle) ) {
+                        return assetKey;
+                    }
+                }
+            }
+            return url;
+        };
+        var importedSet = new Set(this.listKeysFromCustomHostsFiles(externalHostsFiles)),
+            toImportSet = new Set(this.listKeysFromCustomHostsFiles(details.toImport)),
+            iter = toImportSet.values();
+        for (;;) {
+            var entry = iter.next();
+            if ( entry.done ) { break; }
+            if ( importedSet.has(entry.value) ) { continue; }
+            assetKey = assetKeyFromURL(entry.value);
+            if ( assetKey === entry.value ) {
+                importedSet.add(entry.value);
+            }
+            this.liveHostsFiles[assetKey] = {
+                content: 'filters',
+                contentURL: [ assetKey ],
+                title: assetKey
+            };
+        }
+        externalHostsFiles = this.utils.setToArray(importedSet).sort().join('\n');
+    }
+
+    if ( externalHostsFiles !== this.userSettings.externalHostsFiles ) {
+        this.userSettings.externalHostsFiles = externalHostsFiles;
+        vAPI.storage.set({ externalHostsFiles: externalHostsFiles });
+    }
+    vAPI.storage.set({ 'liveHostsFiles': this.liveHostsFiles });
+
+    if ( typeof callback === 'function' ) {
+        callback();
+    }
 };
 
 /******************************************************************************/
@@ -356,16 +414,7 @@
 // revisited.
 
 µMatrix.reloadHostsFiles = function() {
-    var µm = this;
+    this.loadHostsFiles();
-
-    // We are just reloading the filter lists: we do not want assets to update.
-    this.assets.autoUpdate = false;
-
-    var onHostsFilesReady = function() {
-        µm.assets.autoUpdate = µm.userSettings.autoUpdate;
-    };
-
-    this.loadHostsFiles(onHostsFilesReady);
 };
 
 /******************************************************************************/
@@ -382,119 +431,95 @@
         callback();
     };
 
-    this.assets.get(this.pslPath, applyPublicSuffixList);
+    this.assets.get(this.pslAssetKey, applyPublicSuffixList);
 };
 
 /******************************************************************************/
 
-µMatrix.updateStartHandler = function(callback) {
+µMatrix.scheduleAssetUpdater = (function() {
-    var µm = this;
+    var timer, next = 0;
-    var onListsReady = function(lists) {
+    return function(updateDelay) {
-        var assets = {};
+        if ( timer ) {
-        for ( var location in lists ) {
+            clearTimeout(timer);
-            if ( lists.hasOwnProperty(location) === false ) {
+            timer = undefined;
-                continue;
+        }
-            }
+        if ( updateDelay === 0 ) {
-            if ( lists[location].off ) {
+            next = 0;
-                continue;
+            return;
-            }
+        }
-            assets[location] = true;
+        var now = Date.now();
+        // Use the new schedule if and only if it is earlier than the previous
+        // one.
+        if ( next !== 0 ) {
+            updateDelay = Math.min(updateDelay, Math.max(next - now, 0));
         }
-        assets[µm.pslPath] = true;
+        next = now + updateDelay;
-        callback(assets);
+        timer = vAPI.setTimeout(function() {
+            timer = undefined;
+            next = 0;
+            µMatrix.assets.updateStart({ delay: 120000 });
+        }, updateDelay);
     };
-
+})();
-    this.getAvailableHostsFiles(onListsReady);
-};
 
 /******************************************************************************/
 
-µMatrix.assetUpdatedHandler = function(details) {
+µMatrix.assetObserver = function(topic, details) {
-    var path = details.path || '';
+    // Do not update filter list if not in use.
-
+    if ( topic === 'before-asset-updated' ) {
-    if ( path !== '' ) {
+        if (
-        this.logger.writeOne('', 'info', vAPI.i18n('loggerEntryAssetUpdated').replace('{{value}}', path));
+            this.liveHostsFiles.hasOwnProperty(details.assetKey) === false ||
-    }
+            this.liveHostsFiles[details.assetKey].off === true
-
+        ) {
-    if ( this.liveHostsFiles.hasOwnProperty(path) === false ) {
+            return false;
+        }
         return;
     }
-    var entry = this.liveHostsFiles[path];
+
-    if ( entry.off ) {
+    if ( topic === 'after-asset-updated' ) {
+        vAPI.messaging.broadcast({
+            what: 'assetUpdated',
+            key: details.assetKey,
+            cached: true
+        });
         return;
     }
-    // Compile the list while we have the raw version in memory
-    //console.debug('µMatrix.getCompiledFilterList/onRawListLoaded: compiling "%s"', path);
-    //this.assets.put(
-    //    this.getCompiledFilterListPath(path),
-    //    this.compileFilters(details.content)
-    //);
-};
-
-/******************************************************************************/
 
-µMatrix.updateCompleteHandler = function(details) {
+    // Update failed.
-    var µm = this;
+    if ( topic === 'asset-update-failed' ) {
-
+        vAPI.messaging.broadcast({
-    var updatedCount = details.updatedCount;
+            what: 'assetUpdated',
-    if ( updatedCount === 0 ) {
+            key: details.assetKey,
+            failed: true
+        });
         return;
     }
 
-    // Assets are supposed to have been all updated, prevent fetching from
+    // Reload all filter lists if needed.
-    // remote servers.
+    if ( topic === 'after-assets-updated' ) {
-    µm.assets.remoteFetchBarrier += 1;
+        if ( details.assetKeys.length !== 0 ) {
-
+            this.loadHostsFiles();
-    var onFiltersReady = function() {
+        }
-        µm.assets.remoteFetchBarrier -= 1;
+        if ( this.userSettings.autoUpdate ) {
-    };
+            this.scheduleAssetUpdater(25200000);
-
-    var onPSLReady = function() {
-        if ( updatedCount !== 0 ) {
-            //console.debug('storage.js > µMatrix.updateCompleteHandler: reloading filter lists');
-            µm.loadHostsFiles(onFiltersReady);
         } else {
-            onFiltersReady();
+            this.scheduleAssetUpdater(0);
         }
-    };
+        vAPI.messaging.broadcast({
-
+            what: 'assetsUpdated',
-    if ( details.hasOwnProperty(this.pslPath) ) {
+            assetKeys: details.assetKeys
-        //console.debug('storage.js > µMatrix.updateCompleteHandler: reloading PSL');
+        });
-        this.loadPublicSuffixList(onPSLReady);
+        return;
-        updatedCount -= 1;
-    } else {
-        onPSLReady();
     }
-};
-
-/******************************************************************************/
 
-µMatrix.assetCacheRemovedHandler = (function() {
+    // New asset source became available, if it's a filter list, should we
-    var barrier = false;
+    // auto-select it?
-
+    if ( topic === 'builtin-asset-source-added' ) {
-    var handler = function(paths) {
+        if ( details.entry.content === 'filters' ) {
-        if ( barrier ) {
+            if ( details.entry.off !== true ) {
-            return;
+                this.saveSelectedFilterLists([ details.assetKey ], true);
-        }
-        barrier = true;
-        var i = paths.length;
-        var path;
-        while ( i-- ) {
-            path = paths[i];
-            if ( this.liveHostsFiles.hasOwnProperty(path) ) {
-                //console.debug('µMatrix.assetCacheRemovedHandler: decompiling "%s"', path);
-                //this.purgeCompiledFilterList(path);
-                continue;
-            }
-            if ( path === this.pslPath ) {
-                //console.debug('µMatrix.assetCacheRemovedHandler: decompiling "%s"', path);
-                //this.assets.purge('cache://compiled-publicsuffixlist');
-                continue;
             }
         }
-        //this.destroySelfie();
+        return;
-        barrier = false;
+    }
-    };
+};
-
-    return handler;
-})();

src/js/traffic.js

@@ -1,7 +1,7 @@
 /*******************************************************************************
 
     uMatrix - a Chromium browser extension to black/white list requests.
-    Copyright (C) 2014-2016 Raymond Hill
+    Copyright (C) 2014-2017 Raymond Hill
 
     This program is free software: you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
@@ -449,44 +449,33 @@ var requestTypeNormalizer = {
     'font'          : 'css',
     'image'         : 'image',
     'main_frame'    : 'doc',
+    'media'         : 'plugin',
     'object'        : 'plugin',
     'other'         : 'other',
     'ping'          : 'ping',
     'script'        : 'script',
     'stylesheet'    : 'css',
     'sub_frame'     : 'frame',
+    'websocket'     : 'xhr',
     'xmlhttprequest': 'xhr'
 };
 
 /******************************************************************************/
 
 vAPI.net.onBeforeRequest = {
-    urls: [
-        "http://*/*",
-        "https://*/*"
-    ],
     extra: [ 'blocking' ],
     callback: onBeforeRequestHandler
 };
 
 vAPI.net.onBeforeSendHeaders = {
-    urls: [
+    urls: [ 'http://*/*', 'https://*/*' ],
-        "http://*/*",
-        "https://*/*"
-    ],
     extra: [ 'blocking', 'requestHeaders' ],
     callback: onBeforeSendHeadersHandler
 };
 
 vAPI.net.onHeadersReceived = {
-    urls: [
+    urls: [ 'http://*/*', 'https://*/*' ],
-        "http://*/*",
+    types: [ 'main_frame', 'sub_frame' ],
-        "https://*/*"
-    ],
-    types: [
-        "main_frame",
-        "sub_frame"
-    ],
     extra: [ 'blocking', 'responseHeaders' ],
     callback: onHeadersReceived
 };

src/js/utils.js

@@ -1,7 +1,7 @@
 /*******************************************************************************
 
     µMatrix - a Chromium browser extension to black/white list requests.
-    Copyright (C) 2014 Raymond Hill
+    Copyright (C) 2014-2017 Raymond Hill
 
     This program is free software: you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
@@ -19,7 +19,7 @@
     Home: https://github.com/gorhill/uMatrix
 */
 
-/* global chrome, µMatrix */
+'use strict';
 
 /******************************************************************************/
 
@@ -45,9 +45,71 @@ var gotoExtensionURL = function(url) {
 
 /******************************************************************************/
 
+var LineIterator = function(text, offset) {
+    this.text = text;
+    this.textLen = this.text.length;
+    this.offset = offset || 0;
+};
+
+LineIterator.prototype.next = function() {
+    var lineEnd = this.text.indexOf('\n', this.offset);
+    if ( lineEnd === -1 ) {
+        lineEnd = this.text.indexOf('\r', this.offset);
+        if ( lineEnd === -1 ) {
+            lineEnd = this.textLen;
+        }
+    }
+    var line = this.text.slice(this.offset, lineEnd);
+    this.offset = lineEnd + 1;
+    return line;
+};
+
+LineIterator.prototype.rewind = function() {
+    if ( this.offset <= 1 ) {
+        this.offset = 0;
+        return;
+    }
+    var lineEnd = this.text.lastIndexOf('\n', this.offset - 2);
+    if ( lineEnd !== -1 ) {
+        this.offset = lineEnd + 1;
+    } else {
+        lineEnd = this.text.lastIndexOf('\r', this.offset - 2);
+        this.offset = lineEnd !== -1 ? lineEnd + 1 : 0;
+    }
+};
+
+LineIterator.prototype.eot = function() {
+    return this.offset >= this.textLen;
+};
+
+/******************************************************************************/
+
+var setToArray = typeof Array.from === 'function'
+    ? Array.from
+    : function(dict) {
+        var out = [],
+            entries = dict.values(),
+            entry;
+        for (;;) {
+            entry = entries.next();
+            if ( entry.done ) { break; }
+            out.push(entry.value);
+        }
+        return out;
+    };
+
+var setFromArray = function(arr) {
+    return new Set(arr);
+};
+
+/******************************************************************************/
+
 return {
     gotoURL: gotoURL,
-    gotoExtensionURL: gotoExtensionURL
+    gotoExtensionURL: gotoExtensionURL,
+    LineIterator: LineIterator,
+    setToArray: setToArray,
+    setFromArray: setFromArray
 };
 
 /******************************************************************************/

src/popup.html

@@ -19,39 +19,39 @@
 
 <div class="paneHead">
     <a id="gotoDashboard" class="extensionURL" href="#" data-extension-url="dashboard.html" title="popupTipDashboard">uMatrix <span id="version"> </span><span class="fa">&#xf013;</span></a>
-    <div id="toolbarLeft" class="toolbar alignLeft">
+    <div id="toolbarContainer">
-        <button id="scopeCell" class="dropdown-menu-button" tabindex="-1"></button>
+        <div id="toolbarLeft" class="toolbar alignLeft">
-        <div class="dropdown-menu">
+            <button id="scopeCell" class="dropdown-menu-button" tabindex="-1"></button>
-            <ul>
-                <li id="scopeKeyGlobal" class="dropdown-menu-entry">&#x2217;
-                <li id="scopeKeyDomain" class="dropdown-menu-entry">
-                <li id="scopeKeySite" class="dropdown-menu-entry">
-            </ul>
-        </div>
-        <div class="dropdown-menu-capture"></div>
-        <button id="mtxSwitch_matrix-off" type="button" class="fa scopeRel tip-anchor-left" data-i18n-tip="matrixMtxButtonTip">&#xf011;<span class="badge"></span></button>
-        <div style="display: inline-block; position: relative">
-            <button id="buttonMtxSwitches" type="button" class="dropdown-menu-button fa scopeRel" tabindex="-1">&#xf142;<span class="badge"></span></button>
             <div class="dropdown-menu">
-                <ul id="mtxSwitches">
+                <ul>
-                    <li id="mtxSwitch_ua-spoof" class="dropdown-menu-entry" data-i18n="matrixSwitchUASpoof">
+                    <li id="scopeKeyGlobal" class="dropdown-menu-entry">&#x2217;
-                    <li id="mtxSwitch_referrer-spoof" class="dropdown-menu-entry" data-i18n="matrixSwitchReferrerSpoof">
+                    <li id="scopeKeyDomain" class="dropdown-menu-entry">
-                    <li id="mtxSwitch_https-strict" class="dropdown-menu-entry" data-i18n="matrixSwitchNoMixedContent">
+                    <li id="scopeKeySite" class="dropdown-menu-entry">
                 </ul>
             </div>
             <div class="dropdown-menu-capture"></div>
-        </div>
+            <button id="mtxSwitch_matrix-off" type="button" class="fa scopeRel tip-anchor-left" data-i18n-tip="matrixMtxButtonTip">&#xf011;<span class="badge"></span></button>
-
+            <div style="display: inline-block; position: relative">
-        <button id="buttonPersist" type="button" class="fa scopeRel tip-anchor-left" data-i18n-tip="matrixPersistButtonTip">&#xf023;<span class="badge"></span></button>
+                <button id="buttonMtxSwitches" type="button" class="dropdown-menu-button fa scopeRel" tabindex="-1">&#xf142;<span class="badge"></span></button>
-        <button id="buttonRevertScope" type="button" class="fa scopeRel tip-anchor-left" tabindex="-1" data-i18n-tip="matrixRevertButtonTip">&#xf12d;</button>
+                <div class="dropdown-menu">
-        <button id="buttonReload" type="button" class="fa tip-anchor-left" data-i18n-tip="matrixReloadButton">&#xf021;</button>
+                    <ul id="mtxSwitches">
-    </div>
+                        <li id="mtxSwitch_ua-spoof" class="dropdown-menu-entry" data-i18n="matrixSwitchUASpoof">
+                        <li id="mtxSwitch_referrer-spoof" class="dropdown-menu-entry" data-i18n="matrixSwitchReferrerSpoof">
+                        <li id="mtxSwitch_https-strict" class="dropdown-menu-entry" data-i18n="matrixSwitchNoMixedContent">
+                    </ul>
+                </div>
+                <div class="dropdown-menu-capture"></div>
+            </div>
 
-    <div class="toolbar alignRight">
+            <button id="buttonPersist" type="button" class="fa scopeRel tip-anchor-left" data-i18n-tip="matrixPersistButtonTip">&#xf023;<span class="badge"></span></button>
-        <button id="buttonRevertAll" class="fa tip-anchor-right" data-i18n-tip="matrixRevertAllEntry">&#xf122;</button>
+            <button id="buttonRevertScope" type="button" class="fa scopeRel tip-anchor-left" tabindex="-1" data-i18n-tip="matrixRevertButtonTip">&#xf12d;</button>
-        <button type="button" class="fa extensionURL tip-anchor-right" data-extension-url="logger-ui.html" data-i18n-tip="matrixLoggerMenuEntry">&#xf022;</button>
+            <button id="buttonReload" type="button" class="fa tip-anchor-left" data-i18n-tip="matrixReloadButton">&#xf021;</button>
+        </div>
+        <div class="toolbar alignRight">
+            <button id="buttonRevertAll" class="fa tip-anchor-right" data-i18n-tip="matrixRevertAllEntry">&#xf122;</button>
+            <button type="button" class="fa extensionURL tip-anchor-right" data-extension-url="logger-ui.html" data-i18n-tip="matrixLoggerMenuEntry">&#xf022;</button>
+        </div>
     </div>
-
     <div id="matHead" class="matrix collapsible">
         <div class="matRow rw" style="display:none"><div class="matCell" data-req-type="all">all</div><div class="matCell" data-req-type="cookie">cookie</div><div class="matCell" data-req-type="css">css</div><div class="matCell" data-req-type="image">img</div><div class="matCell" data-req-type="plugin">plugin</div><div class="matCell" data-req-type="script">script</div><div class="matCell" data-req-type="xhr">XHR</div><div class="matCell" data-req-type="frame">frame</div><div class="matCell" data-req-type="other">other</div></div>
     </div>

tools/make-firefox.sh

@@ -19,7 +19,7 @@ cp -R src/*                             $DES/
 
 mv    $DES/img/icon_128.png             $DES/icon.png
 cp    platform/firefox/css/*            $DES/css/
-cp    platform/firefox/vapi-*.js        $DES/js/
+cp    platform/firefox/*.js             $DES/js/
 cp    platform/firefox/bootstrap.js     $DES/
 cp    platform/firefox/frame*.js        $DES/
 cp -R platform/chromium/img             $DES/

tools/make-webext-meta.py

@@ -0,0 +1,29 @@
+#!/usr/bin/env python3
+
+import os
+import json
+import sys
+
+if len(sys.argv) == 1 or not sys.argv[1]:
+    raise SystemExit('Build dir missing.')
+
+proj_dir = os.path.join(os.path.split(os.path.abspath(__file__))[0], '..')
+build_dir = os.path.abspath(sys.argv[1])
+
+# Import version number from chromium platform
+chromium_manifest = {}
+webext_manifest = {}
+
+chromium_manifest_file = os.path.join(proj_dir, 'platform', 'chromium', 'manifest.json')
+with open(chromium_manifest_file) as f1:
+    chromium_manifest = json.load(f1)
+
+webext_manifest_file = os.path.join(build_dir, 'manifest.json')
+with open(webext_manifest_file) as f2:
+    webext_manifest = json.load(f2)
+
+webext_manifest['version'] = chromium_manifest['version']
+
+with open(webext_manifest_file, 'w') as f2:
+    json.dump(webext_manifest, f2, indent=2, separators=(',', ': '), sort_keys=True)
+    f2.write('\n')

tools/make-webext.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+#
+# This script assumes a linux environment
+
+echo "*** uMatrix.webext: Creating web store package"
+echo "*** uMatrix.webext: Copying files"
+
+DES=dist/build/uMatrix.webext
+rm -rf $DES
+mkdir -p $DES
+
+cp -R ./assets                   $DES/
+cp -R ./src/*                    $DES/
+cp -R $DES/_locales/nb           $DES/_locales/no
+cp platform/chromium/*.html      $DES/
+cp platform/webext/polyfill.js   $DES/js/
+cp platform/chromium/*.js        $DES/js/
+cp -R platform/chromium/img/*    $DES/img/
+cp platform/webext/manifest.json $DES/
+cp LICENSE.txt                   $DES/
+
+echo "*** uMatrix.webext: Generating meta..."
+python tools/make-webext-meta.py $DES/
+
+if [ "$1" = all ]; then
+    echo "*** uMatrix.webext: Creating package..."
+    pushd $DES > /dev/null
+    zip ../$(basename $DES).zip -qr *
+    popd > /dev/null
+fi
+
+echo "*** uMatrix.webext: Package done."