move 3rd-party assets management to uBlockOrigin/uAssets

assets/assets.json

@@ -12,6 +12,7 @@
 		"updateAfter": 19,
 		"contentURL": [
 			"https://publicsuffix.org/list/public_suffix_list.dat",
+			"assets/thirdparties/publicsuffix.org/list/public_suffix_list.dat",
 			"assets/thirdparties/publicsuffix.org/list/effective_tld_names.dat"
 		]
 	},
@@ -41,7 +42,10 @@
 		"group": "multipurpose",
 		"updateAfter": 11,
 		"title": "Dan Pollock’s hosts file",
-		"contentURL": "http://someonewhocares.org/hosts/hosts",
+		"contentURL": [
+			"http://someonewhocares.org/hosts/hosts",
+			"assets/thirdparties/someonewhocares.org/hosts/hosts.txt"
+		],
 		"supportURL": "http://someonewhocares.org/hosts/"
 	},
 	"hphosts": {
@@ -49,7 +53,10 @@
 		"group": "multipurpose",
 		"updateAfter": 11,
 		"title": "hpHosts’ Ad and tracking servers",
-		"contentURL": "https://hosts-file.net/.%5Cad_servers.txt",
+		"contentURL": [
+			"https://hosts-file.net/.%5Cad_servers.txt",
+			"assets/thirdparties/hosts-file.net/ad_servers.txt"
+		],
 		"supportURL": "https://hosts-file.net/"
 	},
 	"mvps-0": {
@@ -57,7 +64,10 @@
 		"group": "multipurpose",
 		"updateAfter": 11,
 		"title": "MVPS HOSTS",
-		"contentURL": "http://winhelp2002.mvps.org/hosts.txt",
+		"contentURL": [
+			"http://winhelp2002.mvps.org/hosts.txt",
+			"assets/thirdparties/winhelp2002.mvps.org/hosts.txt"
+		],
 		"supportURL": "http://winhelp2002.mvps.org/"
 	},
 	"plowe-0": {

assets/thirdparties/mirror1.malwaredomains.com/files/README.md

@@ -1,3 +0,0 @@
-<http://www.malwaredomains.com/?page_id=1508>:
-
-"This malware block lists provided here are for free for noncommercial use as part of the fight against malware."

assets/thirdparties/pgl.yoyo.org/as/README.md

@@ -1,4 +0,0 @@
-<http://pgl.yoyo.org/as/index.php>:
-
-Site does encourage use of the list, and nowhere could I find terms and
-conditions to use the list. Assuming it can be used freely.

assets/thirdparties/www.malwaredomainlist.com/hostslist/README.md

@@ -1,3 +0,0 @@
-<http://www.malwaredomainlist.com/>:
-
-"Our list can be used for free by anyone. Feel free to use it."

tools/commit-assets.sh

@@ -1,13 +0,0 @@
-#!/bin/bash
-#
-# This script assumes a linux environment
-
-echo "*** uMatrix: git adding changed assets..."
-git add --update --ignore-removal --ignore-errors assets
-echo "*** uMatrix: git committing assets..."
-git commit -m 'update of third-party assets'
-echo "*** uMatrix: git pushing assets to remote master..."
-git push origin master
-
-echo "*** uMatrix: git done."
-

tools/make-assets.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+#
+# This script assumes a linux environment
+
+DES=$1/assets
+
+printf "*** Packaging assets in $DES... "
+
+rm -rf $DES
+mkdir $DES
+cp    ./assets/assets.json $DES/
+
+if [ -n "${TRAVIS_TAG}" ]; then
+  pushd .. > /dev/null
+  git clone --depth 1 https://github.com/uBlockOrigin/uAssets.git
+  popd > /dev/null
+fi
+
+mkdir $DES/thirdparties
+cp -R ../uAssets/thirdparties/hosts-file.net             $DES/thirdparties/
+cp -R ../uAssets/thirdparties/mirror1.malwaredomains.com $DES/thirdparties/
+cp -R ../uAssets/thirdparties/pgl.yoyo.org               $DES/thirdparties/
+cp -R ../uAssets/thirdparties/publicsuffix.org           $DES/thirdparties/
+cp -R ../uAssets/thirdparties/someonewhocares.org        $DES/thirdparties/
+cp -R ../uAssets/thirdparties/winhelp2002.mvps.org       $DES/thirdparties/
+cp -R ../uAssets/thirdparties/www.malwaredomainlist.com  $DES/thirdparties/
+
+echo "done."

tools/make-chromium.sh

@@ -9,7 +9,8 @@ DES=./dist/build/uMatrix.chromium
 rm -rf $DES
 mkdir -p $DES
 
-cp -R ./assets                          $DES/
+bash ./tools/make-assets.sh $DES
+
 cp -R ./src/*                           $DES/
 cp -R $DES/_locales/nb                  $DES/_locales/no # Chrome store quirk
 cp    ./platform/chromium/*.html        $DES/

tools/make-opera.sh

@@ -9,7 +9,8 @@ DES=./dist/build/uMatrix.opera
 rm -r $DES
 mkdir -p $DES
 
-cp -R ./assets                          $DES/
+bash ./tools/make-assets.sh $DES
+
 cp -R ./src/*                           $DES/
 cp    ./platform/chromium/*.html        $DES/
 cp    ./platform/chromium/*.js          $DES/js/

tools/make-webext.sh

@@ -9,7 +9,8 @@ DES=dist/build/uMatrix.webext
 rm -rf $DES
 mkdir -p $DES
 
-cp -R ./assets                             $DES/
+bash ./tools/make-assets.sh $DES
+
 cp -R ./src/*                              $DES/
 cp    platform/chromium/*.html             $DES/
 cp    platform/chromium/*.js               $DES/js/

tools/update-3rdparties.sh

@@ -1,45 +0,0 @@
-#!/bin/bash
-#
-# This script assumes a linux environment
-
-TEMPFILE=/tmp/umatrix-asset
-
-echo "*** uMatrix: updating remote assets..."
-
-THIRDPARTY_REMOTEURLS=(
-    'http://mirror1.malwaredomains.com/files/justdomains'
-    'http://pgl.yoyo.org/adservers/serverlist.php?hostformat=hosts&showintro=1&startdate%5Bday%5D=&startdate%5Bmonth%5D=&startdate%5Byear%5D=&mimetype=plaintext'
-    'http://www.malwaredomainlist.com/hostslist/hosts.txt'
-    'http://hosts-file.net/.%5Cad_servers.txt'
-    'http://someonewhocares.org/hosts/hosts'
-    'http://winhelp2002.mvps.org/hosts.txt'
-    'http://publicsuffix.org/list/effective_tld_names.dat'
-    )
-
-THIRDPARTY_LOCALURLS=(
-    'assets/thirdparties/mirror1.malwaredomains.com/files/justdomains'
-    'assets/thirdparties/pgl.yoyo.org/as/serverlist'
-    'assets/thirdparties/www.malwaredomainlist.com/hostslist/hosts.txt'
-    'assets/thirdparties/hosts-file.net/ad-servers'
-    'assets/thirdparties/someonewhocares.org/hosts/hosts'
-    'assets/thirdparties/winhelp2002.mvps.org/hosts.txt'
-    'assets/thirdparties/publicsuffix.org/list/effective_tld_names.dat'
-    )
-
-ENTRY_INDEX=0
-for THIRDPARTY_REMOTEURL in ${THIRDPARTY_REMOTEURLS[@]}; do
-    THIRDPARTY_LOCALURL=${THIRDPARTY_LOCALURLS[ENTRY_INDEX]}
-    echo "*** Downloading" $THIRDPARTY_REMOTEURL
-    if wget -q -T 30 -O $TEMPFILE -- $THIRDPARTY_REMOTEURL; then
-        if [ -s $TEMPFILE ]; then
-            if ! cmp -s $TEMPFILE $THIRDPARTY_LOCALURL; then
-                echo "    New version found: $THIRDPARTY_LOCALURL"
-                if [ "$1" != "dry" ]; then
-                    mv $TEMPFILE $THIRDPARTY_LOCALURL
-                fi
-            fi
-        fi
-    fi
-    let ENTRY_INDEX+=1
-done
-