Use Github API as a fallback to fetch js dependencies to workaround throttling issues (#6248)

CHANGELOG

@@ -67,6 +67,7 @@ CHANGELOG Roundcube Webmail
 - Use 7bit encoding for ISO-2022-* charsets in sent mail (#5640)
 - Handle inline images also inside multipart/mixed messages (#5905)
 - Allow style tags in HTML editor on composed/reply messages (#5751)
+- Use Github API as a fallback to fetch js dependencies to workaround throttling issues (#6248)
 - Fix handling of forwarded messages inside of a TNEF message (#5632)
 - Fix bug where attachment size wasn't visible when the filename was too long (#6033)
 - Fix checking table columns when there's more schemas/databases in postgres/mysql (#6047)

bin/install-jsdeps.sh

@@ -119,15 +119,30 @@ function fetch_from_source($package, $useCache = true, &$filetype = null)
       die("ERROR: Required program 'wget' or 'curl' not found\n");
     }
 
-    echo "Fetching {$package['url']}\n";
+    $url = str_replace('$v', $package['version'], $package['url']);
+
+    echo "Fetching $url\n";
 
     if ($CURL)
-        exec(sprintf('%s -L -s %s -o %s', $CURL, escapeshellarg($package['url']), $cache_file), $out, $retval);
+        exec(sprintf('%s -L -s %s -o %s', $CURL, escapeshellarg($url), $cache_file), $out, $retval);
     else
-        exec(sprintf('%s -q %s -O %s', $WGET, escapeshellarg($package['url']), $cache_file), $out, $retval);
+        exec(sprintf('%s -q %s -O %s', $WGET, escapeshellarg($url), $cache_file), $out, $retval);
+
+    // Try Github API as a fallback (#6248)
+    if ($retval !== 0 && $package['api_url']) {
+      $url    = str_replace('$v', $package['version'], $package['api_url']);
+      $header = 'Accept:application/vnd.github.v3.raw';
+
+      echo "Fetching failed. Using Github API on $url\n";
+
+      if ($CURL)
+        exec(sprintf('%s -L -H %s -s %s -o %s', $CURL, escapeshellarg($header), escapeshellarg($url), $cache_file), $out, $retval);
+      else
+        exec(sprintf('%s --header %s -q %s -O %s', $WGET, escapeshellarg($header), escapeshellarg($url), $cache_file), $out, $retval);
+    }
 
     if ($retval !== 0) {
-      die("ERROR: Failed to download source file from " . $package['url'] . "\n");
+      die("ERROR: Failed to download source file from " . $url . "\n");
     }
   }
 
@@ -167,7 +182,7 @@ function compose_destfile($package, $srcfile)
   $header = sprintf("/**\n * %s - v%s\n *\n", $package['name'], $package['version']);
 
   if (!empty($package['source'])) {
-    $header .= " * @source " . $package['source'] . "\n";
+    $header .= " * @source " . str_replace('$v', $package['version'], $package['source']) . "\n";
     $header .= " *\n";
   }
 

jsdeps.json

@@ -4,12 +4,12 @@
       "lib": "jquery",
       "name": "jQuery",
       "version": "3.3.1",
-      "url": "https://code.jquery.com/jquery-3.3.1.min.js",
+      "url": "https://code.jquery.com/jquery-$v.min.js",
       "dest": "program/js/jquery.min.js",
       "sha1": "0dc32db4aa9c5f03f3b38c47d883dbd4fed13aae",
       "license": "MIT",
       "copyright": "Copyright JS Foundation and other contributors",
-      "source": "https://github.com/jquery/jquery/tree/3.3.1"
+      "source": "https://github.com/jquery/jquery/tree/$v"
     },
     {
       "lib": "jstz",
@@ -25,8 +25,9 @@
     {
       "lib": "publickey",
       "name": "PublicKey.js",
-      "version": "0e011cb",
-      "url": "https://raw.githubusercontent.com/diafygi/publickeyjs/0e011cb18907a1adc0313aa92e69cd8858e1ef66/publickey.js",
+      "version": "0e011cb1",
+      "url": "https://raw.githubusercontent.com/diafygi/publickeyjs/$v/publickey.js",
+      "api_url": "https://api.github.com/repos/diafygi/publickeyjs/contents/publickey.js?ref=$v",
       "dest": "program/js/publickey.js",
       "sha1": "d0920e190754e024c4be76ad5bbc7e76b2e37a4d",
       "license": "GPLv3",
@@ -36,7 +37,7 @@
     {
       "lib": "tinymce",
       "version": "4.5.8",
-      "url": "http://download.ephox.com/tinymce/community/tinymce_4.5.8.zip",
+      "url": "http://download.ephox.com/tinymce/community/tinymce_$v.zip",
       "dest": "program/js",
       "sha1": "08b0757264adb86066940bbafb7aa9ec0c7c6685",
       "license": "LGPL",
@@ -63,17 +64,18 @@
       "lib": "openpgp",
       "name": "OpenPGP.js",
       "version": "2.6.2",
-      "url": "https://raw.githubusercontent.com/openpgpjs/openpgpjs/v2.6.2/dist/openpgp.min.js",
+      "url": "https://raw.githubusercontent.com/openpgpjs/openpgpjs/v$v/dist/openpgp.min.js",
+      "api_url": "https://api.github.com/repos/openpgpjs/openpgpjs/contents/dist/openpgp.min.js?ref=v$v",
       "dest": "plugins/enigma/openpgp.min.js",
       "sha1": "cce0087eb20328453b9378a2b01cc6339cef97b2",
       "license": "LGPL",
       "copyright": "Copyright (c) OpenPGP Development Team",
-      "source": "https://github.com/openpgpjs/openpgpjs/blob/v2.6.2/dist/openpgp.js"
+      "source": "https://github.com/openpgpjs/openpgpjs/blob/v$v/dist/openpgp.js"
     },
     {
       "lib": "codemirror",
       "version": "5.34.0",
-      "url": "https://codemirror.net/codemirror-5.34.0.zip",
+      "url": "https://codemirror.net/codemirror-$v.zip",
       "dest": "plugins/managesieve/codemirror",
       "sha1": "24d0834c6a5c4b89c4c82ca2ea31e574b0751b32",
       "license": "MIT",
@@ -87,7 +89,7 @@
       "lib": "bootstrap",
       "name": "Bootstrap",
       "version": "4.1.0",
-      "url": "https://github.com/twbs/bootstrap/releases/download/v4.1.0/bootstrap-4.1.0-dist.zip",
+      "url": "https://github.com/twbs/bootstrap/releases/download/v$v/bootstrap-$v-dist.zip",
       "dest": "skins/elastic/deps",
       "sha1": "90abd51cf27a3dc4823eed6c1de620725f068b6c",
       "license": "MIT",
@@ -101,11 +103,12 @@
       "lib": "less",
       "name": "LessJS",
       "version": "2.7.3",
-      "url": "https://raw.githubusercontent.com/less/less.js/v2.7.3/dist/less.min.js",
+      "url": "https://raw.githubusercontent.com/less/less.js/v$v/dist/less.min.js",
+      "api_url": "https://api.github.com/repos/less/less.js/contents/dist/less.min.js?ref=v$v",
       "dest": "skins/elastic/deps/less.min.js",
       "sha1": "45ea4f9fed6c0568ec11faba048c3e3cb7612e9e",
       "license": "Apache-2.0",
-      "source": "https://raw.githubusercontent.com/less/less.js/v2.7.3/dist/less.js"
+      "source": "https://raw.githubusercontent.com/less/less.js/v$v/dist/less.js"
     }
   ]
 }