commit ab62f82cca671717a0fa9cf09c1a1c35d8aa4611 Author: Felix Stupp Date: Sat May 2 14:01:02 2020 +0200 First commit diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..952cb29 --- /dev/null +++ b/LICENSE @@ -0,0 +1,22 @@ +MIT License + +Copyright (c) 2020 Felix Stupp + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + diff --git a/README.md b/README.md new file mode 100644 index 0000000..c9dbe6b --- /dev/null +++ b/README.md @@ -0,0 +1,53 @@ +# Approx Redirector + +This simple Python 3 script does redirect all entries in your `sources.list` and `sources.list.d` files to a given approx instance. +It redirects all entries cached by the approx server +but does not change uncached repositories. + +## Features + +- Looks up cacheable repositories +- Supports ≈99.9 % of all approx configurations out there (except approx forcing https) +- Verboses changes if requested +- Can rewrite sources files if run as `root` +- Does backup old sources files for easy restoring + +### ToDo + +- Support https for approx +- Implement '--mirror' mode + +## Usage + +- Obviously requires a Debian-based system +- Requires `python3` and `python3-request` to be installed + +Assuming `approx` is the hostname of the approx cache you want to use. +You can use an IP address instead. +You can append a port by using `approx:9999`. +By default `http://` will be used +but you can specify https as protocol, too: `https://approx` (**not supported yet**). + +``` +./redirect.py -v approx +``` + +The script will check which entries can be redirected +and report these to stdout. +If you want to approve these changes, run as `root`: + +``` +./redirect.py -vc approx +``` + +Now your system will use the approx cache. +The old entries are commented out. + +## Contribute + +Feel free to contribute to this project. +Please follow the common [style guide for Python](https://www.python.org/dev/peps/pep-0008/). + +## License + +This project is licensed under MIT. diff --git a/redirect.py b/redirect.py new file mode 100755 index 0000000..e509327 --- /dev/null +++ b/redirect.py @@ -0,0 +1,161 @@ +#!/usr/bin/env python3 + +# Imports +import argparse +import os +from pathlib import Path +import re +import requests +import sys +import tempfile + +## Variables + +# Mirrors with multiple addresses (like per country) +multi_mapping = [ + r"(ftp[0-9]*\.[a-z]+|deb)\.debian\.org", + r"[a-z]+\.(archive|releases)\.ubuntu\.com", +] + +# Regexs used +mapRegex = re.compile(r"^[^\"]+)\">[^<>]+.*)\">http[^<>]*$") +allProtoRegex = re.compile(r"^[a-z+]+:/*(?!/)") +protoReplaceRegex = re.compile(r"^https?://") +lineReplaceRegex = re.compile(r"^\s*deb(-src)? ") + +## Code + +multi_mapping = [re.compile(x) for x in multi_mapping] + +verboseLevel = 0 + +def verb(txt, add=0): + global verboseLevel + global verbose_mode + if verbose_mode: + print((" " * (verboseLevel * 4 + add)) + txt) + +def verbLevel(num): + global verboseLevel + verboseLevel += num + if verboseLevel < 0: + verboseLevel = 0 + +def splitDomainPath(url): + if "/" in url: + domain, path = url.split('/', 1) + return domain, '/' + path + else: + return url, '' + +def removeProtocol(url): + return allProtoRegex.sub("", url) + +def remove_slash_suffix(url): + # Removes slash suffix so that also urls in sources without suffix will be matched + return url[:-1] if url.endswith('/') else url + +def discoverMap(server): + global mapRegex + # Request approx server + res = requests.get(server) + # Extract url_map + url_map = {} + for line in res.text.split('\n'): + match = mapRegex.match(line) + if match: + url_map[remove_slash_suffix(match.group('old'))] = server + '/' + remove_slash_suffix(match.group('new')) + return url_map + +def url_to_regex(url, multi_mapping=multi_mapping): + # Check if old path is prepend by https?:// + if protoReplaceRegex.match(url): + url = removeProtocol(url) + domain, path = splitDomainPath(url) + # Check if domain is given in multi_mapping + for m in multi_mapping: + if m.match(domain): + # If given exchange with multi mapped regex + url_pattern = m.sub(m.pattern, domain) + re.escape(path) + break + else: + url_pattern = re.escape(url) + # Prefix protocol again + return r"https?://" + url_pattern + else: + return re.escape(url) + +def modifyMap(d): + return {re.compile(url_to_regex(old)): new for old, new in d.items()} + +def isDebLine(line): + return lineReplaceRegex.search(line) is not None + +def checkFile(file, map): + global write_mode + verb(("Run replacements on" if write_mode else "Check") + f" {file}:") + verbLevel(1) + if write_mode: + newFile = tempfile.NamedTemporaryFile(mode="w", delete=False) + changed = False + for line in open(file, "r"): + line = line[:-1] + if isDebLine(line): + for old_url, new_url in map.items(): + if old_url.search(line): + changed = True + new_line = old_url.sub(new_url, line) + verb(f"{line}") + verb(f"-> {new_line}", add=-3) + line = new_line + break + else: + verb(f"= {line}", add=-2) + if write_mode: + newFile.write(line + "\n") + if write_mode: + origFilePath = Path(file) + backFilePath = Path(file + ".save") + newFilePath = Path(newFile.name) + newFile.close() + newFilePath.chmod(0o644) + if changed: + origFilePath.replace(backFilePath) + newFilePath.rename(origFilePath) + verbLevel(-1) + +def main(argv): + global mirror_mode + global verbose_mode + global write_mode + # Parse arguments + parser = argparse.ArgumentParser(description="Redirects apt sources to a given approx cache if cached by approx. Only files ending with .list in sources.list.d will be changed.") + parser.add_argument('host', help="The URL of the approx cache, uses http:// if protocol is omitted") + parser.add_argument('-c', '--confirm', action='store_true', dest='confirm', help="Does rewrite the source files to redirect to approx, does require run as root") + parser.add_argument('-m', '--mirror', action='store_true', dest='use_mirror', help="Uses mirror lists to allow falling back to direct connection") + parser.add_argument('-p', '--path', dest='path', default='/etc/apt', type=Path, help="Configuration directory of apt containing sources.list files, defaults to /etc/apt") + parser.add_argument('-v', '--verbose', action='store_true', dest='verbose', help="Displays debug information") + args = parser.parse_args(argv) + # Check server host + if not allProtoRegex.match(args.host): + args.host = "http://" + args.host + # Store configuration in global vars + mirror_mode = args.use_mirror + verbose_mode = args.verbose + write_mode = args.confirm + # Retrieve repository map + verb(f"Connect to {args.host} to retrieve repository list") + repo_map = modifyMap(discoverMap(args.host)) + verb("Found following repositories:") + verbLevel(1) + for k, v in repo_map.items(): + verb(k.pattern) + verbLevel(-1) + # Check for sources files + file_list = [args.path / 'sources.list'] + [file for file in (args.path / 'sources.list.d').rglob('*.list') if file.is_file()] + for file in file_list: + if file.exists(): + checkFile(str(file.resolve()), repo_map) + +if __name__ == '__main__': + main(sys.argv[1:])