WIP acme dehydrated

group_vars/all/vars.yml

@@ -54,6 +54,7 @@ global_dns_session_key_algorithm: "hmac-sha512"
 global_dns_update_key_algorithm: "ED25519"
 global_dns_ttl: "{{ 60 * 60 }}" # default if omitted in all cases
 global_dns_debug_ttl: "{{ 60 }}" # mostly used if has_debug_instance to allow short transfer times
+global_dns_key_ttl: "{{ 15 * 60 }}" # Lower TTL for security relevant records / more often updated records
 
 global_ssh_key_directory: "{{ global_public_key_directory }}/ssh"
 global_ssh_host_key_directory: "{{ global_ssh_key_directory }}/hosts"
@@ -117,6 +118,18 @@ global_certbot_certificates_directory: "/etc/letsencrypt/live"
 global_chromium_configuration_directory: "/etc/chromium"
 global_chromium_managed_policies_file: "{{ global_chromium_configuration_directory }}/policies/managed/managed_policies.json"
 
+global_dehydrated_system_user: dehydrated
+# configs
+global_dehydrated_configuration_directory: "/etc/dehydrated"
+global_dehydrated_configuration_file: "{{ global_dehydrated_configuration_directory }}/config"
+global_dehydrated_domains_directory: "{{ global_dehydrated_configuration_directory }}/domains"
+global_dehydrated_hook_script_path: "{{ global_dehydrated_configuration_directory }}/hook.py"
+global_dehydrated_hook_configuration_file: "{{ global_dehydrated_configuration_directory }}/hook.json"
+# data dirs
+global_dehydrated_data_directory: "/var/lib/dehydrated"
+global_dehydrated_domains_main_file: "{{ global_dehydrated_data_directory }}/domains.generated.txt"
+global_dehydrated_certificates_directory: "{{ global_dehydrated_data_directory }}/certificates"
+
 global_dns_upstream_servers:
   - "9.9.9.11"
   - "149.112.112.11"

roles/acme/application/defaults/main.yml

@@ -2,4 +2,21 @@
 
 acme_account_mail: "{{ global_admin_mail }}"
 
+acme_key_algorithm: rsa
 acme_key_size: 4096
+
+acme_user_umask: "0477" # resulting in u=rw,g=,o= being allowed
+
+domain_data_name: "data.txt"
+
+keyfullchain_name: "keyfullchain.pem"
+
+hook_script_path: "{{ global_dehydrated_configuration_directory }}/hook.py"
+hook_config_path: "{{ global_dehydrated_configuration_directory }}/hook.json"
+hook_config:
+  challenge_record_ttl: 20
+  domain_data_name: "{{ domain_data_name }}"
+  domains_directory: "{{ global_dehydrated_domains_directory }}"
+  domains_main_file: "{{ global_dehydrated_domains_main_file }}"
+  key_file_mode: 0o0310 # resulting in u=rw,g=r,o=
+  keyfullchain_name: "{{ keyfullchain_name }}"

roles/acme/application/meta/main.yml

@@ -3,4 +3,7 @@
 allow_duplicates: no
 
 dependencies:
+  - role: misc/system_user
+    system_user: "{{ global_dehydrated_system_user }}"
+    user_directory: "{{ global_dehydrated_data_directory }}"
   - role: nginx/application

roles/acme/application/tasks/main.yml

@@ -4,12 +4,39 @@
   apt:
     state: present
     name:
-      - certbot # main package
+      - dehydrated # main package
 
-- name: Configure certbot
+- name: Create configuration directory
+  file:
+    state: directory
+    path: "{{ global_dehydrated_configuration_directory }}"
+    owner: root
+    group: "{{ global_dehydrated_system_user }}"
+    mode: u=rwx,g=rx,o=
+
+- name: Configure dehydrated
+  template:
+    src: config
+    dest: "{{ configuration_file }}"
+    owner: root
+    group: "{{ global_dehydrated_system_user }}"
+    mode: u=rw,g=r,o=
+    validate: "{{ global_validate_shell_script }}"
+
+- name: Deploy global hook config
+  copy:
+    content: |
+      {{ hook_config | to_nice_json }}
+    dest: "{{ global_dehydrated_hook_configuration_file }}"
+    owner: root
+    group: "{{ global_dehydrated_system_user }}"
+    mode: u=rw,g=r,o=
+
+- name: Deploy global hook script
   template:
-    src: cli.ini
+    src: hook.py
-    dest: "{{ global_certbot_configuration_file }}"
+    dest: "{{ hook_script_path }}"
     owner: root
-    group: root
+    group: "{{ global_dehydrated_system_user }}"
-    mode: u=rw,g=r,o=r
+    mode: u=rwx,g=rx,o=
+    validate: "{{ global_validate_shell_script }}"

roles/acme/application/templates/cli.ini

@@ -1,12 +0,0 @@
-# Accept service terms
-agree-tos
-
-# Default RSA key size
-rsa-key-size = {{ acme_key_size }}
-
-# E-Mail Address for registration
-email = {{ acme_account_mail }}
-
-# Use webroot per default
-authenticator = webroot
-webroot-path = {{ acme_validation_root_directory }}

roles/acme/application/templates/config

@@ -0,0 +1,133 @@
+
+########################################################
+# This is the main config file for dehydrated          #
+#                                                      #
+# This file is looked for in the following locations:  #
+# $SCRIPTDIR/config (next to this script)              #
+# /usr/local/etc/dehydrated/config                     #
+# /etc/dehydrated/config                               #
+# ${PWD}/config (in current working-directory)         #
+#                                                      #
+# Default values of this config are in comments        #
+########################################################
+
+# Which user should dehydrated run as? This will be implicitly enforced when running as root
+#DEHYDRATED_USER=
+
+# Which group should dehydrated run as? This will be implicitly enforced when running as root
+#DEHYDRATED_GROUP=
+
+# Resolve names to addresses of IP version only. (curl)
+# supported values: 4, 6
+# default: <unset>
+#IP_VERSION=
+
+# URL to certificate authority or internal preset
+# Presets: letsencrypt, letsencrypt-test, zerossl, buypass, buypass-test
+# default: letsencrypt
+CA="letsencrypt"
+
+# Path to old certificate authority
+# Set this value to your old CA value when upgrading from ACMEv1 to ACMEv2 under a different endpoint.
+# If dehydrated detects an account-key for the old CA it will automatically reuse that key
+# instead of registering a new one.
+# default: https://acme-v01.api.letsencrypt.org/directory
+#OLDCA="https://acme-v01.api.letsencrypt.org/directory"
+
+# Which challenge should be used? Currently http-01, dns-01 and tls-alpn-01 are supported
+#CHALLENGETYPE="http-01"
+
+# Path to a directory containing additional config files, allowing to override
+# the defaults found in the main configuration file. Additional config files
+# in this directory needs to be named with a '.sh' ending.
+# default: <unset>
+#CONFIG_D=
+
+# Directory for per-domain configuration files.
+# If not set, per-domain configurations are sourced from each certificates output directory.
+# default: <unset>
+#DOMAINS_D=
+
+# Base directory for account key, generated certificates and list of domains (default: $SCRIPTDIR -- uses config directory if undefined)
+BASEDIR={{ global_dehydrated_configuration_directory | quote }}
+
+# File containing the list of domains to request certificates for (default: $BASEDIR/domains.txt)
+DOMAINS_TXT={{ global_dehydrated_domains_file | quote }}
+
+# Output directory for generated certificates
+CERTDIR={{ global_dehydrated_certificates_directory }}
+
+# Output directory for alpn verification certificates
+#ALPNCERTDIR="${BASEDIR}/alpn-certs"
+
+# Directory for account keys and registration information
+#ACCOUNTDIR="${BASEDIR}/accounts"
+
+# Output directory for challenge-tokens to be served by webserver or deployed in HOOK (default: /var/www/dehydrated)
+WELLKNOWN={{ acme_validation_root_directory | quote }}
+
+# Default keysize for private keys (default: 4096)
+KEYSIZE={{ acme_key_size | quote }}
+
+# Path to openssl config file (default: <unset> - tries to figure out system default)
+#OPENSSL_CNF=
+
+# Path to OpenSSL binary (default: "openssl")
+#OPENSSL="openssl"
+
+# Extra options passed to the curl binary (default: <unset>)
+#CURL_OPTS=
+
+# Program or function called in certain situations
+#
+# After generating the challenge-response, or after failed challenge (in this case altname is empty)
+# Given arguments: clean_challenge|deploy_challenge altname token-filename token-content
+#
+# After successfully signing certificate
+# Given arguments: deploy_cert domain path/to/privkey.pem path/to/cert.pem path/to/fullchain.pem
+#
+# BASEDIR and WELLKNOWN variables are exported and can be used in an external program
+# default: <unset>
+HOOK={{ hook_script_path | quote }}
+
+# Chain clean_challenge|deploy_challenge arguments together into one hook call per certificate (default: no)
+#HOOK_CHAIN="no"
+
+# Minimum days before expiration to automatically renew certificate (default: 30)
+#RENEW_DAYS="30"
+
+# Regenerate private keys instead of just signing new certificates on renewal (default: yes)
+PRIVATE_KEY_RENEW="yes"
+
+# Create an extra private key for rollover (default: no)
+#PRIVATE_KEY_ROLLOVER="no"
+
+# Which public key algorithm should be used? Supported: rsa, prime256v1 and secp384r1
+KEY_ALGO={{ acme_key_algorithm | quote }}
+
+# E-mail to use during the registration (default: <unset>)
+CONTACT_EMAIL={{ acme_account_mail | quote }}
+
+# Lockfile location, to prevent concurrent access (default: $BASEDIR/lock)
+#LOCKFILE="${BASEDIR}/lock"
+
+# Option to add CSR-flag indicating OCSP stapling to be mandatory (default: no)
+#OCSP_MUST_STAPLE="no"
+
+# Fetch OCSP responses (default: no)
+OCSP_FETCH="yes"
+
+# OCSP refresh interval (default: 5 days)
+#OCSP_DAYS=5
+
+# Issuer chain cache directory (default: $BASEDIR/chains)
+#CHAINCACHE="${BASEDIR}/chains"
+
+# Automatic cleanup (default: no)
+#AUTO_CLEANUP="no"
+
+# ACME API version (default: auto)
+#API=auto
+
+# Preferred issuer chain (default: <unset> -> uses default chain)
+#PREFERRED_CHAIN=

roles/acme/application/templates/hook.py

@@ -0,0 +1,329 @@
+#!/usr/bin/env python3
+
+import argparse
+import functools
+import json
+import os
+from pathlib import Path
+import subprocess
+import sys
+
+
+# constants
+
+ACME_DOMAIN_PREFIX = "_acme-challenge"
+SUBPROCESS_DEFAULT_KWARGS = {"shell": False}
+SUBPROCESS_ENV = ["/usr/bin/env"]
+
+
+# subprocess helpers
+
+def safe_run(args, **kwargs):
+    return subprocess.run(args, **SUBPROCESS_DEFAULT_KWARGS, check=True, **kwargs)
+
+def safe_call(args, **kwargs):
+    return safe_run(SUBPROCESS_ENV + args, **kwargs)
+
+def safe_popen(args, subprocess_input, **kwargs):
+    proc = subprocess.Popen(SUBPROCESS_ENV + args, **SUBPROCESS_DEFAULT_KWARGS, stdin=subprocess.PIPE, **kwargs)
+    ret = proc.communicate(input=subprocess_input)
+    if proc.returncode != 0:
+        raise subprocess.CalledProcessError(returncode=proc.returncode, cmd=proc.args)
+    return (proc, *ret)
+
+
+# shell replacements
+
+def chmod(mode, *paths):
+    safe_call(["chmod", mode] + [str(path) for path in paths])
+
+def concat(out_path, *in_paths, mode=0o333):
+    with open(out_path, mode="wb", opener=functools.partial(os.open, mode=mode)) as out_file:
+        for in_path in in_paths:
+            with in_path.open(mode="rb") as in_file:
+                out_file.write(in_file.read())
+
+def iterdir_recursive(dir_path, suffix=""):
+    ret = []
+    for entry in dir_path.iterdir():
+        if entry.is_dir():
+            ret.extend(iterdir_recursive(entry, suffix=suffix))
+        elif entry.name.endswith(suffix):
+            ret.append(entry)
+    return ret
+
+def nsupdate(key_path, record_name, record_ttl=60, record_type=None, record_data=None, delete_record=False):
+    if delete_record:
+        record_type = record_type or ""
+        record_data = record_data or ""
+        action = f"update delete {record_name} {record_ttl} IN {record_type} {record_data}"
+    else:
+        if not (record_name and record_ttl and record_type and record_data):
+            raise Exception("args missing")
+        action = f"update add {record_name} {record_ttl} IN {record_type} {record_data}"
+    safe_popen(["nsupdate", "-k", str(key_path)], f"{action}\nsend\n")
+
+
+# hooks code
+
+
+class DomainData:
+
+    def __init__(self, domain):
+        self.__domain = domain
+
+    @property
+    def domain(self):
+        return self.__domain
+
+    @property
+    def domain_dir(self):
+        return config.domains_directory / self.__domain
+
+    @property
+    def domain_data_file(self):
+        return self.domain_dir / config.domain_data_name
+
+    def run_hooks(self, hook_name, **kwargs):
+        hook_dir = self.domain_dir / hook_name
+        if not hook_dir.exists():
+            return
+        for hook in iterdir_recursive(hook_dir):
+            if os.access(hook, os.X_OK):
+                safe_run([str(hook)]) # TODO args
+
+
+def deploy_challenge(domain, token_filename, token_value):
+    # token_filename only for http-01 challenges
+    challenge_name = f"{ACME_DOMAIN_PREFIX}.{domain}."
+    nsupdate(key_path=None, record_name=challenge_name, record_ttl=config.challenge_record_ttl, record_type="TXT", record_data=token_value)
+
+
+def clean_challenge(domain, token_filename, token_value):
+    # token_filename only for http-01 challenges
+    challenge_name = f"{ACME_DOMAIN_PREFIX}.{domain}."
+    nsupdate(key_path=None, record_name=challenge_name, delete_record=True)
+
+
+def sync_cert(domain, key_path, cert_path, fullchain_path, chain_path, request_path):
+    safe_call(["sync", key_path, cert_path, fullchain_path, chain_path, request_path])
+
+
+def deploy_cert(domain, key_path, cert_path, fullchain_path, chain_path, timestamp: int):
+    # make public key files readable for all
+    chmod("+r", cert_path, fullchain_path, chain_path)
+    # create legacy key+cert+chain file
+    keyfullchain_path = key_path / ".." / config.keyfullchain_name
+    concat(keyfullchain_path, key_path, fullchain_path, mode=config.key_file_mode)
+
+
+def deploy_ocsp(domain, ocsp_path, timestamp):
+    # make OCSP readable for all
+    chmod("+r", ocsp_path)
+    pass
+
+
+def unchanged_cert(domain, key_path, cert_path, fullchain_path, chain_path):
+    pass
+
+
+def invalid_challenge(domain, response):
+    pass
+
+
+def request_failure(status_code, reason, req_type, headers):
+    pass
+
+
+def generate_csr(domain, cert_dir, alt_names):
+    pass
+
+
+def startup_hook():
+    # prepare domains list
+    concat(config.domains_main_file, *iterdir_recursive(config.domains_directory))
+
+
+def exit_hook(error):
+    if error is None:
+        pass
+    else:
+        pass
+
+
+# hooks metadata
+
+arg_infos = {
+    "alt_names": {
+        "help": "All domain names for the current certificate as specified in domains.txt.",
+    },
+    "cert_path": {
+        "help": "The path of the file containing the signed certificate.",
+        "type": Path,
+    },
+    "chain_path": {
+        "help": "The path of the file containing the intermediate certificate(s).",
+        "type": Path,
+    },
+    "cert_dir": {
+        "help": "Certificate output directory for this particular certificate.",
+        "type": Path,
+    },
+    "domain": {
+        "help": "The domain name (CN or subject alternative name) being validated.",
+    },
+    "error": {
+        "help": "Contains error message if dehydrated exits with error.",
+        "nargs": "?",
+    },
+    "fullchain_path": {
+        "help": "The path of the file containing the signed full certificate chain.",
+        "type": Path,
+    },
+    "headers": {
+        "help": "HTTP headers returned by the CA",
+    },
+    "key_path": {
+        "help": "The path of the file containing the private key.",
+        "type": Path,
+    },
+    "ocsp_path": {
+        "help": "The path of the ocsp stapling file.",
+        "type": Path,
+    },
+    "req_type": {
+        "help": "The kind of request that was made (GET, POST, …).",
+    },
+    "reason": {
+        "help": "The specific reason for the error.",
+    },
+    "request_path": {
+        "help": "The path of the file containing the certificate signing request.",
+        "type": Path,
+    },
+    "response": {
+        "help": "The response that the verification server returned."
+    },
+    "status_code": {
+        "help": "The HTML status code that originated the error.",
+        "type": int,
+    },
+    "timestamp": {
+        "help": "Timestamp when the specified certificate was created.",
+        "type": int, # TODO to date
+    },
+    "token_filename": {
+        "help": "The name of the file containing the token to be served for HTTP validation.",
+    },
+    "token_value": {
+        "help": "The name of the file containing the token to be served for HTTP validation.",
+    },
+}
+
+hooks = {
+    "deploy_challenge": (deploy_challenge, [
+        "domain",
+        "token_filename",
+        "token_value",
+    ]),
+    "clean_challenge": (clean_challenge, [
+        "domain",
+        "token_filename",
+        "token_value",
+    ]),
+    "sync_cert": (sync_cert, [
+        "domain",
+        "key_path",
+        "cert_path",
+        "fullchain_path",
+        "chain_path",
+        "request_path",
+    ]),
+    "deploy_cert": (deploy_cert, [
+        "domain",
+        "key_path",
+        "cert_path",
+        "fullchain_path",
+        "chain_path",
+        "timestamp",
+    ]),
+    "deploy_ocsp": (deploy_ocsp, [
+        "domain",
+        "ocsp_path",
+        "timestamp",
+    ]),
+    "unchanged_cert": (unchanged_cert, [
+        "domain",
+        "key_path",
+        "cert_path",
+        "fullchain_path",
+        "chain_path",
+    ]),
+    "invalid_challenge": (invalid_challenge, [
+        "domain",
+        "response",
+    ]),
+    "request_failure": (request_failure, [
+        "status_code",
+        "reason",
+        "req_type",
+        "headers",
+    ]),
+    "generate_csr": (generate_csr, [
+        "domain",
+        "cert_dir",
+        "alt_names",
+    ]),
+    "startup_hook": (startup_hook, [
+    ]),
+    "exit_hook": (exit_hook, [
+        "error",
+    ]),
+}
+
+
+# general
+
+
+def read_config(config_path):
+    return json.loads(config_path.read_text())
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-c", "--config", "--config-file", dest='_config', type=Path, help="Path to config file")
+    subparsers = parser.add_subparsers(dest='_hook', required=True)
+    for hook_name, hook_data in hooks.items():
+        hook_fun, hook_args = hook_data
+        hook_parser = subparsers.add_parser(
+            hook_name,
+            prefix_chars='+',
+        )
+        hook_parser.set_defaults(
+            _func=hook_fun,
+            _parser=hook_parser,
+        )
+        for arg_name in hook_args:
+            hook_parser.add_argument(
+                arg_name,
+                **arg_infos[arg_name],
+            )
+    args = parser.parse_args()
+    return (args, {key: val for key, val in args.__dict__.items() if not key.startswith("_")})
+
+
+def external_hook(hook_name, domain, **kwargs):
+    domain_dir = config.domains_directory / domain
+    print(domain)
+
+
+def main(argv):
+    global config
+    args, fun_args = parse_args()
+    #config = read_config(args._config)
+    #args._fun(**fun_args)
+    external_hook(args._hook, **fun_args)
+
+
+if __name__ == "__main__":
+    main(sys.argv[1:])

roles/acme/certificate/defaults/main.yml

@@ -8,6 +8,21 @@ domains:
 
 acme_must_staple: yes
 
+dane_configure: yes
+dane_protocol: tcp
+dane_port: 443 # default for https
+dane_domain: "_{{ dane_port }}._{{ dane_protocol }}.{{ domain }}"
+
+# TODO Requires gnutls-bin to be installed
+dane_command: >-
+  danetool --tlsa-rr
+  --load-pubkey=cert.pem
+  --hash=sha512
+  --host={{ domain | quote }}
+  --proto={{ dane_protocol | quote }}
+  --port={{ dane_port | quote }}
+  --no-domain
+
 certificate_name: "{{ effective_domain }}"
 
 # acme_validation_root_directory from nginx/application

roles/acme/certificate/tasks/main.yml

@@ -1,5 +1,8 @@
 ---
 
+# TODO DANE TLSA
+# Test with https://check.sidnlabs.nl/dane/
+
 - name: Issue certificate for {{ certificate_name }}
   command:
     cmd: >-