Overhaul httptester support in ansible-test. (#39892)

- Works with the --remote option.
- Can be disabled with the --disable-httptester option.
- Change image with the --httptester option.
- Only load and run httptester for targets that require it.

(cherry picked from commit c1f9efabf4)

test/integration/targets/get_url/aliases

@@ -1,2 +1,3 @@
 destructive
 posix/ci/group1
+needs/httptester

test/integration/targets/lookups/aliases

@@ -1 +1,2 @@
 posix/ci/group2
+needs/httptester

test/integration/targets/prepare_http_tests/tasks/main.yml

@@ -46,4 +46,26 @@
       command: update-ca-certificates
       when: ansible_os_family == 'Debian' or ansible_os_family == 'Suse'
 
+    - name: FreeBSD - Retrieve test cacert
+      get_url:
+        url: "http://ansible.http.tests/cacert.pem"
+        dest: "/tmp/ansible.pem"
+      when: ansible_os_family == 'FreeBSD'
+
+    - name: FreeBSD - Add cacert to root certificate store
+      blockinfile:
+        path: "/etc/ssl/cert.pem"
+        block: "{{ lookup('file', '/tmp/ansible.pem') }}"
+      when: ansible_os_family == 'FreeBSD'
+
+    - name: MacOS - Retrieve test cacert
+      get_url:
+        url: "http://ansible.http.tests/cacert.pem"
+        dest: "/usr/local/etc/openssl/certs/ansible.pem"
+      when: ansible_os_family == 'Darwin'
+
+    - name: MacOS - Update ca certificates
+      command: /usr/local/opt/openssl/bin/c_rehash
+      when: ansible_os_family == 'Darwin'
+
   when: has_httptester|bool

test/integration/targets/uri/aliases

@@ -1,2 +1,3 @@
 destructive
 posix/ci/group1
+needs/httptester

test/runner/lib/config.py

@@ -43,7 +43,6 @@ class EnvironmentConfig(CommonConfig):
         self.remote = args.remote  # type: str
 
         self.docker_privileged = args.docker_privileged if 'docker_privileged' in args else False  # type: bool
-        self.docker_util = docker_qualify_image(args.docker_util if 'docker_util' in args else '')  # type: str
         self.docker_pull = args.docker_pull if 'docker_pull' in args else False  # type: bool
         self.docker_keep_git = args.docker_keep_git if 'docker_keep_git' in args else False  # type: bool
         self.docker_memory = args.docker_memory if 'docker_memory' in args else None
@@ -70,6 +69,9 @@ class EnvironmentConfig(CommonConfig):
         if self.delegate:
             self.requirements = True
 
+        self.inject_httptester = args.inject_httptester if 'inject_httptester' in args else False  # type: bool
+        self.httptester = docker_qualify_image(args.httptester if 'httptester' in args else '')  # type: str
+
     @property
     def python_executable(self):
         """

test/runner/lib/delegation.py

@@ -12,7 +12,10 @@ import lib.thread
 
 from lib.executor import (
     SUPPORTED_PYTHON_VERSIONS,
+    HTTPTESTER_HOSTS,
     create_shell_command,
+    run_httptester,
+    start_httptester,
 )
 
 from lib.config import (
@@ -37,6 +40,7 @@ from lib.util import (
     run_command,
     common_environment,
     pass_vars,
+    display,
 )
 
 from lib.docker_util import (
@@ -46,18 +50,24 @@ from lib.docker_util import (
     docker_put,
     docker_rm,
     docker_run,
+    docker_available,
 )
 
 from lib.cloud import (
     get_cloud_providers,
 )
 
+from lib.target import (
+    IntegrationTarget,
+)
+
 
-def delegate(args, exclude, require):
+def delegate(args, exclude, require, integration_targets):
     """
     :type args: EnvironmentConfig
     :type exclude: list[str]
     :type require: list[str]
+    :type integration_targets: tuple[IntegrationTarget]
     :rtype: bool
     """
     if isinstance(args, TestConfig):
@@ -66,40 +76,42 @@ def delegate(args, exclude, require):
             args.metadata.to_file(args.metadata_path)
 
             try:
-                return delegate_command(args, exclude, require)
+                return delegate_command(args, exclude, require, integration_targets)
             finally:
                 args.metadata_path = None
     else:
-        return delegate_command(args, exclude, require)
+        return delegate_command(args, exclude, require, integration_targets)
 
 
-def delegate_command(args, exclude, require):
+def delegate_command(args, exclude, require, integration_targets):
     """
     :type args: EnvironmentConfig
     :type exclude: list[str]
     :type require: list[str]
+    :type integration_targets: tuple[IntegrationTarget]
     :rtype: bool
     """
     if args.tox:
-        delegate_tox(args, exclude, require)
+        delegate_tox(args, exclude, require, integration_targets)
         return True
 
     if args.docker:
-        delegate_docker(args, exclude, require)
+        delegate_docker(args, exclude, require, integration_targets)
         return True
 
     if args.remote:
-        delegate_remote(args, exclude, require)
+        delegate_remote(args, exclude, require, integration_targets)
         return True
 
     return False
 
 
-def delegate_tox(args, exclude, require):
+def delegate_tox(args, exclude, require, integration_targets):
     """
     :type args: EnvironmentConfig
     :type exclude: list[str]
     :type require: list[str]
+    :type integration_targets: tuple[IntegrationTarget]
     """
     if args.python:
         versions = args.python_version,
@@ -109,6 +121,12 @@ def delegate_tox(args, exclude, require):
     else:
         versions = SUPPORTED_PYTHON_VERSIONS
 
+    if args.httptester:
+        needs_httptester = sorted(target.name for target in integration_targets if 'needs/httptester/' in target.aliases)
+
+        if needs_httptester:
+            display.warning('Use --docker or --remote to enable httptester for tests marked "needs/httptester": %s' % ', '.join(needs_httptester))
+
     options = {
         '--tox': args.tox_args,
         '--tox-sitepackages': 0,
@@ -145,22 +163,27 @@ def delegate_tox(args, exclude, require):
         run_command(args, tox + cmd, env=env)
 
 
-def delegate_docker(args, exclude, require):
+def delegate_docker(args, exclude, require, integration_targets):
     """
     :type args: EnvironmentConfig
     :type exclude: list[str]
     :type require: list[str]
+    :type integration_targets: tuple[IntegrationTarget]
     """
-    util_image = args.docker_util
     test_image = args.docker
     privileged = args.docker_privileged
 
-    if util_image:
-        docker_pull(args, util_image)
+    if isinstance(args, ShellConfig):
+        use_httptester = args.httptester
+    else:
+        use_httptester = args.httptester and any('needs/httptester/' in target.aliases for target in integration_targets)
+
+    if use_httptester:
+        docker_pull(args, args.httptester)
 
     docker_pull(args, test_image)
 
-    util_id = None
+    httptester_id = None
     test_id = None
 
     options = {
@@ -196,19 +219,10 @@ def delegate_docker(args, exclude, require):
 
                 lib.pytar.create_tarfile(local_source_fd.name, '.', tar_filter)
 
-            if util_image:
-                util_options = [
-                    '--detach',
-                ]
-
-                util_id, _ = docker_run(args, util_image, options=util_options)
-
-                if args.explain:
-                    util_id = 'util_id'
-                else:
-                    util_id = util_id.strip()
+            if use_httptester:
+                httptester_id = run_httptester(args)
             else:
-                util_id = None
+                httptester_id = None
 
             test_options = [
                 '--detach',
@@ -227,14 +241,11 @@ def delegate_docker(args, exclude, require):
             if os.path.exists(docker_socket):
                 test_options += ['--volume', '%s:%s' % (docker_socket, docker_socket)]
 
-            if util_id:
-                test_options += [
-                    '--link', '%s:ansible.http.tests' % util_id,
-                    '--link', '%s:sni1.ansible.http.tests' % util_id,
-                    '--link', '%s:sni2.ansible.http.tests' % util_id,
-                    '--link', '%s:fail.ansible.http.tests' % util_id,
-                    '--env', 'HTTPTESTER=1',
-                ]
+            if httptester_id:
+                test_options += ['--env', 'HTTPTESTER=1']
+
+                for host in HTTPTESTER_HOSTS:
+                    test_options += ['--link', '%s:%s' % (httptester_id, host)]
 
             if isinstance(args, IntegrationConfig):
                 cloud_platforms = get_cloud_providers(args)
@@ -268,18 +279,19 @@ def delegate_docker(args, exclude, require):
                     docker_get(args, test_id, '/root/results.tgz', local_result_fd.name)
                     run_command(args, ['tar', 'oxzf', local_result_fd.name, '-C', 'test'])
         finally:
-            if util_id:
-                docker_rm(args, util_id)
+            if httptester_id:
+                docker_rm(args, httptester_id)
 
             if test_id:
                 docker_rm(args, test_id)
 
 
-def delegate_remote(args, exclude, require):
+def delegate_remote(args, exclude, require, integration_targets):
     """
     :type args: EnvironmentConfig
     :type exclude: list[str]
     :type require: list[str]
+    :type integration_targets: tuple[IntegrationTarget]
     """
     parts = args.remote.split('/', 1)
 
@@ -289,8 +301,24 @@ def delegate_remote(args, exclude, require):
     core_ci = AnsibleCoreCI(args, platform, version, stage=args.remote_stage, provider=args.remote_provider)
     success = False
 
+    if isinstance(args, ShellConfig):
+        use_httptester = args.httptester
+    else:
+        use_httptester = args.httptester and any('needs/httptester/' in target.aliases for target in integration_targets)
+
+    if use_httptester and not docker_available():
+        display.warning('Assuming --disable-httptester since `docker` is not available.')
+        use_httptester = False
+
+    httptester_id = None
+    ssh_options = []
+
     try:
         core_ci.start()
+
+        if use_httptester:
+            httptester_id, ssh_options = start_httptester(args)
+
         core_ci.wait()
 
         options = {
@@ -299,6 +327,9 @@ def delegate_remote(args, exclude, require):
 
         cmd = generate_command(args, 'ansible/test/runner/test.py', options, exclude, require)
 
+        if httptester_id:
+            cmd += ['--inject-httptester']
+
         if isinstance(args, TestConfig):
             if args.coverage and not args.coverage_label:
                 cmd += ['--coverage-label', 'remote-%s-%s' % (platform, version)]
@@ -314,8 +345,6 @@ def delegate_remote(args, exclude, require):
         manage = ManagePosixCI(core_ci)
         manage.setup()
 
-        ssh_options = []
-
         if isinstance(args, IntegrationConfig):
             cloud_platforms = get_cloud_providers(args)
 
@@ -332,6 +361,9 @@ def delegate_remote(args, exclude, require):
         if args.remote_terminate == 'always' or (args.remote_terminate == 'success' and success):
             core_ci.stop()
 
+        if httptester_id:
+            docker_rm(args, httptester_id)
+
 
 def generate_command(args, path, options, exclude, require):
     """

test/runner/lib/docker_util.py

@@ -15,6 +15,7 @@ from lib.util import (
     run_command,
     common_environment,
     display,
+    find_executable,
 )
 
 from lib.config import (
@@ -24,6 +25,13 @@ from lib.config import (
 BUFFER_SIZE = 256 * 256
 
 
+def docker_available():
+    """
+    :rtype: bool
+    """
+    return find_executable('docker', required=False)
+
+
 def get_docker_container_id():
     """
     :rtype: str | None
@@ -48,6 +56,17 @@ def get_docker_container_id():
     raise ApplicationError('Found multiple container_id candidates: %s\n%s' % (sorted(container_ids), contents))
 
 
+def get_docker_container_ip(args, container_id):
+    """
+    :type args: EnvironmentConfig
+    :type container_id: str
+    :rtype: str
+    """
+    results = docker_inspect(args, container_id)
+    ipaddress = results[0]['NetworkSettings']['IPAddress']
+    return ipaddress
+
+
 def docker_pull(args, image):
     """
     :type args: EnvironmentConfig

test/runner/lib/executor.py

@@ -48,6 +48,14 @@ from lib.util import (
     find_executable,
     raw_command,
     get_coverage_path,
+    get_available_port,
+)
+
+from lib.docker_util import (
+    docker_pull,
+    docker_run,
+    get_docker_container_id,
+    get_docker_container_ip,
 )
 
 from lib.ansible_util import (
@@ -100,6 +108,12 @@ SUPPORTED_PYTHON_VERSIONS = (
     '3.7',
 )
 
+HTTPTESTER_HOSTS = (
+    'ansible.http.tests',
+    'sni1.ansible.http.tests',
+    'fail.ansible.http.tests',
+)
+
 
 def check_startup():
     """Checks to perform at startup before running commands."""
@@ -277,6 +291,9 @@ def command_shell(args):
 
     install_command_requirements(args)
 
+    if args.inject_httptester:
+        inject_httptester(args)
+
     cmd = create_shell_command(['bash', '-i'])
     run_command(args, cmd)
 
@@ -649,7 +666,7 @@ def command_integration_filter(args, targets, init_callback=None):
     cloud_init(args, internal_targets)
 
     if args.delegate:
-        raise Delegate(require=changes, exclude=exclude)
+        raise Delegate(require=changes, exclude=exclude, integration_targets=internal_targets)
 
     install_command_requirements(args)
 
@@ -697,6 +714,9 @@ def command_integration_filtered(args, targets, all_targets):
                 display.warning('SSH service not responding. Waiting %d second(s) before checking again.' % seconds)
                 time.sleep(seconds)
 
+    if args.inject_httptester:
+        inject_httptester(args)
+
     start_at_task = args.start_at_task
 
     results = {}
@@ -815,6 +835,133 @@ def command_integration_filtered(args, targets, all_targets):
             len(failed), len(passed) + len(failed), '\n'.join(target.name for target in failed)))
 
 
+def start_httptester(args):
+    """
+    :type args: EnvironmentConfig
+    :rtype: str, list[str]
+    """
+
+    # map ports from remote -> localhost -> container
+    # passing through localhost is only used when ansible-test is not already running inside a docker container
+    ports = [
+        dict(
+            remote=8080,
+            container=80,
+        ),
+        dict(
+            remote=8443,
+            container=443,
+        ),
+    ]
+
+    container_id = get_docker_container_id()
+
+    if container_id:
+        display.info('Running in docker container: %s' % container_id, verbosity=1)
+    else:
+        for item in ports:
+            item['localhost'] = get_available_port()
+
+    docker_pull(args, args.httptester)
+
+    httptester_id = run_httptester(args, dict((port['localhost'], port['container']) for port in ports if 'localhost' in port))
+
+    if container_id:
+        container_host = get_docker_container_ip(args, httptester_id)
+        display.info('Found httptester container address: %s' % container_host, verbosity=1)
+    else:
+        container_host = 'localhost'
+
+    ssh_options = []
+
+    for port in ports:
+        ssh_options += ['-R', '%d:%s:%d' % (port['remote'], container_host, port.get('localhost', port['container']))]
+
+    return httptester_id, ssh_options
+
+
+def run_httptester(args, ports=None):
+    """
+    :type args: EnvironmentConfig
+    :type ports: dict[int, int] | None
+    :rtype: str
+    """
+    options = [
+        '--detach',
+    ]
+
+    if ports:
+        for localhost_port, container_port in ports.items():
+            options += ['-p', '%d:%d' % (localhost_port, container_port)]
+
+    httptester_id, _ = docker_run(args, args.httptester, options=options)
+
+    if args.explain:
+        httptester_id = 'httptester_id'
+    else:
+        httptester_id = httptester_id.strip()
+
+    return httptester_id
+
+
+def inject_httptester(args):
+    """
+    :type args: CommonConfig
+    """
+    comment = ' # ansible-test httptester\n'
+    append_lines = ['127.0.0.1 %s%s' % (host, comment) for host in HTTPTESTER_HOSTS]
+
+    with open('/etc/hosts', 'r+') as hosts_fd:
+        original_lines = hosts_fd.readlines()
+
+        if not any(line.endswith(comment) for line in original_lines):
+            hosts_fd.writelines(append_lines)
+
+    # determine which forwarding mechanism to use
+    pfctl = find_executable('pfctl', required=False)
+    iptables = find_executable('iptables', required=False)
+
+    if pfctl:
+        kldload = find_executable('kldload', required=False)
+
+        if kldload:
+            try:
+                run_command(args, ['kldload', 'pf'], capture=True)
+            except SubprocessError:
+                pass  # already loaded
+
+        rules = '''
+rdr pass inet proto tcp from any to any port 80 -> 127.0.0.1 port 8080
+rdr pass inet proto tcp from any to any port 443 -> 127.0.0.1 port 8443
+'''
+        cmd = ['pfctl', '-ef', '-']
+
+        try:
+            run_command(args, cmd, capture=True, data=rules)
+        except SubprocessError:
+            pass  # non-zero exit status on success
+
+    elif iptables:
+        ports = [
+            (80, 8080),
+            (443, 8443),
+        ]
+
+        for src, dst in ports:
+            rule = ['-o', 'lo', '-p', 'tcp', '--dport', str(src), '-j', 'REDIRECT', '--to-port', str(dst)]
+
+            try:
+                # check for existing rule
+                cmd = ['iptables', '-t', 'nat', '-C', 'OUTPUT'] + rule
+                run_command(args, cmd, capture=True)
+            except SubprocessError:
+                # append rule when it does not exist
+                cmd = ['iptables', '-t', 'nat', '-A', 'OUTPUT'] + rule
+                run_command(args, cmd, capture=True)
+    else:
+        raise ApplicationError('No supported port forwarding mechanism detected.')
+
+
 def run_setup_targets(args, test_dir, target_names, targets_dict, targets_executed, always):
     """
     :type args: IntegrationConfig
@@ -852,6 +999,11 @@ def integration_environment(args, target, cmd):
     """
     env = ansible_environment(args)
 
+    if args.inject_httptester:
+        env.update(dict(
+            HTTPTESTER='1',
+        ))
+
     integration = dict(
         JUNIT_OUTPUT_DIR=os.path.abspath('test/results/junit'),
         ANSIBLE_CALLBACK_WHITELIST='junit',
@@ -1464,15 +1616,17 @@ class NoTestsForChanges(ApplicationWarning):
 
 class Delegate(Exception):
     """Trigger command delegation."""
-    def __init__(self, exclude=None, require=None):
+    def __init__(self, exclude=None, require=None, integration_targets=None):
         """
         :type exclude: list[str] | None
         :type require: list[str] | None
+        :type integration_targets: tuple[IntegrationTarget] | None
         """
         super(Delegate, self).__init__()
 
         self.exclude = exclude or []
         self.require = require or []
+        self.integration_targets = integration_targets or tuple()
 
 
 class AllTargetsSkipped(ApplicationWarning):

test/runner/lib/util.py

@@ -3,6 +3,7 @@
 from __future__ import absolute_import, print_function
 
 import atexit
+import contextlib
 import errno
 import filecmp
 import fcntl
@@ -14,6 +15,7 @@ import pkgutil
 import random
 import re
 import shutil
+import socket
 import stat
 import string
 import subprocess
@@ -720,6 +722,18 @@ def parse_to_dict(pattern, value):
     return match.groupdict()
 
 
+def get_available_port():
+    """
+    :rtype: int
+    """
+    # this relies on the kernel not reusing previously assigned ports immediately
+    socket_fd = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+
+    with contextlib.closing(socket_fd):
+        socket_fd.bind(('', 0))
+        return socket_fd.getsockname()[1]
+
+
 def get_subclasses(class_type):
     """
     :type class_type: type

test/runner/test.py

@@ -88,7 +88,7 @@ def main():
         try:
             args.func(config)
         except Delegate as ex:
-            delegate(config, ex.exclude, ex.require)
+            delegate(config, ex.exclude, ex.require, ex.integration_targets)
 
         display.review_warnings()
     except ApplicationWarning as ex:
@@ -278,6 +278,7 @@ def parse_args():
                                    config=PosixIntegrationConfig)
 
     add_extra_docker_options(posix_integration)
+    add_httptester_options(posix_integration, argparse)
 
     network_integration = subparsers.add_parser('network-integration',
                                                 parents=[integration],
@@ -380,6 +381,7 @@ def parse_args():
 
     add_environments(shell, tox_version=True)
     add_extra_docker_options(shell)
+    add_httptester_options(shell, argparse)
 
     coverage_common = argparse.ArgumentParser(add_help=False, parents=[common])
 
@@ -606,6 +608,29 @@ def add_extra_coverage_options(parser):
                         help='generate empty report of all python source files')
 
 
+def add_httptester_options(parser, argparse):
+    """
+    :type parser: argparse.ArgumentParser
+    :type argparse: argparse
+    """
+    group = parser.add_mutually_exclusive_group()
+
+    group.add_argument('--httptester',
+                       metavar='IMAGE',
+                       default='quay.io/ansible/http-test-container:1.0.0',
+                       help='docker image to use for the httptester container')
+
+    group.add_argument('--disable-httptester',
+                       dest='httptester',
+                       action='store_const',
+                       const='',
+                       help='do not use the httptester container')
+
+    parser.add_argument('--inject-httptester',
+                        action='store_true',
+                        help=argparse.SUPPRESS)  # internal use only
+
+
 def add_extra_docker_options(parser, integration=True):
     """
     :type parser: argparse.ArgumentParser
@@ -625,11 +650,6 @@ def add_extra_docker_options(parser, integration=True):
     if not integration:
         return
 
-    docker.add_argument('--docker-util',
-                        metavar='IMAGE',
-                        default='quay.io/ansible/http-test-container:1.0.0',
-                        help='docker utility image to provide test services')
-
     docker.add_argument('--docker-privileged',
                         action='store_true',
                         help='run docker container in privileged mode')