diff --git a/.ci/ansible_tests.py b/.ci/ansible_tests.py index 51eab874..4df2dc70 100755 --- a/.ci/ansible_tests.py +++ b/.ci/ansible_tests.py @@ -20,11 +20,17 @@ def pause_if_interactive(): signal.pause() +interesting = ci_lib.get_interesting_procs() + + with ci_lib.Fold('unit_tests'): os.environ['SKIP_MITOGEN'] = '1' ci_lib.run('./run_tests -v') +ci_lib.check_stray_processes(interesting) + + with ci_lib.Fold('docker_setup'): containers = ci_lib.make_containers() ci_lib.start_containers(containers) @@ -75,4 +81,7 @@ with ci_lib.Fold('ansible'): pause_if_interactive() raise + +ci_lib.check_stray_processes(interesting, containers) + pause_if_interactive() diff --git a/.ci/ci_lib.py b/.ci/ci_lib.py index 34a6faba..9225723e 100644 --- a/.ci/ci_lib.py +++ b/.ci/ci_lib.py @@ -215,6 +215,44 @@ def make_containers(name_prefix='', port_offset=0): return lst +INTERESTING_COMMS = ('python', 'ssh', 'sudo', 'su', 'doas') + + +def proc_is_docker(pid): + try: + fp = open('/proc/%s/cgroup' % (pid,), 'rb') + except IOError: + return False + + try: + return 'docker' in fp.read() + finally: + fp.close() + + +def get_interesting_procs(container_name=None): + args = ['ps', '-a', '-x', '-oppid=', '-opid=', '-ocomm=', '-ocommand='] + if container_name is not None: + args = ['docker', 'exec', container_name] + args + + out = [] + for line in subprocess__check_output(args).splitlines(): + ppid, pid, comm, rest = line.split(None, 3) + if ( + ( + any(comm.startswith(s) for s in INTERESTING_COMMS) or + 'mitogen:' in rest + ) and + ( + container_name is not None or + (not proc_is_docker(pid)) + ) + ): + out.append((int(pid), line)) + + return sorted(out) + + def start_containers(containers): if os.environ.get('KEEP'): return @@ -236,9 +274,44 @@ def start_containers(containers): ] for container in containers ]) + + for container in containers: + container['interesting'] = get_interesting_procs(container['name']) + return containers +def verify_procs(hostname, old, new): + oldpids = set(pid for pid, _ in old) + if any(pid not in oldpids for pid, _ in new): + print('%r had stray processes running:' % (hostname,)) + for pid, line in new: + if pid not in oldpids: + print('New process:', line) + + print() + return False + + return True + + +def check_stray_processes(old, containers=None): + ok = True + + new = get_interesting_procs() + if old is not None: + ok &= verify_procs('test host machine', old, new) + + for container in containers or (): + ok &= verify_procs( + container['name'], + container['interesting'], + get_interesting_procs(container['name']) + ) + + assert ok, 'stray processes were found' + + def dump_file(path): print() print('--- %s ---' % (path,)) diff --git a/.ci/debops_common_tests.py b/.ci/debops_common_tests.py index 8b35de1e..e8f2907b 100755 --- a/.ci/debops_common_tests.py +++ b/.ci/debops_common_tests.py @@ -68,9 +68,15 @@ with ci_lib.Fold('job_setup'): os.environ['ANSIBLE_HOST_KEY_CHECKING'] = 'False' +interesting = ci_lib.get_interesting_procs() + with ci_lib.Fold('first_run'): ci_lib.run('debops common %s', ' '.join(sys.argv[1:])) +ci_lib.check_stray_processes(interesting, containers) + with ci_lib.Fold('second_run'): ci_lib.run('debops common %s', ' '.join(sys.argv[1:])) + +ci_lib.check_stray_processes(interesting, containers) diff --git a/.ci/mitogen_install.py b/.ci/mitogen_install.py index 72bc75e3..b8862f89 100755 --- a/.ci/mitogen_install.py +++ b/.ci/mitogen_install.py @@ -14,4 +14,5 @@ if ci_lib.have_docker(): 'docker pull %s' % (ci_lib.image_for_distro(ci_lib.DISTRO),), ]) + ci_lib.run_batches(batches) diff --git a/.ci/mitogen_tests.py b/.ci/mitogen_tests.py index 36928ac9..4de94b4c 100755 --- a/.ci/mitogen_tests.py +++ b/.ci/mitogen_tests.py @@ -14,4 +14,6 @@ os.environ.update({ if not ci_lib.have_docker(): os.environ['SKIP_DOCKER_TESTS'] = '1' +interesting = ci_lib.get_interesting_procs() ci_lib.run('./run_tests -v') +ci_lib.check_stray_processes(interesting)