issue #482: ci: add stray process checks to all jobs

List of interesting processes can probably expand more over time.
pull/618/head
David Wilson 5 years ago
parent 1e3621a88b
commit 7ca073cdf8

@ -20,11 +20,17 @@ def pause_if_interactive():
signal.pause() signal.pause()
interesting = ci_lib.get_interesting_procs()
with ci_lib.Fold('unit_tests'): with ci_lib.Fold('unit_tests'):
os.environ['SKIP_MITOGEN'] = '1' os.environ['SKIP_MITOGEN'] = '1'
ci_lib.run('./run_tests -v') ci_lib.run('./run_tests -v')
ci_lib.check_stray_processes(interesting)
with ci_lib.Fold('docker_setup'): with ci_lib.Fold('docker_setup'):
containers = ci_lib.make_containers() containers = ci_lib.make_containers()
ci_lib.start_containers(containers) ci_lib.start_containers(containers)
@ -75,4 +81,7 @@ with ci_lib.Fold('ansible'):
pause_if_interactive() pause_if_interactive()
raise raise
ci_lib.check_stray_processes(interesting, containers)
pause_if_interactive() pause_if_interactive()

@ -215,6 +215,44 @@ def make_containers(name_prefix='', port_offset=0):
return lst return lst
INTERESTING_COMMS = ('python', 'ssh', 'sudo', 'su', 'doas')
def proc_is_docker(pid):
try:
fp = open('/proc/%s/cgroup' % (pid,), 'rb')
except IOError:
return False
try:
return 'docker' in fp.read()
finally:
fp.close()
def get_interesting_procs(container_name=None):
args = ['ps', '-a', '-x', '-oppid=', '-opid=', '-ocomm=', '-ocommand=']
if container_name is not None:
args = ['docker', 'exec', container_name] + args
out = []
for line in subprocess__check_output(args).splitlines():
ppid, pid, comm, rest = line.split(None, 3)
if (
(
any(comm.startswith(s) for s in INTERESTING_COMMS) or
'mitogen:' in rest
) and
(
container_name is not None or
(not proc_is_docker(pid))
)
):
out.append((int(pid), line))
return sorted(out)
def start_containers(containers): def start_containers(containers):
if os.environ.get('KEEP'): if os.environ.get('KEEP'):
return return
@ -236,9 +274,44 @@ def start_containers(containers):
] ]
for container in containers for container in containers
]) ])
for container in containers:
container['interesting'] = get_interesting_procs(container['name'])
return containers return containers
def verify_procs(hostname, old, new):
oldpids = set(pid for pid, _ in old)
if any(pid not in oldpids for pid, _ in new):
print('%r had stray processes running:' % (hostname,))
for pid, line in new:
if pid not in oldpids:
print('New process:', line)
print()
return False
return True
def check_stray_processes(old, containers=None):
ok = True
new = get_interesting_procs()
if old is not None:
ok &= verify_procs('test host machine', old, new)
for container in containers or ():
ok &= verify_procs(
container['name'],
container['interesting'],
get_interesting_procs(container['name'])
)
assert ok, 'stray processes were found'
def dump_file(path): def dump_file(path):
print() print()
print('--- %s ---' % (path,)) print('--- %s ---' % (path,))

@ -68,9 +68,15 @@ with ci_lib.Fold('job_setup'):
os.environ['ANSIBLE_HOST_KEY_CHECKING'] = 'False' os.environ['ANSIBLE_HOST_KEY_CHECKING'] = 'False'
interesting = ci_lib.get_interesting_procs()
with ci_lib.Fold('first_run'): with ci_lib.Fold('first_run'):
ci_lib.run('debops common %s', ' '.join(sys.argv[1:])) ci_lib.run('debops common %s', ' '.join(sys.argv[1:]))
ci_lib.check_stray_processes(interesting, containers)
with ci_lib.Fold('second_run'): with ci_lib.Fold('second_run'):
ci_lib.run('debops common %s', ' '.join(sys.argv[1:])) ci_lib.run('debops common %s', ' '.join(sys.argv[1:]))
ci_lib.check_stray_processes(interesting, containers)

@ -14,4 +14,5 @@ if ci_lib.have_docker():
'docker pull %s' % (ci_lib.image_for_distro(ci_lib.DISTRO),), 'docker pull %s' % (ci_lib.image_for_distro(ci_lib.DISTRO),),
]) ])
ci_lib.run_batches(batches) ci_lib.run_batches(batches)

@ -14,4 +14,6 @@ os.environ.update({
if not ci_lib.have_docker(): if not ci_lib.have_docker():
os.environ['SKIP_DOCKER_TESTS'] = '1' os.environ['SKIP_DOCKER_TESTS'] = '1'
interesting = ci_lib.get_interesting_procs()
ci_lib.run('./run_tests -v') ci_lib.run('./run_tests -v')
ci_lib.check_stray_processes(interesting)

Loading…
Cancel
Save