diff --git a/tests/ansible/README.md b/tests/ansible/README.md index 8051ef85..46320951 100644 --- a/tests/ansible/README.md +++ b/tests/ansible/README.md @@ -1,5 +1,5 @@ -# ``tests/ansible`` Directory +# `tests/ansible` Directory This is an an organically growing collection of integration and regression tests used for development and end-user bug reports. @@ -10,10 +10,10 @@ demonstrator for what does and doesn't work. ## Preparation -For OS X, run the ``osx_setup.yml`` script to create a bunch of users. +See `../image_prep/README.md`. -## ``run_ansible_playbook.sh`` +## `run_ansible_playbook.sh` This is necessary to set some environment variables used by future tests, as there appears to be no better way to inject them into the top-level process @@ -22,12 +22,10 @@ environment before the Mitogen connection process forks. ## Running Everything -``` -ANSIBLE_STRATEGY=mitogen_linear ./run_ansible_playbook.sh all.yml -``` +`ANSIBLE_STRATEGY=mitogen_linear ./run_ansible_playbook.sh all.yml` -## ``hosts/`` and ``common-hosts`` +## `hosts/` and `common-hosts` To support running the tests against a dev machine that has the requisite user accounts, the the default inventory is a directory containing a 'localhost' @@ -35,7 +33,7 @@ file that defines 'localhost' to be named 'target' in Ansible inventory, and a symlink to 'common-hosts', which defines additional targets that all derive from 'target'. -This allows ``ansible_tests.sh`` to reuse the common-hosts definitions while +This allows `ansible_tests.sh` to reuse the common-hosts definitions while replacing localhost as the test target by creating a new directory that similarly symlinks in common-hosts. diff --git a/tests/ansible/osx_setup.yml b/tests/ansible/osx_setup.yml deleted file mode 100644 index 7a6ff23f..00000000 --- a/tests/ansible/osx_setup.yml +++ /dev/null @@ -1,155 +0,0 @@ - -# -# Add users expected by tests to an OS X machine. Assumes passwordless sudo to -# root. -# -# WARNING: this creates non-privilged accounts with pre-set passwords! -# - -- hosts: test-targets - gather_facts: true - become: true - tasks: - - name: Disable non-localhost SSH for Mitogen users - blockinfile: - path: /etc/ssh/sshd_config - block: | - Match User mitogen__* Address !127.0.0.1 - DenyUsers * - - # - # Hashed passwords. - # - - name: Create Mitogen test group - group: - name: "mitogen__group" - - - name: Create Mitogen test users - user: - name: "mitogen__{{item}}" - shell: /bin/bash - groups: mitogen__group - password: "{{ (item + '_password') | password_hash('sha256') }}" - with_items: - - has_sudo - - has_sudo_pubkey - - require_tty - - pw_required - - readonly_homedir - - require_tty_pw_required - - slow_user - when: ansible_system != 'Darwin' - - - name: Create Mitogen test users - user: - name: "mitogen__user{{item}}" - shell: /bin/bash - password: "{{ ('user' + item + '_password') | password_hash('sha256') }}" - with_sequence: start=1 end=21 - when: ansible_system != 'Darwin' - - # - # Plaintext passwords - # - - name: Create Mitogen test users - user: - name: "mitogen__{{item}}" - shell: /bin/bash - groups: mitogen__group - password: "{{item}}_password" - with_items: - - has_sudo - - has_sudo_pubkey - - require_tty - - pw_required - - require_tty_pw_required - - readonly_homedir - - slow_user - when: ansible_system == 'Darwin' - - - name: Create Mitogen test users - user: - name: "mitogen__user{{item}}" - shell: /bin/bash - password: "user{{item}}_password" - with_sequence: start=1 end=21 - when: ansible_system == 'Darwin' - - - name: Hide test users from login window. - shell: > - defaults - write - /Library/Preferences/com.apple.loginwindow - HiddenUsersList - -array-add '{{item}}' - with_items: - - mitogen__require_tty - - mitogen__pw_required - - mitogen__require_tty_pw_required - when: ansible_system == 'Darwin' - - - name: Hide test users from login window. - shell: > - defaults - write - /Library/Preferences/com.apple.loginwindow - HiddenUsersList - -array-add 'mitogen__user{{item}}' - with_sequence: start=1 end=21 - when: ansible_distribution == 'MacOSX' - - - name: Readonly homedir for one account - shell: "chown -R root: ~mitogen__readonly_homedir" - - - name: Slow bash profile for one account - copy: - dest: ~mitogen__slow_user/.{{item}} - src: ../data/docker/mitogen__slow_user.profile - with_items: - - bashrc - - profile - - - name: Install pubkey for one account - file: - path: ~mitogen__has_sudo_pubkey/.ssh - state: directory - mode: go= - owner: mitogen__has_sudo_pubkey - - - name: Install pubkey for one account - copy: - dest: ~mitogen__has_sudo_pubkey/.ssh/authorized_keys - src: ../data/docker/mitogen__has_sudo_pubkey.key.pub - mode: go= - owner: mitogen__has_sudo_pubkey - - - name: Require a TTY for two accounts - lineinfile: - path: /etc/sudoers - line: "{{item}}" - with_items: - - Defaults>mitogen__pw_required targetpw - - Defaults>mitogen__require_tty requiretty - - Defaults>mitogen__require_tty_pw_required requiretty,targetpw - - - name: Require password for two accounts - lineinfile: - path: /etc/sudoers - line: "{{lookup('pipe', 'whoami')}} ALL = ({{item}}) ALL" - with_items: - - mitogen__pw_required - - mitogen__require_tty_pw_required - - - name: Allow passwordless for two accounts - lineinfile: - path: /etc/sudoers - line: "{{lookup('pipe', 'whoami')}} ALL = ({{item}}) NOPASSWD:ALL" - with_items: - - mitogen__require_tty - - mitogen__readonly_homedir - - - name: Allow passwordless for many accounts - lineinfile: - path: /etc/sudoers - line: "{{lookup('pipe', 'whoami')}} ALL = (mitogen__user{{item}}) NOPASSWD:ALL" - with_sequence: start=1 end=21 diff --git a/tests/build_docker_images.py b/tests/build_docker_images.py deleted file mode 100755 index 7f856b2b..00000000 --- a/tests/build_docker_images.py +++ /dev/null @@ -1,120 +0,0 @@ -#!/usr/bin/env python - -""" -Build the Docker images used for testing. -""" - -import commands -import os -import shlex -import subprocess -import tempfile - - -DEBIAN_DOCKERFILE = r""" -FROM debian:stretch -RUN apt-get update -RUN \ - apt-get install -y python2.7 openssh-server sudo rsync git strace \ - libjson-perl python-virtualenv && \ - apt-get clean && \ - rm -rf /var/cache/apt -""" - -CENTOS6_DOCKERFILE = r""" -FROM centos:6 -RUN yum clean all && \ - yum -y install -y python2.6 openssh-server sudo rsync git strace sudo \ - perl-JSON python-virtualenv && \ - yum clean all && \ - groupadd sudo && \ - ssh-keygen -t rsa -f /etc/ssh/ssh_host_rsa_key - -""" - -CENTOS7_DOCKERFILE = r""" -FROM centos:7 -RUN yum clean all && \ - yum -y install -y python2.7 openssh-server sudo rsync git strace sudo \ - perl-JSON python-virtualenv && \ - yum clean all && \ - groupadd sudo && \ - ssh-keygen -t rsa -f /etc/ssh/ssh_host_rsa_key - -""" - -DOCKERFILE = r""" -COPY data/001-mitogen.sudo /etc/sudoers.d/001-mitogen -COPY data/docker/ssh_login_banner.txt /etc/ssh/banner.txt -RUN \ - chsh -s /bin/bash && \ - mkdir -p /var/run/sshd && \ - echo i-am-mitogen-test-docker-image > /etc/sentinel && \ - echo "Banner /etc/ssh/banner.txt" >> /etc/ssh/sshd_config && \ - groupadd mitogen__sudo_nopw && \ - useradd -s /bin/bash -m mitogen__has_sudo -G SUDO_GROUP && \ - useradd -s /bin/bash -m mitogen__has_sudo_pubkey -G SUDO_GROUP && \ - useradd -s /bin/bash -m mitogen__has_sudo_nopw -G mitogen__sudo_nopw && \ - useradd -s /bin/bash -m mitogen__webapp && \ - useradd -s /bin/bash -m mitogen__pw_required && \ - useradd -s /bin/bash -m mitogen__require_tty && \ - useradd -s /bin/bash -m mitogen__require_tty_pw_required && \ - useradd -s /bin/bash -m mitogen__readonly_homedir && \ - useradd -s /bin/bash -m mitogen__slow_user && \ - chown -R root: ~mitogen__readonly_homedir && \ - ( for i in `seq 1 21`; do useradd -s /bin/bash -m mitogen__user${i}; done; ) && \ - ( for i in `seq 1 21`; do echo mitogen__user${i}:user${i}_password | chpasswd; done; ) && \ - ( echo 'root:rootpassword' | chpasswd; ) && \ - ( echo 'mitogen__has_sudo:has_sudo_password' | chpasswd; ) && \ - ( echo 'mitogen__has_sudo_pubkey:has_sudo_pubkey_password' | chpasswd; ) && \ - ( echo 'mitogen__has_sudo_nopw:has_sudo_nopw_password' | chpasswd; ) && \ - ( echo 'mitogen__webapp:webapp_password' | chpasswd; ) && \ - ( echo 'mitogen__pw_required:pw_required_password' | chpasswd; ) && \ - ( echo 'mitogen__require_tty:require_tty_password' | chpasswd; ) && \ - ( echo 'mitogen__require_tty_pw_required:require_tty_pw_required_password' | chpasswd; ) && \ - ( echo 'mitogen__readonly_homedir:readonly_homedir_password' | chpasswd; ) && \ - ( echo 'mitogen__slow_user:slow_user_password' | chpasswd; ) && \ - mkdir ~mitogen__has_sudo_pubkey/.ssh && \ - ( echo '#!/bin/bash\nexec strace -ff -o /tmp/pywrap$$.trace python2.7 "$@"' > /usr/local/bin/pywrap; chmod +x /usr/local/bin/pywrap; ) - -COPY data/docker/mitogen__has_sudo_pubkey.key.pub /home/mitogen__has_sudo_pubkey/.ssh/authorized_keys -COPY data/docker/mitogen__slow_user.profile /home/mitogen__slow_user/.profile -COPY data/docker/mitogen__slow_user.profile /home/mitogen__slow_user/.bashrc - -RUN \ - chown -R mitogen__has_sudo_pubkey ~mitogen__has_sudo_pubkey && \ - chmod -R go= ~mitogen__has_sudo_pubkey - -RUN sed -i 's/PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config -RUN sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd - -ENV NOTVISIBLE "in users profile" -RUN echo "export VISIBLE=now" >> /etc/profile - -EXPOSE 22 -CMD ["/usr/sbin/sshd", "-D"] -""" - - -def sh(s, *args): - if args: - s %= tuple(map(commands.mkarg, args)) - return shlex.split(s) - - -for (distro, wheel, prefix) in ( - ('debian', 'sudo', DEBIAN_DOCKERFILE), - ('centos6', 'wheel', CENTOS6_DOCKERFILE), - ('centos7', 'wheel', CENTOS7_DOCKERFILE), - ): - mydir = os.path.abspath(os.path.dirname(__file__)) - with tempfile.NamedTemporaryFile(dir=mydir) as dockerfile_fp: - dockerfile_fp.write(prefix) - dockerfile_fp.write(DOCKERFILE.replace('SUDO_GROUP', wheel)) - dockerfile_fp.flush() - - subprocess.check_call(sh('docker build %s -t %s -f %s', - mydir, - 'mitogen/%s-test' % (distro,), - dockerfile_fp.name - )) diff --git a/tests/data/001-mitogen.sudo b/tests/data/docker/001-mitogen.sudo similarity index 68% rename from tests/data/001-mitogen.sudo rename to tests/data/docker/001-mitogen.sudo index 71e20e6a..95b36f3b 100644 --- a/tests/data/001-mitogen.sudo +++ b/tests/data/docker/001-mitogen.sudo @@ -7,3 +7,8 @@ mitogen__has_sudo_nopw ALL = (mitogen__require_tty_pw_required) ALL Defaults>mitogen__pw_required targetpw Defaults>mitogen__require_tty requiretty Defaults>mitogen__require_tty_pw_required requiretty,targetpw + +mitogen__condel1 ALL=(ALL:ALL) NOPASSWD:ALL +mitogen__condel2 ALL=(ALL:ALL) NOPASSWD:ALL +mitogen__condel3 ALL=(ALL:ALL) NOPASSWD:ALL +mitogen__condel4 ALL=(ALL:ALL) NOPASSWD:ALL diff --git a/tests/image_prep/README.md b/tests/image_prep/README.md new file mode 100644 index 00000000..d275672f --- /dev/null +++ b/tests/image_prep/README.md @@ -0,0 +1,25 @@ + +# `image_prep` + +This directory contains Ansible playbooks for building the Docker containers +used for testing, or for setting up an OS X laptop so the tests can (mostly) +run locally. + +The Docker config is more heavily jinxed to trigger adverse conditions in the +code, the OS X config just has the user accounts. + +See ../README.md for a (mostly) description of the accounts created. + + +## Building the containers + +``./build_docker_images.sh`` + + +## Preparing an OS X box + +WARNING: this creates a ton of accounts with preconfigured passwords. It is +generally impossible to restrict remote access to these, so your only option is +to disable remote login and sharing. + +``ansible-playbook -b -c local -i localhost, -l localhost setup.yml`` diff --git a/tests/image_prep/_container_setup.yml b/tests/image_prep/_container_setup.yml new file mode 100644 index 00000000..6e1416df --- /dev/null +++ b/tests/image_prep/_container_setup.yml @@ -0,0 +1,116 @@ + +- hosts: all + strategy: linear + gather_facts: false + tasks: + - raw: > + if ! python -c ''; then + if type -p yum; then + yum -y install python; + else + apt-get -y update && apt-get -y install python; + fi; + fi + +- hosts: all + strategy: mitogen_linear + # Can't gather facts before here. + gather_facts: true + vars: + distro: "{{ansible_distribution}}" + ver: "{{ansible_distribution_major_version}}" + + packages: + common: + - git + - openssh-server + - rsync + - strace + - sudo + Debian: + "9": + - libjson-perl + - python-virtualenv + CentOS: + "6": + - perl-JSON + "7": + - perl-JSON + - python-virtualenv + + tasks: + - when: ansible_virtualization_type != "docker" + meta: end_play + + - apt: + name: "{{packages.common + packages[distro][ver]}}" + state: installed + update_cache: true + when: distro == "Debian" + + - yum: + name: "{{packages.common + packages[distro][ver]}}" + state: installed + update_cache: true + when: distro == "CentOS" + + - command: apt-get clean + when: distro == "Debian" + + - command: yum clean all + when: distro == "CentOS" + + - file: + path: /var/cache/apt + state: absent + when: distro == "Debian" + + - user: + name: root + password: "{{ 'rootpassword' | password_hash('sha256') }}" + shell: /bin/bash + + - file: + path: /var/run/sshd + state: directory + + - command: ssh-keygen -t rsa -f /etc/ssh/ssh_host_rsa_key + args: + creates: /etc/ssh/ssh_host_rsa_key + + - group: + name: "{{sudo_group[distro]}}" + + - copy: + dest: /etc/sentinel + content: | + i-am-mitogen-test-docker-image + + - copy: + dest: /etc/ssh/banner.txt + src: ../data/docker/ssh_login_banner.txt + + - copy: + dest: /etc/sudoers.d/001-mitogen + src: ../data/docker/001-mitogen.sudo + + - lineinfile: + path: /etc/ssh/sshd_config + line: Banner /etc/ssh/banner.txt + + - lineinfile: + path: /etc/ssh/sshd_config + line: PermitRootLogin yes + regexp: '.*PermitRootLogin.*' + + - lineinfile: + path: /etc/pam.d/sshd + regexp: '.*session.*required.*pam_loginuid.so' + line: session optional pam_loginuid.so + + - copy: + mode: 'u+rwx,go=rx' + dest: /usr/local/bin/pywrap + content: | + #!/bin/bash + exec strace -ff -o /tmp/pywrap$$.trace python2.7 "$@"' diff --git a/tests/image_prep/_user_accounts.yml b/tests/image_prep/_user_accounts.yml new file mode 100644 index 00000000..1ed2c61d --- /dev/null +++ b/tests/image_prep/_user_accounts.yml @@ -0,0 +1,139 @@ +# +# Add users expected by tests. Assumes passwordless sudo to root. +# +# WARNING: this creates non-privilged accounts with pre-set passwords! +# + +- hosts: all + gather_facts: true + strategy: mitogen_linear + become: true + vars: + special_users: + - has_sudo + - has_sudo_pubkey + - pw_required + - readonly_homedir + - require_tty + - require_tty_pw_required + - slow_user + - webapp + + groups: + - has_sudo: ['mitogen__group', '{{sudo_group[distro]}}'] + - has_sudo_pubkey: ['mitogen__group', '{{sudo_group[distro]}}'] + - has_sudo_nopw: ['mitogen__group', 'mitogen__sudo_nopw'] + + normal_users: "{{ + lookup('sequence', 'start=1 end=5 format=user%d', wantlist=True) + }}" + + all_users: "{{ + special_users + + normal_users + }}" + tasks: + - name: Disable non-localhost SSH for Mitogen users + blockinfile: + path: /etc/ssh/sshd_config + block: | + Match User mitogen__* Address !127.0.0.1 + DenyUsers * + + - name: Create Mitogen test groups + group: + name: "mitogen__{{item}}" + with_items: + - group + - sudo_nopw + + - name: Create user accounts + block: + - user: + name: "mitogen__{{item}}" + shell: /bin/bash + groups: "{{groups[item]|default(['mitogen__group'])}}" + password: "{{ (item + '_password') | password_hash('sha256') }}" + loop: "{{all_users}}" + when: ansible_system != 'Darwin' + - user: + name: "mitogen__{{item}}" + shell: /bin/bash + groups: "{{groups[item]|default(['mitogen__group'])}}" + password: "{{item}}_password" + loop: "{{all_users}}" + when: ansible_system == 'Darwin' + + - name: Hide users from login window. + loop: "{{all_users}}" + when: ansible_system == 'Darwin' + osx_defaults: + array_add: true + domain: /Library/Preferences/com.apple.loginwindow + type: array + key: HiddenUsersList + value: ['mitogen_{{item}}'] + + - name: Readonly homedir for one account + shell: "chown -R root: ~mitogen__readonly_homedir" + + - name: Slow bash profile for one account + copy: + dest: ~mitogen__slow_user/.{{item}} + src: ../data/docker/mitogen__slow_user.profile + with_items: + - bashrc + - profile + + - name: Install pubkey for mitogen__has_sudo_pubkey + block: + - file: + path: ~mitogen__has_sudo_pubkey/.ssh + state: directory + mode: go= + owner: mitogen__has_sudo_pubkey + - copy: + dest: ~mitogen__has_sudo_pubkey/.ssh/authorized_keys + src: ../data/docker/mitogen__has_sudo_pubkey.key.pub + mode: go= + owner: mitogen__has_sudo_pubkey + + - name: Install slow profile for one account + block: + - copy: + dest: ~mitogen__slow_user/.profile + src: ../data/docker/mitogen__slow_user.profile + - copy: + dest: ~mitogen__slow_user/.bashrc + src: ../data/docker/mitogen__slow_user.profile + + - name: Require a TTY for two accounts + lineinfile: + path: /etc/sudoers + line: "{{item}}" + with_items: + - Defaults>mitogen__pw_required targetpw + - Defaults>mitogen__require_tty requiretty + - Defaults>mitogen__require_tty_pw_required requiretty,targetpw + + - name: Require password for two accounts + lineinfile: + path: /etc/sudoers + line: "{{lookup('pipe', 'whoami')}} ALL = ({{item}}) ALL" + with_items: + - mitogen__pw_required + - mitogen__require_tty_pw_required + + - name: Allow passwordless sudo for require_tty/readonly_homedir + lineinfile: + path: /etc/sudoers + line: "{{lookup('pipe', 'whoami')}} ALL = ({{item}}) NOPASSWD:ALL" + with_items: + - mitogen__require_tty + - mitogen__readonly_homedir + + - name: Allow passwordless for many accounts + lineinfile: + path: /etc/sudoers + line: "{{lookup('pipe', 'whoami')}} ALL = (mitogen__{{item}}) NOPASSWD:ALL" + loop: "{{normal_users}}" diff --git a/tests/image_prep/ansible.cfg b/tests/image_prep/ansible.cfg new file mode 100644 index 00000000..a3937825 --- /dev/null +++ b/tests/image_prep/ansible.cfg @@ -0,0 +1,4 @@ + +[defaults] +strategy_plugins = ../../ansible_mitogen/plugins/strategy +retry_files_enabled = false diff --git a/tests/image_prep/build_docker_images.py b/tests/image_prep/build_docker_images.py new file mode 100755 index 00000000..8a4161db --- /dev/null +++ b/tests/image_prep/build_docker_images.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python + +""" +Build the Docker images used for testing. +""" + +import commands +import os +import shlex +import subprocess + + +BASEDIR = os.path.dirname(os.path.abspath(__file__)) + + +def sh(s, *args): + if args: + s %= args + return shlex.split(s) + + +for base_image, name in [('debian:stretch', 'debian'), + ('centos:6', 'centos6'), + ('centos:7', 'centos7')]: + args = sh('docker run --rm -it -d %s /bin/bash', base_image) + container_id = subprocess.check_output(args).strip() + try: + subprocess.check_call( + cwd=BASEDIR, + args=sh(''' + ansible-playbook -i %s, -c docker setup.yml -vvv + ''', container_id) + ) + + subprocess.check_call(sh(''' + docker commit + --change 'EXPOSE 22' + --change 'CMD ["/usr/sbin/sshd", "-D"]' + %s + mitogen/%s-test + ''', container_id, name)) + finally: + subprocess.check_call(sh('docker rm -f %s', container_id)) diff --git a/tests/image_prep/setup.yml b/tests/image_prep/setup.yml new file mode 100644 index 00000000..168d583c --- /dev/null +++ b/tests/image_prep/setup.yml @@ -0,0 +1,13 @@ + +- hosts: all + gather_facts: false + tasks: + - set_fact: + # Hacktacular.. but easiest place for it with current structure. + sudo_group: + MacOSX: admin + Debian: wheel + CentOS: sudo + +- import_playbook: _container_setup.yml +- import_playbook: _user_accounts.yml