tests: merge build_docker_images.py with osx_setup.yml

Hooray!

tests/ansible/README.md

@@ -1,5 +1,5 @@
 
-# ``tests/ansible`` Directory
+# `tests/ansible` Directory
 
 This is an an organically growing collection of integration and regression
 tests used for development and end-user bug reports.
@@ -10,10 +10,10 @@ demonstrator for what does and doesn't work.
 
 ## Preparation
 
-For OS X, run the ``osx_setup.yml`` script to create a bunch of users.
+See `../image_prep/README.md`.
 
 
-## ``run_ansible_playbook.sh``
+## `run_ansible_playbook.sh`
 
 This is necessary to set some environment variables used by future tests, as
 there appears to be no better way to inject them into the top-level process
@@ -22,12 +22,10 @@ environment before the Mitogen connection process forks.
 
 ## Running Everything
 
-```
+`ANSIBLE_STRATEGY=mitogen_linear ./run_ansible_playbook.sh all.yml`
-ANSIBLE_STRATEGY=mitogen_linear ./run_ansible_playbook.sh all.yml
-```
 
 
-## ``hosts/`` and ``common-hosts``
+## `hosts/` and `common-hosts`
 
 To support running the tests against a dev machine that has the requisite user
 accounts, the the default inventory is a directory containing a 'localhost'
@@ -35,7 +33,7 @@ file that defines 'localhost' to be named 'target' in Ansible inventory, and a
 symlink to 'common-hosts', which defines additional targets that all derive
 from 'target'.
 
-This allows ``ansible_tests.sh`` to reuse the common-hosts definitions while
+This allows `ansible_tests.sh` to reuse the common-hosts definitions while
 replacing localhost as the test target by creating a new directory that
 similarly symlinks in common-hosts.
 

tests/ansible/osx_setup.yml

@@ -1,155 +0,0 @@
-
-#
-# Add users expected by tests to an OS X machine. Assumes passwordless sudo to
-# root.
-#
-# WARNING: this creates non-privilged accounts with pre-set passwords!
-#
-
-- hosts: test-targets
-  gather_facts: true
-  become: true
-  tasks:
-    - name: Disable non-localhost SSH for Mitogen users
-      blockinfile:
-        path: /etc/ssh/sshd_config
-        block: |
-          Match User mitogen__* Address !127.0.0.1
-            DenyUsers *
-
-    #
-    # Hashed passwords.
-    #
-    - name: Create Mitogen test group
-      group:
-        name: "mitogen__group"
-
-    - name: Create Mitogen test users
-      user:
-        name: "mitogen__{{item}}"
-        shell: /bin/bash
-        groups: mitogen__group
-        password: "{{ (item + '_password') | password_hash('sha256') }}"
-      with_items:
-        - has_sudo
-        - has_sudo_pubkey
-        - require_tty
-        - pw_required
-        - readonly_homedir
-        - require_tty_pw_required
-        - slow_user
-      when: ansible_system != 'Darwin'
-
-    - name: Create Mitogen test users
-      user:
-        name: "mitogen__user{{item}}"
-        shell: /bin/bash
-        password: "{{ ('user' + item + '_password') | password_hash('sha256') }}"
-      with_sequence: start=1 end=21
-      when: ansible_system != 'Darwin'
-
-    #
-    # Plaintext passwords
-    #
-    - name: Create Mitogen test users
-      user:
-        name: "mitogen__{{item}}"
-        shell: /bin/bash
-        groups: mitogen__group
-        password: "{{item}}_password"
-      with_items:
-        - has_sudo
-        - has_sudo_pubkey
-        - require_tty
-        - pw_required
-        - require_tty_pw_required
-        - readonly_homedir
-        - slow_user
-      when: ansible_system == 'Darwin'
-
-    - name: Create Mitogen test users
-      user:
-        name: "mitogen__user{{item}}"
-        shell: /bin/bash
-        password: "user{{item}}_password"
-      with_sequence: start=1 end=21
-      when: ansible_system == 'Darwin'
-
-    - name: Hide test users from login window.
-      shell: >
-        defaults
-        write
-        /Library/Preferences/com.apple.loginwindow
-        HiddenUsersList
-        -array-add '{{item}}'
-      with_items:
-        - mitogen__require_tty
-        - mitogen__pw_required
-        - mitogen__require_tty_pw_required
-      when: ansible_system == 'Darwin'
-
-    - name: Hide test users from login window.
-      shell: >
-        defaults
-        write
-        /Library/Preferences/com.apple.loginwindow
-        HiddenUsersList
-        -array-add 'mitogen__user{{item}}'
-      with_sequence: start=1 end=21
-      when: ansible_distribution == 'MacOSX'
-
-    - name: Readonly homedir for one account
-      shell: "chown -R root: ~mitogen__readonly_homedir"
-
-    - name: Slow bash profile for one account
-      copy:
-        dest: ~mitogen__slow_user/.{{item}}
-        src: ../data/docker/mitogen__slow_user.profile
-      with_items:
-      - bashrc
-      - profile
-
-    - name: Install pubkey for one account
-      file:
-        path: ~mitogen__has_sudo_pubkey/.ssh
-        state: directory
-        mode: go=
-        owner: mitogen__has_sudo_pubkey
-
-    - name: Install pubkey for one account
-      copy:
-        dest: ~mitogen__has_sudo_pubkey/.ssh/authorized_keys
-        src: ../data/docker/mitogen__has_sudo_pubkey.key.pub
-        mode: go=
-        owner: mitogen__has_sudo_pubkey
-
-    - name: Require a TTY for two accounts
-      lineinfile:
-        path: /etc/sudoers
-        line: "{{item}}"
-      with_items:
-        - Defaults>mitogen__pw_required targetpw
-        - Defaults>mitogen__require_tty requiretty
-        - Defaults>mitogen__require_tty_pw_required requiretty,targetpw
-
-    - name: Require password for two accounts
-      lineinfile:
-        path: /etc/sudoers
-        line: "{{lookup('pipe', 'whoami')}} ALL = ({{item}}) ALL"
-      with_items:
-        - mitogen__pw_required
-        - mitogen__require_tty_pw_required
-
-    - name: Allow passwordless for two accounts
-      lineinfile:
-        path: /etc/sudoers
-        line: "{{lookup('pipe', 'whoami')}} ALL = ({{item}}) NOPASSWD:ALL"
-      with_items:
-        - mitogen__require_tty
-        - mitogen__readonly_homedir
-
-    - name: Allow passwordless for many accounts
-      lineinfile:
-        path: /etc/sudoers
-        line: "{{lookup('pipe', 'whoami')}} ALL = (mitogen__user{{item}}) NOPASSWD:ALL"
-      with_sequence: start=1 end=21

tests/build_docker_images.py

@@ -1,120 +0,0 @@
-#!/usr/bin/env python
-
-"""
-Build the Docker images used for testing.
-"""
-
-import commands
-import os
-import shlex
-import subprocess
-import tempfile
-
-
-DEBIAN_DOCKERFILE = r"""
-FROM debian:stretch
-RUN apt-get update
-RUN \
-    apt-get install -y python2.7 openssh-server sudo rsync git strace \
-                       libjson-perl python-virtualenv && \
-    apt-get clean && \
-    rm -rf /var/cache/apt
-"""
-
-CENTOS6_DOCKERFILE = r"""
-FROM centos:6
-RUN yum clean all && \
-    yum -y install -y python2.6 openssh-server sudo rsync git strace sudo \
-                      perl-JSON python-virtualenv && \
-    yum clean all && \
-    groupadd sudo && \
-    ssh-keygen -t rsa -f /etc/ssh/ssh_host_rsa_key
-
-"""
-
-CENTOS7_DOCKERFILE = r"""
-FROM centos:7
-RUN yum clean all && \
-    yum -y install -y python2.7 openssh-server sudo rsync git strace sudo \
-                      perl-JSON python-virtualenv && \
-    yum clean all && \
-    groupadd sudo && \
-    ssh-keygen -t rsa -f /etc/ssh/ssh_host_rsa_key
-
-"""
-
-DOCKERFILE = r"""
-COPY data/001-mitogen.sudo /etc/sudoers.d/001-mitogen
-COPY data/docker/ssh_login_banner.txt /etc/ssh/banner.txt
-RUN \
-    chsh -s /bin/bash && \
-    mkdir -p /var/run/sshd && \
-    echo i-am-mitogen-test-docker-image > /etc/sentinel && \
-    echo "Banner /etc/ssh/banner.txt" >> /etc/ssh/sshd_config && \
-    groupadd mitogen__sudo_nopw && \
-    useradd -s /bin/bash -m mitogen__has_sudo -G SUDO_GROUP && \
-    useradd -s /bin/bash -m mitogen__has_sudo_pubkey -G SUDO_GROUP && \
-    useradd -s /bin/bash -m mitogen__has_sudo_nopw -G mitogen__sudo_nopw && \
-    useradd -s /bin/bash -m mitogen__webapp && \
-    useradd -s /bin/bash -m mitogen__pw_required && \
-    useradd -s /bin/bash -m mitogen__require_tty && \
-    useradd -s /bin/bash -m mitogen__require_tty_pw_required && \
-    useradd -s /bin/bash -m mitogen__readonly_homedir && \
-    useradd -s /bin/bash -m mitogen__slow_user && \
-    chown -R root: ~mitogen__readonly_homedir && \
-    ( for i in `seq 1 21`; do useradd -s /bin/bash -m mitogen__user${i}; done; ) && \
-    ( for i in `seq 1 21`; do echo mitogen__user${i}:user${i}_password | chpasswd; done; ) && \
-    ( echo 'root:rootpassword' | chpasswd; ) && \
-    ( echo 'mitogen__has_sudo:has_sudo_password' | chpasswd; ) && \
-    ( echo 'mitogen__has_sudo_pubkey:has_sudo_pubkey_password' | chpasswd; ) && \
-    ( echo 'mitogen__has_sudo_nopw:has_sudo_nopw_password' | chpasswd; ) && \
-    ( echo 'mitogen__webapp:webapp_password' | chpasswd; ) && \
-    ( echo 'mitogen__pw_required:pw_required_password' | chpasswd; ) && \
-    ( echo 'mitogen__require_tty:require_tty_password' | chpasswd; ) && \
-    ( echo 'mitogen__require_tty_pw_required:require_tty_pw_required_password' | chpasswd; ) && \
-    ( echo 'mitogen__readonly_homedir:readonly_homedir_password' | chpasswd; ) && \
-    ( echo 'mitogen__slow_user:slow_user_password' | chpasswd; ) && \
-    mkdir ~mitogen__has_sudo_pubkey/.ssh && \
-    ( echo '#!/bin/bash\nexec strace -ff -o /tmp/pywrap$$.trace python2.7 "$@"' > /usr/local/bin/pywrap; chmod +x /usr/local/bin/pywrap; )
-
-COPY data/docker/mitogen__has_sudo_pubkey.key.pub /home/mitogen__has_sudo_pubkey/.ssh/authorized_keys
-COPY data/docker/mitogen__slow_user.profile /home/mitogen__slow_user/.profile
-COPY data/docker/mitogen__slow_user.profile /home/mitogen__slow_user/.bashrc
-
-RUN \
-    chown -R mitogen__has_sudo_pubkey ~mitogen__has_sudo_pubkey && \
-    chmod -R go= ~mitogen__has_sudo_pubkey
-
-RUN sed -i 's/PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config
-RUN sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd
-
-ENV NOTVISIBLE "in users profile"
-RUN echo "export VISIBLE=now" >> /etc/profile
-
-EXPOSE 22
-CMD ["/usr/sbin/sshd", "-D"]
-"""
-
-
-def sh(s, *args):
-    if args:
-        s %= tuple(map(commands.mkarg, args))
-    return shlex.split(s)
-
-
-for (distro, wheel, prefix) in (
-        ('debian', 'sudo', DEBIAN_DOCKERFILE),
-        ('centos6', 'wheel', CENTOS6_DOCKERFILE),
-        ('centos7', 'wheel', CENTOS7_DOCKERFILE),
-    ):
-    mydir = os.path.abspath(os.path.dirname(__file__))
-    with tempfile.NamedTemporaryFile(dir=mydir) as dockerfile_fp:
-        dockerfile_fp.write(prefix)
-        dockerfile_fp.write(DOCKERFILE.replace('SUDO_GROUP', wheel))
-        dockerfile_fp.flush()
-
-        subprocess.check_call(sh('docker build %s -t %s -f %s',
-            mydir,
-            'mitogen/%s-test' % (distro,),
-            dockerfile_fp.name
-        ))

tests/data/docker/001-mitogen.sudo

@@ -7,3 +7,8 @@ mitogen__has_sudo_nopw ALL = (mitogen__require_tty_pw_required) ALL
 Defaults>mitogen__pw_required targetpw
 Defaults>mitogen__require_tty requiretty
 Defaults>mitogen__require_tty_pw_required requiretty,targetpw
+
+mitogen__condel1 ALL=(ALL:ALL) NOPASSWD:ALL
+mitogen__condel2 ALL=(ALL:ALL) NOPASSWD:ALL
+mitogen__condel3 ALL=(ALL:ALL) NOPASSWD:ALL
+mitogen__condel4 ALL=(ALL:ALL) NOPASSWD:ALL

tests/image_prep/README.md

@@ -0,0 +1,25 @@
+
+# `image_prep`
+
+This directory contains Ansible playbooks for building the Docker containers
+used for testing, or for setting up an OS X laptop so the tests can (mostly)
+run locally.
+
+The Docker config is more heavily jinxed to trigger adverse conditions in the
+code, the OS X config just has the user accounts.
+
+See ../README.md for a (mostly) description of the accounts created.
+
+
+## Building the containers
+
+``./build_docker_images.sh``
+
+
+## Preparing an OS X box
+
+WARNING: this creates a ton of accounts with preconfigured passwords. It is
+generally impossible to restrict remote access to these, so your only option is
+to disable remote login and sharing.
+
+``ansible-playbook -b -c local -i localhost, -l localhost setup.yml``

tests/image_prep/_container_setup.yml

@@ -0,0 +1,116 @@
+
+- hosts: all
+  strategy: linear
+  gather_facts: false
+  tasks:
+    - raw: >
+        if ! python -c ''; then
+          if type -p yum; then
+            yum -y install python;
+          else
+            apt-get -y update && apt-get -y install python;
+          fi;
+        fi
+
+- hosts: all
+  strategy: mitogen_linear
+  # Can't gather facts before here.
+  gather_facts: true
+  vars:
+    distro: "{{ansible_distribution}}"
+    ver: "{{ansible_distribution_major_version}}"
+
+    packages:
+      common:
+        - git
+        - openssh-server
+        - rsync
+        - strace
+        - sudo
+      Debian:
+        "9":
+          - libjson-perl
+          - python-virtualenv
+      CentOS:
+        "6":
+          - perl-JSON
+        "7":
+          - perl-JSON
+          - python-virtualenv
+
+  tasks:
+    - when: ansible_virtualization_type != "docker"
+      meta: end_play
+
+    - apt:
+        name: "{{packages.common + packages[distro][ver]}}"
+        state: installed
+        update_cache: true
+      when: distro == "Debian"
+
+    - yum:
+        name: "{{packages.common + packages[distro][ver]}}"
+        state: installed
+        update_cache: true
+      when: distro == "CentOS"
+
+    - command: apt-get clean
+      when: distro == "Debian"
+
+    - command: yum clean all
+      when: distro == "CentOS"
+
+    - file:
+        path: /var/cache/apt
+        state: absent
+      when: distro == "Debian"
+
+    - user:
+        name: root
+        password: "{{ 'rootpassword' | password_hash('sha256') }}"
+        shell: /bin/bash
+
+    - file:
+        path: /var/run/sshd
+        state: directory
+
+    - command: ssh-keygen -t rsa -f /etc/ssh/ssh_host_rsa_key
+      args:
+        creates: /etc/ssh/ssh_host_rsa_key
+
+    - group:
+        name: "{{sudo_group[distro]}}"
+
+    - copy:
+        dest: /etc/sentinel
+        content: |
+          i-am-mitogen-test-docker-image
+
+    - copy:
+        dest: /etc/ssh/banner.txt
+        src: ../data/docker/ssh_login_banner.txt
+
+    - copy:
+        dest: /etc/sudoers.d/001-mitogen
+        src: ../data/docker/001-mitogen.sudo
+
+    - lineinfile:
+        path: /etc/ssh/sshd_config
+        line: Banner /etc/ssh/banner.txt
+
+    - lineinfile:
+        path: /etc/ssh/sshd_config
+        line: PermitRootLogin yes
+        regexp: '.*PermitRootLogin.*'
+
+    - lineinfile:
+        path: /etc/pam.d/sshd
+        regexp: '.*session.*required.*pam_loginuid.so'
+        line: session optional pam_loginuid.so
+
+    - copy:
+        mode: 'u+rwx,go=rx'
+        dest: /usr/local/bin/pywrap
+        content: |
+         #!/bin/bash
+         exec strace -ff -o /tmp/pywrap$$.trace python2.7 "$@"'

tests/image_prep/_user_accounts.yml

@@ -0,0 +1,139 @@
+#
+# Add users expected by tests. Assumes passwordless sudo to root.
+#
+# WARNING: this creates non-privilged accounts with pre-set passwords!
+#
+
+- hosts: all
+  gather_facts: true
+  strategy: mitogen_linear
+  become: true
+  vars:
+    special_users:
+      - has_sudo
+      - has_sudo_pubkey
+      - pw_required
+      - readonly_homedir
+      - require_tty
+      - require_tty_pw_required
+      - slow_user
+      - webapp
+
+    groups:
+      - has_sudo: ['mitogen__group', '{{sudo_group[distro]}}']
+      - has_sudo_pubkey: ['mitogen__group', '{{sudo_group[distro]}}']
+      - has_sudo_nopw: ['mitogen__group', 'mitogen__sudo_nopw']
+
+    normal_users: "{{
+      lookup('sequence', 'start=1 end=5 format=user%d', wantlist=True)
+    }}"
+
+    all_users: "{{
+      special_users +
+      normal_users
+    }}"
+  tasks:
+    - name: Disable non-localhost SSH for Mitogen users
+      blockinfile:
+        path: /etc/ssh/sshd_config
+        block: |
+          Match User mitogen__* Address !127.0.0.1
+            DenyUsers *
+
+    - name: Create Mitogen test groups
+      group:
+        name: "mitogen__{{item}}"
+      with_items:
+      - group
+      - sudo_nopw
+
+    - name: Create user accounts
+      block:
+      - user:
+          name: "mitogen__{{item}}"
+          shell: /bin/bash
+          groups: "{{groups[item]|default(['mitogen__group'])}}"
+          password: "{{ (item + '_password') | password_hash('sha256') }}"
+        loop: "{{all_users}}"
+        when: ansible_system != 'Darwin'
+      - user:
+          name: "mitogen__{{item}}"
+          shell: /bin/bash
+          groups: "{{groups[item]|default(['mitogen__group'])}}"
+          password: "{{item}}_password"
+        loop: "{{all_users}}"
+        when: ansible_system == 'Darwin'
+
+    - name: Hide users from login window.
+      loop: "{{all_users}}"
+      when: ansible_system == 'Darwin'
+      osx_defaults:
+        array_add: true
+        domain: /Library/Preferences/com.apple.loginwindow
+        type: array
+        key: HiddenUsersList
+        value: ['mitogen_{{item}}']
+
+    - name: Readonly homedir for one account
+      shell: "chown -R root: ~mitogen__readonly_homedir"
+
+    - name: Slow bash profile for one account
+      copy:
+        dest: ~mitogen__slow_user/.{{item}}
+        src: ../data/docker/mitogen__slow_user.profile
+      with_items:
+      - bashrc
+      - profile
+
+    - name: Install pubkey for mitogen__has_sudo_pubkey
+      block:
+        - file:
+            path: ~mitogen__has_sudo_pubkey/.ssh
+            state: directory
+            mode: go=
+            owner: mitogen__has_sudo_pubkey
+        - copy:
+            dest: ~mitogen__has_sudo_pubkey/.ssh/authorized_keys
+            src: ../data/docker/mitogen__has_sudo_pubkey.key.pub
+            mode: go=
+            owner: mitogen__has_sudo_pubkey
+
+    - name: Install slow profile for one account
+      block:
+        - copy:
+            dest: ~mitogen__slow_user/.profile
+            src: ../data/docker/mitogen__slow_user.profile
+        - copy:
+            dest: ~mitogen__slow_user/.bashrc
+            src: ../data/docker/mitogen__slow_user.profile
+
+    - name: Require a TTY for two accounts
+      lineinfile:
+        path: /etc/sudoers
+        line: "{{item}}"
+      with_items:
+        - Defaults>mitogen__pw_required targetpw
+        - Defaults>mitogen__require_tty requiretty
+        - Defaults>mitogen__require_tty_pw_required requiretty,targetpw
+
+    - name: Require password for two accounts
+      lineinfile:
+        path: /etc/sudoers
+        line: "{{lookup('pipe', 'whoami')}} ALL = ({{item}}) ALL"
+      with_items:
+        - mitogen__pw_required
+        - mitogen__require_tty_pw_required
+
+    - name: Allow passwordless sudo for require_tty/readonly_homedir
+      lineinfile:
+        path: /etc/sudoers
+        line: "{{lookup('pipe', 'whoami')}} ALL = ({{item}}) NOPASSWD:ALL"
+      with_items:
+        - mitogen__require_tty
+        - mitogen__readonly_homedir
+
+    - name: Allow passwordless for many accounts
+      lineinfile:
+        path: /etc/sudoers
+        line: "{{lookup('pipe', 'whoami')}} ALL = (mitogen__{{item}}) NOPASSWD:ALL"
+      loop: "{{normal_users}}"

tests/image_prep/ansible.cfg

@@ -0,0 +1,4 @@
+
+[defaults]
+strategy_plugins = ../../ansible_mitogen/plugins/strategy
+retry_files_enabled = false

tests/image_prep/build_docker_images.py

@@ -0,0 +1,43 @@
+#!/usr/bin/env python
+
+"""
+Build the Docker images used for testing.
+"""
+
+import commands
+import os
+import shlex
+import subprocess
+
+
+BASEDIR = os.path.dirname(os.path.abspath(__file__))
+
+
+def sh(s, *args):
+    if args:
+        s %= args
+    return shlex.split(s)
+
+
+for base_image, name in [('debian:stretch', 'debian'),
+                         ('centos:6', 'centos6'),
+                         ('centos:7', 'centos7')]:
+    args = sh('docker run --rm -it -d %s /bin/bash', base_image)
+    container_id = subprocess.check_output(args).strip()
+    try:
+        subprocess.check_call(
+            cwd=BASEDIR,
+            args=sh('''
+                ansible-playbook -i %s, -c docker setup.yml -vvv
+            ''', container_id)
+        )
+
+        subprocess.check_call(sh('''
+            docker commit
+            --change 'EXPOSE 22'
+            --change 'CMD ["/usr/sbin/sshd", "-D"]'
+            %s
+            mitogen/%s-test
+        ''', container_id, name))
+    finally:
+        subprocess.check_call(sh('docker rm -f %s', container_id))

tests/image_prep/setup.yml

@@ -0,0 +1,13 @@
+
+- hosts: all
+  gather_facts: false
+  tasks:
+  - set_fact:
+      # Hacktacular.. but easiest place for it with current structure.
+      sudo_group:
+        MacOSX: admin
+        Debian: wheel
+        CentOS: sudo
+
+- import_playbook: _container_setup.yml
+- import_playbook: _user_accounts.yml