tests: merge build_docker_images.py with osx_setup.yml

Hooray!
pull/350/head
David Wilson 6 years ago
parent 053c594d65
commit a192935daf

@ -1,5 +1,5 @@
# ``tests/ansible`` Directory
# `tests/ansible` Directory
This is an an organically growing collection of integration and regression
tests used for development and end-user bug reports.
@ -10,10 +10,10 @@ demonstrator for what does and doesn't work.
## Preparation
For OS X, run the ``osx_setup.yml`` script to create a bunch of users.
See `../image_prep/README.md`.
## ``run_ansible_playbook.sh``
## `run_ansible_playbook.sh`
This is necessary to set some environment variables used by future tests, as
there appears to be no better way to inject them into the top-level process
@ -22,12 +22,10 @@ environment before the Mitogen connection process forks.
## Running Everything
```
ANSIBLE_STRATEGY=mitogen_linear ./run_ansible_playbook.sh all.yml
```
`ANSIBLE_STRATEGY=mitogen_linear ./run_ansible_playbook.sh all.yml`
## ``hosts/`` and ``common-hosts``
## `hosts/` and `common-hosts`
To support running the tests against a dev machine that has the requisite user
accounts, the the default inventory is a directory containing a 'localhost'
@ -35,7 +33,7 @@ file that defines 'localhost' to be named 'target' in Ansible inventory, and a
symlink to 'common-hosts', which defines additional targets that all derive
from 'target'.
This allows ``ansible_tests.sh`` to reuse the common-hosts definitions while
This allows `ansible_tests.sh` to reuse the common-hosts definitions while
replacing localhost as the test target by creating a new directory that
similarly symlinks in common-hosts.

@ -1,155 +0,0 @@
#
# Add users expected by tests to an OS X machine. Assumes passwordless sudo to
# root.
#
# WARNING: this creates non-privilged accounts with pre-set passwords!
#
- hosts: test-targets
gather_facts: true
become: true
tasks:
- name: Disable non-localhost SSH for Mitogen users
blockinfile:
path: /etc/ssh/sshd_config
block: |
Match User mitogen__* Address !127.0.0.1
DenyUsers *
#
# Hashed passwords.
#
- name: Create Mitogen test group
group:
name: "mitogen__group"
- name: Create Mitogen test users
user:
name: "mitogen__{{item}}"
shell: /bin/bash
groups: mitogen__group
password: "{{ (item + '_password') | password_hash('sha256') }}"
with_items:
- has_sudo
- has_sudo_pubkey
- require_tty
- pw_required
- readonly_homedir
- require_tty_pw_required
- slow_user
when: ansible_system != 'Darwin'
- name: Create Mitogen test users
user:
name: "mitogen__user{{item}}"
shell: /bin/bash
password: "{{ ('user' + item + '_password') | password_hash('sha256') }}"
with_sequence: start=1 end=21
when: ansible_system != 'Darwin'
#
# Plaintext passwords
#
- name: Create Mitogen test users
user:
name: "mitogen__{{item}}"
shell: /bin/bash
groups: mitogen__group
password: "{{item}}_password"
with_items:
- has_sudo
- has_sudo_pubkey
- require_tty
- pw_required
- require_tty_pw_required
- readonly_homedir
- slow_user
when: ansible_system == 'Darwin'
- name: Create Mitogen test users
user:
name: "mitogen__user{{item}}"
shell: /bin/bash
password: "user{{item}}_password"
with_sequence: start=1 end=21
when: ansible_system == 'Darwin'
- name: Hide test users from login window.
shell: >
defaults
write
/Library/Preferences/com.apple.loginwindow
HiddenUsersList
-array-add '{{item}}'
with_items:
- mitogen__require_tty
- mitogen__pw_required
- mitogen__require_tty_pw_required
when: ansible_system == 'Darwin'
- name: Hide test users from login window.
shell: >
defaults
write
/Library/Preferences/com.apple.loginwindow
HiddenUsersList
-array-add 'mitogen__user{{item}}'
with_sequence: start=1 end=21
when: ansible_distribution == 'MacOSX'
- name: Readonly homedir for one account
shell: "chown -R root: ~mitogen__readonly_homedir"
- name: Slow bash profile for one account
copy:
dest: ~mitogen__slow_user/.{{item}}
src: ../data/docker/mitogen__slow_user.profile
with_items:
- bashrc
- profile
- name: Install pubkey for one account
file:
path: ~mitogen__has_sudo_pubkey/.ssh
state: directory
mode: go=
owner: mitogen__has_sudo_pubkey
- name: Install pubkey for one account
copy:
dest: ~mitogen__has_sudo_pubkey/.ssh/authorized_keys
src: ../data/docker/mitogen__has_sudo_pubkey.key.pub
mode: go=
owner: mitogen__has_sudo_pubkey
- name: Require a TTY for two accounts
lineinfile:
path: /etc/sudoers
line: "{{item}}"
with_items:
- Defaults>mitogen__pw_required targetpw
- Defaults>mitogen__require_tty requiretty
- Defaults>mitogen__require_tty_pw_required requiretty,targetpw
- name: Require password for two accounts
lineinfile:
path: /etc/sudoers
line: "{{lookup('pipe', 'whoami')}} ALL = ({{item}}) ALL"
with_items:
- mitogen__pw_required
- mitogen__require_tty_pw_required
- name: Allow passwordless for two accounts
lineinfile:
path: /etc/sudoers
line: "{{lookup('pipe', 'whoami')}} ALL = ({{item}}) NOPASSWD:ALL"
with_items:
- mitogen__require_tty
- mitogen__readonly_homedir
- name: Allow passwordless for many accounts
lineinfile:
path: /etc/sudoers
line: "{{lookup('pipe', 'whoami')}} ALL = (mitogen__user{{item}}) NOPASSWD:ALL"
with_sequence: start=1 end=21

@ -1,120 +0,0 @@
#!/usr/bin/env python
"""
Build the Docker images used for testing.
"""
import commands
import os
import shlex
import subprocess
import tempfile
DEBIAN_DOCKERFILE = r"""
FROM debian:stretch
RUN apt-get update
RUN \
apt-get install -y python2.7 openssh-server sudo rsync git strace \
libjson-perl python-virtualenv && \
apt-get clean && \
rm -rf /var/cache/apt
"""
CENTOS6_DOCKERFILE = r"""
FROM centos:6
RUN yum clean all && \
yum -y install -y python2.6 openssh-server sudo rsync git strace sudo \
perl-JSON python-virtualenv && \
yum clean all && \
groupadd sudo && \
ssh-keygen -t rsa -f /etc/ssh/ssh_host_rsa_key
"""
CENTOS7_DOCKERFILE = r"""
FROM centos:7
RUN yum clean all && \
yum -y install -y python2.7 openssh-server sudo rsync git strace sudo \
perl-JSON python-virtualenv && \
yum clean all && \
groupadd sudo && \
ssh-keygen -t rsa -f /etc/ssh/ssh_host_rsa_key
"""
DOCKERFILE = r"""
COPY data/001-mitogen.sudo /etc/sudoers.d/001-mitogen
COPY data/docker/ssh_login_banner.txt /etc/ssh/banner.txt
RUN \
chsh -s /bin/bash && \
mkdir -p /var/run/sshd && \
echo i-am-mitogen-test-docker-image > /etc/sentinel && \
echo "Banner /etc/ssh/banner.txt" >> /etc/ssh/sshd_config && \
groupadd mitogen__sudo_nopw && \
useradd -s /bin/bash -m mitogen__has_sudo -G SUDO_GROUP && \
useradd -s /bin/bash -m mitogen__has_sudo_pubkey -G SUDO_GROUP && \
useradd -s /bin/bash -m mitogen__has_sudo_nopw -G mitogen__sudo_nopw && \
useradd -s /bin/bash -m mitogen__webapp && \
useradd -s /bin/bash -m mitogen__pw_required && \
useradd -s /bin/bash -m mitogen__require_tty && \
useradd -s /bin/bash -m mitogen__require_tty_pw_required && \
useradd -s /bin/bash -m mitogen__readonly_homedir && \
useradd -s /bin/bash -m mitogen__slow_user && \
chown -R root: ~mitogen__readonly_homedir && \
( for i in `seq 1 21`; do useradd -s /bin/bash -m mitogen__user${i}; done; ) && \
( for i in `seq 1 21`; do echo mitogen__user${i}:user${i}_password | chpasswd; done; ) && \
( echo 'root:rootpassword' | chpasswd; ) && \
( echo 'mitogen__has_sudo:has_sudo_password' | chpasswd; ) && \
( echo 'mitogen__has_sudo_pubkey:has_sudo_pubkey_password' | chpasswd; ) && \
( echo 'mitogen__has_sudo_nopw:has_sudo_nopw_password' | chpasswd; ) && \
( echo 'mitogen__webapp:webapp_password' | chpasswd; ) && \
( echo 'mitogen__pw_required:pw_required_password' | chpasswd; ) && \
( echo 'mitogen__require_tty:require_tty_password' | chpasswd; ) && \
( echo 'mitogen__require_tty_pw_required:require_tty_pw_required_password' | chpasswd; ) && \
( echo 'mitogen__readonly_homedir:readonly_homedir_password' | chpasswd; ) && \
( echo 'mitogen__slow_user:slow_user_password' | chpasswd; ) && \
mkdir ~mitogen__has_sudo_pubkey/.ssh && \
( echo '#!/bin/bash\nexec strace -ff -o /tmp/pywrap$$.trace python2.7 "$@"' > /usr/local/bin/pywrap; chmod +x /usr/local/bin/pywrap; )
COPY data/docker/mitogen__has_sudo_pubkey.key.pub /home/mitogen__has_sudo_pubkey/.ssh/authorized_keys
COPY data/docker/mitogen__slow_user.profile /home/mitogen__slow_user/.profile
COPY data/docker/mitogen__slow_user.profile /home/mitogen__slow_user/.bashrc
RUN \
chown -R mitogen__has_sudo_pubkey ~mitogen__has_sudo_pubkey && \
chmod -R go= ~mitogen__has_sudo_pubkey
RUN sed -i 's/PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config
RUN sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd
ENV NOTVISIBLE "in users profile"
RUN echo "export VISIBLE=now" >> /etc/profile
EXPOSE 22
CMD ["/usr/sbin/sshd", "-D"]
"""
def sh(s, *args):
if args:
s %= tuple(map(commands.mkarg, args))
return shlex.split(s)
for (distro, wheel, prefix) in (
('debian', 'sudo', DEBIAN_DOCKERFILE),
('centos6', 'wheel', CENTOS6_DOCKERFILE),
('centos7', 'wheel', CENTOS7_DOCKERFILE),
):
mydir = os.path.abspath(os.path.dirname(__file__))
with tempfile.NamedTemporaryFile(dir=mydir) as dockerfile_fp:
dockerfile_fp.write(prefix)
dockerfile_fp.write(DOCKERFILE.replace('SUDO_GROUP', wheel))
dockerfile_fp.flush()
subprocess.check_call(sh('docker build %s -t %s -f %s',
mydir,
'mitogen/%s-test' % (distro,),
dockerfile_fp.name
))

@ -7,3 +7,8 @@ mitogen__has_sudo_nopw ALL = (mitogen__require_tty_pw_required) ALL
Defaults>mitogen__pw_required targetpw
Defaults>mitogen__require_tty requiretty
Defaults>mitogen__require_tty_pw_required requiretty,targetpw
mitogen__condel1 ALL=(ALL:ALL) NOPASSWD:ALL
mitogen__condel2 ALL=(ALL:ALL) NOPASSWD:ALL
mitogen__condel3 ALL=(ALL:ALL) NOPASSWD:ALL
mitogen__condel4 ALL=(ALL:ALL) NOPASSWD:ALL

@ -0,0 +1,25 @@
# `image_prep`
This directory contains Ansible playbooks for building the Docker containers
used for testing, or for setting up an OS X laptop so the tests can (mostly)
run locally.
The Docker config is more heavily jinxed to trigger adverse conditions in the
code, the OS X config just has the user accounts.
See ../README.md for a (mostly) description of the accounts created.
## Building the containers
``./build_docker_images.sh``
## Preparing an OS X box
WARNING: this creates a ton of accounts with preconfigured passwords. It is
generally impossible to restrict remote access to these, so your only option is
to disable remote login and sharing.
``ansible-playbook -b -c local -i localhost, -l localhost setup.yml``

@ -0,0 +1,116 @@
- hosts: all
strategy: linear
gather_facts: false
tasks:
- raw: >
if ! python -c ''; then
if type -p yum; then
yum -y install python;
else
apt-get -y update && apt-get -y install python;
fi;
fi
- hosts: all
strategy: mitogen_linear
# Can't gather facts before here.
gather_facts: true
vars:
distro: "{{ansible_distribution}}"
ver: "{{ansible_distribution_major_version}}"
packages:
common:
- git
- openssh-server
- rsync
- strace
- sudo
Debian:
"9":
- libjson-perl
- python-virtualenv
CentOS:
"6":
- perl-JSON
"7":
- perl-JSON
- python-virtualenv
tasks:
- when: ansible_virtualization_type != "docker"
meta: end_play
- apt:
name: "{{packages.common + packages[distro][ver]}}"
state: installed
update_cache: true
when: distro == "Debian"
- yum:
name: "{{packages.common + packages[distro][ver]}}"
state: installed
update_cache: true
when: distro == "CentOS"
- command: apt-get clean
when: distro == "Debian"
- command: yum clean all
when: distro == "CentOS"
- file:
path: /var/cache/apt
state: absent
when: distro == "Debian"
- user:
name: root
password: "{{ 'rootpassword' | password_hash('sha256') }}"
shell: /bin/bash
- file:
path: /var/run/sshd
state: directory
- command: ssh-keygen -t rsa -f /etc/ssh/ssh_host_rsa_key
args:
creates: /etc/ssh/ssh_host_rsa_key
- group:
name: "{{sudo_group[distro]}}"
- copy:
dest: /etc/sentinel
content: |
i-am-mitogen-test-docker-image
- copy:
dest: /etc/ssh/banner.txt
src: ../data/docker/ssh_login_banner.txt
- copy:
dest: /etc/sudoers.d/001-mitogen
src: ../data/docker/001-mitogen.sudo
- lineinfile:
path: /etc/ssh/sshd_config
line: Banner /etc/ssh/banner.txt
- lineinfile:
path: /etc/ssh/sshd_config
line: PermitRootLogin yes
regexp: '.*PermitRootLogin.*'
- lineinfile:
path: /etc/pam.d/sshd
regexp: '.*session.*required.*pam_loginuid.so'
line: session optional pam_loginuid.so
- copy:
mode: 'u+rwx,go=rx'
dest: /usr/local/bin/pywrap
content: |
#!/bin/bash
exec strace -ff -o /tmp/pywrap$$.trace python2.7 "$@"'

@ -0,0 +1,139 @@
#
# Add users expected by tests. Assumes passwordless sudo to root.
#
# WARNING: this creates non-privilged accounts with pre-set passwords!
#
- hosts: all
gather_facts: true
strategy: mitogen_linear
become: true
vars:
special_users:
- has_sudo
- has_sudo_pubkey
- pw_required
- readonly_homedir
- require_tty
- require_tty_pw_required
- slow_user
- webapp
groups:
- has_sudo: ['mitogen__group', '{{sudo_group[distro]}}']
- has_sudo_pubkey: ['mitogen__group', '{{sudo_group[distro]}}']
- has_sudo_nopw: ['mitogen__group', 'mitogen__sudo_nopw']
normal_users: "{{
lookup('sequence', 'start=1 end=5 format=user%d', wantlist=True)
}}"
all_users: "{{
special_users +
normal_users
}}"
tasks:
- name: Disable non-localhost SSH for Mitogen users
blockinfile:
path: /etc/ssh/sshd_config
block: |
Match User mitogen__* Address !127.0.0.1
DenyUsers *
- name: Create Mitogen test groups
group:
name: "mitogen__{{item}}"
with_items:
- group
- sudo_nopw
- name: Create user accounts
block:
- user:
name: "mitogen__{{item}}"
shell: /bin/bash
groups: "{{groups[item]|default(['mitogen__group'])}}"
password: "{{ (item + '_password') | password_hash('sha256') }}"
loop: "{{all_users}}"
when: ansible_system != 'Darwin'
- user:
name: "mitogen__{{item}}"
shell: /bin/bash
groups: "{{groups[item]|default(['mitogen__group'])}}"
password: "{{item}}_password"
loop: "{{all_users}}"
when: ansible_system == 'Darwin'
- name: Hide users from login window.
loop: "{{all_users}}"
when: ansible_system == 'Darwin'
osx_defaults:
array_add: true
domain: /Library/Preferences/com.apple.loginwindow
type: array
key: HiddenUsersList
value: ['mitogen_{{item}}']
- name: Readonly homedir for one account
shell: "chown -R root: ~mitogen__readonly_homedir"
- name: Slow bash profile for one account
copy:
dest: ~mitogen__slow_user/.{{item}}
src: ../data/docker/mitogen__slow_user.profile
with_items:
- bashrc
- profile
- name: Install pubkey for mitogen__has_sudo_pubkey
block:
- file:
path: ~mitogen__has_sudo_pubkey/.ssh
state: directory
mode: go=
owner: mitogen__has_sudo_pubkey
- copy:
dest: ~mitogen__has_sudo_pubkey/.ssh/authorized_keys
src: ../data/docker/mitogen__has_sudo_pubkey.key.pub
mode: go=
owner: mitogen__has_sudo_pubkey
- name: Install slow profile for one account
block:
- copy:
dest: ~mitogen__slow_user/.profile
src: ../data/docker/mitogen__slow_user.profile
- copy:
dest: ~mitogen__slow_user/.bashrc
src: ../data/docker/mitogen__slow_user.profile
- name: Require a TTY for two accounts
lineinfile:
path: /etc/sudoers
line: "{{item}}"
with_items:
- Defaults>mitogen__pw_required targetpw
- Defaults>mitogen__require_tty requiretty
- Defaults>mitogen__require_tty_pw_required requiretty,targetpw
- name: Require password for two accounts
lineinfile:
path: /etc/sudoers
line: "{{lookup('pipe', 'whoami')}} ALL = ({{item}}) ALL"
with_items:
- mitogen__pw_required
- mitogen__require_tty_pw_required
- name: Allow passwordless sudo for require_tty/readonly_homedir
lineinfile:
path: /etc/sudoers
line: "{{lookup('pipe', 'whoami')}} ALL = ({{item}}) NOPASSWD:ALL"
with_items:
- mitogen__require_tty
- mitogen__readonly_homedir
- name: Allow passwordless for many accounts
lineinfile:
path: /etc/sudoers
line: "{{lookup('pipe', 'whoami')}} ALL = (mitogen__{{item}}) NOPASSWD:ALL"
loop: "{{normal_users}}"

@ -0,0 +1,4 @@
[defaults]
strategy_plugins = ../../ansible_mitogen/plugins/strategy
retry_files_enabled = false

@ -0,0 +1,43 @@
#!/usr/bin/env python
"""
Build the Docker images used for testing.
"""
import commands
import os
import shlex
import subprocess
BASEDIR = os.path.dirname(os.path.abspath(__file__))
def sh(s, *args):
if args:
s %= args
return shlex.split(s)
for base_image, name in [('debian:stretch', 'debian'),
('centos:6', 'centos6'),
('centos:7', 'centos7')]:
args = sh('docker run --rm -it -d %s /bin/bash', base_image)
container_id = subprocess.check_output(args).strip()
try:
subprocess.check_call(
cwd=BASEDIR,
args=sh('''
ansible-playbook -i %s, -c docker setup.yml -vvv
''', container_id)
)
subprocess.check_call(sh('''
docker commit
--change 'EXPOSE 22'
--change 'CMD ["/usr/sbin/sshd", "-D"]'
%s
mitogen/%s-test
''', container_id, name))
finally:
subprocess.check_call(sh('docker rm -f %s', container_id))

@ -0,0 +1,13 @@
- hosts: all
gather_facts: false
tasks:
- set_fact:
# Hacktacular.. but easiest place for it with current structure.
sudo_group:
MacOSX: admin
Debian: wheel
CentOS: sudo
- import_playbook: _container_setup.yml
- import_playbook: _user_accounts.yml
Loading…
Cancel
Save