From 4ced885619ee9c2893dca57eac25c9e9907e7196 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 30 Jul 2019 22:11:04 +0100 Subject: [PATCH 01/11] issue #543: localhost_ansible scripts. --- .ci/localhost_ansible_install.py | 16 ++++++++++++ .ci/localhost_ansible_tests.py | 43 ++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+) create mode 100755 .ci/localhost_ansible_install.py create mode 100755 .ci/localhost_ansible_tests.py diff --git a/.ci/localhost_ansible_install.py b/.ci/localhost_ansible_install.py new file mode 100755 index 00000000..0cb47374 --- /dev/null +++ b/.ci/localhost_ansible_install.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python + +import ci_lib + +batches = [ + [ + # Must be installed separately, as PyNACL indirect requirement causes + # newer version to be installed if done in a single pip run. + 'pip install "pycparser<2.19" "idna<2.7"', + 'pip install ' + '-r tests/requirements.txt ' + '-r tests/ansible/requirements.txt', + ] +] + +ci_lib.run_batches(batches) diff --git a/.ci/localhost_ansible_tests.py b/.ci/localhost_ansible_tests.py new file mode 100755 index 00000000..71542d21 --- /dev/null +++ b/.ci/localhost_ansible_tests.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python +# Run tests/ansible/all.yml under Ansible and Ansible-Mitogen + +import glob +import os +import sys + +import ci_lib +from ci_lib import run + + +TESTS_DIR = os.path.join(ci_lib.GIT_ROOT, 'tests/ansible') +IMAGE_PREP_DIR = os.path.join(ci_lib.GIT_ROOT, 'tests/image_prep') +HOSTS_DIR = os.path.join(TESTS_DIR, 'hosts') + + +with ci_lib.Fold('unit_tests'): + os.environ['SKIP_MITOGEN'] = '1' + ci_lib.run('./run_tests -v') + + +with ci_lib.Fold('job_setup'): + # Don't set -U as that will upgrade Paramiko to a non-2.6 compatible version. + run("pip install -q ansible==%s", ci_lib.ANSIBLE_VERSION) + + os.chdir(TESTS_DIR) + os.chmod('../data/docker/mitogen__has_sudo_pubkey.key', int('0600', 7)) + + if not ci_lib.exists_in_path('sshpass'): + run("brew install sshpass") + + +with ci_lib.Fold('machine_prep'): + if os.path.expanduser('~mitogen__user1') == '~mitogen_user1': + os.chdir(IMAGE_PREP_DIR) + run("ansible-playbook -i localhost, _user_accounts.yml") + + +with ci_lib.Fold('ansible'): + os.chdir(TESTS_DIR) + playbook = os.environ.get('PLAYBOOK', 'all.yml') + run('./run_ansible_playbook.py %s -l target %s', + playbook, HOSTS_DIR, ' '.join(sys.argv[1:])) From 57203aef531f24c0065853d3c7327c425d2177ed Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 30 Jul 2019 22:12:06 +0100 Subject: [PATCH 02/11] issue #543: add Ansible job to Azure matrix --- .ci/azure-pipelines.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.ci/azure-pipelines.yml b/.ci/azure-pipelines.yml index 95f239ff..920e82a1 100644 --- a/.ci/azure-pipelines.yml +++ b/.ci/azure-pipelines.yml @@ -15,6 +15,9 @@ jobs: Mito27_27: python.version: '2.7' MODE: mitogen + Ans280_27: + python.version: '2.7' + MODE: localhost_ansible - job: Linux From 501cfca82b50af3576390ca69ad436bbba74a483 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 30 Jul 2019 22:52:26 +0100 Subject: [PATCH 03/11] issue #543: make localhost_ansible_tests run locally --- .ci/localhost_ansible_tests.py | 15 ++++++++++++--- tests/ansible/tests/affinity_test.py | 6 ++++++ tests/image_prep/_container_setup.yml | 4 ++++ tests/image_prep/_user_accounts.yml | 2 ++ tests/image_prep/setup.yml | 11 ----------- tests/image_prep/shared_vars.yml | 5 +++++ 6 files changed, 29 insertions(+), 14 deletions(-) create mode 100644 tests/image_prep/shared_vars.yml diff --git a/.ci/localhost_ansible_tests.py b/.ci/localhost_ansible_tests.py index 71542d21..b795da48 100755 --- a/.ci/localhost_ansible_tests.py +++ b/.ci/localhost_ansible_tests.py @@ -27,11 +27,20 @@ with ci_lib.Fold('job_setup'): os.chmod('../data/docker/mitogen__has_sudo_pubkey.key', int('0600', 7)) if not ci_lib.exists_in_path('sshpass'): - run("brew install sshpass") + run("brew install http://git.io/sshpass.rb") with ci_lib.Fold('machine_prep'): - if os.path.expanduser('~mitogen__user1') == '~mitogen_user1': + key_path = os.path.expanduser('~/.ssh/id_rsa') + if not os.path.exists(key_path): + run("ssh-keygen -N '' -f %s", key_path) + + auth_path = os.path.expanduser('~/.ssh/authorized_keys') + with open(auth_path, 'a') as fp: + fp.write(open(key_path + '.pub').read()) + os.chmod(auth_path, int('0600', 8)) + + if os.path.expanduser('~mitogen__user1') == '~mitogen__user1': os.chdir(IMAGE_PREP_DIR) run("ansible-playbook -i localhost, _user_accounts.yml") @@ -40,4 +49,4 @@ with ci_lib.Fold('ansible'): os.chdir(TESTS_DIR) playbook = os.environ.get('PLAYBOOK', 'all.yml') run('./run_ansible_playbook.py %s -l target %s', - playbook, HOSTS_DIR, ' '.join(sys.argv[1:])) + playbook, ' '.join(sys.argv[1:])) diff --git a/tests/ansible/tests/affinity_test.py b/tests/ansible/tests/affinity_test.py index 641455bd..ccd72243 100644 --- a/tests/ansible/tests/affinity_test.py +++ b/tests/ansible/tests/affinity_test.py @@ -1,6 +1,7 @@ import multiprocessing import os +import sys import tempfile import mock @@ -221,6 +222,11 @@ class MockLinuxPolicyTest(testlib.TestCase): for x in range(1, 4096, 32): policy.assign_subprocess() +MockLinuxPolicyTest = unittest2.skipIf( + condition=(not sys.platform.startswith('linuxPolicy')), + reason='select.select() not supported' +)(MockLinuxPolicyTest) + if __name__ == '__main__': unittest2.main() diff --git a/tests/image_prep/_container_setup.yml b/tests/image_prep/_container_setup.yml index 9d001f48..2a30f49d 100644 --- a/tests/image_prep/_container_setup.yml +++ b/tests/image_prep/_container_setup.yml @@ -1,5 +1,7 @@ - hosts: all + vars_files: + - shared_vars.yml strategy: linear gather_facts: false tasks: @@ -13,6 +15,8 @@ fi - hosts: all + vars_files: + - shared_vars.yml strategy: mitogen_free # Can't gather facts before here. gather_facts: true diff --git a/tests/image_prep/_user_accounts.yml b/tests/image_prep/_user_accounts.yml index 5f1bf0dc..e6ace82f 100644 --- a/tests/image_prep/_user_accounts.yml +++ b/tests/image_prep/_user_accounts.yml @@ -5,6 +5,8 @@ # - hosts: all + vars_files: + - shared_vars.yml gather_facts: true strategy: mitogen_free become: true diff --git a/tests/image_prep/setup.yml b/tests/image_prep/setup.yml index 760da0f6..2c37c6bb 100644 --- a/tests/image_prep/setup.yml +++ b/tests/image_prep/setup.yml @@ -1,14 +1,3 @@ -- hosts: all - gather_facts: false - tasks: - - set_fact: - # Hacktacular.. but easiest place for it with current structure. - sudo_group: - MacOSX: admin - Debian: sudo - Ubuntu: sudo - CentOS: wheel - - include: _container_setup.yml - include: _user_accounts.yml diff --git a/tests/image_prep/shared_vars.yml b/tests/image_prep/shared_vars.yml new file mode 100644 index 00000000..4be7babe --- /dev/null +++ b/tests/image_prep/shared_vars.yml @@ -0,0 +1,5 @@ +sudo_group: + MacOSX: admin + Debian: sudo + Ubuntu: sudo + CentOS: wheel From 0e55bb3eb7d3dd972165910847ebebf3a0a5b659 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Wed, 31 Jul 2019 00:13:15 +0100 Subject: [PATCH 04/11] image_prep: ensure Mac users can SSH without manual intervention --- tests/image_prep/_user_accounts.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/image_prep/_user_accounts.yml b/tests/image_prep/_user_accounts.yml index e6ace82f..70f5d0eb 100644 --- a/tests/image_prep/_user_accounts.yml +++ b/tests/image_prep/_user_accounts.yml @@ -75,7 +75,11 @@ - user: name: "mitogen__{{item}}" shell: /bin/bash - groups: "{{user_groups[item]|default(['mitogen__group'])}}" + groups: | + {{ + ['com.apple.access_ssh'] + + (user_groups[item] | default(['mitogen__group'])) + }} password: "{{item}}_password" with_items: "{{all_users}}" when: ansible_system == 'Darwin' From 92de2abeea611cb1ed98f0d493c0fabe91db1c9f Mon Sep 17 00:00:00 2001 From: David Wilson Date: Wed, 31 Jul 2019 00:13:50 +0100 Subject: [PATCH 05/11] issue #543: use key from Git, newer ssh-keygen unsupported by Paramiko --- .ci/localhost_ansible_tests.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/.ci/localhost_ansible_tests.py b/.ci/localhost_ansible_tests.py index b795da48..662c7c73 100755 --- a/.ci/localhost_ansible_tests.py +++ b/.ci/localhost_ansible_tests.py @@ -3,6 +3,7 @@ import glob import os +import shutil import sys import ci_lib @@ -12,6 +13,7 @@ from ci_lib import run TESTS_DIR = os.path.join(ci_lib.GIT_ROOT, 'tests/ansible') IMAGE_PREP_DIR = os.path.join(ci_lib.GIT_ROOT, 'tests/image_prep') HOSTS_DIR = os.path.join(TESTS_DIR, 'hosts') +KEY_PATH = os.path.join(TESTS_DIR, '../data/docker/mitogen__has_sudo_pubkey.key') with ci_lib.Fold('unit_tests'): @@ -23,21 +25,17 @@ with ci_lib.Fold('job_setup'): # Don't set -U as that will upgrade Paramiko to a non-2.6 compatible version. run("pip install -q ansible==%s", ci_lib.ANSIBLE_VERSION) - os.chdir(TESTS_DIR) - os.chmod('../data/docker/mitogen__has_sudo_pubkey.key', int('0600', 7)) - + os.chmod(KEY_PATH, int('0600', 8)) if not ci_lib.exists_in_path('sshpass'): run("brew install http://git.io/sshpass.rb") with ci_lib.Fold('machine_prep'): key_path = os.path.expanduser('~/.ssh/id_rsa') - if not os.path.exists(key_path): - run("ssh-keygen -N '' -f %s", key_path) + shutil.copy(KEY_PATH, key_path) auth_path = os.path.expanduser('~/.ssh/authorized_keys') - with open(auth_path, 'a') as fp: - fp.write(open(key_path + '.pub').read()) + os.system('ssh-keygen -y -f %s >> %s' % (key_path, auth_path)) os.chmod(auth_path, int('0600', 8)) if os.path.expanduser('~mitogen__user1') == '~mitogen__user1': From 17d0e1b31534ccdfeb54973f24f441845abd04c6 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Wed, 31 Jul 2019 00:14:33 +0100 Subject: [PATCH 06/11] issue #543: skip test that's hard to do on Mac --- tests/ansible/integration/context_service/remote_name.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/ansible/integration/context_service/remote_name.yml b/tests/ansible/integration/context_service/remote_name.yml index 827abaee..d7116ec1 100644 --- a/tests/ansible/integration/context_service/remote_name.yml +++ b/tests/ansible/integration/context_service/remote_name.yml @@ -7,6 +7,10 @@ - meta: end_play when: not is_mitogen + # Too much hassle to make this work for OSX + - meta: end_play + when: ansible_system != 'Linux' + - shell: 'cat /proc/$PPID/cmdline | tr \\0 \\n' register: out - debug: var=out From 0741876392c4983aacc27e7688ca8dbe95aeb746 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Wed, 31 Jul 2019 00:04:49 +0100 Subject: [PATCH 07/11] issue #543: Hide Mitogen test users from gdm --- tests/image_prep/_user_accounts.yml | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/tests/image_prep/_user_accounts.yml b/tests/image_prep/_user_accounts.yml index 70f5d0eb..e5b5722d 100644 --- a/tests/image_prep/_user_accounts.yml +++ b/tests/image_prep/_user_accounts.yml @@ -84,9 +84,9 @@ with_items: "{{all_users}}" when: ansible_system == 'Darwin' - - name: Hide users from login window. - with_items: "{{all_users}}" + - name: Hide users from login window (Darwin). when: ansible_system == 'Darwin' + with_items: "{{all_users}}" osx_defaults: array_add: true domain: /Library/Preferences/com.apple.loginwindow @@ -94,6 +94,26 @@ key: HiddenUsersList value: ['mitogen_{{item}}'] + - name: Check if AccountsService is used + file: + path: /var/lib/AccountsService/users + register: out + + - name: Hide users from login window (Linux). + when: ansible_system == 'Linux' and out.stat.exists + with_items: "{{all_users}}" + copy: + dest: /var/lib/AccountsService/users/mitogen__{{item}} + content: | + [User] + SystemAccount=true + + - name: Restart AccountsService (Linux). + when: ansible_system == 'Linux' and out.stat.exists + service: + name: accounts-daemon + restarted: true + - name: Readonly homedir for one account shell: "chown -R root: ~mitogen__readonly_homedir" From edb745f4348c2867d3e0b477796723409ca0aaa5 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Wed, 31 Jul 2019 00:22:59 +0100 Subject: [PATCH 08/11] issue #543: create ~/.ssh if it doesn't exist --- .ci/localhost_ansible_tests.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.ci/localhost_ansible_tests.py b/.ci/localhost_ansible_tests.py index 662c7c73..11b1faa1 100755 --- a/.ci/localhost_ansible_tests.py +++ b/.ci/localhost_ansible_tests.py @@ -31,6 +31,10 @@ with ci_lib.Fold('job_setup'): with ci_lib.Fold('machine_prep'): + ssh_dir = os.path.expanduser('~/.ssh') + if not os.path.exists(ssh_dir): + os.makedirs(ssh_dir, int('0700', 8)) + key_path = os.path.expanduser('~/.ssh/id_rsa') shutil.copy(KEY_PATH, key_path) From f3915b5f4021220b303759d07f61a02fde3e0b13 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Wed, 31 Jul 2019 00:50:22 +0100 Subject: [PATCH 09/11] issue #543: disable host key checking --- .ci/localhost_ansible_tests.py | 2 +- tests/image_prep/ansible.cfg | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.ci/localhost_ansible_tests.py b/.ci/localhost_ansible_tests.py index 11b1faa1..f9465899 100755 --- a/.ci/localhost_ansible_tests.py +++ b/.ci/localhost_ansible_tests.py @@ -44,7 +44,7 @@ with ci_lib.Fold('machine_prep'): if os.path.expanduser('~mitogen__user1') == '~mitogen__user1': os.chdir(IMAGE_PREP_DIR) - run("ansible-playbook -i localhost, _user_accounts.yml") + run("ansible-playbook -c local -i localhost, _user_accounts.yml") with ci_lib.Fold('ansible'): diff --git a/tests/image_prep/ansible.cfg b/tests/image_prep/ansible.cfg index 8a8c47fa..60f2975e 100644 --- a/tests/image_prep/ansible.cfg +++ b/tests/image_prep/ansible.cfg @@ -4,3 +4,4 @@ strategy_plugins = ../../ansible_mitogen/plugins/strategy retry_files_enabled = false display_args_to_stdout = True no_target_syslog = True +host_key_checking = False From ebb4a7ca6a23bdc91689c5982d540cf2cc067eec Mon Sep 17 00:00:00 2001 From: David Wilson Date: Wed, 31 Jul 2019 00:57:32 +0100 Subject: [PATCH 10/11] issue #543: dumb fix for file vs. stat :( --- tests/image_prep/_user_accounts.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/image_prep/_user_accounts.yml b/tests/image_prep/_user_accounts.yml index e5b5722d..fbefd9c3 100644 --- a/tests/image_prep/_user_accounts.yml +++ b/tests/image_prep/_user_accounts.yml @@ -95,7 +95,7 @@ value: ['mitogen_{{item}}'] - name: Check if AccountsService is used - file: + stat: path: /var/lib/AccountsService/users register: out From 57db3a36e1d468789abc731d95949e4a52251ed0 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Wed, 31 Jul 2019 01:03:23 +0100 Subject: [PATCH 11/11] issue #543: install virtualenv for Azure --- .ci/localhost_ansible_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/localhost_ansible_tests.py b/.ci/localhost_ansible_tests.py index f9465899..f7e1ecbd 100755 --- a/.ci/localhost_ansible_tests.py +++ b/.ci/localhost_ansible_tests.py @@ -23,7 +23,7 @@ with ci_lib.Fold('unit_tests'): with ci_lib.Fold('job_setup'): # Don't set -U as that will upgrade Paramiko to a non-2.6 compatible version. - run("pip install -q ansible==%s", ci_lib.ANSIBLE_VERSION) + run("pip install -q virtualenv ansible==%s", ci_lib.ANSIBLE_VERSION) os.chmod(KEY_PATH, int('0600', 8)) if not ci_lib.exists_in_path('sshpass'):