Merge remote-tracking branch 'origin/543-darwin-ansible-ci'

* origin/543-darwin-ansible-ci:
  issue #543: install virtualenv for Azure
  issue #543: dumb fix for file vs. stat :(
  issue #543: disable host key checking
  issue #543: create ~/.ssh if it doesn't exist
  issue #543: Hide Mitogen test users from gdm
  issue #543: skip test that's hard to do on Mac
  issue #543: use key from Git, newer ssh-keygen unsupported by Paramiko
  image_prep: ensure Mac users can SSH without manual intervention
  issue #543: make localhost_ansible_tests run locally
  issue #543: add Ansible job to Azure matrix
  issue #543: localhost_ansible scripts.

.ci/azure-pipelines.yml

@@ -15,6 +15,9 @@ jobs:
       Mito27_27:
         python.version: '2.7'
         MODE: mitogen
+      Ans280_27:
+        python.version: '2.7'
+        MODE: localhost_ansible
 
 
 - job: Linux

.ci/localhost_ansible_install.py

@@ -0,0 +1,16 @@
+#!/usr/bin/env python
+
+import ci_lib
+
+batches = [
+    [
+        # Must be installed separately, as PyNACL indirect requirement causes
+        # newer version to be installed if done in a single pip run.
+        'pip install "pycparser<2.19" "idna<2.7"',
+        'pip install '
+            '-r tests/requirements.txt '
+            '-r tests/ansible/requirements.txt',
+    ]
+]
+
+ci_lib.run_batches(batches)

.ci/localhost_ansible_tests.py

@@ -0,0 +1,54 @@
+#!/usr/bin/env python
+# Run tests/ansible/all.yml under Ansible and Ansible-Mitogen
+
+import glob
+import os
+import shutil
+import sys
+
+import ci_lib
+from ci_lib import run
+
+
+TESTS_DIR = os.path.join(ci_lib.GIT_ROOT, 'tests/ansible')
+IMAGE_PREP_DIR = os.path.join(ci_lib.GIT_ROOT, 'tests/image_prep')
+HOSTS_DIR = os.path.join(TESTS_DIR, 'hosts')
+KEY_PATH = os.path.join(TESTS_DIR, '../data/docker/mitogen__has_sudo_pubkey.key')
+
+
+with ci_lib.Fold('unit_tests'):
+    os.environ['SKIP_MITOGEN'] = '1'
+    ci_lib.run('./run_tests -v')
+
+
+with ci_lib.Fold('job_setup'):
+    # Don't set -U as that will upgrade Paramiko to a non-2.6 compatible version.
+    run("pip install -q virtualenv ansible==%s", ci_lib.ANSIBLE_VERSION)
+
+    os.chmod(KEY_PATH, int('0600', 8))
+    if not ci_lib.exists_in_path('sshpass'):
+        run("brew install http://git.io/sshpass.rb")
+
+
+with ci_lib.Fold('machine_prep'):
+    ssh_dir = os.path.expanduser('~/.ssh')
+    if not os.path.exists(ssh_dir):
+        os.makedirs(ssh_dir, int('0700', 8))
+
+    key_path = os.path.expanduser('~/.ssh/id_rsa')
+    shutil.copy(KEY_PATH, key_path)
+
+    auth_path = os.path.expanduser('~/.ssh/authorized_keys')
+    os.system('ssh-keygen -y -f %s >> %s' % (key_path, auth_path))
+    os.chmod(auth_path, int('0600', 8))
+
+    if os.path.expanduser('~mitogen__user1') == '~mitogen__user1':
+        os.chdir(IMAGE_PREP_DIR)
+        run("ansible-playbook -c local -i localhost, _user_accounts.yml")
+
+
+with ci_lib.Fold('ansible'):
+    os.chdir(TESTS_DIR)
+    playbook = os.environ.get('PLAYBOOK', 'all.yml')
+    run('./run_ansible_playbook.py %s -l target %s',
+        playbook, ' '.join(sys.argv[1:]))

tests/ansible/integration/context_service/remote_name.yml

@@ -7,6 +7,10 @@
   - meta: end_play
     when: not is_mitogen
 
+  # Too much hassle to make this work for OSX
+  - meta: end_play
+    when: ansible_system != 'Linux'
+
   - shell: 'cat /proc/$PPID/cmdline | tr \\0 \\n'
     register: out
   - debug: var=out

tests/ansible/tests/affinity_test.py

@@ -1,6 +1,7 @@
 
 import multiprocessing
 import os
+import sys
 import tempfile
 
 import mock
@@ -221,6 +222,11 @@ class MockLinuxPolicyTest(testlib.TestCase):
         for x in range(1, 4096, 32):
             policy.assign_subprocess()
 
+MockLinuxPolicyTest = unittest2.skipIf(
+    condition=(not sys.platform.startswith('linuxPolicy')),
+    reason='select.select() not supported'
+)(MockLinuxPolicyTest)
+
 
 if __name__ == '__main__':
     unittest2.main()

tests/image_prep/_container_setup.yml

@@ -1,5 +1,7 @@
 
 - hosts: all
+  vars_files:
+  - shared_vars.yml
   strategy: linear
   gather_facts: false
   tasks:
@@ -13,6 +15,8 @@
         fi
 
 - hosts: all
+  vars_files:
+  - shared_vars.yml
   strategy: mitogen_free
   # Can't gather facts before here.
   gather_facts: true

tests/image_prep/_user_accounts.yml

@@ -5,6 +5,8 @@
 #
 
 - hosts: all
+  vars_files:
+  - shared_vars.yml
   gather_facts: true
   strategy: mitogen_free
   become: true
@@ -73,14 +75,18 @@
       - user:
           name: "mitogen__{{item}}"
           shell: /bin/bash
-          groups: "{{user_groups[item]|default(['mitogen__group'])}}"
+          groups: |
+            {{
+                ['com.apple.access_ssh'] +
+                (user_groups[item] | default(['mitogen__group']))
+            }}
           password: "{{item}}_password"
         with_items: "{{all_users}}"
         when: ansible_system == 'Darwin'
 
-    - name: Hide users from login window.
+    - name: Hide users from login window (Darwin).
-      with_items: "{{all_users}}"
       when: ansible_system == 'Darwin'
+      with_items: "{{all_users}}"
       osx_defaults:
         array_add: true
         domain: /Library/Preferences/com.apple.loginwindow
@@ -88,6 +94,26 @@
         key: HiddenUsersList
         value: ['mitogen_{{item}}']
 
+    - name: Check if AccountsService is used
+      stat:
+        path: /var/lib/AccountsService/users
+      register: out
+
+    - name: Hide users from login window (Linux).
+      when: ansible_system == 'Linux' and out.stat.exists
+      with_items: "{{all_users}}"
+      copy:
+        dest: /var/lib/AccountsService/users/mitogen__{{item}}
+        content: |
+          [User]
+          SystemAccount=true
+
+    - name: Restart AccountsService (Linux).
+      when: ansible_system == 'Linux' and out.stat.exists
+      service:
+        name: accounts-daemon
+        restarted: true
+
     - name: Readonly homedir for one account
       shell: "chown -R root: ~mitogen__readonly_homedir"
 

tests/image_prep/ansible.cfg

@@ -4,3 +4,4 @@ strategy_plugins = ../../ansible_mitogen/plugins/strategy
 retry_files_enabled = false
 display_args_to_stdout = True
 no_target_syslog = True
+host_key_checking = False

tests/image_prep/setup.yml

@@ -1,14 +1,3 @@
 
-- hosts: all
-  gather_facts: false
-  tasks:
-  - set_fact:
-      # Hacktacular.. but easiest place for it with current structure.
-      sudo_group:
-        MacOSX: admin
-        Debian: sudo
-        Ubuntu: sudo
-        CentOS: wheel
-
 - include: _container_setup.yml
 - include: _user_accounts.yml

tests/image_prep/shared_vars.yml

@@ -0,0 +1,5 @@
+sudo_group:
+  MacOSX: admin
+  Debian: sudo
+  Ubuntu: sudo
+  CentOS: wheel