diff --git a/test/utils/ansible-playbook_integration_runner/ansible.cfg b/test/utils/ansible-playbook_integration_runner/ansible.cfg deleted file mode 100644 index 14c80651521..00000000000 --- a/test/utils/ansible-playbook_integration_runner/ansible.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[defaults] -host_key_checking = False diff --git a/test/utils/ansible-playbook_integration_runner/ec2.yml b/test/utils/ansible-playbook_integration_runner/ec2.yml deleted file mode 100644 index 387b0c7e8de..00000000000 --- a/test/utils/ansible-playbook_integration_runner/ec2.yml +++ /dev/null @@ -1,40 +0,0 @@ -- name: Launch Instance - ec2: - group_id: 'sg-07bb906d' # jenkins-slave_new - count: 1 - instance_type: 'm3.large' - image: '{{ item.image }}' - wait: true - region: 'us-east-1' - keypair: '{{ keypair }}' - aws_access_key: "{{ aws_access_key|default(lookup('env', 'AWS_ACCESS_KEY')) }}" - aws_secret_key: "{{ aws_secret_key|default(lookup('env', 'AWS_SECRET_KEY')) }}" - instance_tags: - jenkins: jenkins_ansible_pr_test - register: ec2 - with_items: slaves -# We could do an async here, that would speed things up - - -- name: Wait for SSH - wait_for: - host: "{{ item['instances'][0]['public_ip'] }}" - port: 22 - delay: 10 - timeout: 320 - state: started - with_items: ec2.results - -- name: Wait a little longer for centos - pause: seconds=20 - -- name: Add hosts group temporary inventory group with pem path - add_host: - name: "{{ item.1.platform }}-{{ ec2.results[item.0]['instances'][0]['public_ip'] }}" - groups: dynamic_hosts - ansible_ssh_host: "{{ ec2.results[item.0]['instances'][0]['public_ip'] }}" - ansible_ssh_private_key_file: '{{ pem_path }}' - ansible_ssh_user: "{{ item.1.ssh_user }}" - ec2_vars: "{{ ec2.results[item.0]['instances'][0] }}" - ec2_instance_ids: "{{ ec2.results[item.0]['instance_ids'] }}" - with_indexed_items: slaves diff --git a/test/utils/ansible-playbook_integration_runner/inventory b/test/utils/ansible-playbook_integration_runner/inventory deleted file mode 100644 index 2302edae31b..00000000000 --- a/test/utils/ansible-playbook_integration_runner/inventory +++ /dev/null @@ -1 +0,0 @@ -localhost ansible_connection=local diff --git a/test/utils/ansible-playbook_integration_runner/main.yml b/test/utils/ansible-playbook_integration_runner/main.yml deleted file mode 100644 index 82ec9ec9bf6..00000000000 --- a/test/utils/ansible-playbook_integration_runner/main.yml +++ /dev/null @@ -1,77 +0,0 @@ -- hosts: all - connection: local - vars: - slaves: - - distribution: "Ubuntu" - version: "12.04" - image: "ami-309ddf5a" - ssh_user: "ubuntu" - platform: "ubuntu-12.04-x86_64" - - distribution: "Ubuntu" - version: "14.04" - image: "ami-d06632ba" - ssh_user: "ubuntu" - platform: "ubuntu-14.04-x86_64" - - distribution: "CentOS" - version: "6.5" - image: "ami-8997afe0" - ssh_user: "root" - platform: "centos-6.5-x86_64" - - distribution: "CentOS" - version: "7" - image: "ami-61bbf104" - ssh_user: "centos" - platform: "centos-7-x86_64" - - distribution: "Fedora" - version: "23" - image: "ami-518bfb3b" - ssh_user: "fedora" - platform: "fedora-23-x86_64" - - tasks: - - debug: var=ansible_version - - include: ec2.yml - when: groups['dynamic_hosts'] is not defined - -# Have to hardcode these per-slave. We can't even run setup yet so we can't -# introspect what they have. -- hosts: dynamic_hosts - sudo: true - gather_facts: False - tasks: - - name: Install packages that let setup and package manager modules run - raw: dnf install -y python2 python2-dnf libselinux-python - when: "'fedora-23' in '{{ inventory_hostname }}'" - -- hosts: dynamic_hosts - sudo: true - vars: - credentials_file: '' - test_flags: "" - make_target: "non_destructive" - #pre_tasks: - roles: - - { role: ansible_test_deps, tags: ansible_test_deps } - - { role: run_integration, - tags: run_integration, - run_integration_test_flags: "{{ test_flags }}", - run_integration_credentials_file: "{{ credentials_file }}", - run_integration_make_target: "{{ make_target }}", } - tasks: - - - name: Kill ec2 instances - sudo: false - local_action: - module: ec2 - state: absent - region: 'us-east-1' - instance_ids: "{{ hostvars[item]['ec2_instance_ids'] }}" - when: hostvars[item]['ec2_instance_ids'] is defined and item == inventory_hostname - with_items: "{{groups['dynamic_hosts']}}" - - - set_fact: - ansible_connection: local - - - name: Fail - shell: 'echo "{{ inventory_hostname }}, Failed" && exit 1' - when: "'rc' not in test_results or test_results.rc != 0" diff --git a/test/utils/ansible-playbook_integration_runner/roles/ansible_test_deps/.travis.yml b/test/utils/ansible-playbook_integration_runner/roles/ansible_test_deps/.travis.yml deleted file mode 100644 index 2264f0b20a7..00000000000 --- a/test/utils/ansible-playbook_integration_runner/roles/ansible_test_deps/.travis.yml +++ /dev/null @@ -1,37 +0,0 @@ -sudo: required -dist: trusty -language: python -python: - - "2.7" -services: - - docker -env: - global: - - PATH="/usr/bin:$PATH" - -before_install: - # Ansible doesn't play well with virtualenv - - deactivate - - sudo apt-get update -qq - - sudo apt-get install docker-engine - -install: - - sudo pip install docker-py - # software-properties-common for ubuntu 14.04 - # python-software-properties for ubuntu 12.04 - - sudo apt-get install -y sshpass software-properties-common python-software-properties - - sudo apt-add-repository -y ppa:ansible/ansible - - sudo apt-get update -qq - - sudo apt-get install -y ansible - - sudo rm /usr/bin/python && sudo ln -s /usr/bin/python2.7 /usr/bin/python - - ansible-galaxy install -r test/requirements.yml -p test/roles/ - -script: - # Ensure any invocation of ansible-playbook (i.e. sudo) results in host_key_checking disabled - - sudo ansible all -i "127.0.0.1," -m lineinfile -a "regexp=^#host_key_checking dest=/etc/ansible/ansible.cfg line='host_key_checking = False'" -c local - - ansible-playbook -i test/inventory test/main.yml --syntax-check - - sudo ansible-playbook -i test/inventory test/main.yml - -notifications: - # notify ansible galaxy of results - webhooks: http://goo.gl/nSuq9h diff --git a/test/utils/ansible-playbook_integration_runner/roles/ansible_test_deps/README.md b/test/utils/ansible-playbook_integration_runner/roles/ansible_test_deps/README.md deleted file mode 100644 index 09ffacacaf5..00000000000 --- a/test/utils/ansible-playbook_integration_runner/roles/ansible_test_deps/README.md +++ /dev/null @@ -1,6 +0,0 @@ -[![Build Status](https://travis-ci.org/chrismeyersfsu/ansible_test_deps.svg)](https://travis-ci.org/chrismeyersfsu/ansible_test_deps) - -ansible_test_deps -========= - -Install needed packages to run ansible integration tests. diff --git a/test/utils/ansible-playbook_integration_runner/roles/ansible_test_deps/defaults/main.yml b/test/utils/ansible-playbook_integration_runner/roles/ansible_test_deps/defaults/main.yml deleted file mode 100644 index c7837fc56b1..00000000000 --- a/test/utils/ansible-playbook_integration_runner/roles/ansible_test_deps/defaults/main.yml +++ /dev/null @@ -1,2 +0,0 @@ ---- -# defaults file for . diff --git a/test/utils/ansible-playbook_integration_runner/roles/ansible_test_deps/handlers/main.yml b/test/utils/ansible-playbook_integration_runner/roles/ansible_test_deps/handlers/main.yml deleted file mode 100644 index 050cdd12342..00000000000 --- a/test/utils/ansible-playbook_integration_runner/roles/ansible_test_deps/handlers/main.yml +++ /dev/null @@ -1,2 +0,0 @@ ---- -# handlers file for . diff --git a/test/utils/ansible-playbook_integration_runner/roles/ansible_test_deps/meta/.galaxy_install_info b/test/utils/ansible-playbook_integration_runner/roles/ansible_test_deps/meta/.galaxy_install_info deleted file mode 100644 index ffc298fff6f..00000000000 --- a/test/utils/ansible-playbook_integration_runner/roles/ansible_test_deps/meta/.galaxy_install_info +++ /dev/null @@ -1 +0,0 @@ -{install_date: 'Tue Dec 8 15:06:28 2015', version: master} diff --git a/test/utils/ansible-playbook_integration_runner/roles/ansible_test_deps/meta/main.yml b/test/utils/ansible-playbook_integration_runner/roles/ansible_test_deps/meta/main.yml deleted file mode 100644 index 47e181bb68a..00000000000 --- a/test/utils/ansible-playbook_integration_runner/roles/ansible_test_deps/meta/main.yml +++ /dev/null @@ -1,22 +0,0 @@ ---- -galaxy_info: - author: Chris Meyers - description: install ansible integration test dependencies - company: Ansible - license: license (GPLv2, CC-BY, etc) - min_ansible_version: 1.2 - platforms: - - name: EL - versions: - - 6 - - 7 - - name: Ubuntu - versions: - - precise - - trusty - galaxy_tags: - - testing - - integration - - ansible - - dependencies -dependencies: [] diff --git a/test/utils/ansible-playbook_integration_runner/roles/ansible_test_deps/tasks/main.yml b/test/utils/ansible-playbook_integration_runner/roles/ansible_test_deps/tasks/main.yml deleted file mode 100644 index 3ca58c6e5d4..00000000000 --- a/test/utils/ansible-playbook_integration_runner/roles/ansible_test_deps/tasks/main.yml +++ /dev/null @@ -1,100 +0,0 @@ ---- - -- apt: update_cache=yes - when: ansible_os_family == 'Debian' - -- name: Install sudo - package: name=sudo state=installed - ignore_errors: true - -- name: Install RH epel - yum: name="epel-release" state=installed - sudo: true - when: ansible_distribution in ('CentOS', 'RedHat') - -- name: Install RH ansible dependencies - package: name="{{ item }}" state=installed - sudo: true - with_items: - - python-pip - - python-httplib2 - - rsync - - subversion - - mercurial - - git - - rubygems - - unzip - - openssl - - make - - gcc - - gawk - - python-devel - - libselinux-python - - python-virtualenv - - yum - - yum-metadata-parser - - redhat-rpm-config - when: ansible_os_family == 'RedHat' - -- name: Install Debian ansible dependencies - apt: name="{{ item }}" state=installed update_cache=yes - sudo: true - with_items: - - python-pip - - python-httplib2 - - rsync - - subversion - - mercurial - - git - - unzip - - gawk - - python-dev - - python-virtualenv - when: ansible_os_family == 'Debian' - -- name: update ca certificates - sudo: true - yum: name=ca-certificates state=latest - when: ansible_os_family == 'RedHat' - -- name: Install ubuntu 12.04 ansible dependencies - apt: name="{{ item }}" state=installed update_cache=yes - sudo: true - with_items: - - rubygems - when: ansible_distribution == 'Ubuntu' and ansible_distribution_version == "12.04" - -- name: Install ubuntu 14.04 ansible dependencies - apt: name="{{ item }}" state=installed update_cache=yes - sudo: true - with_items: - - rubygems-integration - when: ansible_distribution == 'Ubuntu' and ansible_distribution_version == "14.04" - -# Not sure why CentOS 6 is working without this.... -#- name: Install Red Hat 6 ansible dependencies -# yum: name="{{ item }}" state=installed -# sudo: true -# with_items: -# - python-crypto2.6 -# when: ansible_distribution in ('CentOS', 'RedHat') and ansible_distribution_major_version == "6" - -- name: Install ansible pip deps - sudo: true - pip: name="{{ item }}" - with_items: - - PyYAML - - Jinja2 - - paramiko - -- name: Install ubuntu 12.04 ansible pip deps - sudo: true - pip: name="{{ item }}" state=latest - with_items: - - pycrypto - when: ansible_distribution == 'Ubuntu' and ansible_distribution_version == "12.04" - -- name: Remove tty sudo requirement - sudo: true - lineinfile: "dest=/etc/sudoers regexp='^Defaults[ , ]*requiretty' line='#Defaults requiretty'" - when: ansible_os_family == 'RedHat' diff --git a/test/utils/ansible-playbook_integration_runner/roles/ansible_test_deps/test/inventory b/test/utils/ansible-playbook_integration_runner/roles/ansible_test_deps/test/inventory deleted file mode 100644 index 2302edae31b..00000000000 --- a/test/utils/ansible-playbook_integration_runner/roles/ansible_test_deps/test/inventory +++ /dev/null @@ -1 +0,0 @@ -localhost ansible_connection=local diff --git a/test/utils/ansible-playbook_integration_runner/roles/ansible_test_deps/test/main.yml b/test/utils/ansible-playbook_integration_runner/roles/ansible_test_deps/test/main.yml deleted file mode 100644 index 5410c1f29c3..00000000000 --- a/test/utils/ansible-playbook_integration_runner/roles/ansible_test_deps/test/main.yml +++ /dev/null @@ -1,28 +0,0 @@ ---- -- name: Bring up docker containers - hosts: localhost - gather_facts: false - vars: - inventory: - - name: ansible_test_deps_host_1 - image: "chrismeyers/centos6" - - name: ansible_test_deps_host_2 - image: "chrismeyers/ubuntu12.04" - - name: ansible_test_deps_host_3 - image: "ubuntu-upstart:14.04" - roles: - - { role: provision_docker, provision_docker_company: 'ansible', provision_docker_inventory: "{{ inventory }}" } - -- name: Run ansible_test_deps Tests - hosts: docker_containers - vars: - git_dir: "/tmp/ansible" - roles: - - { role: ansible_test_deps } - tasks: - - name: Clone ansible - git: - repo: "https://github.com/ansible/ansible.git" - dest: "{{ git_dir }}" - - name: Invoke ansible in hacking mode - shell: "cd {{ git_dir }} && . hacking/env-setup && ansible --version && ansible-playbook --version" diff --git a/test/utils/ansible-playbook_integration_runner/roles/ansible_test_deps/test/requirements.yml b/test/utils/ansible-playbook_integration_runner/roles/ansible_test_deps/test/requirements.yml deleted file mode 100644 index fa10641a72e..00000000000 --- a/test/utils/ansible-playbook_integration_runner/roles/ansible_test_deps/test/requirements.yml +++ /dev/null @@ -1,2 +0,0 @@ -- src: chrismeyersfsu.provision_docker - name: provision_docker diff --git a/test/utils/ansible-playbook_integration_runner/roles/ansible_test_deps/test/roles/ansible_test_deps b/test/utils/ansible-playbook_integration_runner/roles/ansible_test_deps/test/roles/ansible_test_deps deleted file mode 120000 index eb6d9edda4b..00000000000 --- a/test/utils/ansible-playbook_integration_runner/roles/ansible_test_deps/test/roles/ansible_test_deps +++ /dev/null @@ -1 +0,0 @@ -../../../ansible_test_deps \ No newline at end of file diff --git a/test/utils/ansible-playbook_integration_runner/roles/ansible_test_deps/vars/main.yml b/test/utils/ansible-playbook_integration_runner/roles/ansible_test_deps/vars/main.yml deleted file mode 100644 index a38c5fb0425..00000000000 --- a/test/utils/ansible-playbook_integration_runner/roles/ansible_test_deps/vars/main.yml +++ /dev/null @@ -1,2 +0,0 @@ ---- -# vars file for . diff --git a/test/utils/ansible-playbook_integration_runner/roles/run_integration/tasks/main.yml b/test/utils/ansible-playbook_integration_runner/roles/run_integration/tasks/main.yml deleted file mode 100644 index f214949c4bf..00000000000 --- a/test/utils/ansible-playbook_integration_runner/roles/run_integration/tasks/main.yml +++ /dev/null @@ -1,43 +0,0 @@ ---- -- name: Sync ansible repo to ec2 instance - synchronize: - src: "{{ sync_dir }}/" - dest: "~/ansible" - no_log: true - -- name: Get ansible source dir - sudo: false - shell: "cd ~/ansible && pwd" - register: results - -- shell: "ls -la && . hacking/env-setup && cd test/integration && TEST_FLAGS='-e clean_working_dir=no' make {{ run_integration_make_target }}" - args: - chdir: "{{ results.stdout }}" - async: 3600 - poll: 0 - register: async_test_results - sudo: true - environment: - TEST_FLAGS: "{{ run_integration_test_flags|default(lookup('env', 'TEST_FLAGS')) }}" - CREDENTIALS_FILE: "{{ run_integration_credentials_file|default(lookup('env', 'CREDENTIALS_FILE')) }}" - -- name: poll for test results - async_status: jid="{{async_test_results.ansible_job_id}}" - register: test_results - until: test_results.finished - retries: 120 - delay: 30 - ignore_errors: true - no_log: true - -- name: save stdout test results for each host - local_action: copy - args: - dest: "{{sync_dir}}/{{inventory_hostname}}.stdout_results.txt" - content: "{{test_results.stdout}}" - -- name: save stderr test results for each host - local_action: copy - args: - dest: "{{sync_dir}}/{{inventory_hostname}}.stderr_results.txt" - content: "{{test_results.stderr}}" diff --git a/test/utils/shippable/ci.sh b/test/utils/shippable/ci.sh deleted file mode 100755 index 5c0f847e661..00000000000 --- a/test/utils/shippable/ci.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -eux - -set -o pipefail - -source_root=$(python -c "from os import path; print(path.abspath(path.join(path.dirname('$0'), '../../..')))") - -"${source_root}/test/utils/shippable/${TEST}.sh" 2>&1 | gawk '{ print strftime("%Y-%m-%d %H:%M:%S"), $0; fflush(); }' diff --git a/test/utils/shippable/code-smell-requirements.txt b/test/utils/shippable/code-smell-requirements.txt deleted file mode 100644 index b2c729ca4de..00000000000 --- a/test/utils/shippable/code-smell-requirements.txt +++ /dev/null @@ -1 +0,0 @@ -yamllint diff --git a/test/utils/shippable/code-smell.sh b/test/utils/shippable/code-smell.sh deleted file mode 100755 index da66564bd67..00000000000 --- a/test/utils/shippable/code-smell.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -eux - -source_root=$(python -c "from os import path; print(path.abspath(path.join(path.dirname('$0'), '../../..')))") - -install_deps="${INSTALL_DEPS:-}" - -cd "${source_root}" - -if [ "${install_deps}" != "" ]; then - apt-add-repository 'deb http://archive.ubuntu.com/ubuntu trusty-backports universe' - apt-get update -qq - apt-get install shellcheck - - pip install -r "${source_root}/test/utils/shippable/code-smell-requirements.txt" --upgrade - pip list -fi - -yamllint ./test - -test/sanity/code-smell/replace-urlopen.sh -test/sanity/code-smell/use-compat-six.sh -test/sanity/code-smell/boilerplate.sh -test/sanity/code-smell/required-and-default-attributes.sh -test/sanity/code-smell/shebang.sh -test/sanity/code-smell/line-endings.sh -test/sanity/code-smell/empty-init.sh -test/sanity/code-smell/no-basestring.sh - -shellcheck \ - test/integration/targets/*/*.sh \ - test/utils/shippable/*.sh diff --git a/test/utils/shippable/modules/generate-tests b/test/utils/shippable/modules/generate-tests deleted file mode 100755 index fdc73e84502..00000000000 --- a/test/utils/shippable/modules/generate-tests +++ /dev/null @@ -1,288 +0,0 @@ -#!/usr/bin/env python - -# (c) 2016 Matt Clay -# -# This file is part of Ansible -# -# Ansible is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Ansible is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Ansible. If not, see . - -from __future__ import print_function - -import os -import subprocess -import sys - -from os import path -from argparse import ArgumentParser - -import ansible.constants as C - -from ansible.playbook import Playbook -from ansible.vars import VariableManager -from ansible.parsing.dataloader import DataLoader - - -def main(): - """Generate an integration test script for changed modules.""" - - C.DEPRECATION_WARNINGS = False - - C.DEFAULT_ROLES_PATH = [os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), - '..', '..', '..', 'integration', 'targets'))] - - posix_targets = [ - 'non_destructive', - 'destructive', - ] - - windows_targets = [ - 'test_win_group1', - 'test_win_group2', - 'test_win_group3', - ] - - parser = ArgumentParser(description='Generate an integration test script for changed modules.') - parser.add_argument('module_group', choices=['core', 'extras'], help='module group to test') - parser.add_argument('-v', '--verbose', dest='verbose', action='store_true', help='write verbose output to stderr') - parser.add_argument('--changes', dest='changes', default=None, - help='file listing changed paths (default: query git)') - parser.add_argument('--image', dest='image', default=os.environ.get('IMAGE'), - help='image to run tests with') - parser.add_argument('--privileged', dest='privileged', action='store_true', - default=os.environ.get('PRIVILEGED') == 'true', - help='run container in privileged mode') - parser.add_argument('--python3', dest='python3', action='store_true', - default=os.environ.get('PYTHON3', '') != '', - help='run tests using python3') - parser.add_argument('--platform', dest='platform', default=os.environ.get('PLATFORM'), - help='platform to run tests on') - parser.add_argument('--version', dest='version', default=os.environ.get('VERSION'), - help='version of platform to run tests on') - parser.add_argument('--output', dest='output', required=True, - help='path to write output script to') - - args = parser.parse_args() - - targets = posix_targets - - if args.image is not None: - script = 'integration' - options = '' - if args.privileged: - options += ' PRIVILEGED=true' - if args.python3: - options += ' PYTHON3=1' - jobs = ['IMAGE=%s%s' % (args.image, options)] - elif args.platform is not None and args.version is not None: - script = 'remote' - jobs = ['PLATFORM=%s VERSION=%s' % (args.platform, args.version)] - - if args.platform == 'windows': - targets = windows_targets - else: - raise Exception('job parameters not specified') - - generate_test_commands(args.module_group, targets, script, args.output, jobs=jobs, verbose=args.verbose, changes=args.changes) - - -def generate_test_commands(module_group, targets, script, output, jobs=None, verbose=False, changes=None): - """Generate test commands for the given module group and test targets. - - Args: - module_group: The module group (core, extras) to examine. - targets: The test targets to examine. - script: The script used to execute the test targets. - output: The path to write the output script to. - jobs: The test jobs to execute, or None to auto-detect. - verbose: True to write detailed output to stderr. - changes: Path to file containing list of changed files, or None to query git. - """ - - base_dir = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', '..', '..', '..')) - - job_config_path = path.join(base_dir, 'shippable.yml') - module_dir = os.path.join(base_dir, 'lib', 'ansible', 'modules', module_group) - - if verbose: - print_stderr(' config: %s' % job_config_path) - print_stderr('modules: %s' % module_dir) - print_stderr('targets: %s' % ' '.join(targets)) - print_stderr() - - if changes is None: - paths_changed = get_changed_paths(module_dir) - else: - with open(changes, 'r') as f: - paths_changed = f.read().strip().split('\n') - - if len(paths_changed) == 0: - print_stderr('No changes to files detected.') - exit() - - if verbose: - dump_stderr('paths_changed', paths_changed) - - modules_changed = get_modules(paths_changed) - - if len(modules_changed) == 0: - print_stderr('No changes to modules detected.') - exit() - - if verbose: - dump_stderr('modules_changed', modules_changed) - - module_tags = get_module_test_tags(modules_changed) - - if verbose: - dump_stderr('module_tags', module_tags) - - available_tags = get_target_tags(base_dir, targets) - - if verbose: - dump_stderr('available_tags', available_tags) - - use_tags = module_tags & available_tags - - if len(use_tags) == 0: - print_stderr('No tagged test roles found for changed modules.') - exit() - - if verbose: - dump_stderr('use_tags', use_tags) - - target = ' '.join(targets) - tags = ','.join(use_tags) - script_path = 'test/utils/shippable/%s.sh' % script - - commands = ['TARGET="%s" TEST_FLAGS="-t %s" %s %s' % (target, tags, j, script_path) for j in jobs] - - with open(output, 'w') as f: - f.writelines(commands) - f.write('\n') - - -def print_stderr(*args, **kwargs): - """Print to stderr.""" - - print(*args, file=sys.stderr, **kwargs) - sys.stderr.flush() - - -def dump_stderr(label, l): - """Write a label and list contents to stderr. - - Args: - label: The label to print for this list. - l: The list to dump to stderr. - """ - - print_stderr('[%s:%s]\n%s\n' % (label, len(l), '\n'.join(l))) - - -def get_target_tags(base_dir, targets): - """Get role tags from the integration tests for the given test targets. - - Args: - base_dir: The root of the ansible source code. - targets: The test targets to scan for tags. - - Returns: Set of role tags. - """ - - playbook_dir = os.path.join(base_dir, 'test', 'integration') - - tags = set() - - for target in targets: - playbook_path = os.path.join(playbook_dir, target + '.yml') - tags |= get_role_tags(playbook_path) - - return tags - - -def get_role_tags(playbook_path): - """Get role tags from the given playbook. - - Args: - playbook_path: Path to the playbook to get role tags from. - - Returns: Set of role tags. - """ - - variable_manager = VariableManager() - loader = DataLoader() - playbook = Playbook.load(playbook_path, variable_manager=variable_manager, loader=loader) - tags = set() - - for play in playbook.get_plays(): - for role in play.get_roles(): - for tag in role.tags: - tags.add(tag) - - return tags - - -def get_changed_paths(git_root, branch='devel'): - """Get file paths changed in current branch vs given branch. - - Args: - git_root: The root of the git clone. - branch: The branch to compare against (default: devel) - - Returns: List of file paths changed. - """ - - paths = subprocess.check_output(['git', 'diff', '--name-only', branch], cwd=git_root).strip().split('\n') - - return paths - - -def get_modules(paths): - """Get module names from file paths. - - Args: - paths: List of paths to extract module names from. - - Returns: List of module names. - """ - - module_extensions = [ - '.py', - '.ps1', - ] - - modules = [path.splitext(path.basename(c))[0].strip('_') for c in paths if - path.splitext(c)[1] in module_extensions and - '/' in c and - not c.startswith('test/') and - not path.basename(c)[0] == '__init__.py'] - - return modules - - -def get_module_test_tags(modules): - """Get test tags from module names. - - Args: - modules: List of module names to get test tags for. - - Returns: Set of test tags. - """ - - tags = set(['test_' + m for m in modules]) - return tags - - -if __name__ == '__main__': - main() diff --git a/test/utils/shippable/modules/generate-tests-requirements.txt b/test/utils/shippable/modules/generate-tests-requirements.txt deleted file mode 100644 index 4e859bb8c71..00000000000 --- a/test/utils/shippable/modules/generate-tests-requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -jinja2 -pyyaml diff --git a/test/utils/shippable/remote-integration.sh b/test/utils/shippable/remote-integration.sh deleted file mode 100644 index 4609be26c2b..00000000000 --- a/test/utils/shippable/remote-integration.sh +++ /dev/null @@ -1,169 +0,0 @@ -#!/bin/sh - -set -eux -env - -container="${PLATFORM}" -build_dir="${HOME}/ansible" - -test_target="${TARGET:-}" -test_flags="${TEST_FLAGS:-}" - -# Force ansible color output by default. -# To disable color force mode use FORCE_COLOR=0 -force_color="${FORCE_COLOR:-1}" - -# FIXME: these tests fail -skip_tags='test_service,test_postgresql,test_mysql_db,test_mysql_user,test_mysql_variables,test_uri,test_get_url' - -cd ~/ - -make="make" - -if [ "${container}" = "freebsd" ]; then - make="gmake" - - pkg install -y curl - - if [ ! -f bootstrap.sh ]; then - curl "https://raw.githubusercontent.com/mattclay/ansible-hacking/master/bootstrap.sh" -o bootstrap.sh - fi - - chmod +x bootstrap.sh - ./bootstrap.sh pip -y -q - - # tests require these packages - # TODO: bootstrap.sh should be capable of installing these - pkg install -y \ - bash \ - devel/ruby-gems \ - gtar \ - mercurial \ - rsync \ - ruby \ - subversion \ - sudo \ - zip -fi - -# TODO: bootstrap.sh should install these -pip install \ - junit-xml \ - virtualenv \ - jmespath - -# Tests assume loopback addresses other than 127.0.0.1 will work. -# Add aliases for loopback addresses used by tests. - -for i in 3 4 254; do - ifconfig lo0 alias "127.0.0.${i}" up -done - -ifconfig lo0 - -# Since tests run as root, we also need to be able to ssh to localhost as root. -sed -i '' 's/^# *PermitRootLogin.*$/PermitRootLogin yes/;' /etc/ssh/sshd_config - -if [ "${container}" = "freebsd" ]; then - # Restart sshd for configuration changes and loopback aliases to work. - service sshd restart -fi - -# Generate our ssh key and add it to our authorized_keys file. -# We also need to add localhost's server keys to known_hosts. - -if [ ! -f "${HOME}/.ssh/id_rsa.pub" ]; then - ssh-keygen -q -t rsa -N '' -f "${HOME}/.ssh/id_rsa" - cp "${HOME}/.ssh/id_rsa.pub" "${HOME}/.ssh/authorized_keys" - for key in /etc/ssh/ssh_host_*_key.pub; do - pk=$(cat "${key}") - echo "localhost ${pk}" >> "${HOME}/.ssh/known_hosts" - done -fi - -repo_name="${REPO_NAME:-ansible}" - -if [ -d "${build_dir}" ]; then - cd "${build_dir}" -elif [ "${repo_name}" = "ansible" ]; then - git clone "${REPOSITORY_URL:-https://github.com/ansible/ansible.git}" "${build_dir}" - cd "${build_dir}" - - if [ "${PULL_REQUEST:-false}" = "false" ]; then - git checkout -f "${BRANCH:-devel}" -- - git reset --hard "${COMMIT:-HEAD}" - else - git fetch origin "pull/${PULL_REQUEST}/head" - git checkout -f FETCH_HEAD - git merge "origin/${BRANCH}" - fi - - git submodule init - git submodule sync - git submodule update -else - case "${repo_name}" in - "ansible-modules-core") - this_module_group="core" - ;; - "ansible-modules-extras") - this_module_group="extras" - ;; - *) - echo "Unsupported repo name: ${repo_name}" - exit 1 - ;; - esac - - git clone "https://github.com/ansible/ansible.git" "${build_dir}" - - cd "${build_dir}" - - git submodule init - git submodule sync - git submodule update - - cd "${build_dir}/lib/ansible/modules/${this_module_group}" - - if [ "${PULL_REQUEST:-false}" = "false" ]; then - echo "Only pull requests are supported for module repositories." - exit - else - git fetch origin "pull/${PULL_REQUEST}/head" - git checkout -f FETCH_HEAD - git merge "origin/${BRANCH}" - fi - - cd "${build_dir}" -fi - -set +u -. hacking/env-setup -set -u - -cd test/integration - -if [ "${container}" = "osx" ]; then - # FIXME: these test targets fail - sed -i '' 's/ test_gathering_facts / /;' Makefile - - # FIXME: these tests fail - skip_tags="${skip_tags},test_iterators,test_git" - - # test_template assumes the group 'root' exists if this variable is not set - export GROUP=wheel -fi - -# TODO: support httptester via reverse ssh tunnel - -rm -rf "/tmp/shippable" -mkdir -p "/tmp/shippable/testresults" - -# TODO: enable jail test -# shellcheck disable=SC2086 -JUNIT_OUTPUT_DIR="/tmp/shippable/testresults" \ - ANSIBLE_FORCE_COLOR="${force_color}" \ - ANSIBLE_CALLBACK_WHITELIST=junit \ - TEST_FLAGS="-e ansible_python_interpreter=/usr/local/bin/python2 --skip-tags '${skip_tags}' ${test_flags}" \ - container="${container}" \ - ${make} ${test_target} diff --git a/test/utils/shippable/remote-requirements.txt b/test/utils/shippable/remote-requirements.txt deleted file mode 100644 index 23a9d3a2a81..00000000000 --- a/test/utils/shippable/remote-requirements.txt +++ /dev/null @@ -1,8 +0,0 @@ -cryptography -junit-xml -ndg-httpsclient -pyasn1 -pyopenssl -requests -pywinrm -xmltodict diff --git a/test/utils/shippable/remote.sh b/test/utils/shippable/remote.sh deleted file mode 100755 index 9901f8e20fb..00000000000 --- a/test/utils/shippable/remote.sh +++ /dev/null @@ -1,191 +0,0 @@ -#!/bin/bash -eux - -set -o pipefail - -source_root=$(python -c "from os import path; print(path.abspath(path.join(path.dirname('$0'), '../../..')))") - -test_flags="${TEST_FLAGS:-}" -test_platform="${PLATFORM}" -test_version="${VERSION}" - -test_target=(${TARGET}) - -instance_id="${INSTANCE_ID:-}" -start_instance= - -if [ "${instance_id}" == "" ]; then - instance_id=$(python -c 'import uuid; print(uuid.uuid4())') - start_instance=1 -fi - -# Set this to a non-empty value to skip immediate termination of the remote instance after tests finish. -# The remote instance will still be auto-terminated when the remote time limit is reached. -keep_instance="${KEEP_INSTANCE:-}" - -# Force ansible color output by default. -# To disable color force mode use FORCE_COLOR=0 -force_color="${FORCE_COLOR:-1}" - -if [ "${SHIPPABLE:-}" = "true" ]; then - test_auth="shippable" -else - test_auth="remote" -fi - -case "${test_platform}" in - "windows") - ci_endpoint="https://14blg63h2i.execute-api.us-east-1.amazonaws.com" - ;; - "freebsd") - ci_endpoint="https://14blg63h2i.execute-api.us-east-1.amazonaws.com" - ;; - "osx") - ci_endpoint="https://osx.testing.ansible.com" - ;; - *) - echo "unsupported platform: ${test_platform}" - exit 1 - ;; -esac - -env - -case "${test_platform}" in - "windows") - args="" - ;; - *) - ssh_key="${HOME}/.ssh/id_rsa" - args="--public-key=${ssh_key}.pub" - if [ ! -f "${ssh_key}.pub" ]; then - ssh-keygen -q -t rsa -N '' -f "${ssh_key}" - fi - ;; -esac - -pre_cleanup= - -function cleanup -{ - if [ "${pre_cleanup}" != '' ]; then - "${pre_cleanup}" - fi - - if [ "${keep_instance}" = '' ]; then - "${source_root}/test/utils/shippable/ansible-core-ci" --endpoint "${ci_endpoint}" -v stop "${instance_id}" - fi - - echo "instance_id: ${instance_id}" -} - -trap cleanup EXIT INT TERM - -if [ ${start_instance} ]; then - # shellcheck disable=SC2086 - "${source_root}/test/utils/shippable/ansible-core-ci" --endpoint "${ci_endpoint}" -v \ - start --id "${instance_id}" "${test_auth}" "${test_platform}" "${test_version}" ${args} -fi - -pip install "${source_root}" --upgrade -pip install -r "${source_root}/test/utils/shippable/remote-requirements.txt" --upgrade -pip list - -cd "${source_root}" -source hacking/env-setup -cd test/integration - -case "${test_platform}" in - "windows") - inventory_template="${source_root}/test/integration/inventory.winrm.template" - inventory_file="${source_root}/test/integration/inventory.winrm" - ping_module="win_ping" - ping_host="windows" - test_function="test_windows" - ;; - *) - inventory_template="${source_root}/test/integration/inventory.remote.template" - inventory_file="${source_root}/test/integration/inventory.remote" - ping_module="ping" - ping_host="remote" - test_function="test_remote" - ;; -esac - -"${source_root}/test/utils/shippable/ansible-core-ci" --endpoint "${ci_endpoint}" -v \ - get "${instance_id}" \ - --template "${inventory_template}" \ - > "${inventory_file}" \ - -# hack to make sure instance is responding before beginning tests -n=60 -for i in $(seq 1 ${n}); do - echo "Verifying host is responding ($i of $n)" - if \ - ANSIBLE_SSH_ARGS='' \ - ANSIBLE_HOST_KEY_CHECKING=False \ - ANSIBLE_FORCE_COLOR="${force_color}" \ - ansible -m "${ping_module}" -i "${inventory_file}" "${ping_host}"; then - break - fi - sleep 5 -done - -test_windows() { - JUNIT_OUTPUT_DIR="${source_root}/shippable/testresults" \ - ANSIBLE_FORCE_COLOR="${force_color}" \ - ANSIBLE_CALLBACK_WHITELIST=junit \ - TEST_FLAGS="${test_flags}" \ - LC_ALL=en_US.utf-8 \ - make "${test_target[@]}" -} - -test_remote() { - endpoint=$("${source_root}/test/utils/shippable/ansible-core-ci" --endpoint "${ci_endpoint}" get \ - "${instance_id}" \ - --template <(echo "@ansible_user@@ansible_host")) - ssh_port=$("${source_root}/test/utils/shippable/ansible-core-ci" --endpoint "${ci_endpoint}" get \ - "${instance_id}" \ - --template <(echo "@ansible_port")) - -( -cat < /tmp/remote-script.sh - -( -cat <