From ea7f24a1d58557b3ecdde9f7de765eb931c17230 Mon Sep 17 00:00:00 2001 From: James Milligan Date: Thu, 24 Mar 2022 17:56:21 +0000 Subject: [PATCH] Support ignoring of certificates for ansible-galaxy during SCM cloning (#67616) * Support ignoring of certificates for ansible-galaxy during SCM cloning * Add integration tests installing a role from an untrusted repository Test installing the role without --ignore-certs fails Test installing the role with --ignore-certs is successful --- .../67616-galaxy-scm-cloning-ignore-certs.yml | 2 + lib/ansible/utils/galaxy.py | 15 +++- .../ansible-galaxy/cleanup-default.yml | 10 ++- .../ansible-galaxy/cleanup-freebsd.yml | 8 +- .../targets/ansible-galaxy/cleanup.yml | 7 ++ .../ansible-galaxy/files/testserver.py | 20 +++++ .../targets/ansible-galaxy/runme.sh | 87 ++++++++++++++----- .../targets/ansible-galaxy/setup.yml | 54 +++++++++++- 8 files changed, 176 insertions(+), 27 deletions(-) create mode 100644 changelogs/fragments/67616-galaxy-scm-cloning-ignore-certs.yml create mode 100644 test/integration/targets/ansible-galaxy/files/testserver.py diff --git a/changelogs/fragments/67616-galaxy-scm-cloning-ignore-certs.yml b/changelogs/fragments/67616-galaxy-scm-cloning-ignore-certs.yml new file mode 100644 index 00000000000..f42712805cf --- /dev/null +++ b/changelogs/fragments/67616-galaxy-scm-cloning-ignore-certs.yml @@ -0,0 +1,2 @@ +minor_changes: +- ansible-galaxy - the option to skip certificate verification now also applies when cloning via SCM (git/hg) (https://github.com/ansible/ansible/issues/41077) diff --git a/lib/ansible/utils/galaxy.py b/lib/ansible/utils/galaxy.py index cb1f125be1f..bbb26fb1115 100644 --- a/lib/ansible/utils/galaxy.py +++ b/lib/ansible/utils/galaxy.py @@ -25,6 +25,7 @@ from subprocess import Popen, PIPE import tarfile import ansible.constants as C +from ansible import context from ansible.errors import AnsibleError from ansible.utils.display import Display from ansible.module_utils.common.process import get_bin_path @@ -62,7 +63,19 @@ def scm_archive_resource(src, scm='git', name=None, version='HEAD', keep_scm_met raise AnsibleError("could not find/use %s, it is required to continue with installing %s" % (scm, src)) tempdir = tempfile.mkdtemp(dir=C.DEFAULT_LOCAL_TMP) - clone_cmd = [scm_path, 'clone', src, name] + clone_cmd = [scm_path, 'clone'] + + # Add specific options for ignoring certificates if requested + ignore_certs = context.CLIARGS['ignore_certs'] + + if ignore_certs: + if scm == 'git': + clone_cmd.extend(['-c', 'http.sslVerify=false']) + elif scm == 'hg': + clone_cmd.append('--insecure') + + clone_cmd.extend([src, name]) + run_scm_cmd(clone_cmd, tempdir) if scm == 'git' and version: diff --git a/test/integration/targets/ansible-galaxy/cleanup-default.yml b/test/integration/targets/ansible-galaxy/cleanup-default.yml index f2265c093db..80600792bcb 100644 --- a/test/integration/targets/ansible-galaxy/cleanup-default.yml +++ b/test/integration/targets/ansible-galaxy/cleanup-default.yml @@ -1,5 +1,13 @@ -- name: remove unwanted packages +- name: remove git package package: name: git state: absent when: git_install.changed +- name: remove openssl package + package: + name: openssl + state: absent + when: ansible_distribution not in ["MacOSX", "Alpine"] and openssl_install.changed +- name: remove openssl package + command: apk del openssl + when: ansible_distribution == "Alpine" and openssl_install.changed diff --git a/test/integration/targets/ansible-galaxy/cleanup-freebsd.yml b/test/integration/targets/ansible-galaxy/cleanup-freebsd.yml index fa224d830de..87b987d1312 100644 --- a/test/integration/targets/ansible-galaxy/cleanup-freebsd.yml +++ b/test/integration/targets/ansible-galaxy/cleanup-freebsd.yml @@ -1,6 +1,12 @@ -- name: remove auto-installed packages from FreeBSD +- name: remove git from FreeBSD pkgng: name: git state: absent autoremove: yes when: git_install.changed +- name: remove openssl from FreeBSD + pkgng: + name: openssl + state: absent + autoremove: yes + when: openssl_install.changed diff --git a/test/integration/targets/ansible-galaxy/cleanup.yml b/test/integration/targets/ansible-galaxy/cleanup.yml index 57442631f3a..e80eeefbae8 100644 --- a/test/integration/targets/ansible-galaxy/cleanup.yml +++ b/test/integration/targets/ansible-galaxy/cleanup.yml @@ -1,6 +1,8 @@ - hosts: localhost vars: git_install: '{{ lookup("file", lookup("env", "OUTPUT_DIR") + "/git_install.json") | from_json }}' + openssl_install: '{{ lookup("file", lookup("env", "OUTPUT_DIR") + "/openssl_install.json") | from_json }}' + ws_dir: '{{ lookup("file", lookup("env", "OUTPUT_DIR") + "/ws_dir.json") | from_json }}' tasks: - name: cleanup include_tasks: "{{ cleanup_filename }}" @@ -17,3 +19,8 @@ loop: - "~/.ansible/collections/ansible_collections" - /usr/share/ansible/collections/ansible_collections + + - name: Remove webserver directory + file: + path: "{{ ws_dir }}" + state: absent diff --git a/test/integration/targets/ansible-galaxy/files/testserver.py b/test/integration/targets/ansible-galaxy/files/testserver.py new file mode 100644 index 00000000000..135985075ab --- /dev/null +++ b/test/integration/targets/ansible-galaxy/files/testserver.py @@ -0,0 +1,20 @@ +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import sys +import ssl + +if __name__ == '__main__': + if sys.version_info[0] >= 3: + import http.server + import socketserver + Handler = http.server.SimpleHTTPRequestHandler + httpd = socketserver.TCPServer(("", 4443), Handler) + else: + import BaseHTTPServer + import SimpleHTTPServer + Handler = SimpleHTTPServer.SimpleHTTPRequestHandler + httpd = BaseHTTPServer.HTTPServer(("", 4443), Handler) + + httpd.socket = ssl.wrap_socket(httpd.socket, certfile='./cert.pem', keyfile='./key.pem', server_side=True) + httpd.serve_forever() diff --git a/test/integration/targets/ansible-galaxy/runme.sh b/test/integration/targets/ansible-galaxy/runme.sh index 1f2d56b311e..4bb459588ad 100755 --- a/test/integration/targets/ansible-galaxy/runme.sh +++ b/test/integration/targets/ansible-galaxy/runme.sh @@ -2,6 +2,20 @@ set -eux -o pipefail +galaxy_testdir="${OUTPUT_DIR}/galaxy-test-dir" +role_testdir="${OUTPUT_DIR}/role-test-dir" +# Prep the local git repos with role and make a tar archive so we can test +# different things +galaxy_local_test_role="test-role" +galaxy_local_test_role_dir="${OUTPUT_DIR}/galaxy-role-test-root" +galaxy_local_test_role_git_repo="${galaxy_local_test_role_dir}/${galaxy_local_test_role}" +galaxy_local_test_role_tar="${galaxy_local_test_role_dir}/${galaxy_local_test_role}.tar" +galaxy_webserver_root="${OUTPUT_DIR}/ansible-galaxy-webserver" + +mkdir -p "${galaxy_local_test_role_dir}" +mkdir -p "${role_testdir}" +mkdir -p "${galaxy_webserver_root}" + ansible-playbook setup.yml "$@" trap 'ansible-playbook ${ANSIBLE_PLAYBOOK_DIR}/cleanup.yml' EXIT @@ -51,22 +65,26 @@ f_ansible_galaxy_create_role_repo_post() --format=tar \ --prefix="${repo_name}/" \ master > "${repo_tar}" + # Configure basic (insecure) HTTPS-accessible repository + galaxy_local_test_role_http_repo="${galaxy_webserver_root}/${galaxy_local_test_role}.git" + if [[ ! -d "${galaxy_local_test_role_http_repo}" ]]; then + git clone --bare "${galaxy_local_test_role_git_repo}" "${galaxy_local_test_role_http_repo}" + pushd "${galaxy_local_test_role_http_repo}" + touch "git-daemon-export-ok" + git --bare update-server-info + mv "hooks/post-update.sample" "hooks/post-update" + popd # ${galaxy_local_test_role_http_repo} + fi popd # "${repo_name}" popd # "${repo_dir}" } -# Prep the local git repos with role and make a tar archive so we can test -# different things -galaxy_local_test_role="test-role" -galaxy_local_test_role_dir=$(mktemp -d) -galaxy_local_test_role_git_repo="${galaxy_local_test_role_dir}/${galaxy_local_test_role}" -galaxy_local_test_role_tar="${galaxy_local_test_role_dir}/${galaxy_local_test_role}.tar" - f_ansible_galaxy_create_role_repo_pre "${galaxy_local_test_role}" "${galaxy_local_test_role_dir}" f_ansible_galaxy_create_role_repo_post "${galaxy_local_test_role}" "${galaxy_local_test_role_tar}" galaxy_local_parent_role="parent-role" -galaxy_local_parent_role_dir=$(mktemp -d) +galaxy_local_parent_role_dir="${OUTPUT_DIR}/parent-role" +mkdir -p "${galaxy_local_parent_role_dir}" galaxy_local_parent_role_git_repo="${galaxy_local_parent_role_dir}/${galaxy_local_parent_role}" galaxy_local_parent_role_tar="${galaxy_local_parent_role_dir}/${galaxy_local_parent_role}.tar" @@ -82,7 +100,7 @@ f_ansible_galaxy_create_role_repo_post "${galaxy_local_parent_role}" "${galaxy_l # # Install local git repo f_ansible_galaxy_status "install of local git repo" -galaxy_testdir=$(mktemp -d) +mkdir -p "${galaxy_testdir}" pushd "${galaxy_testdir}" ansible-galaxy install git+file:///"${galaxy_local_test_role_git_repo}" "$@" @@ -97,7 +115,7 @@ rm -fr "${HOME}/.ansible/roles/${galaxy_local_test_role}" # # Install local git repo and ensure that if a role_path is passed, it is in fact used f_ansible_galaxy_status "install of local git repo with -p \$role_path" -galaxy_testdir=$(mktemp -d) +mkdir -p "${galaxy_testdir}" pushd "${galaxy_testdir}" mkdir -p "${galaxy_relative_rolespath}" @@ -108,11 +126,40 @@ pushd "${galaxy_testdir}" popd # ${galaxy_testdir} rm -fr "${galaxy_testdir}" +# Galaxy install test case - skipping cert verification +# +# Install from remote git repo and ensure that cert validation is skipped +# +# Protect against regression (GitHub Issue #41077) +# https://github.com/ansible/ansible/issues/41077 +f_ansible_galaxy_status "install of role from untrusted repository" +mkdir -p "${galaxy_testdir}" +pushd "${galaxy_testdir}" + mkdir -p "${galaxy_relative_rolespath}" + + # Without --ignore-certs, installing a role from an untrusted repository should fail + set +e + ansible-galaxy install --verbose git+https://localhost:4443/"${galaxy_local_test_role}.git" -p "${galaxy_relative_rolespath}" "$@" 2>&1 | tee out.txt + ansible_exit_code="$?" + set -e + cat out.txt + + if [[ "$ansible_exit_code" -ne 1 ]]; then echo "Exit code ($ansible_exit_code) is expected to be 1" && exit "$ansible_exit_code"; fi + [[ $(grep -c 'ERROR' out.txt) -eq 1 ]] + [[ ! -d "${galaxy_relative_rolespath}/${galaxy_local_test_role}" ]] + + ansible-galaxy install --verbose --ignore-certs git+https://localhost:4443/"${galaxy_local_test_role}.git" -p "${galaxy_relative_rolespath}" "$@" + + # Test that the role was installed to the expected directory + [[ -d "${galaxy_relative_rolespath}/${galaxy_local_test_role}" ]] +popd # ${galaxy_testdir} +rm -fr "${galaxy_testdir}" + # Galaxy install test case # # Install local git repo with a meta/requirements.yml f_ansible_galaxy_status "install of local git repo with meta/requirements.yml" -galaxy_testdir=$(mktemp -d) +mkdir -p "${galaxy_testdir}" pushd "${galaxy_testdir}" ansible-galaxy install git+file:///"${galaxy_local_parent_role_git_repo}" "$@" @@ -132,7 +179,7 @@ rm -fr "${HOME}/.ansible/roles/${galaxy_local_test_role}" # # Install local git repo with a meta/requirements.yml + --no-deps argument f_ansible_galaxy_status "install of local git repo with meta/requirements.yml + --no-deps argument" -galaxy_testdir=$(mktemp -d) +mkdir -p "${galaxy_testdir}" pushd "${galaxy_testdir}" ansible-galaxy install git+file:///"${galaxy_local_parent_role_git_repo}" --no-deps "$@" @@ -158,7 +205,7 @@ rm -fr "${HOME}/.ansible/roles/${galaxy_local_test_role}" f_ansible_galaxy_status \ "install of local git repo and local tarball with -p \$role_path and -r \$role_file" \ "Protect against regression (Issue #35217)" -galaxy_testdir=$(mktemp -d) +mkdir -p "${galaxy_testdir}" pushd "${galaxy_testdir}" git clone "${galaxy_local_test_role_git_repo}" "${galaxy_local_test_role}" @@ -189,7 +236,7 @@ rm -fr "${galaxy_testdir}" # Basic tests to ensure listing roles works f_ansible_galaxy_status "role list" -galaxy_testdir=$(mktemp -d) +mkdir -p "${galaxy_testdir}" pushd "${galaxy_testdir}" ansible-galaxy install git+file:///"${galaxy_local_test_role_git_repo}" "$@" @@ -207,7 +254,7 @@ popd # ${galaxy_testdir} f_ansible_galaxy_status \ "list specific role not in the first path in ANSIBLE_ROLES_PATH" -role_testdir=$(mktemp -d) +mkdir -p "${role_testdir}" pushd "${role_testdir}" mkdir testroles @@ -229,7 +276,7 @@ rm -fr "${role_testdir}" # Get info about role that is not installed f_ansible_galaxy_status "role info" -galaxy_testdir=$(mktemp -d) +mkdir -p "${galaxy_testdir}" pushd "${galaxy_testdir}" ansible-galaxy role info samdoran.fish | tee out.txt @@ -241,7 +288,7 @@ popd # ${galaxy_testdir} f_ansible_galaxy_status \ "role info non-existant role" -role_testdir=$(mktemp -d) +mkdir -p "${role_testdir}" pushd "${role_testdir}" ansible-galaxy role info notaroll | tee out.txt @@ -293,7 +340,7 @@ rm -fr "${role_testdir}" f_ansible_galaxy_status \ "list roles where the role name is the same or a subset of the role path (#67365)" -role_testdir=$(mktemp -d) +mkdir -p "${role_testdir}" pushd "${role_testdir}" mkdir parrot @@ -312,7 +359,7 @@ rm -rf "${role_testdir}" f_ansible_galaxy_status \ "Test role with non-ascii characters" -role_testdir=$(mktemp -d) +mkdir -p "${role_testdir}" pushd "${role_testdir}" mkdir nonascii @@ -342,7 +389,7 @@ rm -rf "${role_testdir}" ################################# # TODO: Move these to ansible-galaxy-collection -galaxy_testdir=$(mktemp -d) +mkdir -p "${galaxy_testdir}" pushd "${galaxy_testdir}" ## ansible-galaxy collection list tests diff --git a/test/integration/targets/ansible-galaxy/setup.yml b/test/integration/targets/ansible-galaxy/setup.yml index ebd5a1c08be..b4fb6d3b60c 100644 --- a/test/integration/targets/ansible-galaxy/setup.yml +++ b/test/integration/targets/ansible-galaxy/setup.yml @@ -1,11 +1,57 @@ - hosts: localhost + vars: + ws_dir: '{{ lookup("env", "OUTPUT_DIR") }}/ansible-galaxy-webserver' tasks: - - name: install git + - name: install git & OpenSSL package: name: git when: ansible_distribution not in ["MacOSX", "Alpine"] register: git_install - - name: save install result + + - name: install OpenSSL + package: + name: openssl + when: ansible_distribution not in ["MacOSX", "Alpine"] + register: openssl_install + + - name: install OpenSSL + command: apk add openssl + when: ansible_distribution == "Alpine" + register: openssl_install + + - name: setup webserver dir + file: + state: directory + path: "{{ ws_dir }}" + + - name: copy webserver + copy: + src: testserver.py + dest: "{{ ws_dir }}" + + - name: Create rand file + command: dd if=/dev/urandom of="{{ ws_dir }}/.rnd" bs=256 count=1 + + - name: Create self-signed cert + shell: RANDFILE={{ ws_dir }}/.rnd openssl req -x509 -newkey rsa:2048 \ + -nodes -days 365 -keyout "{{ ws_dir }}/key.pem" -out "{{ ws_dir }}/cert.pem" \ + -subj "/C=GB/O=Red Hat/OU=Ansible/CN=ansible-test-cert" + + - name: start SimpleHTTPServer + shell: cd {{ ws_dir }} && {{ ansible_python.executable }} {{ ws_dir }}/testserver.py + async: 120 # this test set can take ~1m to run on FreeBSD (via Shippable) + poll: 0 + + - wait_for: port=4443 + + - name: save results copy: - content: '{{ git_install }}' - dest: '{{ lookup("env", "OUTPUT_DIR") }}/git_install.json' + content: "{{ item.content }}" + dest: '{{ lookup("env", "OUTPUT_DIR") }}/{{ item.key }}.json' + loop: + - key: git_install + content: "{{ git_install }}" + - key: openssl_install + content: "{{ openssl_install }}" + - key: ws_dir + content: "{{ ws_dir | to_json }}"