Support ignoring of certificates for ansible-galaxy during SCM cloning (#67616)

* Support ignoring of certificates for ansible-galaxy during SCM cloning

* Add integration tests installing a role from an untrusted repository

Test installing the role without --ignore-certs fails
Test installing the role with --ignore-certs is successful

changelogs/fragments/67616-galaxy-scm-cloning-ignore-certs.yml

@@ -0,0 +1,2 @@
+minor_changes:
+- ansible-galaxy - the option to skip certificate verification now also applies when cloning via SCM (git/hg) (https://github.com/ansible/ansible/issues/41077)

lib/ansible/utils/galaxy.py

@@ -25,6 +25,7 @@ from subprocess import Popen, PIPE
 import tarfile
 
 import ansible.constants as C
+from ansible import context
 from ansible.errors import AnsibleError
 from ansible.utils.display import Display
 from ansible.module_utils.common.process import get_bin_path
@@ -62,7 +63,19 @@ def scm_archive_resource(src, scm='git', name=None, version='HEAD', keep_scm_met
         raise AnsibleError("could not find/use %s, it is required to continue with installing %s" % (scm, src))
 
     tempdir = tempfile.mkdtemp(dir=C.DEFAULT_LOCAL_TMP)
-    clone_cmd = [scm_path, 'clone', src, name]
+    clone_cmd = [scm_path, 'clone']
+
+    # Add specific options for ignoring certificates if requested
+    ignore_certs = context.CLIARGS['ignore_certs']
+
+    if ignore_certs:
+        if scm == 'git':
+            clone_cmd.extend(['-c', 'http.sslVerify=false'])
+        elif scm == 'hg':
+            clone_cmd.append('--insecure')
+
+    clone_cmd.extend([src, name])
+
     run_scm_cmd(clone_cmd, tempdir)
 
     if scm == 'git' and version:

test/integration/targets/ansible-galaxy/cleanup-default.yml

@@ -1,5 +1,13 @@
-- name: remove unwanted packages
+- name: remove git package
   package:
     name: git
     state: absent
   when: git_install.changed
+- name: remove openssl package
+  package:
+    name: openssl
+    state: absent
+  when: ansible_distribution not in ["MacOSX", "Alpine"] and openssl_install.changed
+- name: remove openssl package
+  command: apk del openssl
+  when: ansible_distribution == "Alpine" and openssl_install.changed

test/integration/targets/ansible-galaxy/cleanup-freebsd.yml

@@ -1,6 +1,12 @@
-- name: remove auto-installed packages from FreeBSD
+- name: remove git from FreeBSD
   pkgng:
     name: git
     state: absent
     autoremove: yes
   when: git_install.changed
+- name: remove openssl from FreeBSD
+  pkgng:
+    name: openssl
+    state: absent
+    autoremove: yes
+  when: openssl_install.changed

test/integration/targets/ansible-galaxy/cleanup.yml

@@ -1,6 +1,8 @@
 - hosts: localhost
   vars:
     git_install: '{{ lookup("file", lookup("env", "OUTPUT_DIR") + "/git_install.json") | from_json }}'
+    openssl_install: '{{ lookup("file", lookup("env", "OUTPUT_DIR") + "/openssl_install.json") | from_json }}'
+    ws_dir: '{{ lookup("file", lookup("env", "OUTPUT_DIR") + "/ws_dir.json") | from_json }}'
   tasks:
     - name: cleanup
       include_tasks: "{{ cleanup_filename }}"
@@ -17,3 +19,8 @@
       loop:
         - "~/.ansible/collections/ansible_collections"
         - /usr/share/ansible/collections/ansible_collections
+
+    - name: Remove webserver directory
+      file:
+        path: "{{ ws_dir }}"
+        state: absent

test/integration/targets/ansible-galaxy/files/testserver.py

@@ -0,0 +1,20 @@
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import sys
+import ssl
+
+if __name__ == '__main__':
+    if sys.version_info[0] >= 3:
+        import http.server
+        import socketserver
+        Handler = http.server.SimpleHTTPRequestHandler
+        httpd = socketserver.TCPServer(("", 4443), Handler)
+    else:
+        import BaseHTTPServer
+        import SimpleHTTPServer
+        Handler = SimpleHTTPServer.SimpleHTTPRequestHandler
+        httpd = BaseHTTPServer.HTTPServer(("", 4443), Handler)
+
+    httpd.socket = ssl.wrap_socket(httpd.socket, certfile='./cert.pem', keyfile='./key.pem', server_side=True)
+    httpd.serve_forever()

test/integration/targets/ansible-galaxy/runme.sh

@@ -2,6 +2,20 @@
 
 set -eux -o pipefail
 
+galaxy_testdir="${OUTPUT_DIR}/galaxy-test-dir"
+role_testdir="${OUTPUT_DIR}/role-test-dir"
+# Prep the local git repos with role and make a tar archive so we can test
+# different things
+galaxy_local_test_role="test-role"
+galaxy_local_test_role_dir="${OUTPUT_DIR}/galaxy-role-test-root"
+galaxy_local_test_role_git_repo="${galaxy_local_test_role_dir}/${galaxy_local_test_role}"
+galaxy_local_test_role_tar="${galaxy_local_test_role_dir}/${galaxy_local_test_role}.tar"
+galaxy_webserver_root="${OUTPUT_DIR}/ansible-galaxy-webserver"
+
+mkdir -p "${galaxy_local_test_role_dir}"
+mkdir -p "${role_testdir}"
+mkdir -p "${galaxy_webserver_root}"
+
 ansible-playbook setup.yml "$@"
 
 trap 'ansible-playbook ${ANSIBLE_PLAYBOOK_DIR}/cleanup.yml' EXIT
@@ -51,22 +65,26 @@ f_ansible_galaxy_create_role_repo_post()
                 --format=tar \
                 --prefix="${repo_name}/" \
                 master > "${repo_tar}"
+            # Configure basic (insecure) HTTPS-accessible repository
+            galaxy_local_test_role_http_repo="${galaxy_webserver_root}/${galaxy_local_test_role}.git"
+            if [[ ! -d "${galaxy_local_test_role_http_repo}" ]]; then
+                git clone --bare "${galaxy_local_test_role_git_repo}" "${galaxy_local_test_role_http_repo}"
+                pushd "${galaxy_local_test_role_http_repo}"
+                    touch "git-daemon-export-ok"
+                    git --bare update-server-info
+                    mv "hooks/post-update.sample" "hooks/post-update"
+                popd # ${galaxy_local_test_role_http_repo}
+            fi
         popd # "${repo_name}"
     popd # "${repo_dir}"
 }
 
-# Prep the local git repos with role and make a tar archive so we can test
-# different things
-galaxy_local_test_role="test-role"
-galaxy_local_test_role_dir=$(mktemp -d)
-galaxy_local_test_role_git_repo="${galaxy_local_test_role_dir}/${galaxy_local_test_role}"
-galaxy_local_test_role_tar="${galaxy_local_test_role_dir}/${galaxy_local_test_role}.tar"
-
 f_ansible_galaxy_create_role_repo_pre "${galaxy_local_test_role}" "${galaxy_local_test_role_dir}"
 f_ansible_galaxy_create_role_repo_post "${galaxy_local_test_role}" "${galaxy_local_test_role_tar}"
 
 galaxy_local_parent_role="parent-role"
-galaxy_local_parent_role_dir=$(mktemp -d)
+galaxy_local_parent_role_dir="${OUTPUT_DIR}/parent-role"
+mkdir -p "${galaxy_local_parent_role_dir}"
 galaxy_local_parent_role_git_repo="${galaxy_local_parent_role_dir}/${galaxy_local_parent_role}"
 galaxy_local_parent_role_tar="${galaxy_local_parent_role_dir}/${galaxy_local_parent_role}.tar"
 
@@ -82,7 +100,7 @@ f_ansible_galaxy_create_role_repo_post "${galaxy_local_parent_role}" "${galaxy_l
 #
 # Install local git repo
 f_ansible_galaxy_status "install of local git repo"
-galaxy_testdir=$(mktemp -d)
+mkdir -p "${galaxy_testdir}"
 pushd "${galaxy_testdir}"
 
     ansible-galaxy install git+file:///"${galaxy_local_test_role_git_repo}" "$@"
@@ -97,7 +115,7 @@ rm -fr "${HOME}/.ansible/roles/${galaxy_local_test_role}"
 #
 # Install local git repo and ensure that if a role_path is passed, it is in fact used
 f_ansible_galaxy_status "install of local git repo with -p \$role_path"
-galaxy_testdir=$(mktemp -d)
+mkdir -p "${galaxy_testdir}"
 pushd "${galaxy_testdir}"
     mkdir -p "${galaxy_relative_rolespath}"
 
@@ -108,11 +126,40 @@ pushd "${galaxy_testdir}"
 popd # ${galaxy_testdir}
 rm -fr "${galaxy_testdir}"
 
+# Galaxy install test case - skipping cert verification
+#
+# Install from remote git repo and ensure that cert validation is skipped
+#
+# Protect against regression (GitHub Issue #41077)
+#   https://github.com/ansible/ansible/issues/41077
+f_ansible_galaxy_status "install of role from untrusted repository"
+mkdir -p "${galaxy_testdir}"
+pushd "${galaxy_testdir}"
+    mkdir -p "${galaxy_relative_rolespath}"
+
+    # Without --ignore-certs, installing a role from an untrusted repository should fail
+    set +e
+    ansible-galaxy install --verbose git+https://localhost:4443/"${galaxy_local_test_role}.git" -p "${galaxy_relative_rolespath}" "$@" 2>&1 | tee out.txt
+    ansible_exit_code="$?"
+    set -e
+    cat out.txt
+
+    if [[ "$ansible_exit_code" -ne 1 ]]; then echo "Exit code ($ansible_exit_code) is expected to be 1" && exit "$ansible_exit_code"; fi
+    [[ $(grep -c 'ERROR' out.txt) -eq 1 ]]
+    [[ ! -d "${galaxy_relative_rolespath}/${galaxy_local_test_role}" ]]
+
+    ansible-galaxy install --verbose --ignore-certs git+https://localhost:4443/"${galaxy_local_test_role}.git" -p "${galaxy_relative_rolespath}" "$@"
+
+    # Test that the role was installed to the expected directory
+    [[ -d "${galaxy_relative_rolespath}/${galaxy_local_test_role}" ]]
+popd # ${galaxy_testdir}
+rm -fr "${galaxy_testdir}"
+
 # Galaxy install test case
 #
 # Install local git repo with a meta/requirements.yml
 f_ansible_galaxy_status "install of local git repo with meta/requirements.yml"
-galaxy_testdir=$(mktemp -d)
+mkdir -p "${galaxy_testdir}"
 pushd "${galaxy_testdir}"
 
     ansible-galaxy install git+file:///"${galaxy_local_parent_role_git_repo}" "$@"
@@ -132,7 +179,7 @@ rm -fr "${HOME}/.ansible/roles/${galaxy_local_test_role}"
 #
 # Install local git repo with a meta/requirements.yml + --no-deps argument
 f_ansible_galaxy_status "install of local git repo with meta/requirements.yml + --no-deps argument"
-galaxy_testdir=$(mktemp -d)
+mkdir -p "${galaxy_testdir}"
 pushd "${galaxy_testdir}"
 
     ansible-galaxy install git+file:///"${galaxy_local_parent_role_git_repo}" --no-deps "$@"
@@ -158,7 +205,7 @@ rm -fr "${HOME}/.ansible/roles/${galaxy_local_test_role}"
 f_ansible_galaxy_status \
     "install of local git repo and local tarball with -p \$role_path and -r \$role_file" \
     "Protect against regression (Issue #35217)"
-galaxy_testdir=$(mktemp -d)
+mkdir -p "${galaxy_testdir}"
 pushd "${galaxy_testdir}"
 
     git clone "${galaxy_local_test_role_git_repo}" "${galaxy_local_test_role}"
@@ -189,7 +236,7 @@ rm -fr "${galaxy_testdir}"
 # Basic tests to ensure listing roles works
 
 f_ansible_galaxy_status "role list"
-galaxy_testdir=$(mktemp -d)
+mkdir -p "${galaxy_testdir}"
 pushd "${galaxy_testdir}"
     ansible-galaxy install git+file:///"${galaxy_local_test_role_git_repo}" "$@"
 
@@ -207,7 +254,7 @@ popd # ${galaxy_testdir}
 f_ansible_galaxy_status \
     "list specific role not in the first path in ANSIBLE_ROLES_PATH"
 
-role_testdir=$(mktemp -d)
+mkdir -p "${role_testdir}"
 pushd "${role_testdir}"
 
     mkdir testroles
@@ -229,7 +276,7 @@ rm -fr "${role_testdir}"
 # Get info about role that is not installed
 
 f_ansible_galaxy_status "role info"
-galaxy_testdir=$(mktemp -d)
+mkdir -p "${galaxy_testdir}"
 pushd "${galaxy_testdir}"
     ansible-galaxy role info samdoran.fish | tee out.txt
 
@@ -241,7 +288,7 @@ popd # ${galaxy_testdir}
 f_ansible_galaxy_status \
     "role info non-existant role"
 
-role_testdir=$(mktemp -d)
+mkdir -p "${role_testdir}"
 pushd "${role_testdir}"
 
     ansible-galaxy role info notaroll | tee out.txt
@@ -293,7 +340,7 @@ rm -fr "${role_testdir}"
 f_ansible_galaxy_status \
     "list roles where the role name is the same or a subset of the role path (#67365)"
 
-role_testdir=$(mktemp -d)
+mkdir -p "${role_testdir}"
 pushd "${role_testdir}"
 
     mkdir parrot
@@ -312,7 +359,7 @@ rm -rf "${role_testdir}"
 f_ansible_galaxy_status \
     "Test role with non-ascii characters"
 
-role_testdir=$(mktemp -d)
+mkdir -p "${role_testdir}"
 pushd "${role_testdir}"
 
     mkdir nonascii
@@ -342,7 +389,7 @@ rm -rf "${role_testdir}"
 #################################
 # TODO: Move these to ansible-galaxy-collection
 
-galaxy_testdir=$(mktemp -d)
+mkdir -p "${galaxy_testdir}"
 pushd "${galaxy_testdir}"
 
 ## ansible-galaxy collection list tests

test/integration/targets/ansible-galaxy/setup.yml

@@ -1,11 +1,57 @@
 - hosts: localhost
+  vars:
+    ws_dir: '{{ lookup("env", "OUTPUT_DIR") }}/ansible-galaxy-webserver'
   tasks:
-    - name: install git
+    - name: install git & OpenSSL
       package:
         name: git
       when: ansible_distribution not in ["MacOSX", "Alpine"]
       register: git_install
-    - name: save install result
+
+    - name: install OpenSSL
+      package:
+        name: openssl
+      when: ansible_distribution not in ["MacOSX", "Alpine"]
+      register: openssl_install
+
+    - name: install OpenSSL
+      command: apk add openssl
+      when: ansible_distribution == "Alpine"
+      register: openssl_install
+
+    - name: setup webserver dir
+      file:
+        state: directory
+        path: "{{ ws_dir }}"
+
+    - name: copy webserver
+      copy:
+        src: testserver.py
+        dest: "{{ ws_dir }}"
+
+    - name: Create rand file
+      command: dd if=/dev/urandom of="{{ ws_dir }}/.rnd" bs=256 count=1
+
+    - name: Create self-signed cert
+      shell: RANDFILE={{ ws_dir }}/.rnd openssl req -x509 -newkey rsa:2048 \
+        -nodes -days 365 -keyout "{{ ws_dir }}/key.pem" -out "{{ ws_dir }}/cert.pem" \
+        -subj "/C=GB/O=Red Hat/OU=Ansible/CN=ansible-test-cert"
+
+    - name: start SimpleHTTPServer
+      shell: cd {{ ws_dir }} && {{ ansible_python.executable }} {{ ws_dir }}/testserver.py
+      async: 120 # this test set can take ~1m to run on FreeBSD (via Shippable)
+      poll: 0
+
+    - wait_for: port=4443
+
+    - name: save results
       copy:
-        content: '{{ git_install }}'
+        content: "{{ item.content }}"
-        dest: '{{ lookup("env", "OUTPUT_DIR") }}/git_install.json'
+        dest: '{{ lookup("env", "OUTPUT_DIR") }}/{{ item.key }}.json'
+      loop:
+        - key: git_install
+          content: "{{ git_install }}"
+        - key: openssl_install
+          content: "{{ openssl_install }}"
+        - key: ws_dir
+          content: "{{ ws_dir | to_json }}"