Add test support for Python 3.8 on RHEL and FreeBSD. (#73393)

* Support Python 3.8 on RHEL 8.2+ in ansible-test.

* Support Python 3.8 on FreeBSD in ansible-test.

* Use libyaml with PyYAML on FreeBSD.

changelogs/fragments/ansible-test-freebsd-python-3.8.yml

@@ -0,0 +1,2 @@
+minor_changes:
+  - ansible-test - FreeBSD 11.4 and 12.2 provisioning can now be used with the ``--python 3.8`` option.

changelogs/fragments/ansible-test-freebsd-pyyaml-libyaml.yml

@@ -0,0 +1,2 @@
+minor_changes:
+  - ansible-test - FreeBSD instances provisioned with ``--remote`` now make ``libyaml`` available for use with PyYAML installation.

changelogs/fragments/ansible-test-rhel-python-3.8.yml

@@ -0,0 +1,2 @@
+minor_changes:
+  - ansible-test - RHEL 8.2+ provisioning can now be used with the ``--python 3.8`` option, taking advantage of the Python 3.8 AppStream.

test/lib/ansible_test/_data/completion/remote.txt

@@ -1,7 +1,7 @@
 freebsd/11.1 python=2.7,3.6 python_dir=/usr/local/bin
-freebsd/11.4 python=2.7,3.7 python_dir=/usr/local/bin
+freebsd/11.4 python=2.7,3.7,3.8 python_dir=/usr/local/bin
 freebsd/12.1 python=3.6,2.7 python_dir=/usr/local/bin
-freebsd/12.2 python=3.7,2.7 python_dir=/usr/local/bin
+freebsd/12.2 python=3.7,2.7,3.8 python_dir=/usr/local/bin
 osx/10.11 python=2.7 python_dir=/usr/local/bin
 macos/10.15 python=3.8 python_dir=/usr/local/bin
 macos/11.1 python=3.9 python_dir=/usr/local/bin
@@ -9,7 +9,7 @@ rhel/7.6 python=2.7
 rhel/7.8 python=2.7
 rhel/7.9 python=2.7
 rhel/8.1 python=3.6
-rhel/8.2 python=3.6
-rhel/8.3 python=3.6
+rhel/8.2 python=3.6,3.8
+rhel/8.3 python=3.6,3.8
 aix/7.2 python=2.7 httptester=disabled temp-unicode=disabled pip-check=disabled
 power/centos/7 python=2.7

test/lib/ansible_test/_data/setup/remote.sh

@@ -3,7 +3,8 @@
 set -eu
 
 platform="$1"
-python_version="$2"
+platform_version="$2"
+python_version="$3"
 python_interpreter="python${python_version}"
 
 cd ~/
@@ -32,15 +33,41 @@ if [ "${platform}" = "freebsd" ]; then
         virtualenv_pkg=""
     fi
 
+    # Declare platform/python version combinations which do not have supporting OS packages available.
+    # For these combinations ansible-test will use pip to install the requirements instead.
+    case "${platform_version}/${python_version}" in
+        "11.4/3.8")
+            have_os_packages=""
+            ;;
+        "12.2/3.8")
+            have_os_packages=""
+            ;;
+        *)
+            have_os_packages="yes"
+            ;;
+    esac
+
+    # PyYAML is never installed with an OS package since it does not include libyaml support.
+    # Instead, ansible-test will always install it using pip.
+    if [ "${have_os_packages}" ]; then
+        jinja2_pkg="py${py_version}-Jinja2"
+        cryptography_pkg="py${py_version}-cryptography"
+    else
+        jinja2_pkg=""
+        cryptography_pkg=""
+    fi
+
     while true; do
+        # shellcheck disable=SC2086
         env ASSUME_ALWAYS_YES=YES pkg bootstrap && \
         pkg install -q -y \
             bash \
             curl \
             gtar \
+            libyaml \
             "python${py_version}" \
-            "py${py_version}-Jinja2" \
-            "py${py_version}-cryptography" \
+            ${jinja2_pkg} \
+            ${cryptography_pkg} \
             ${virtualenv_pkg} \
             sudo \
         && break
@@ -56,13 +83,21 @@ if [ "${platform}" = "freebsd" ]; then
     fi
 elif [ "${platform}" = "rhel" ]; then
     if grep '8\.' /etc/redhat-release; then
+        py_version="$(echo "${python_version}" | tr -d '.')"
+
+        if [ "${py_version}" = "36" ]; then
+            py_pkg_prefix="python3"
+        else
+            py_pkg_prefix="python${py_version}"
+        fi
+
         while true; do
-            yum module install -q -y python36 && \
+            yum module install -q -y "python${py_version}" && \
             yum install -q -y \
                 gcc \
-                python3-devel \
-                python3-jinja2 \
-                python3-cryptography \
+                "${py_pkg_prefix}-devel" \
+                "${py_pkg_prefix}-jinja2" \
+                "${py_pkg_prefix}-cryptography" \
                 iptables \
             && break
             echo "Failed to install packages. Sleeping before trying again..."

test/lib/ansible_test/_internal/manage_ci.py

@@ -269,7 +269,7 @@ class ManagePosixCI:
         :type python_version: str
         """
         self.upload(os.path.join(ANSIBLE_TEST_DATA_ROOT, 'setup', 'remote.sh'), '/tmp')
-        self.ssh('chmod +x /tmp/remote.sh && /tmp/remote.sh %s %s' % (self.core_ci.platform, python_version))
+        self.ssh('chmod +x /tmp/remote.sh && /tmp/remote.sh %s %s %s' % (self.core_ci.platform, self.core_ci.version, python_version))
 
     def upload_source(self):
         """Upload and extract source."""

test/lib/ansible_test/_internal/util.py

@@ -386,6 +386,14 @@ def common_environment():
         'CFLAGS',
     )
 
+    # FreeBSD Compatibility
+    # This is required to include libyaml support in PyYAML.
+    # The header /usr/local/include/yaml.h isn't in the default include path for the compiler.
+    # It is included here so that tests can take advantage of it, rather than only ansible-test during managed pip installs.
+    # If CFLAGS has been set in the environment that value will take precedence due to being an optional var when calling pass_vars.
+    if os.path.exists('/etc/freebsd-update.conf'):
+        env.update(CFLAGS='-I/usr/local/include/')
+
     env.update(pass_vars(required=required, optional=optional))
 
     return env