Refactor ansible-test integration tests. (#78168)

* Relocate update-ignore.py for easier re-use.

* Add script to ease collection testing.

* Skip ignore rewrite if file does not exist.

* Add integration test for the shebang sanity test.

* Fix ansible-test-no-tty integration test.

Previously the test only verified a TTY was not used if a TTY already existed.
This prevented the test from verifying behavior when run in CI.
Now the test creates a PTY before invoking ansible-test.

* Clean up ansible-test-docker integration test.
pull/76306/merge
Matt Clay 2 years ago committed by GitHub
parent bcdc2e167a
commit f70cc2fb7e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -1,2 +1,3 @@
shippable/generic/group1 # Runs in the default test container so access to tools like pwsh
context/controller
needs/target/collection

@ -1,18 +0,0 @@
#!/usr/bin/env bash
set -eux -o pipefail
cp -a "${TEST_DIR}/ansible_collections" "${WORK_DIR}"
cd "${WORK_DIR}/ansible_collections/ns/col"
# common args for all tests
# because we are running in shippable/generic/ we are already in the default docker container
common=(--python "${ANSIBLE_TEST_PYTHON_VERSION}" --venv --venv-system-site-packages --color --truncate 0 "${@}")
# prime the venv to work around issue with PyYAML detection in ansible-test
ansible-test sanity "${common[@]}" --test ignores
# tests
ansible-test sanity "${common[@]}"
ansible-test units "${common[@]}"
ansible-test integration "${common[@]}"

@ -1,24 +1,14 @@
#!/usr/bin/env bash
set -eu -o pipefail
source ../collection/setup.sh
# tests must be executed outside of the ansible source tree
# otherwise ansible-test will test the ansible source instead of the test collection
# the temporary directory provided by ansible-test resides within the ansible source tree
tmp_dir=$(mktemp -d)
set -x
trap 'rm -rf "${tmp_dir}"' EXIT
# common args for all tests
# because we are running in shippable/generic/ we are already in the default docker container
common=(--python "${ANSIBLE_TEST_PYTHON_VERSION}" --venv --venv-system-site-packages --color --truncate 0 "${@}")
export TEST_DIR
export WORK_DIR
TEST_DIR="$PWD"
for test in collection-tests/*.sh; do
WORK_DIR="${tmp_dir}/$(basename "${test}" ".sh")"
mkdir "${WORK_DIR}"
echo "**********************************************************************"
echo "TEST: ${test}: STARTING"
"${test}" "${@}" || (echo "TEST: ${test}: FAILED" && exit 1)
echo "TEST: ${test}: PASSED"
done
# tests
ansible-test sanity "${common[@]}"
ansible-test units "${common[@]}"
ansible-test integration "${common[@]}"

@ -1,2 +1,4 @@
context/controller
shippable/posix/group1
shippable/posix/group1 # runs in the distro test containers
shippable/generic/group1 # runs in the default test container
needs/target/collection

@ -0,0 +1,11 @@
#!/usr/bin/env python
"""Run a command using a PTY."""
import sys
if sys.version_info < (3, 10):
import vendored_pty as pty
else:
import pty
sys.exit(1 if pty.spawn(sys.argv[1:]) else 0)

@ -0,0 +1,13 @@
#!/usr/bin/env python
"""Assert no TTY is available."""
import sys
status = 0
for handle in sys.stdin, sys.stdout, sys.stderr:
if handle.isatty():
print(f'{handle} is a TTY', file=sys.stderr)
status += 1
sys.exit(status)

@ -0,0 +1,189 @@
# Vendored copy of https://github.com/python/cpython/blob/3680ebed7f3e529d01996dd0318601f9f0d02b4b/Lib/pty.py
# PSF License (see licenses/PSF-license.txt or https://opensource.org/licenses/Python-2.0)
"""Pseudo terminal utilities."""
# Bugs: No signal handling. Doesn't set slave termios and window size.
# Only tested on Linux, FreeBSD, and macOS.
# See: W. Richard Stevens. 1992. Advanced Programming in the
# UNIX Environment. Chapter 19.
# Author: Steen Lumholt -- with additions by Guido.
from select import select
import os
import sys
import tty
# names imported directly for test mocking purposes
from os import close, waitpid
from tty import setraw, tcgetattr, tcsetattr
__all__ = ["openpty", "fork", "spawn"]
STDIN_FILENO = 0
STDOUT_FILENO = 1
STDERR_FILENO = 2
CHILD = 0
def openpty():
"""openpty() -> (master_fd, slave_fd)
Open a pty master/slave pair, using os.openpty() if possible."""
try:
return os.openpty()
except (AttributeError, OSError):
pass
master_fd, slave_name = _open_terminal()
slave_fd = slave_open(slave_name)
return master_fd, slave_fd
def master_open():
"""master_open() -> (master_fd, slave_name)
Open a pty master and return the fd, and the filename of the slave end.
Deprecated, use openpty() instead."""
try:
master_fd, slave_fd = os.openpty()
except (AttributeError, OSError):
pass
else:
slave_name = os.ttyname(slave_fd)
os.close(slave_fd)
return master_fd, slave_name
return _open_terminal()
def _open_terminal():
"""Open pty master and return (master_fd, tty_name)."""
for x in 'pqrstuvwxyzPQRST':
for y in '0123456789abcdef':
pty_name = '/dev/pty' + x + y
try:
fd = os.open(pty_name, os.O_RDWR)
except OSError:
continue
return (fd, '/dev/tty' + x + y)
raise OSError('out of pty devices')
def slave_open(tty_name):
"""slave_open(tty_name) -> slave_fd
Open the pty slave and acquire the controlling terminal, returning
opened filedescriptor.
Deprecated, use openpty() instead."""
result = os.open(tty_name, os.O_RDWR)
try:
from fcntl import ioctl, I_PUSH
except ImportError:
return result
try:
ioctl(result, I_PUSH, "ptem")
ioctl(result, I_PUSH, "ldterm")
except OSError:
pass
return result
def fork():
"""fork() -> (pid, master_fd)
Fork and make the child a session leader with a controlling terminal."""
try:
pid, fd = os.forkpty()
except (AttributeError, OSError):
pass
else:
if pid == CHILD:
try:
os.setsid()
except OSError:
# os.forkpty() already set us session leader
pass
return pid, fd
master_fd, slave_fd = openpty()
pid = os.fork()
if pid == CHILD:
# Establish a new session.
os.setsid()
os.close(master_fd)
# Slave becomes stdin/stdout/stderr of child.
os.dup2(slave_fd, STDIN_FILENO)
os.dup2(slave_fd, STDOUT_FILENO)
os.dup2(slave_fd, STDERR_FILENO)
if slave_fd > STDERR_FILENO:
os.close(slave_fd)
# Explicitly open the tty to make it become a controlling tty.
tmp_fd = os.open(os.ttyname(STDOUT_FILENO), os.O_RDWR)
os.close(tmp_fd)
else:
os.close(slave_fd)
# Parent and child process.
return pid, master_fd
def _writen(fd, data):
"""Write all the data to a descriptor."""
while data:
n = os.write(fd, data)
data = data[n:]
def _read(fd):
"""Default read function."""
return os.read(fd, 1024)
def _copy(master_fd, master_read=_read, stdin_read=_read):
"""Parent copy loop.
Copies
pty master -> standard output (master_read)
standard input -> pty master (stdin_read)"""
fds = [master_fd, STDIN_FILENO]
while fds:
rfds, _wfds, _xfds = select(fds, [], [])
if master_fd in rfds:
# Some OSes signal EOF by returning an empty byte string,
# some throw OSErrors.
try:
data = master_read(master_fd)
except OSError:
data = b""
if not data: # Reached EOF.
return # Assume the child process has exited and is
# unreachable, so we clean up.
else:
os.write(STDOUT_FILENO, data)
if STDIN_FILENO in rfds:
data = stdin_read(STDIN_FILENO)
if not data:
fds.remove(STDIN_FILENO)
else:
_writen(master_fd, data)
def spawn(argv, master_read=_read, stdin_read=_read):
"""Create a spawned process."""
if isinstance(argv, str):
argv = (argv,)
sys.audit('pty.spawn', argv)
pid, master_fd = fork()
if pid == CHILD:
os.execlp(argv[0], *argv)
try:
mode = tcgetattr(STDIN_FILENO)
setraw(STDIN_FILENO)
restore = True
except tty.error: # This is the same as termios.error
restore = False
try:
_copy(master_fd, master_read, stdin_read)
finally:
if restore:
tcsetattr(STDIN_FILENO, tty.TCSAFLUSH, mode)
close(master_fd)
return waitpid(pid, 0)[1]

@ -1,7 +0,0 @@
#!/usr/bin/env python
import sys
assert not sys.stdin.isatty()
assert not sys.stdout.isatty()
assert not sys.stderr.isatty()

@ -1,5 +1,13 @@
#!/usr/bin/env bash
# Verify that ansible-test runs integration tests without a TTY.
set -eux
source ../collection/setup.sh
./runme.py
set -x
if ./run-with-pty.py tests/integration/targets/no-tty/assert-no-tty.py > /dev/null; then
echo "PTY assertion did not fail. Either PTY creation failed or PTY detection is broken."
exit 1
fi
./run-with-pty.py ansible-test integration --color "${@}"

@ -0,0 +1,4 @@
shippable/posix/group1 # runs in the distro test containers
shippable/generic/group1 # runs in the default test container
context/controller
needs/target/collection

@ -0,0 +1,9 @@
plugins/modules/no-shebang-executable.py:0:0: file without shebang should not be executable
plugins/modules/python-executable.py:0:0: module should not be executable
plugins/modules/python-wrong-shebang.py:1:1: expected module shebang "b'#!/usr/bin/python'" but found: b'#!invalid'
plugins/modules/utf-16-be-bom.py:0:0: file starts with a UTF-16 (BE) byte order mark
plugins/modules/utf-16-le-bom.py:0:0: file starts with a UTF-16 (LE) byte order mark
plugins/modules/utf-32-be-bom.py:0:0: file starts with a UTF-32 (BE) byte order mark
plugins/modules/utf-32-le-bom.py:0:0: file starts with a UTF-32 (LE) byte order mark
plugins/modules/utf-8-bom.py:0:0: file starts with a UTF-8 byte order mark
scripts/unexpected-shebang:1:1: unexpected non-module shebang: b'#!/usr/bin/custom'

@ -0,0 +1,47 @@
#!/usr/bin/env bash
set -eu
# Create test scenarios at runtime that do not pass sanity tests.
# This avoids the need to create ignore entries for the tests.
(
cd ansible_collections/ns/col/plugins/modules
touch no-shebang-executable.py && chmod +x no-shebang-executable.py # file without shebang should not be executable
python -c "open('utf-32-be-bom.py', 'wb').write(b'\x00\x00\xFE\xFF')" # file starts with a UTF-32 (BE) byte order mark
python -c "open('utf-32-le-bom.py', 'wb').write(b'\xFF\xFE\x00\x00')" # file starts with a UTF-32 (LE) byte order mark
python -c "open('utf-16-be-bom.py', 'wb').write(b'\xFE\xFF')" # file starts with a UTF-16 (BE) byte order mark
python -c "open('utf-16-le-bom.py', 'wb').write(b'\xFF\xFE')" # file starts with a UTF-16 (LE) byte order mark
python -c "open('utf-8-bom.py', 'wb').write(b'\xEF\xBB\xBF')" # file starts with a UTF-8 byte order mark
echo '#!/usr/bin/python' > python-executable.py && chmod +x python-executable.py # module should not be executable
echo '#!invalid' > python-wrong-shebang.py # expected module shebang "b'#!/usr/bin/python'" but found: b'#!invalid'
)
(
cd ansible_collections/ns/col/scripts
echo '#!/usr/bin/custom' > unexpected-shebang # unexpected non-module shebang: b'#!/usr/bin/custom'
echo '#!/usr/bin/make -f' > Makefile && chmod +x Makefile # pass
echo '#!/bin/bash -eu' > bash_eu.sh && chmod +x bash_eu.sh # pass
echo '#!/bin/bash -eux' > bash_eux.sh && chmod +x bash_eux.sh # pass
echo '#!/usr/bin/env fish' > env_fish.fish && chmod +x env_fish.fish # pass
echo '#!/usr/bin/env pwsh' > env_pwsh.ps1 && chmod +x env_pwsh.ps1 # pass
)
mkdir ansible_collections/ns/col/examples
(
cd ansible_collections/ns/col/examples
echo '#!/usr/bin/custom' > unexpected-shebang # pass
)
source ../collection/setup.sh
set -x
ansible-test sanity --test shebang --color --lint --failure-ok "${@}" > actual.txt
diff -u "${TEST_DIR}/expected.txt" actual.txt

@ -1,4 +1,5 @@
shippable/posix/group1 # runs in the distro test containers
shippable/generic/group1 # runs in the default test container
context/controller
needs/target/collection
destructive # adds and then removes packages into lib/ansible/_vendor/

@ -5,7 +5,7 @@ set -eux -o pipefail
cp -a "${TEST_DIR}/ansible_collections" "${WORK_DIR}"
cd "${WORK_DIR}/ansible_collections/ns/col"
"${TEST_DIR}/collection-tests/update-ignore.py"
"${TEST_DIR}/../collection/update-ignore.py"
# common args for all tests
common=(--venv --color --truncate 0 "${@}")

@ -5,7 +5,7 @@ set -eux -o pipefail
cp -a "${TEST_DIR}/ansible_collections" "${WORK_DIR}"
cd "${WORK_DIR}/ansible_collections/ns/col"
"${TEST_DIR}/collection-tests/update-ignore.py"
"${TEST_DIR}/../collection/update-ignore.py"
vendor_dir="$(python -c 'import pathlib, ansible._vendor; print(pathlib.Path(ansible._vendor.__file__).parent)')"

@ -5,6 +5,6 @@ set -eux -o pipefail
cp -a "${TEST_DIR}/ansible_collections" "${WORK_DIR}"
cd "${WORK_DIR}/ansible_collections/ns/col"
"${TEST_DIR}/collection-tests/update-ignore.py"
"${TEST_DIR}/../collection/update-ignore.py"
ansible-test sanity --color --truncate 0 "${@}"

@ -0,0 +1,29 @@
#!/usr/bin/env bash
# Source this file from collection integration tests.
#
# It simplifies several aspects of collection testing:
#
# 1) Collection tests must be executed outside of the ansible source tree.
# Otherwise ansible-test will test the ansible source instead of the test collection.
# The temporary directory provided by ansible-test resides within the ansible source tree.
#
# 2) Sanity test ignore files for collections must be versioned based on the ansible-core version being used.
# This script generates an ignore file with the correct filename for the current ansible-core version.
#
# 3) Sanity tests which are multi-version require an ignore entry per Python version.
# This script replicates these ignore entries for each supported Python version based on the ignored path.
set -eu -o pipefail
export TEST_DIR
export WORK_DIR
TEST_DIR="$PWD"
WORK_DIR="$(mktemp -d)"
trap 'rm -rf "${WORK_DIR}"' EXIT
cp -a "${TEST_DIR}/ansible_collections" "${WORK_DIR}"
cd "${WORK_DIR}/ansible_collections/ns/col"
"${TEST_DIR}/../collection/update-ignore.py"

@ -16,6 +16,11 @@ def main():
from ansible_test._internal import constants
src_path = 'tests/sanity/ignore.txt'
if not os.path.exists(src_path):
print(f'Skipping updates on non-existent ignore file: {src_path}')
return
directory = os.path.dirname(src_path)
name, ext = os.path.splitext(os.path.basename(src_path))
major_minor = '.'.join(release.__version__.split('.')[:2])

@ -128,6 +128,7 @@ test/integration/targets/ansible-test/ansible_collections/ns/col/tests/unit/plug
test/integration/targets/ansible-test-docker/ansible_collections/ns/col/plugins/modules/hello.py pylint:relative-beyond-top-level
test/integration/targets/ansible-test-docker/ansible_collections/ns/col/tests/unit/plugins/modules/test_hello.py pylint:relative-beyond-top-level
test/integration/targets/ansible-test-docker/ansible_collections/ns/col/tests/unit/plugins/module_utils/test_my_util.py pylint:relative-beyond-top-level
test/integration/targets/ansible-test-no-tty/ansible_collections/ns/col/vendored_pty.py pep8!skip # vendored code
test/integration/targets/collections_relative_imports/collection_root/ansible_collections/my_ns/my_col/plugins/modules/my_module.py pylint:relative-beyond-top-level
test/integration/targets/collections_relative_imports/collection_root/ansible_collections/my_ns/my_col/plugins/module_utils/my_util2.py pylint:relative-beyond-top-level
test/integration/targets/collections_relative_imports/collection_root/ansible_collections/my_ns/my_col/plugins/module_utils/my_util3.py pylint:relative-beyond-top-level

Loading…
Cancel
Save