mirror of https://github.com/ansible/ansible.git
Refactor ansible-test integration tests. (#78168)
* Relocate update-ignore.py for easier re-use. * Add script to ease collection testing. * Skip ignore rewrite if file does not exist. * Add integration test for the shebang sanity test. * Fix ansible-test-no-tty integration test. Previously the test only verified a TTY was not used if a TTY already existed. This prevented the test from verifying behavior when run in CI. Now the test creates a PTY before invoking ansible-test. * Clean up ansible-test-docker integration test.pull/76306/merge
Matt Clay
2 years ago
committed by
GitHub
parent
bcdc2e167a
commit
f70cc2fb7e
@ -1,2 +1,3 @@
|
||||
shippable/generic/group1 # Runs in the default test container so access to tools like pwsh
|
||||
context/controller
|
||||
needs/target/collection
|
||||
|
||||
@ -0,0 +1 @@
|
||||
context/controller
|
||||
@ -1,18 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -eux -o pipefail
|
||||
|
||||
cp -a "${TEST_DIR}/ansible_collections" "${WORK_DIR}"
|
||||
cd "${WORK_DIR}/ansible_collections/ns/col"
|
||||
|
||||
# common args for all tests
|
||||
# because we are running in shippable/generic/ we are already in the default docker container
|
||||
common=(--python "${ANSIBLE_TEST_PYTHON_VERSION}" --venv --venv-system-site-packages --color --truncate 0 "${@}")
|
||||
|
||||
# prime the venv to work around issue with PyYAML detection in ansible-test
|
||||
ansible-test sanity "${common[@]}" --test ignores
|
||||
|
||||
# tests
|
||||
ansible-test sanity "${common[@]}"
|
||||
ansible-test units "${common[@]}"
|
||||
ansible-test integration "${common[@]}"
|
||||
@ -1,24 +1,14 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -eu -o pipefail
|
||||
source ../collection/setup.sh
|
||||
|
||||
# tests must be executed outside of the ansible source tree
|
||||
# otherwise ansible-test will test the ansible source instead of the test collection
|
||||
# the temporary directory provided by ansible-test resides within the ansible source tree
|
||||
tmp_dir=$(mktemp -d)
|
||||
set -x
|
||||
|
||||
trap 'rm -rf "${tmp_dir}"' EXIT
|
||||
# common args for all tests
|
||||
# because we are running in shippable/generic/ we are already in the default docker container
|
||||
common=(--python "${ANSIBLE_TEST_PYTHON_VERSION}" --venv --venv-system-site-packages --color --truncate 0 "${@}")
|
||||
|
||||
export TEST_DIR
|
||||
export WORK_DIR
|
||||
|
||||
TEST_DIR="$PWD"
|
||||
|
||||
for test in collection-tests/*.sh; do
|
||||
WORK_DIR="${tmp_dir}/$(basename "${test}" ".sh")"
|
||||
mkdir "${WORK_DIR}"
|
||||
echo "**********************************************************************"
|
||||
echo "TEST: ${test}: STARTING"
|
||||
"${test}" "${@}" || (echo "TEST: ${test}: FAILED" && exit 1)
|
||||
echo "TEST: ${test}: PASSED"
|
||||
done
|
||||
# tests
|
||||
ansible-test sanity "${common[@]}"
|
||||
ansible-test units "${common[@]}"
|
||||
ansible-test integration "${common[@]}"
|
||||
|
||||
@ -1,2 +1,4 @@
|
||||
context/controller
|
||||
shippable/posix/group1
|
||||
shippable/posix/group1 # runs in the distro test containers
|
||||
shippable/generic/group1 # runs in the default test container
|
||||
needs/target/collection
|
||||
|
||||
@ -0,0 +1,11 @@
|
||||
#!/usr/bin/env python
|
||||
"""Run a command using a PTY."""
|
||||
|
||||
import sys
|
||||
|
||||
if sys.version_info < (3, 10):
|
||||
import vendored_pty as pty
|
||||
else:
|
||||
import pty
|
||||
|
||||
sys.exit(1 if pty.spawn(sys.argv[1:]) else 0)
|
||||
@ -0,0 +1 @@
|
||||
context/controller
|
||||
@ -0,0 +1,13 @@
|
||||
#!/usr/bin/env python
|
||||
"""Assert no TTY is available."""
|
||||
|
||||
import sys
|
||||
|
||||
status = 0
|
||||
|
||||
for handle in sys.stdin, sys.stdout, sys.stderr:
|
||||
if handle.isatty():
|
||||
print(f'{handle} is a TTY', file=sys.stderr)
|
||||
status += 1
|
||||
|
||||
sys.exit(status)
|
||||
@ -0,0 +1,5 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -eux
|
||||
|
||||
./assert-no-tty.py
|
||||
@ -0,0 +1,189 @@
|
||||
# Vendored copy of https://github.com/python/cpython/blob/3680ebed7f3e529d01996dd0318601f9f0d02b4b/Lib/pty.py
|
||||
# PSF License (see licenses/PSF-license.txt or https://opensource.org/licenses/Python-2.0)
|
||||
"""Pseudo terminal utilities."""
|
||||
|
||||
# Bugs: No signal handling. Doesn't set slave termios and window size.
|
||||
# Only tested on Linux, FreeBSD, and macOS.
|
||||
# See: W. Richard Stevens. 1992. Advanced Programming in the
|
||||
# UNIX Environment. Chapter 19.
|
||||
# Author: Steen Lumholt -- with additions by Guido.
|
||||
|
||||
from select import select
|
||||
import os
|
||||
import sys
|
||||
import tty
|
||||
|
||||
# names imported directly for test mocking purposes
|
||||
from os import close, waitpid
|
||||
from tty import setraw, tcgetattr, tcsetattr
|
||||
|
||||
__all__ = ["openpty", "fork", "spawn"]
|
||||
|
||||
STDIN_FILENO = 0
|
||||
STDOUT_FILENO = 1
|
||||
STDERR_FILENO = 2
|
||||
|
||||
CHILD = 0
|
||||
|
||||
def openpty():
|
||||
"""openpty() -> (master_fd, slave_fd)
|
||||
Open a pty master/slave pair, using os.openpty() if possible."""
|
||||
|
||||
try:
|
||||
return os.openpty()
|
||||
except (AttributeError, OSError):
|
||||
pass
|
||||
master_fd, slave_name = _open_terminal()
|
||||
slave_fd = slave_open(slave_name)
|
||||
return master_fd, slave_fd
|
||||
|
||||
def master_open():
|
||||
"""master_open() -> (master_fd, slave_name)
|
||||
Open a pty master and return the fd, and the filename of the slave end.
|
||||
Deprecated, use openpty() instead."""
|
||||
|
||||
try:
|
||||
master_fd, slave_fd = os.openpty()
|
||||
except (AttributeError, OSError):
|
||||
pass
|
||||
else:
|
||||
slave_name = os.ttyname(slave_fd)
|
||||
os.close(slave_fd)
|
||||
return master_fd, slave_name
|
||||
|
||||
return _open_terminal()
|
||||
|
||||
def _open_terminal():
|
||||
"""Open pty master and return (master_fd, tty_name)."""
|
||||
for x in 'pqrstuvwxyzPQRST':
|
||||
for y in '0123456789abcdef':
|
||||
pty_name = '/dev/pty' + x + y
|
||||
try:
|
||||
fd = os.open(pty_name, os.O_RDWR)
|
||||
except OSError:
|
||||
continue
|
||||
return (fd, '/dev/tty' + x + y)
|
||||
raise OSError('out of pty devices')
|
||||
|
||||
def slave_open(tty_name):
|
||||
"""slave_open(tty_name) -> slave_fd
|
||||
Open the pty slave and acquire the controlling terminal, returning
|
||||
opened filedescriptor.
|
||||
Deprecated, use openpty() instead."""
|
||||
|
||||
result = os.open(tty_name, os.O_RDWR)
|
||||
try:
|
||||
from fcntl import ioctl, I_PUSH
|
||||
except ImportError:
|
||||
return result
|
||||
try:
|
||||
ioctl(result, I_PUSH, "ptem")
|
||||
ioctl(result, I_PUSH, "ldterm")
|
||||
except OSError:
|
||||
pass
|
||||
return result
|
||||
|
||||
def fork():
|
||||
"""fork() -> (pid, master_fd)
|
||||
Fork and make the child a session leader with a controlling terminal."""
|
||||
|
||||
try:
|
||||
pid, fd = os.forkpty()
|
||||
except (AttributeError, OSError):
|
||||
pass
|
||||
else:
|
||||
if pid == CHILD:
|
||||
try:
|
||||
os.setsid()
|
||||
except OSError:
|
||||
# os.forkpty() already set us session leader
|
||||
pass
|
||||
return pid, fd
|
||||
|
||||
master_fd, slave_fd = openpty()
|
||||
pid = os.fork()
|
||||
if pid == CHILD:
|
||||
# Establish a new session.
|
||||
os.setsid()
|
||||
os.close(master_fd)
|
||||
|
||||
# Slave becomes stdin/stdout/stderr of child.
|
||||
os.dup2(slave_fd, STDIN_FILENO)
|
||||
os.dup2(slave_fd, STDOUT_FILENO)
|
||||
os.dup2(slave_fd, STDERR_FILENO)
|
||||
if slave_fd > STDERR_FILENO:
|
||||
os.close(slave_fd)
|
||||
|
||||
# Explicitly open the tty to make it become a controlling tty.
|
||||
tmp_fd = os.open(os.ttyname(STDOUT_FILENO), os.O_RDWR)
|
||||
os.close(tmp_fd)
|
||||
else:
|
||||
os.close(slave_fd)
|
||||
|
||||
# Parent and child process.
|
||||
return pid, master_fd
|
||||
|
||||
def _writen(fd, data):
|
||||
"""Write all the data to a descriptor."""
|
||||
while data:
|
||||
n = os.write(fd, data)
|
||||
data = data[n:]
|
||||
|
||||
def _read(fd):
|
||||
"""Default read function."""
|
||||
return os.read(fd, 1024)
|
||||
|
||||
def _copy(master_fd, master_read=_read, stdin_read=_read):
|
||||
"""Parent copy loop.
|
||||
Copies
|
||||
pty master -> standard output (master_read)
|
||||
standard input -> pty master (stdin_read)"""
|
||||
fds = [master_fd, STDIN_FILENO]
|
||||
while fds:
|
||||
rfds, _wfds, _xfds = select(fds, [], [])
|
||||
|
||||
if master_fd in rfds:
|
||||
# Some OSes signal EOF by returning an empty byte string,
|
||||
# some throw OSErrors.
|
||||
try:
|
||||
data = master_read(master_fd)
|
||||
except OSError:
|
||||
data = b""
|
||||
if not data: # Reached EOF.
|
||||
return # Assume the child process has exited and is
|
||||
# unreachable, so we clean up.
|
||||
else:
|
||||
os.write(STDOUT_FILENO, data)
|
||||
|
||||
if STDIN_FILENO in rfds:
|
||||
data = stdin_read(STDIN_FILENO)
|
||||
if not data:
|
||||
fds.remove(STDIN_FILENO)
|
||||
else:
|
||||
_writen(master_fd, data)
|
||||
|
||||
def spawn(argv, master_read=_read, stdin_read=_read):
|
||||
"""Create a spawned process."""
|
||||
if isinstance(argv, str):
|
||||
argv = (argv,)
|
||||
sys.audit('pty.spawn', argv)
|
||||
|
||||
pid, master_fd = fork()
|
||||
if pid == CHILD:
|
||||
os.execlp(argv[0], *argv)
|
||||
|
||||
try:
|
||||
mode = tcgetattr(STDIN_FILENO)
|
||||
setraw(STDIN_FILENO)
|
||||
restore = True
|
||||
except tty.error: # This is the same as termios.error
|
||||
restore = False
|
||||
|
||||
try:
|
||||
_copy(master_fd, master_read, stdin_read)
|
||||
finally:
|
||||
if restore:
|
||||
tcsetattr(STDIN_FILENO, tty.TCSAFLUSH, mode)
|
||||
|
||||
close(master_fd)
|
||||
return waitpid(pid, 0)[1]
|
||||
@ -1,7 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
import sys
|
||||
|
||||
assert not sys.stdin.isatty()
|
||||
assert not sys.stdout.isatty()
|
||||
assert not sys.stderr.isatty()
|
||||
@ -1,5 +1,13 @@
|
||||
#!/usr/bin/env bash
|
||||
# Verify that ansible-test runs integration tests without a TTY.
|
||||
|
||||
set -eux
|
||||
source ../collection/setup.sh
|
||||
|
||||
./runme.py
|
||||
set -x
|
||||
|
||||
if ./run-with-pty.py tests/integration/targets/no-tty/assert-no-tty.py > /dev/null; then
|
||||
echo "PTY assertion did not fail. Either PTY creation failed or PTY detection is broken."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
./run-with-pty.py ansible-test integration --color "${@}"
|
||||
|
||||
@ -0,0 +1,4 @@
|
||||
shippable/posix/group1 # runs in the distro test containers
|
||||
shippable/generic/group1 # runs in the default test container
|
||||
context/controller
|
||||
needs/target/collection
|
||||
@ -0,0 +1 @@
|
||||
#!powershell
|
||||
@ -0,0 +1 @@
|
||||
#!/usr/bin/python
|
||||
@ -0,0 +1 @@
|
||||
#!/usr/bin/env bash
|
||||
@ -0,0 +1 @@
|
||||
#!/usr/bin/env python
|
||||
@ -0,0 +1 @@
|
||||
#!/bin/sh
|
||||
@ -0,0 +1 @@
|
||||
#!/usr/bin/env bash
|
||||
@ -0,0 +1 @@
|
||||
#!/usr/bin/env python
|
||||
@ -0,0 +1 @@
|
||||
#!/bin/sh
|
||||
@ -0,0 +1,9 @@
|
||||
plugins/modules/no-shebang-executable.py:0:0: file without shebang should not be executable
|
||||
plugins/modules/python-executable.py:0:0: module should not be executable
|
||||
plugins/modules/python-wrong-shebang.py:1:1: expected module shebang "b'#!/usr/bin/python'" but found: b'#!invalid'
|
||||
plugins/modules/utf-16-be-bom.py:0:0: file starts with a UTF-16 (BE) byte order mark
|
||||
plugins/modules/utf-16-le-bom.py:0:0: file starts with a UTF-16 (LE) byte order mark
|
||||
plugins/modules/utf-32-be-bom.py:0:0: file starts with a UTF-32 (BE) byte order mark
|
||||
plugins/modules/utf-32-le-bom.py:0:0: file starts with a UTF-32 (LE) byte order mark
|
||||
plugins/modules/utf-8-bom.py:0:0: file starts with a UTF-8 byte order mark
|
||||
scripts/unexpected-shebang:1:1: unexpected non-module shebang: b'#!/usr/bin/custom'
|
||||
@ -0,0 +1,47 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -eu
|
||||
|
||||
# Create test scenarios at runtime that do not pass sanity tests.
|
||||
# This avoids the need to create ignore entries for the tests.
|
||||
|
||||
(
|
||||
cd ansible_collections/ns/col/plugins/modules
|
||||
|
||||
touch no-shebang-executable.py && chmod +x no-shebang-executable.py # file without shebang should not be executable
|
||||
python -c "open('utf-32-be-bom.py', 'wb').write(b'\x00\x00\xFE\xFF')" # file starts with a UTF-32 (BE) byte order mark
|
||||
python -c "open('utf-32-le-bom.py', 'wb').write(b'\xFF\xFE\x00\x00')" # file starts with a UTF-32 (LE) byte order mark
|
||||
python -c "open('utf-16-be-bom.py', 'wb').write(b'\xFE\xFF')" # file starts with a UTF-16 (BE) byte order mark
|
||||
python -c "open('utf-16-le-bom.py', 'wb').write(b'\xFF\xFE')" # file starts with a UTF-16 (LE) byte order mark
|
||||
python -c "open('utf-8-bom.py', 'wb').write(b'\xEF\xBB\xBF')" # file starts with a UTF-8 byte order mark
|
||||
echo '#!/usr/bin/python' > python-executable.py && chmod +x python-executable.py # module should not be executable
|
||||
echo '#!invalid' > python-wrong-shebang.py # expected module shebang "b'#!/usr/bin/python'" but found: b'#!invalid'
|
||||
)
|
||||
|
||||
(
|
||||
cd ansible_collections/ns/col/scripts
|
||||
|
||||
echo '#!/usr/bin/custom' > unexpected-shebang # unexpected non-module shebang: b'#!/usr/bin/custom'
|
||||
|
||||
echo '#!/usr/bin/make -f' > Makefile && chmod +x Makefile # pass
|
||||
echo '#!/bin/bash -eu' > bash_eu.sh && chmod +x bash_eu.sh # pass
|
||||
echo '#!/bin/bash -eux' > bash_eux.sh && chmod +x bash_eux.sh # pass
|
||||
echo '#!/usr/bin/env fish' > env_fish.fish && chmod +x env_fish.fish # pass
|
||||
echo '#!/usr/bin/env pwsh' > env_pwsh.ps1 && chmod +x env_pwsh.ps1 # pass
|
||||
)
|
||||
|
||||
mkdir ansible_collections/ns/col/examples
|
||||
|
||||
(
|
||||
cd ansible_collections/ns/col/examples
|
||||
|
||||
echo '#!/usr/bin/custom' > unexpected-shebang # pass
|
||||
)
|
||||
|
||||
source ../collection/setup.sh
|
||||
|
||||
set -x
|
||||
|
||||
ansible-test sanity --test shebang --color --lint --failure-ok "${@}" > actual.txt
|
||||
|
||||
diff -u "${TEST_DIR}/expected.txt" actual.txt
|
||||
@ -1,4 +1,5 @@
|
||||
shippable/posix/group1 # runs in the distro test containers
|
||||
shippable/generic/group1 # runs in the default test container
|
||||
context/controller
|
||||
needs/target/collection
|
||||
destructive # adds and then removes packages into lib/ansible/_vendor/
|
||||
|
||||
@ -0,0 +1 @@
|
||||
hidden
|
||||
@ -0,0 +1,29 @@
|
||||
#!/usr/bin/env bash
|
||||
# Source this file from collection integration tests.
|
||||
#
|
||||
# It simplifies several aspects of collection testing:
|
||||
#
|
||||
# 1) Collection tests must be executed outside of the ansible source tree.
|
||||
# Otherwise ansible-test will test the ansible source instead of the test collection.
|
||||
# The temporary directory provided by ansible-test resides within the ansible source tree.
|
||||
#
|
||||
# 2) Sanity test ignore files for collections must be versioned based on the ansible-core version being used.
|
||||
# This script generates an ignore file with the correct filename for the current ansible-core version.
|
||||
#
|
||||
# 3) Sanity tests which are multi-version require an ignore entry per Python version.
|
||||
# This script replicates these ignore entries for each supported Python version based on the ignored path.
|
||||
|
||||
set -eu -o pipefail
|
||||
|
||||
export TEST_DIR
|
||||
export WORK_DIR
|
||||
|
||||
TEST_DIR="$PWD"
|
||||
WORK_DIR="$(mktemp -d)"
|
||||
|
||||
trap 'rm -rf "${WORK_DIR}"' EXIT
|
||||
|
||||
cp -a "${TEST_DIR}/ansible_collections" "${WORK_DIR}"
|
||||
cd "${WORK_DIR}/ansible_collections/ns/col"
|
||||
|
||||
"${TEST_DIR}/../collection/update-ignore.py"
|
||||
Loading…
Reference in New Issue