Fix ansible-test submodule handling. (#68759)

* Refactor ansible-test integration test.
* Add env --list-files option.
* Add tests for collection files tracked by git.
* Fix ansible-test submodule usage on older git.
* Fix submodule directory detection as files.
* Improve handling of nested source control.
pull/68774/head
Matt Clay 5 years ago committed by GitHub
parent d4d2c95819
commit 148e83f832
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -0,0 +1,2 @@
minor_changes:
- ansible-test now has a ``--list-files`` option to list files using the ``env`` command.

@ -0,0 +1,2 @@
bugfixes:
- ansible-test now supports submodules using older ``git`` versions which require querying status from the top level directory of the repo.

@ -0,0 +1,2 @@
bugfixes:
- ansible-test now ignores version control within subdirectories of collections. Previously this condition was an error.

@ -0,0 +1,2 @@
minor_changes:
- ansible-test no longer detects ``git`` submodule directories as files.

@ -0,0 +1,10 @@
#!/usr/bin/env bash
set -eux -o pipefail
export GIT_TOP_LEVEL SUBMODULE_DST
GIT_TOP_LEVEL="${WORK_DIR}/super/ansible_collections/ns/col"
SUBMODULE_DST="sub"
source collection-tests/git-common.bash

@ -0,0 +1,10 @@
#!/usr/bin/env bash
set -eux -o pipefail
export GIT_TOP_LEVEL SUBMODULE_DST
GIT_TOP_LEVEL="${WORK_DIR}/super"
SUBMODULE_DST="ansible_collections/ns/col/sub"
source collection-tests/git-common.bash

@ -0,0 +1,43 @@
#!/usr/bin/env bash
set -eux -o pipefail
# make sure git is installed
git --version || ansible-playbook collection-tests/install-git.yml -i ../../inventory "$@"
# init sub project
mkdir "${WORK_DIR}/sub"
cd "${WORK_DIR}/sub"
touch "README.md"
git init
git config user.name 'Ansible Test'
git config user.email 'ansible-test@ansible.com'
git add "README.md"
git commit -m "Initial commit."
# init super project
rm -rf "${WORK_DIR}/super" # needed when re-creating in place
mkdir "${WORK_DIR}/super"
cp -a "${TEST_DIR}/ansible_collections" "${WORK_DIR}/super"
cd "${GIT_TOP_LEVEL}"
git init
# add submodule
git submodule add "${WORK_DIR}/sub" "${SUBMODULE_DST}"
# prepare for tests
expected="${WORK_DIR}/expected.txt"
actual="${WORK_DIR}/actual.txt"
cd "${WORK_DIR}/super/ansible_collections/ns/col"
mkdir tests/.git
touch tests/.git/keep.txt # make sure ansible-test correctly ignores version control within collection subdirectories
find . -type f ! -path '*/.git/*' ! -name .git | sed 's|^\./||' | sort >"${expected}"
set -x
# test at the collection base
ansible-test env --list-files | sort >"${actual}"
diff --unified "${expected}" "${actual}"
# test at the submodule base
(cd sub && ansible-test env --list-files | sort >"${actual}")
diff --unified "${expected}" "${actual}"

@ -0,0 +1,5 @@
- hosts: localhost
tasks:
- name: Make sure git is installed
package:
name: git

@ -0,0 +1,17 @@
#!/usr/bin/env bash
set -eux -o pipefail
cp -a "${TEST_DIR}/ansible_collections" "${WORK_DIR}"
cd "${WORK_DIR}/ansible_collections/ns/col"
# common args for all tests
common=(--venv --python "${ANSIBLE_TEST_PYTHON_VERSION}" --color --truncate 0 "${@}")
# prime the venv to work around issue with PyYAML detection in ansible-test
ansible-test sanity "${common[@]}" --test ignores
# tests
ansible-test sanity "${common[@]}"
ansible-test units "${common[@]}"
ansible-test integration "${common[@]}"

@ -1,6 +1,6 @@
#!/usr/bin/env bash
set -eux -o pipefail
set -eu -o pipefail
# tests must be executed outside of the ansible source tree
# otherwise ansible-test will test the ansible source instead of the test collection
@ -9,16 +9,16 @@ tmp_dir=$(mktemp -d)
trap 'rm -rf "${tmp_dir}"' EXIT
cp -a ansible_collections "${tmp_dir}"
cd "${tmp_dir}/ansible_collections/ns/col"
export TEST_DIR
export WORK_DIR
# common args for all tests
common=(--venv --python "${ANSIBLE_TEST_PYTHON_VERSION}" --color --truncate 0 "${@}")
TEST_DIR="$PWD"
# prime the venv to work around issue with PyYAML detection in ansible-test
ansible-test sanity "${common[@]}" --test ignores
# tests
ansible-test sanity "${common[@]}"
ansible-test units "${common[@]}"
ansible-test integration "${common[@]}"
for test in collection-tests/*.sh; do
WORK_DIR="${tmp_dir}/$(basename "${test}" ".sh")"
mkdir "${WORK_DIR}"
echo "**********************************************************************"
echo "TEST: ${test}: STARTING"
"${test}" "${@}" || (echo "TEST: ${test}: FAILED" && exit 1)
echo "TEST: ${test}: PASSED"
done

@ -636,6 +636,10 @@ def parse_args():
action='store_true',
help='dump environment to disk')
env.add_argument('--list-files',
action='store_true',
help='list files on stdout')
# noinspection PyTypeChecker
env.add_argument('--timeout',
type=int,

@ -40,15 +40,6 @@ from .provider.layout import (
)
class UnexpectedSourceRoot(ApplicationError):
"""Exception generated when a source root is found below a layout root."""
def __init__(self, source_root, layout_root): # type: (str, str) -> None
super(UnexpectedSourceRoot, self).__init__('Source root "%s" cannot be below layout root "%s".' % (source_root, layout_root))
self.source_root = source_root
self.layout_root = layout_root
class DataContext:
"""Data context providing details about the current execution environment for ansible-test."""
def __init__(self):
@ -121,13 +112,14 @@ class DataContext:
layout_provider = find_path_provider(LayoutProvider, layout_providers, root, walk)
try:
source_provider = find_path_provider(SourceProvider, source_providers, root, walk)
# Begin the search for the source provider at the layout provider root.
# This intentionally ignores version control within subdirectories of the layout root, a condition which was previously an error.
# Doing so allows support for older git versions for which it is difficult to distinguish between a super project and a sub project.
# It also provides a better user experience, since the solution for the user would effectively be the same -- to remove the nested version control.
source_provider = find_path_provider(SourceProvider, source_providers, layout_provider.root, walk)
except ProviderNotFoundForPath:
source_provider = UnversionedSource(layout_provider.root)
if source_provider.root != layout_provider.root and is_subdir(source_provider.root, layout_provider.root):
raise UnexpectedSourceRoot(source_provider.root, layout_provider.root)
layout = layout_provider.create(layout_provider.root, source_provider.get_paths(layout_provider.root))
return layout

@ -31,6 +31,7 @@ from .util import (
)
from .util_common import (
data_context,
write_json_test_results,
ResultType,
)
@ -72,8 +73,9 @@ class EnvConfig(CommonConfig):
self.show = args.show
self.dump = args.dump
self.timeout = args.timeout
self.list_files = args.list_files
if not self.show and not self.dump and self.timeout is None:
if not self.show and not self.dump and self.timeout is None and not self.list_files:
# default to --show if no options were given
self.show = True
@ -83,6 +85,7 @@ def command_env(args):
:type args: EnvConfig
"""
show_dump_env(args)
list_files_env(args)
set_timeout(args)
@ -130,6 +133,15 @@ def show_dump_env(args):
write_json_test_results(ResultType.BOT, 'data-environment.json', data)
def list_files_env(args): # type: (EnvConfig) -> None
"""List files on stdout."""
if not args.list_files:
return
for path in data_context().content.all_files():
display.info(path)
def set_timeout(args):
"""
:type args: EnvConfig

@ -14,6 +14,10 @@ from ...encoding import (
to_bytes,
)
from ...util import (
SubprocessError,
)
from . import (
SourceProvider,
)
@ -28,15 +32,30 @@ class GitSource(SourceProvider):
def get_paths(self, path): # type: (str) -> t.List[str]
"""Return the list of available content paths under the given path."""
git = Git(path)
paths = self.__get_paths(path)
submodule_paths = git.get_submodule_paths()
try:
submodule_paths = Git(path).get_submodule_paths()
except SubprocessError:
if path == self.root:
raise
# older versions of git require submodule commands to be executed from the top level of the working tree
# git version 2.18.1 (centos8) does not have this restriction
# git version 1.8.3.1 (centos7) does
# fall back to using the top level directory of the working tree only when needed
# this avoids penalizing newer git versions with a potentially slower analysis due to additional submodules
rel_path = os.path.relpath(path, self.root) + os.path.sep
submodule_paths = Git(self.root).get_submodule_paths()
submodule_paths = [os.path.relpath(p, rel_path) for p in submodule_paths if p.startswith(rel_path)]
for submodule_path in submodule_paths:
paths.extend(os.path.join(submodule_path, p) for p in self.__get_paths(os.path.join(path, submodule_path)))
# git reports submodule directories as regular files
paths = [p for p in paths if p not in submodule_paths]
return paths
@staticmethod

Loading…
Cancel
Save