diff --git a/MANIFEST.in b/MANIFEST.in index 863b286037c..5ec2feb94b2 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,25 +3,36 @@ include COPYING include SYMLINK_CACHE.json include requirements.txt include shippable.yml -include bin/ansible-test +recursive-include docs * +exclude docs/docsite/rst_warnings +recursive-exclude docs/docsite/_build * +recursive-exclude docs/docsite/_extensions *.pyc *.pyo include examples/hosts include examples/ansible.cfg include examples/scripts/ConfigureRemotingForAnsible.ps1 include examples/scripts/upgrade_to_ps3.ps1 -recursive-include lib/ansible/executor/powershell * -recursive-include lib/ansible/module_utils/csharp * -recursive-include lib/ansible/module_utils/powershell * -recursive-include lib/ansible/modules * -recursive-include lib/ansible/galaxy/data * -recursive-include docs * -recursive-include licenses * +recursive-include lib/ansible/executor/powershell *.ps1 +recursive-include lib/ansible/module_utils/csharp *.cs +recursive-include lib/ansible/module_utils/powershell *.psm1 +recursive-include lib/ansible/modules/windows *.ps1 +recursive-include lib/ansible/galaxy/data *.yml *.j2 README.md ansible.cfg inventory .git_keep +recursive-include lib/ansible/config *.yml +recursive-include licenses *.txt recursive-include packaging * -recursive-include test * +recursive-include test/cache .keep +recursive-include test/integration * +recursive-include test/lib/ansible_test/config *.template +recursive-include test/lib/ansible_test/_data * +recursive-include test/lib/ansible_test/tests * +recursive-exclude test/lib/ansible_test *.pyc *.pyo *.bak *.orig *~ *.rej +recursive-include test/results .keep +recursive-include test/sanity *.json *.py *.txt +exclude test/sanity/code-smell/botmeta.* +recursive-include test/units * include Makefile include MANIFEST.in include changelogs/CHANGELOG*.rst include contrib/README.md -recursive-include contrib/inventory * -exclude test/sanity/code-smell/botmeta.* +recursive-include contrib/inventory *.py *.ini *.yml *.yaml recursive-include hacking/build_library *.py include hacking/build-ansible.py diff --git a/Makefile b/Makefile index 8c0919d1696..7762d068980 100644 --- a/Makefile +++ b/Makefile @@ -19,6 +19,7 @@ NAME = ansible OS = $(shell uname -s) PREFIX ?= '/usr/local' +SDIST_DIR ?= 'dist' # This doesn't evaluate until it's called. The -D argument is the # directory of the target file ($@), kinda like `dirname`. @@ -226,14 +227,14 @@ sdist_check: .PHONY: sdist sdist: sdist_check clean docs - _ANSIBLE_SDIST_FROM_MAKEFILE=1 $(PYTHON) setup.py sdist + _ANSIBLE_SDIST_FROM_MAKEFILE=1 $(PYTHON) setup.py sdist --dist-dir=$(SDIST_DIR) # Official releases generate the changelog as the last commit before the release. # Snapshots shouldn't result in new checkins so the changelog is generated as # part of creating the tarball. .PHONY: snapshot snapshot: sdist_check clean docs changelog - _ANSIBLE_SDIST_FROM_MAKEFILE=1 $(PYTHON) setup.py sdist + _ANSIBLE_SDIST_FROM_MAKEFILE=1 $(PYTHON) setup.py sdist --dist-dir=$(SDIST_DIR) .PHONY: sdist_upload sdist_upload: clean docs diff --git a/docs/docsite/Makefile b/docs/docsite/Makefile index cd70faa35d7..6db2eb0403d 100644 --- a/docs/docsite/Makefile +++ b/docs/docsite/Makefile @@ -77,6 +77,7 @@ clean: rm -f rst/reference_appendices/config.rst rm -f rst/reference_appendices/playbooks_keywords.rst rm -f rst/dev_guide/collections_galaxy_meta.rst + rm -f rst/cli/*.rst .PHONY: docs clean diff --git a/setup.py b/setup.py index c607de5fa80..e302bcdae96 100644 --- a/setup.py +++ b/setup.py @@ -36,6 +36,13 @@ def _find_symlinks(topdir, extension=''): Maintained symlinks exist in the bin dir or are modules which have aliases. Our heuristic is that they are a link in a certain path which point to a file in the same directory. + + .. warn:: + + We want the symlinks in :file:`bin/` that link into :file:`lib/ansible/*` (currently, + :command:`ansible`, :command:`ansible-test`, and :command:`ansible-connection`) to become + real files on install. Updates to the heuristic here *must not* add them to the symlink + cache. """ symlinks = defaultdict(list) for base_path, dirs, files in os.walk(topdir): @@ -43,11 +50,36 @@ def _find_symlinks(topdir, extension=''): filepath = os.path.join(base_path, filename) if os.path.islink(filepath) and filename.endswith(extension): target = os.readlink(filepath) + if target.startswith('/'): + # We do not support absolute symlinks at all + continue + if os.path.dirname(target) == '': link = filepath[len(topdir):] if link.startswith('/'): link = link[1:] symlinks[os.path.basename(target)].append(link) + else: + # Count how many directory levels from the topdir we are + levels_deep = os.path.dirname(filepath).count('/') + + # Count the number of directory levels higher we walk up the tree in target + target_depth = 0 + for path_component in target.split('/'): + if path_component == '..': + target_depth += 1 + # If we walk past the topdir, then don't store + if target_depth >= levels_deep: + break + else: + target_depth -= 1 + else: + # If we managed to stay within the tree, store the symlink + link = filepath[len(topdir):] + if link.startswith('/'): + link = link[1:] + symlinks[target].append(link) + return symlinks @@ -69,8 +101,11 @@ def _maintain_symlinks(symlink_type, base_path): # SYMLINKS_CACHE doesn't exist. Fallback to trying to create the # cache now. Will work if we're running directly from a git # checkout or from an sdist created earlier. + library_symlinks = _find_symlinks('lib', '.py') + library_symlinks.update(_find_symlinks('test/lib')) + symlink_data = {'script': _find_symlinks('bin'), - 'library': _find_symlinks('lib', '.py'), + 'library': library_symlinks, } # Sanity check that something we know should be a symlink was @@ -129,8 +164,11 @@ class SDistCommand(SDist): def run(self): # have to generate the cache of symlinks for release as sdist is the # only command that has access to symlinks from the git repo + library_symlinks = _find_symlinks('lib', '.py') + library_symlinks.update(_find_symlinks('test/lib')) + symlinks = {'script': _find_symlinks('bin'), - 'library': _find_symlinks('lib', '.py'), + 'library': library_symlinks, } _cache_symlinks(symlinks) @@ -254,28 +292,10 @@ static_setup_params = dict( # Ansible will also make use of a system copy of python-six and # python-selectors2 if installed but use a Bundled copy if it's not. python_requires='>=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*', - package_dir={'': 'lib'}, - packages=find_packages('lib'), - package_data={ - '': [ - 'executor/powershell/*.ps1', - 'module_utils/csharp/*.cs', - 'module_utils/csharp/*/*.cs', - 'module_utils/powershell/*.psm1', - 'module_utils/powershell/*/*.psm1', - 'modules/windows/*.ps1', - 'modules/windows/*/*.ps1', - 'galaxy/data/*.*', - 'galaxy/data/*/*.*', - 'galaxy/data/*/.*', - 'galaxy/data/*/*/.*', - 'galaxy/data/*/*/*.*', - 'galaxy/data/*/tests/inventory', - 'galaxy/data/*/role/tests/inventory', - 'config/base.yml', - 'config/module_defaults.yml', - ], - }, + package_dir={'': 'lib', + 'ansible_test': 'test/lib/ansible_test'}, + packages=find_packages('lib') + find_packages('test/lib'), + include_package_data=True, classifiers=[ 'Development Status :: 5 - Production/Stable', 'Environment :: Console', @@ -306,6 +326,7 @@ static_setup_params = dict( 'bin/ansible-vault', 'bin/ansible-config', 'bin/ansible-inventory', + 'bin/ansible-test', ], data_files=[], # Installing as zip files would break due to references to __file__ diff --git a/test/sanity/code-smell/package-data.json b/test/sanity/code-smell/package-data.json index 22b0a15da70..0aa70a3c9b7 100644 --- a/test/sanity/code-smell/package-data.json +++ b/test/sanity/code-smell/package-data.json @@ -1,8 +1,5 @@ { "disabled": true, "all_targets": true, - "prefixes": [ - "lib/ansible/" - ], "output": "path-message" } diff --git a/test/sanity/code-smell/package-data.py b/test/sanity/code-smell/package-data.py index 6b59d9702db..5b0392c22cd 100755 --- a/test/sanity/code-smell/package-data.py +++ b/test/sanity/code-smell/package-data.py @@ -2,45 +2,371 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type +import contextlib import fnmatch +import glob import os import re +import shutil +import subprocess import sys +import tarfile import tempfile -import subprocess -def main(): +def assemble_files_to_ship(complete_file_list): + """ + This looks for all files which should be shipped in the sdist + """ + # All files which are in the repository except these: + ignore_patterns = ( + # Developer-only tools + '.github/*', + '.github/*/*', + 'changelogs/fragments/*', + 'hacking/aws_config/*', + 'hacking/aws_config/*/*', + 'hacking/tests/*', + 'hacking/ticket_stubs/*', + 'test/legacy/*', + 'test/legacy/*/*', + 'test/legacy/*/*/*', + 'test/legacy/*/*/*/*', + 'test/legacy/*/*/*/*/*', + 'test/legacy/*/*/*/*/*/*', + 'test/sanity/code-smell/botmeta.*', + 'test/utils/*', + 'test/utils/*/*', + 'test/utils/*/*/*', + '.git*', + # Consciously left out + 'examples/playbooks/*', + # Possibly should be included + 'contrib/vault/*', + ) ignore_files = frozenset(( - '*/galaxy/data/default/*/.git_keep', - '*/galaxy/data/default/role/*/main.yml.j2', - '*/galaxy/data/default/role/*/test.yml.j2', - '*/galaxy/data/default/collection/plugins/README.md.j2', + # Developer-only tools + 'changelogs/config.yaml', + 'changelogs/.changes.yaml', + 'hacking/README.md', + 'hacking/ansible-profile', + 'hacking/cgroup_perf_recap_graph.py', + 'hacking/create_deprecated_issues.py', + 'hacking/deprecated_issue_template.md', + 'hacking/fix_test_syntax.py', + 'hacking/get_library.py', + 'hacking/metadata-tool.py', + 'hacking/report.py', + 'hacking/return_skeleton_generator.py', + 'hacking/test-module', + 'hacking/test-module.py', + '.cherry_picker.toml', + '.mailmap', + # Possibly should be included + 'examples/scripts/uptime.py', + 'examples/DOCUMENTATION.yml', + 'examples/hosts.yaml', + 'examples/hosts.yml', + 'examples/inventory_script_schema.json', + 'examples/plugin_filters.yml', + 'hacking/env-setup', + 'hacking/env-setup.fish', + 'CODING_GUIDELINES.md', + 'MANIFEST', + 'MODULE_GUIDELINES.md', )) - non_py_files = [] - for path in sys.argv[1:] or sys.stdin.read().splitlines(): - if os.path.splitext(path)[1] != '.py': - add = True - for ignore in ignore_files: + # These files are generated and then intentionally added to the sdist + + # Manpages + manpages = ['docs/man/man1/ansible.1'] + for dirname, dummy, files in os.walk('bin'): + for filename in files: + path = os.path.join(dirname, filename) + if os.path.islink(path): + if os.readlink(path) == 'ansible': + manpages.append('docs/man/man1/%s.1' % filename) + + # Misc + misc_generated_files = [ + 'SYMLINK_CACHE.json', + 'PKG-INFO', + ] + + shipped_files = manpages + misc_generated_files + + for path in complete_file_list: + if path not in ignore_files: + for ignore in ignore_patterns: if fnmatch.fnmatch(path, ignore): - add = False - if add: - non_py_files.append(os.path.relpath(path, 'lib/ansible')) - - with tempfile.TemporaryDirectory() as tmp_dir: - stdout, _dummy = subprocess.Popen( - ['python', 'setup.py', 'install', '--root=%s' % tmp_dir], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - universal_newlines=True, - ).communicate() - match = re.search('^creating (%s/.*?/(?:site|dist)-packages/ansible)$' % tmp_dir, stdout, flags=re.M) - - for filename in non_py_files: - path = os.path.join(match.group(1), filename) - if not os.path.exists(path): - print('%s: File not installed' % os.path.join('lib', 'ansible', filename)) + break + else: + shipped_files.append(path) + + return shipped_files + + +def assemble_files_to_install(complete_file_list): + """ + This looks for all of the files which should show up in an installation of ansible + """ + ignore_patterns = tuple() + + pkg_data_files = [] + for path in complete_file_list: + + if path.startswith("lib/ansible"): + prefix = 'lib' + elif path.startswith("test/lib/ansible_test"): + prefix = 'test/lib' + else: + continue + + for ignore in ignore_patterns: + if fnmatch.fnmatch(path, ignore): + break + else: + pkg_data_files.append(os.path.relpath(path, prefix)) + + return pkg_data_files + + +@contextlib.contextmanager +def clean_repository(file_list): + """Copy the repository to clean it of artifacts""" + # Create a tempdir that will be the clean repo + with tempfile.TemporaryDirectory() as repo_root: + directories = set((repo_root + os.path.sep,)) + + for filename in file_list: + # Determine if we need to create the directory + directory = os.path.dirname(filename) + dest_dir = os.path.join(repo_root, directory) + if dest_dir not in directories: + os.makedirs(dest_dir) + + # Keep track of all the directories that now exist + path_components = directory.split(os.path.sep) + path = repo_root + for component in path_components: + path = os.path.join(path, component) + if path not in directories: + directories.add(path) + + # Copy the file + shutil.copy2(filename, dest_dir, follow_symlinks=False) + + yield repo_root + + +def create_sdist(tmp_dir): + """Create an sdist in the repository""" + dummy = subprocess.Popen( + ['make', 'snapshot', 'SDIST_DIR=%s' % tmp_dir], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ).communicate() + + # Determine path to sdist + tmp_dir_files = os.listdir(tmp_dir) + + if not tmp_dir_files: + raise Exception('sdist was not created in the temp dir') + elif len(tmp_dir_files) > 1: + raise Exception('Unexpected extra files in the temp dir') + + return os.path.join(tmp_dir, tmp_dir_files[0]) + + +def extract_sdist(sdist_path, tmp_dir): + """Untar the sdist""" + # Untar the sdist from the tmp_dir + with tarfile.open(os.path.join(tmp_dir, sdist_path), 'r|*') as sdist: + sdist.extractall(path=tmp_dir) + + # Determine the sdist directory name + sdist_filename = os.path.basename(sdist_path) + tmp_dir_files = os.listdir(tmp_dir) + try: + tmp_dir_files.remove(sdist_filename) + except ValueError: + # Unexpected could not find original sdist in temp dir + raise + + if len(tmp_dir_files) > 1: + raise Exception('Unexpected extra files in the temp dir') + elif len(tmp_dir_files) < 1: + raise Exception('sdist extraction did not occur i nthe temp dir') + + return os.path.join(tmp_dir, tmp_dir_files[0]) + + +def install_sdist(tmp_dir, sdist_dir): + """Install the extracted sdist into the temporary directory""" + stdout, _dummy = subprocess.Popen( + ['python', 'setup.py', 'install', '--root=%s' % tmp_dir], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + cwd=os.path.join(tmp_dir, sdist_dir), + ).communicate() + + # Determine the prefix for the installed files + match = re.search('^creating (%s/.*?/(?:site|dist)-packages)/ansible$' % + tmp_dir, stdout, flags=re.M) + return match.group(1) + + +def check_sdist_contains_expected(sdist_dir, to_ship_files): + """Check that the files we expect to ship are present in the sdist""" + results = [] + for filename in to_ship_files: + path = os.path.join(sdist_dir, filename) + if not os.path.exists(path): + results.append('%s: File was not added to sdist' % filename) + + # Also changelog + changelog_files = glob.glob(os.path.join(sdist_dir, 'changelogs/CHANGELOG-v2.[0-9]*.rst')) + if not changelog_files: + results.append('changelogs/CHANGELOG-v2.*.rst: Changelog file was not added to the sdist') + elif len(changelog_files) > 1: + results.append('changelogs/CHANGELOG-v2.*.rst: Too many changelog files: %s' + % changelog_files) + + return results + + +def check_sdist_files_are_wanted(sdist_dir, to_ship_files): + """Check that all files in the sdist are desired""" + results = [] + for dirname, dummy, files in os.walk(sdist_dir): + dirname = os.path.relpath(dirname, start=sdist_dir) + if dirname == '.': + dirname = '' + + for filename in files: + path = os.path.join(dirname, filename) + if path not in to_ship_files: + if fnmatch.fnmatch(path, 'changelogs/CHANGELOG-v2.[0-9]*.rst'): + # changelog files are expected + continue + + # FIXME: ansible-test doesn't pass the paths of symlinks to us so we aren't + # checking those + if not os.path.islink(os.path.join(sdist_dir, path)): + results.append('%s: File in sdist was not in the repository' % path) + + return results + + +def check_installed_contains_expected(install_dir, to_install_files): + """Check that all the files we expect to be installed are""" + results = [] + for filename in to_install_files: + path = os.path.join(install_dir, filename) + if not os.path.exists(path): + results.append('%s: File not installed' % os.path.join('lib', filename)) + + return results + + +EGG_RE = re.compile('ansible[^/]+\\.egg-info/(PKG-INFO|SOURCES.txt|' + 'dependency_links.txt|not-zip-safe|requires.txt|top_level.txt)$') + + +def check_installed_files_are_wanted(install_dir, to_install_files): + """Check that all installed files were desired""" + results = [] + + for dirname, dummy, files in os.walk(install_dir): + dirname = os.path.relpath(dirname, start=install_dir) + if dirname == '.': + dirname = '' + + for filename in files: + # If this is a byte code cache, look for the python file's name + directory = dirname + if filename.endswith('.pyc') or filename.endswith('.pyo'): + # Remove the trailing "o" or c" + filename = filename[:-1] + + if directory.endswith('%s__pycache__' % os.path.sep): + # Python3 byte code cache, look for the basename of + # __pycache__/__init__.cpython-36.py + segments = filename.rsplit('.', 2) + if len(segments) >= 3: + filename = '.'.join((segments[0], segments[2])) + directory = os.path.dirname(directory) + + path = os.path.join(directory, filename) + + # Test that the file was listed for installation + if path not in to_install_files: + # FIXME: ansible-test doesn't pass the paths of symlinks to us so we + # aren't checking those + if not os.path.islink(os.path.join(install_dir, path)): + if not EGG_RE.match(path): + results.append('%s: File was installed but was not supposed to be' % path) + + return results + + +def _find_symlinks(): + symlink_list = [] + for dirname, directories, filenames in os.walk('.'): + for filename in filenames: + path = os.path.join(dirname, filename) + # Strip off "./" from the front + path = path[2:] + if os.path.islink(path): + symlink_list.append(path) + + return symlink_list + + +def main(): + """All of the files in the repository""" + complete_file_list = [] + for path in sys.argv[1:] or sys.stdin.read().splitlines(): + complete_file_list.append(path) + + # ansible-test isn't currently passing symlinks to us so construct those ourselves for now + for filename in _find_symlinks(): + if filename not in complete_file_list: + # For some reason ansible-test is passing us lib/ansible/module_utils/ansible_release.py + # which is a symlink even though it doesn't pass any others + complete_file_list.append(filename) + + # We may run this after docs sanity tests so get a clean repository to run in + with clean_repository(complete_file_list) as clean_repo_dir: + os.chdir(clean_repo_dir) + + to_ship_files = assemble_files_to_ship(complete_file_list) + to_install_files = assemble_files_to_install(complete_file_list) + + results = [] + with tempfile.TemporaryDirectory() as tmp_dir: + sdist_path = create_sdist(tmp_dir) + sdist_dir = extract_sdist(sdist_path, tmp_dir) + + # Check that the files that are supposed to be in the sdist are there + results.extend(check_sdist_contains_expected(sdist_dir, to_ship_files)) + + # Check that the files that are in the sdist are in the repository + results.extend(check_sdist_files_are_wanted(sdist_dir, to_ship_files)) + + # install the sdist + install_dir = install_sdist(tmp_dir, sdist_dir) + + # Check that the files that are supposed to be installed are there + results.extend(check_installed_contains_expected(install_dir, to_install_files)) + + # Check that the files that are installed are supposed to be installed + results.extend(check_installed_files_are_wanted(install_dir, to_install_files)) + + for message in results: + print(message) if __name__ == '__main__':