diff --git a/MANIFEST.in b/MANIFEST.in index b87e4c388b5..22c8ce86785 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -7,6 +7,10 @@ include docs/docsite/rst/collections/all_plugins.rst exclude docs/docsite/rst_warnings exclude docs/docsite/rst/conf.py exclude docs/docsite/rst/index.rst +exclude docs/docsite/rst/dev_guide/testing/sanity/bin-symlinks.rst +exclude docs/docsite/rst/dev_guide/testing/sanity/botmeta.rst +exclude docs/docsite/rst/dev_guide/testing/sanity/integration-aliases.rst +exclude docs/docsite/rst/dev_guide/testing/sanity/release-names.rst recursive-exclude docs/docsite/_build * recursive-exclude docs/docsite/_extensions *.pyc *.pyo include examples/hosts @@ -31,6 +35,9 @@ recursive-include test/lib/ansible_test/_util/controller/sanity/validate-modules recursive-include test/sanity *.json *.py *.txt recursive-include test/support *.py *.ps1 *.psm1 *.cs exclude test/sanity/code-smell/botmeta.* +exclude test/sanity/code-smell/release-names.* +exclude test/lib/ansible_test/_internal/commands/sanity/bin_symlinks.py +exclude test/lib/ansible_test/_internal/commands/sanity/integration_aliases.py recursive-include test/units * include Makefile include MANIFEST.in @@ -38,3 +45,5 @@ include changelogs/CHANGELOG*.rst include changelogs/changelog.yaml recursive-include hacking/build_library *.py include hacking/build-ansible.py +include hacking/test-module.py +include bin/* diff --git a/bin/ansible b/bin/ansible index 1acbe23019a..1f096cc4cb0 120000 --- a/bin/ansible +++ b/bin/ansible @@ -1 +1 @@ -../lib/ansible/cli/scripts/ansible_cli_stub.py \ No newline at end of file +../lib/ansible/cli/adhoc.py \ No newline at end of file diff --git a/bin/ansible-config b/bin/ansible-config index cabb1f519aa..d451b210b52 120000 --- a/bin/ansible-config +++ b/bin/ansible-config @@ -1 +1 @@ -ansible \ No newline at end of file +../lib/ansible/cli/config.py \ No newline at end of file diff --git a/bin/ansible-console b/bin/ansible-console index cabb1f519aa..7f5789e2ab6 120000 --- a/bin/ansible-console +++ b/bin/ansible-console @@ -1 +1 @@ -ansible \ No newline at end of file +../lib/ansible/cli/console.py \ No newline at end of file diff --git a/bin/ansible-doc b/bin/ansible-doc index cabb1f519aa..23d4e3912de 120000 --- a/bin/ansible-doc +++ b/bin/ansible-doc @@ -1 +1 @@ -ansible \ No newline at end of file +../lib/ansible/cli/doc.py \ No newline at end of file diff --git a/bin/ansible-galaxy b/bin/ansible-galaxy index cabb1f519aa..3771430a1a3 120000 --- a/bin/ansible-galaxy +++ b/bin/ansible-galaxy @@ -1 +1 @@ -ansible \ No newline at end of file +../lib/ansible/cli/galaxy.py \ No newline at end of file diff --git a/bin/ansible-inventory b/bin/ansible-inventory index cabb1f519aa..989f06a2d88 120000 --- a/bin/ansible-inventory +++ b/bin/ansible-inventory @@ -1 +1 @@ -ansible \ No newline at end of file +../lib/ansible/cli/inventory.py \ No newline at end of file diff --git a/bin/ansible-playbook b/bin/ansible-playbook index cabb1f519aa..3a25cc82165 120000 --- a/bin/ansible-playbook +++ b/bin/ansible-playbook @@ -1 +1 @@ -ansible \ No newline at end of file +../lib/ansible/cli/playbook.py \ No newline at end of file diff --git a/bin/ansible-pull b/bin/ansible-pull index cabb1f519aa..f0c87d47fb0 120000 --- a/bin/ansible-pull +++ b/bin/ansible-pull @@ -1 +1 @@ -ansible \ No newline at end of file +../lib/ansible/cli/pull.py \ No newline at end of file diff --git a/bin/ansible-vault b/bin/ansible-vault index cabb1f519aa..1a166e8fe54 120000 --- a/bin/ansible-vault +++ b/bin/ansible-vault @@ -1 +1 @@ -ansible \ No newline at end of file +../lib/ansible/cli/vault.py \ No newline at end of file diff --git a/changelogs/fragments/modernize-install.yaml b/changelogs/fragments/modernize-install.yaml new file mode 100644 index 00000000000..f21077161c7 --- /dev/null +++ b/changelogs/fragments/modernize-install.yaml @@ -0,0 +1,3 @@ +minor_changes: +- Installation - modernize our python installation, to reduce dynamic code in setup.py, and migrate + what is feasible to setup.cfg. This will enable shipping wheels in the future. diff --git a/hacking/env-setup b/hacking/env-setup index c9fbdbd55f8..4741423b4d3 100644 --- a/hacking/env-setup +++ b/hacking/env-setup @@ -48,10 +48,12 @@ FULL_PATH=$($PYTHON_BIN -c "import os; print(os.path.realpath('$HACKING_DIR'))") export ANSIBLE_HOME="$(dirname "$FULL_PATH")" PREFIX_PYTHONPATH="$ANSIBLE_HOME/lib" +ANSIBLE_TEST_PREFIX_PYTHONPATH="$ANSIBLE_HOME/test/lib" PREFIX_PATH="$ANSIBLE_HOME/bin" PREFIX_MANPATH="$ANSIBLE_HOME/docs/man" expr "$PYTHONPATH" : "${PREFIX_PYTHONPATH}.*" > /dev/null || prepend_path PYTHONPATH "$PREFIX_PYTHONPATH" +expr "$PYTHONPATH" : "${ANSIBLE_TEST_PREFIX_PYTHONPATH}.*" > /dev/null || prepend_path PYTHONPATH "$ANSIBLE_TEST_PREFIX_PYTHONPATH" expr "$PATH" : "${PREFIX_PATH}.*" > /dev/null || prepend_path PATH "$PREFIX_PATH" expr "$MANPATH" : "${PREFIX_MANPATH}.*" > /dev/null || prepend_path MANPATH "$PREFIX_MANPATH" diff --git a/hacking/env-setup.fish b/hacking/env-setup.fish index 94126cad406..944c029ff9b 100644 --- a/hacking/env-setup.fish +++ b/hacking/env-setup.fish @@ -5,6 +5,7 @@ set HACKING_DIR (dirname (status -f)) set FULL_PATH (python -c "import os; print(os.path.realpath('$HACKING_DIR'))") set ANSIBLE_HOME (dirname $FULL_PATH) set PREFIX_PYTHONPATH $ANSIBLE_HOME/lib +set ANSIBLE_TEST_PREFIX_PYTHONPATH $ANSIBLE_HOME/test/lib set PREFIX_PATH $ANSIBLE_HOME/bin set PREFIX_MANPATH $ANSIBLE_HOME/docs/man @@ -31,6 +32,16 @@ else end end +# Set ansible_test PYTHONPATH +switch PYTHONPATH + case "$ANSIBLE_TEST_PREFIX_PYTHONPATH*" + case "*" + if not [ $QUIET ] + echo "Appending PYTHONPATH" + end + set -gx PYTHONPATH "$ANSIBLE_TEST_PREFIX_PYTHONPATH:$PYTHONPATH" +end + # Set PATH if not contains $PREFIX_PATH $PATH set -gx PATH $PREFIX_PATH $PATH diff --git a/lib/ansible/__main__.py b/lib/ansible/__main__.py new file mode 100644 index 00000000000..f670d9adfde --- /dev/null +++ b/lib/ansible/__main__.py @@ -0,0 +1,41 @@ +# Copyright: (c) 2021, Matt Martz +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +import argparse +import importlib +import os +import sys + +from importlib.metadata import distribution + + +def _short_name(name): + return name.replace('ansible-', '').replace('ansible', 'adhoc') + + +def main(): + dist = distribution('ansible-core') + ep_map = {_short_name(ep.name): ep for ep in dist.entry_points if ep.group == 'console_scripts'} + + parser = argparse.ArgumentParser(prog='python -m ansible', add_help=False) + parser.add_argument('entry_point', choices=list(ep_map) + ['test']) + args, extra = parser.parse_known_args() + + if args.entry_point == 'test': + ansible_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + source_root = os.path.join(ansible_root, 'test', 'lib') + + if os.path.exists(os.path.join(source_root, 'ansible_test', '_internal', '__init__.py')): + # running from source, use that version of ansible-test instead of any version that may already be installed + sys.path.insert(0, source_root) + + module = importlib.import_module('ansible_test._util.target.cli.ansible_test_cli_stub') + main = module.main + else: + main = ep_map[args.entry_point].load() + + main([args.entry_point] + extra) + + +if __name__ == '__main__': + main() diff --git a/lib/ansible/cli/__init__.py b/lib/ansible/cli/__init__.py index 515822e23ab..4ab4b84c5ed 100644 --- a/lib/ansible/cli/__init__.py +++ b/lib/ansible/cli/__init__.py @@ -7,17 +7,36 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type +import sys + +# Used for determining if the system is running a new enough python version +# and should only restrict on our documented minimum versions +if sys.version_info < (3, 8): + raise SystemExit( + 'ERROR: Ansible requires Python 3.8 or newer on the controller. ' + 'Current version: %s' % ''.join(sys.version.splitlines()) + ) + +import errno import getpass import os import subprocess -import sys - +import traceback from abc import ABCMeta, abstractmethod +from pathlib import Path + +try: + from ansible import constants as C + from ansible.utils.display import Display, initialize_locale + initialize_locale() + display = Display() +except Exception as e: + print('ERROR: %s' % e, file=sys.stderr) + sys.exit(5) -from ansible.cli.arguments import option_helpers as opt_help -from ansible import constants as C from ansible import context -from ansible.errors import AnsibleError +from ansible.cli.arguments import option_helpers as opt_help +from ansible.errors import AnsibleError, AnsibleOptionsError, AnsibleParserError from ansible.inventory.manager import InventoryManager from ansible.module_utils.six import with_metaclass, string_types, PY3 from ansible.module_utils._text import to_bytes, to_text @@ -27,7 +46,6 @@ from ansible.plugins.loader import add_all_plugin_dirs from ansible.release import __version__ from ansible.utils.collection_loader import AnsibleCollectionConfig from ansible.utils.collection_loader._collection_finder import _get_collection_name_from_path -from ansible.utils.display import Display from ansible.utils.path import unfrackpath from ansible.utils.unsafe_proxy import to_unsafe_text from ansible.vars.manager import VariableManager @@ -39,9 +57,6 @@ except ImportError: HAS_ARGCOMPLETE = False -display = Display() - - class CLI(with_metaclass(ABCMeta, object)): ''' code behind bin/ansible* programs ''' @@ -292,7 +307,7 @@ class CLI(with_metaclass(ABCMeta, object)): ansible.arguments.option_helpers.add_runas_options(self.parser) self.parser.add_option('--my-option', dest='my_option', action='store') """ - self.parser = opt_help.create_base_parser(os.path.basename(self.args[0]), usage=usage, desc=desc, epilog=epilog, ) + self.parser = opt_help.create_base_parser(self.name, usage=usage, desc=desc, epilog=epilog) @abstractmethod def post_process_args(self, options): @@ -532,3 +547,74 @@ class CLI(with_metaclass(ABCMeta, object)): raise AnsibleError('Empty password was provided from file (%s)' % pwd_file) return to_unsafe_text(secret) + + @classmethod + def cli_executor(cls, args=None): + if args is None: + args = sys.argv + + try: + display.debug("starting run") + + ansible_dir = Path("~/.ansible").expanduser() + try: + ansible_dir.mkdir(mode=0o700) + except OSError as exc: + if exc.errno != errno.EEXIST: + display.warning( + "Failed to create the directory '%s': %s" % (ansible_dir, to_text(exc, errors='surrogate_or_replace')) + ) + else: + display.debug("Created the '%s' directory" % ansible_dir) + + try: + args = [to_text(a, errors='surrogate_or_strict') for a in args] + except UnicodeError: + display.error('Command line args are not in utf-8, unable to continue. Ansible currently only understands utf-8') + display.display(u"The full traceback was:\n\n%s" % to_text(traceback.format_exc())) + exit_code = 6 + else: + cli = cls(args) + exit_code = cli.run() + + except AnsibleOptionsError as e: + cli.parser.print_help() + display.error(to_text(e), wrap_text=False) + exit_code = 5 + except AnsibleParserError as e: + display.error(to_text(e), wrap_text=False) + exit_code = 4 + # TQM takes care of these, but leaving comment to reserve the exit codes + # except AnsibleHostUnreachable as e: + # display.error(str(e)) + # exit_code = 3 + # except AnsibleHostFailed as e: + # display.error(str(e)) + # exit_code = 2 + except AnsibleError as e: + display.error(to_text(e), wrap_text=False) + exit_code = 1 + except KeyboardInterrupt: + display.error("User interrupted execution") + exit_code = 99 + except Exception as e: + if C.DEFAULT_DEBUG: + # Show raw stacktraces in debug mode, It also allow pdb to + # enter post mortem mode. + raise + have_cli_options = bool(context.CLIARGS) + display.error("Unexpected Exception, this is probably a bug: %s" % to_text(e), wrap_text=False) + if not have_cli_options or have_cli_options and context.CLIARGS['verbosity'] > 2: + log_only = False + if hasattr(e, 'orig_exc'): + display.vvv('\nexception type: %s' % to_text(type(e.orig_exc))) + why = to_text(e.orig_exc) + if to_text(e) != why: + display.vvv('\noriginal msg: %s' % why) + else: + display.display("to see the full traceback, use -vvv") + log_only = True + display.display(u"the full traceback was:\n\n%s" % to_text(traceback.format_exc()), log_only=log_only) + exit_code = 250 + + sys.exit(exit_code) diff --git a/lib/ansible/cli/adhoc.py b/lib/ansible/cli/adhoc.py old mode 100644 new mode 100755 index 4ce876a6f76..de19e0e1fd1 --- a/lib/ansible/cli/adhoc.py +++ b/lib/ansible/cli/adhoc.py @@ -1,13 +1,16 @@ +#!/usr/bin/env python # Copyright: (c) 2012, Michael DeHaan # Copyright: (c) 2018, Ansible Project # GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +# PYTHON_ARGCOMPLETE_OK from __future__ import (absolute_import, division, print_function) __metaclass__ = type +# ansible.cli needs to be imported first, to ensure the source bin/* scripts run that code first +from ansible.cli import CLI from ansible import constants as C from ansible import context -from ansible.cli import CLI from ansible.cli.arguments import option_helpers as opt_help from ansible.errors import AnsibleError, AnsibleOptionsError from ansible.executor.task_queue_manager import TaskQueueManager @@ -25,6 +28,8 @@ class AdHocCLI(CLI): this command allows you to define and run a single task 'playbook' against a set of hosts ''' + name = 'ansible' + def init_parser(self): ''' create an options parser for bin/ansible ''' super(AdHocCLI, self).init_parser(usage='%prog [options]', @@ -179,3 +184,11 @@ class AdHocCLI(CLI): loader.cleanup_all_tmp_files() return result + + +def main(args=None): + AdHocCLI.cli_executor(args) + + +if __name__ == '__main__': + main() diff --git a/lib/ansible/cli/config.py b/lib/ansible/cli/config.py old mode 100644 new mode 100755 index 328b3c1110f..59041ba3b30 --- a/lib/ansible/cli/config.py +++ b/lib/ansible/cli/config.py @@ -1,9 +1,14 @@ +#!/usr/bin/env python # Copyright: (c) 2017, Ansible Project # GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +# PYTHON_ARGCOMPLETE_OK from __future__ import (absolute_import, division, print_function) __metaclass__ = type +# ansible.cli needs to be imported first, to ensure the source bin/* scripts run that code first +from ansible.cli import CLI + import os import shlex import subprocess @@ -13,7 +18,6 @@ from ansible import context import ansible.plugins.loader as plugin_loader from ansible import constants as C -from ansible.cli import CLI from ansible.cli.arguments import option_helpers as opt_help from ansible.config.manager import ConfigManager, Setting from ansible.errors import AnsibleError, AnsibleOptionsError @@ -33,6 +37,8 @@ display = Display() class ConfigCLI(CLI): """ Config command line class """ + name = 'ansible-config' + def __init__(self, args, callback=None): self.config_file = None @@ -469,3 +475,11 @@ class ConfigCLI(CLI): text = self._get_plugin_configs(context.CLIARGS['type'], context.CLIARGS['args']) self.pager(to_text('\n'.join(text), errors='surrogate_or_strict')) + + +def main(args=None): + ConfigCLI.cli_executor(args) + + +if __name__ == '__main__': + main() diff --git a/lib/ansible/cli/console.py b/lib/ansible/cli/console.py old mode 100644 new mode 100755 index e28748961bc..bedd0e9257f --- a/lib/ansible/cli/console.py +++ b/lib/ansible/cli/console.py @@ -1,11 +1,16 @@ +#!/usr/bin/env python # Copyright: (c) 2014, Nandor Sivok # Copyright: (c) 2016, Redhat Inc # Copyright: (c) 2018, Ansible Project # GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +# PYTHON_ARGCOMPLETE_OK from __future__ import (absolute_import, division, print_function) __metaclass__ = type +# ansible.cli needs to be imported first, to ensure the source bin/* scripts run that code first +from ansible.cli import CLI + import atexit import cmd import getpass @@ -15,7 +20,6 @@ import sys from ansible import constants as C from ansible import context -from ansible.cli import CLI from ansible.cli.arguments import option_helpers as opt_help from ansible.executor.task_queue_manager import TaskQueueManager from ansible.module_utils._text import to_native, to_text @@ -56,6 +60,7 @@ class ConsoleCLI(CLI, cmd.Cmd): - `exit`: exit ansible-console ''' + name = 'ansible-console' modules = [] ARGUMENTS = {'host-pattern': 'A name of a group in the inventory, a shell-like glob ' 'selecting hosts in inventory or any combination of the two separated by commas.'} @@ -406,7 +411,7 @@ class ConsoleCLI(CLI, cmd.Cmd): if module_name in self.modules: in_path = module_loader.find_plugin(module_name) if in_path: - oc, a, _, _ = plugin_docs.get_docstring(in_path, fragment_loader) + oc, a, _dummy1, _dummy2 = plugin_docs.get_docstring(in_path, fragment_loader) if oc: display.display(oc['short_description']) display.display('Parameters:') @@ -438,7 +443,7 @@ class ConsoleCLI(CLI, cmd.Cmd): def module_args(self, module_name): in_path = module_loader.find_plugin(module_name) - oc, a, _, _ = plugin_docs.get_docstring(in_path, fragment_loader, is_module=True) + oc, a, _dummy1, _dummy2 = plugin_docs.get_docstring(in_path, fragment_loader, is_module=True) return list(oc['options'].keys()) def run(self): @@ -494,3 +499,11 @@ class ConsoleCLI(CLI, cmd.Cmd): atexit.register(readline.write_history_file, histfile) self.set_prompt() self.cmdloop() + + +def main(args=None): + ConsoleCLI.cli_executor(args) + + +if __name__ == '__main__': + main() diff --git a/lib/ansible/cli/doc.py b/lib/ansible/cli/doc.py old mode 100644 new mode 100755 index 2ed1cbbfcf2..78db1e6f0cf --- a/lib/ansible/cli/doc.py +++ b/lib/ansible/cli/doc.py @@ -1,10 +1,15 @@ +#!/usr/bin/env python # Copyright: (c) 2014, James Tanner # Copyright: (c) 2018, Ansible Project # GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +# PYTHON_ARGCOMPLETE_OK from __future__ import (absolute_import, division, print_function) __metaclass__ = type +# ansible.cli needs to be imported first, to ensure the source bin/* scripts run that code first +from ansible.cli import CLI + import datetime import json import pkgutil @@ -19,7 +24,6 @@ import ansible.plugins.loader as plugin_loader from ansible import constants as C from ansible import context -from ansible.cli import CLI from ansible.cli.arguments import option_helpers as opt_help from ansible.collections.list import list_collection_dirs from ansible.errors import AnsibleError, AnsibleOptionsError, AnsibleParserError @@ -335,6 +339,8 @@ class DocCLI(CLI, RoleMixin): provides a printout of their DOCUMENTATION strings, and it can create a short "snippet" which can be pasted into a playbook. ''' + name = 'ansible-doc' + # default ignore list for detailed views IGNORE = ('module', 'docuri', 'version_added', 'short_description', 'now_date', 'plainexamples', 'returndocs', 'collection') @@ -1383,3 +1389,11 @@ def _do_lookup_snippet(doc): text.append(snippet) return text + + +def main(args=None): + DocCLI.cli_executor(args) + + +if __name__ == '__main__': + main() diff --git a/lib/ansible/cli/galaxy.py b/lib/ansible/cli/galaxy.py old mode 100644 new mode 100755 index bdaadb5d4c6..b9b5eca9976 --- a/lib/ansible/cli/galaxy.py +++ b/lib/ansible/cli/galaxy.py @@ -1,10 +1,15 @@ +#!/usr/bin/env python # Copyright: (c) 2013, James Cammarata # Copyright: (c) 2018-2021, Ansible Project # GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +# PYTHON_ARGCOMPLETE_OK from __future__ import (absolute_import, division, print_function) __metaclass__ = type +# ansible.cli needs to be imported first, to ensure the source bin/* scripts run that code first +from ansible.cli import CLI + import json import os.path import re @@ -17,7 +22,6 @@ from yaml.error import YAMLError import ansible.constants as C from ansible import context -from ansible.cli import CLI from ansible.cli.arguments import option_helpers as opt_help from ansible.errors import AnsibleError, AnsibleOptionsError from ansible.galaxy import Galaxy, get_collections_galaxy_meta_info @@ -134,6 +138,8 @@ def _get_collection_widths(collections): class GalaxyCLI(CLI): '''command to manage Ansible roles in shared repositories, the default of which is Ansible Galaxy *https://galaxy.ansible.com*.''' + name = 'ansible-galaxy' + SKIP_INFO_KEYS = ("name", "description", "readme_html", "related", "summary_fields", "average_aw_composite", "average_aw_score", "url") def __init__(self, args): @@ -1679,3 +1685,11 @@ class GalaxyCLI(CLI): display.display(resp['status']) return True + + +def main(args=None): + GalaxyCLI.cli_executor(args) + + +if __name__ == '__main__': + main() diff --git a/lib/ansible/cli/inventory.py b/lib/ansible/cli/inventory.py old mode 100644 new mode 100755 index 3fbea734e69..e7d871eaa07 --- a/lib/ansible/cli/inventory.py +++ b/lib/ansible/cli/inventory.py @@ -1,10 +1,15 @@ +#!/usr/bin/env python # Copyright: (c) 2017, Brian Coca # Copyright: (c) 2018, Ansible Project # GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +# PYTHON_ARGCOMPLETE_OK from __future__ import (absolute_import, division, print_function) __metaclass__ = type +# ansible.cli needs to be imported first, to ensure the source bin/* scripts run that code first +from ansible.cli import CLI + import sys import argparse @@ -12,7 +17,6 @@ from operator import attrgetter from ansible import constants as C from ansible import context -from ansible.cli import CLI from ansible.cli.arguments import option_helpers as opt_help from ansible.errors import AnsibleError, AnsibleOptionsError from ansible.module_utils._text import to_bytes, to_native, to_text @@ -46,6 +50,8 @@ INTERNAL_VARS = frozenset(['ansible_diff_mode', class InventoryCLI(CLI): ''' used to display or dump the configured inventory as Ansible sees it ''' + name = 'ansible-inventory' + ARGUMENTS = {'host': 'The name of a host to match in the inventory, relevant when using --list', 'group': 'The name of a group in the inventory, relevant when using --graph', } @@ -402,3 +408,11 @@ class InventoryCLI(CLI): results = format_group(top) return results + + +def main(args=None): + InventoryCLI.cli_executor(args) + + +if __name__ == '__main__': + main() diff --git a/lib/ansible/cli/playbook.py b/lib/ansible/cli/playbook.py old mode 100644 new mode 100755 index c3ec8321ec2..c94cf0ff771 --- a/lib/ansible/cli/playbook.py +++ b/lib/ansible/cli/playbook.py @@ -1,16 +1,20 @@ +#!/usr/bin/env python # (c) 2012, Michael DeHaan # Copyright: (c) 2018, Ansible Project # GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +# PYTHON_ARGCOMPLETE_OK from __future__ import (absolute_import, division, print_function) __metaclass__ = type +# ansible.cli needs to be imported first, to ensure the source bin/* scripts run that code first +from ansible.cli import CLI + import os import stat from ansible import constants as C from ansible import context -from ansible.cli import CLI from ansible.cli.arguments import option_helpers as opt_help from ansible.errors import AnsibleError from ansible.executor.playbook_executor import PlaybookExecutor @@ -29,6 +33,8 @@ class PlaybookCLI(CLI): ''' the tool to run *Ansible playbooks*, which are a configuration and multinode deployment system. See the project home page (https://docs.ansible.com) for more information. ''' + name = 'ansible-playbook' + def init_parser(self): # create parser for CLI options @@ -215,3 +221,11 @@ class PlaybookCLI(CLI): for host in inventory.list_hosts(): hostname = host.get_name() variable_manager.clear_facts(hostname) + + +def main(args=None): + PlaybookCLI.cli_executor(args) + + +if __name__ == '__main__': + main() diff --git a/lib/ansible/cli/pull.py b/lib/ansible/cli/pull.py old mode 100644 new mode 100755 index c92eef760c9..bff636363a0 --- a/lib/ansible/cli/pull.py +++ b/lib/ansible/cli/pull.py @@ -1,10 +1,15 @@ +#!/usr/bin/env python # Copyright: (c) 2012, Michael DeHaan # Copyright: (c) 2018, Ansible Project # GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +# PYTHON_ARGCOMPLETE_OK from __future__ import (absolute_import, division, print_function) __metaclass__ = type +# ansible.cli needs to be imported first, to ensure the source bin/* scripts run that code first +from ansible.cli import CLI + import datetime import os import platform @@ -16,7 +21,6 @@ import time from ansible import constants as C from ansible import context -from ansible.cli import CLI from ansible.cli.arguments import option_helpers as opt_help from ansible.errors import AnsibleOptionsError from ansible.module_utils._text import to_native, to_text @@ -40,6 +44,8 @@ class PullCLI(CLI): excellent way to gather and analyze remote logs from ansible-pull. ''' + name = 'ansible-pull' + DEFAULT_REPO_TYPE = 'git' DEFAULT_PLAYBOOK = 'local.yml' REPO_CHOICES = ('git', 'subversion', 'hg', 'bzr') @@ -341,3 +347,11 @@ class PullCLI(CLI): if playbook is None: display.warning("\n".join(errors)) return playbook + + +def main(args=None): + PullCLI.cli_executor(args) + + +if __name__ == '__main__': + main() diff --git a/lib/ansible/cli/scripts/ansible_cli_stub.py b/lib/ansible/cli/scripts/ansible_cli_stub.py deleted file mode 100755 index 622152c4155..00000000000 --- a/lib/ansible/cli/scripts/ansible_cli_stub.py +++ /dev/null @@ -1,170 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# (c) 2012, Michael DeHaan -# -# This file is part of Ansible -# -# Ansible is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Ansible is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Ansible. If not, see . - -# PYTHON_ARGCOMPLETE_OK - -from __future__ import (absolute_import, division, print_function) -__metaclass__ = type - - -import errno -import os -import shutil -import sys -import traceback - -# Used for determining if the system is running a new enough python version -# and should only restrict on our documented minimum versions -_PY38_MIN = sys.version_info[:2] >= (3, 8) -if not _PY38_MIN: - raise SystemExit( - 'ERROR: Ansible requires Python 3.8 or newer on the controller. ' - 'Current version: %s' % ''.join(sys.version.splitlines()) - ) - - -# These lines appear after the PY38 check, to ensure the "friendly" error happens before -# any invalid syntax appears in other files that may get imported -from ansible import context -from ansible.errors import AnsibleError, AnsibleOptionsError, AnsibleParserError -from ansible.module_utils._text import to_text - -from pathlib import Path - - -class LastResort(object): - # OUTPUT OF LAST RESORT - def display(self, msg, log_only=None): - print(msg, file=sys.stderr) - - def error(self, msg, wrap_text=None): - print(msg, file=sys.stderr) - - -if __name__ == '__main__': - - display = LastResort() - - try: # bad ANSIBLE_CONFIG or config options can force ugly stacktrace - import ansible.constants as C - from ansible.utils.display import Display, initialize_locale - except AnsibleOptionsError as e: - display.error(to_text(e), wrap_text=False) - sys.exit(5) - - initialize_locale() - - cli = None - me = Path(sys.argv[0]).name - - try: - display = Display() - display.debug("starting run") - - sub = None - target = me.split('-') - if target[-1][0].isdigit(): - # Remove any version or python version info as downstreams - # sometimes add that - target = target[:-1] - - if len(target) > 1: - sub = target[1] - myclass = "%sCLI" % sub.capitalize() - elif target[0] == 'ansible': - sub = 'adhoc' - myclass = 'AdHocCLI' - else: - raise AnsibleError("Unknown Ansible alias: %s" % me) - - try: - mycli = getattr(__import__("ansible.cli.%s" % sub, fromlist=[myclass]), myclass) - except ImportError as e: - # ImportError members have changed in py3 - if 'msg' in dir(e): - msg = e.msg - else: - msg = e.message - if msg.endswith(' %s' % sub): - raise AnsibleError("Ansible sub-program not implemented: %s" % me) - else: - raise - - ansible_dir = Path("~/.ansible").expanduser() - try: - ansible_dir.mkdir(mode=0o700) - except OSError as exc: - if exc.errno != errno.EEXIST: - display.warning( - "Failed to create the directory '%s': %s" % (ansible_dir, to_text(exc, errors='surrogate_or_replace')) - ) - else: - display.debug("Created the '%s' directory" % ansible_dir) - - try: - args = [to_text(a, errors='surrogate_or_strict') for a in sys.argv] - except UnicodeError: - display.error('Command line args are not in utf-8, unable to continue. Ansible currently only understands utf-8') - display.display(u"The full traceback was:\n\n%s" % to_text(traceback.format_exc())) - exit_code = 6 - else: - cli = mycli(args) - exit_code = cli.run() - - except AnsibleOptionsError as e: - cli.parser.print_help() - display.error(to_text(e), wrap_text=False) - exit_code = 5 - except AnsibleParserError as e: - display.error(to_text(e), wrap_text=False) - exit_code = 4 -# TQM takes care of these, but leaving comment to reserve the exit codes -# except AnsibleHostUnreachable as e: -# display.error(str(e)) -# exit_code = 3 -# except AnsibleHostFailed as e: -# display.error(str(e)) -# exit_code = 2 - except AnsibleError as e: - display.error(to_text(e), wrap_text=False) - exit_code = 1 - except KeyboardInterrupt: - display.error("User interrupted execution") - exit_code = 99 - except Exception as e: - if C.DEFAULT_DEBUG: - # Show raw stacktraces in debug mode, It also allow pdb to - # enter post mortem mode. - raise - have_cli_options = bool(context.CLIARGS) - display.error("Unexpected Exception, this is probably a bug: %s" % to_text(e), wrap_text=False) - if not have_cli_options or have_cli_options and context.CLIARGS['verbosity'] > 2: - log_only = False - if hasattr(e, 'orig_exc'): - display.vvv('\nexception type: %s' % to_text(type(e.orig_exc))) - why = to_text(e.orig_exc) - if to_text(e) != why: - display.vvv('\noriginal msg: %s' % why) - else: - display.display("to see the full traceback, use -vvv") - log_only = True - display.display(u"the full traceback was:\n\n%s" % to_text(traceback.format_exc()), log_only=log_only) - exit_code = 250 - - sys.exit(exit_code) diff --git a/lib/ansible/cli/scripts/ansible_connection_cli_stub.py b/lib/ansible/cli/scripts/ansible_connection_cli_stub.py index 31047d96c5f..f3cd5816dd5 100755 --- a/lib/ansible/cli/scripts/ansible_connection_cli_stub.py +++ b/lib/ansible/cli/scripts/ansible_connection_cli_stub.py @@ -6,6 +6,7 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type +import argparse import fcntl import hashlib import os @@ -20,6 +21,7 @@ import json from contextlib import contextmanager from ansible import constants as C +from ansible.cli.arguments.option_helpers import AnsibleVersion from ansible.module_utils._text import to_bytes, to_text from ansible.module_utils.six import PY3 from ansible.module_utils.six.moves import cPickle, StringIO @@ -32,6 +34,8 @@ from ansible.utils.path import unfrackpath, makedirs_safe from ansible.utils.display import Display from ansible.utils.jsonrpc import JsonRpcServer +display = Display() + def read_stream(byte_stream): size = int(byte_stream.readline().strip()) @@ -217,9 +221,15 @@ class ConnectionProcess(object): display.display('shutdown complete', log_only=True) -def main(): +def main(args=None): """ Called to initiate the connect to the remote device """ + parser = argparse.ArgumentParser(prog='ansible-connection', add_help=False) + parser.add_argument('--version', action=AnsibleVersion, nargs=0) + parser.add_argument('playbook_pid') + parser.add_argument('task_uuid') + args = parser.parse_args(args[1:] if args is not None else args) + rc = 0 result = {} messages = list() @@ -260,8 +270,8 @@ def main(): if rc == 0: ssh = connection_loader.get('ssh', class_only=True) - ansible_playbook_pid = sys.argv[1] - task_uuid = sys.argv[2] + ansible_playbook_pid = args.playbook_pid + task_uuid = args.task_uuid cp = ssh._create_control_path(play_context.remote_addr, play_context.port, play_context.remote_user, play_context.connection, ansible_playbook_pid) # create the persistent connection dir if need be and create the paths # which we will be using later @@ -345,5 +355,4 @@ def main(): if __name__ == '__main__': - display = Display() main() diff --git a/lib/ansible/cli/vault.py b/lib/ansible/cli/vault.py old mode 100644 new mode 100755 index e08a3bb4613..830d22395d6 --- a/lib/ansible/cli/vault.py +++ b/lib/ansible/cli/vault.py @@ -1,16 +1,20 @@ +#!/usr/bin/env python # (c) 2014, James Tanner # Copyright: (c) 2018, Ansible Project # GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +# PYTHON_ARGCOMPLETE_OK from __future__ import (absolute_import, division, print_function) __metaclass__ = type +# ansible.cli needs to be imported first, to ensure the source bin/* scripts run that code first +from ansible.cli import CLI + import os import sys from ansible import constants as C from ansible import context -from ansible.cli import CLI from ansible.cli.arguments import option_helpers as opt_help from ansible.errors import AnsibleOptionsError from ansible.module_utils._text import to_text, to_bytes @@ -32,6 +36,8 @@ class VaultCLI(CLI): If you'd like to not expose what variables you are using, you can keep an individual task file entirely encrypted. ''' + name = 'ansible-vault' + FROM_STDIN = "stdin" FROM_ARGS = "the command line args" FROM_PROMPT = "the interactive prompt" @@ -462,3 +468,11 @@ class VaultCLI(CLI): self.new_encrypt_vault_id) display.display("Rekey successful", stderr=True) + + +def main(args=None): + VaultCLI.cli_executor(args) + + +if __name__ == '__main__': + main() diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000000..38c5a47d622 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["setuptools >= 39.2.0", "wheel"] +build-backend = "setuptools.build_meta" diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 00000000000..873dfa72e11 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,60 @@ +# Minimum target setuptools 39.2.0 + +[metadata] +name = ansible-core +version = attr: ansible.release.__version__ +description = Radically simple IT automation +long_description = file: README.rst +author = Ansible, Inc. +author_email = info@ansible.com +url = https://ansible.com/ +project_urls = + Bug Tracker=https://github.com/ansible/ansible/issues + CI: Azure Pipelines=https://dev.azure.com/ansible/ansible/ + Code of Conduct=https://docs.ansible.com/ansible/latest/community/code_of_conduct.html + Documentation=https://docs.ansible.com/ansible-core/ + Mailing lists=https://docs.ansible.com/ansible/latest/community/communication.html#mailing-list-information + Source Code=https://github.com/ansible/ansible +license = GPLv3+ +classifiers = + Development Status :: 5 - Production/Stable + Environment :: Console + Intended Audience :: Developers + Intended Audience :: Information Technology + Intended Audience :: System Administrators + License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+) + Natural Language :: English + Operating System :: POSIX + Programming Language :: Python :: 3 + Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.9 + Programming Language :: Python :: 3.10 + Programming Language :: Python :: 3 :: Only + Topic :: System :: Installation/Setup + Topic :: System :: Systems Administration + Topic :: Utilities + +[options] +zip_safe = False +python_requires = >=3.8 +include_package_data = True +# keep ansible-test as a verbatim script to work with editable installs, since it needs to do its +# own package redirection magic that's beyond the scope of the normal `ansible` path redirection +# done by setuptools `develop` +scripts = + bin/ansible-test + +# setuptools 51.0.0 +# [options.entry_points] +# console_scripts = +# ansible = ansible.cli.adhoc:main +# ansible-config = ansible.cli.config:main +# ansible-console = ansible.cli.console:main +# ansible-doc = ansible.cli.doc:main +# ansible-galaxy = ansible.cli.galaxy:main +# ansible-inventory = ansible.cli.inventory:main +# ansible-playbook = ansible.cli.playbook:main +# ansible-pull = ansible.cli.pull:main +# ansible-vault = ansible.cli.vault:main +# ansible-connection = ansible.cli.scripts.ansible_connection_cli_stub:main +# ansible-test = ansible_test._util.target.cli.ansible_test_cli_stub:main diff --git a/setup.py b/setup.py index 11f88c6afaa..b17ae8db832 100644 --- a/setup.py +++ b/setup.py @@ -1,394 +1,31 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type -import json -import os -import os.path -import re -import sys -import warnings +import pathlib -from collections import defaultdict +from setuptools import find_packages, setup -try: - from setuptools import setup, find_packages - from setuptools.command.build_py import build_py as BuildPy - from setuptools.command.install_lib import install_lib as InstallLib - from setuptools.command.install_scripts import install_scripts as InstallScripts -except ImportError: - print("Ansible now needs setuptools in order to build. Install it using" - " your package manager (usually python-setuptools) or via pip (pip" - " install setuptools).", file=sys.stderr) - sys.exit(1) +here = pathlib.Path(__file__).parent.resolve() -# `distutils` must be imported after `setuptools` or it will cause explosions -# with `setuptools >=48.0.0, <49.1`. -# Refs: -# * https://github.com/ansible/ansible/issues/70456 -# * https://github.com/pypa/setuptools/issues/2230 -# * https://github.com/pypa/setuptools/commit/bd110264 -from distutils.command.build_scripts import build_scripts as BuildScripts -from distutils.command.sdist import sdist as SDist +install_requires = (here / 'requirements.txt').read_text(encoding='utf-8').splitlines() - -def find_package_info(*file_paths): - try: - with open(os.path.join(*file_paths), 'r') as f: - info_file = f.read() - except Exception: - raise RuntimeError("Unable to find package info.") - - # The version line must have the form - # __version__ = 'ver' - version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]", - info_file, re.M) - author_match = re.search(r"^__author__ = ['\"]([^'\"]*)['\"]", - info_file, re.M) - - if version_match and author_match: - return version_match.group(1), author_match.group(1) - raise RuntimeError("Unable to find package info.") - - -def _validate_install_ansible_core(): - """Validate that we can install ansible-core. This checks if - ansible<=2.9 or ansible-base>=2.10 are installed. - """ - # Skip common commands we can ignore - # Do NOT add bdist_wheel here, we don't ship wheels - # and bdist_wheel is the only place we can prevent pip - # from installing, as pip creates a wheel, and installs the wheel - # and we have no influence over installation within a wheel - if set(('sdist', 'egg_info')).intersection(sys.argv): - return - - if os.getenv('ANSIBLE_SKIP_CONFLICT_CHECK', '') not in ('', '0'): - return - - # Save these for later restoring things to pre invocation - sys_modules = sys.modules.copy() - sys_modules_keys = set(sys_modules) - - # Make sure `lib` isn't in `sys.path` that could confuse this - sys_path = sys.path[:] - abspath = os.path.abspath - sys.path[:] = [p for p in sys.path if abspath(p) != abspath('lib')] - - try: - from ansible.release import __version__ - except ImportError: - pass - else: - version_tuple = tuple(int(v) for v in __version__.split('.')[:2]) - if version_tuple >= (2, 11): - return - elif version_tuple == (2, 10): - ansible_name = 'ansible-base' - else: - ansible_name = 'ansible' - - stars = '*' * 76 - raise RuntimeError( - ''' - - %s - - Cannot install ansible-core with a pre-existing %s==%s - installation. - - Installing ansible-core with ansible-2.9 or older, or ansible-base-2.10 - currently installed with pip is known to cause problems. Please uninstall - %s and install the new version: - - pip uninstall %s - pip install ansible-core - - If you want to skip the conflict checks and manually resolve any issues - afterwards, set the ANSIBLE_SKIP_CONFLICT_CHECK environment variable: - - ANSIBLE_SKIP_CONFLICT_CHECK=1 pip install ansible-core - - %s - ''' % (stars, ansible_name, __version__, ansible_name, ansible_name, stars)) - finally: - sys.path[:] = sys_path - for key in sys_modules_keys.symmetric_difference(sys.modules): - sys.modules.pop(key, None) - sys.modules.update(sys_modules) - - -_validate_install_ansible_core() - - -SYMLINK_CACHE = 'SYMLINK_CACHE.json' - - -def _find_symlinks(topdir, extension=''): - """Find symlinks that should be maintained - - Maintained symlinks exist in the bin dir or are modules which have - aliases. Our heuristic is that they are a link in a certain path which - point to a file in the same directory. - - .. warn:: - - We want the symlinks in :file:`bin/` that link into :file:`lib/ansible/*` (currently, - :command:`ansible`, :command:`ansible-test`, and :command:`ansible-connection`) to become - real files on install. Updates to the heuristic here *must not* add them to the symlink - cache. - """ - symlinks = defaultdict(list) - for base_path, dirs, files in os.walk(topdir): - for filename in files: - filepath = os.path.join(base_path, filename) - if os.path.islink(filepath) and filename.endswith(extension): - target = os.readlink(filepath) - if target.startswith('/'): - # We do not support absolute symlinks at all - continue - - if os.path.dirname(target) == '': - link = filepath[len(topdir):] - if link.startswith('/'): - link = link[1:] - symlinks[os.path.basename(target)].append(link) - else: - # Count how many directory levels from the topdir we are - levels_deep = os.path.dirname(filepath).count('/') - - # Count the number of directory levels higher we walk up the tree in target - target_depth = 0 - for path_component in target.split('/'): - if path_component == '..': - target_depth += 1 - # If we walk past the topdir, then don't store - if target_depth >= levels_deep: - break - else: - target_depth -= 1 - else: - # If we managed to stay within the tree, store the symlink - link = filepath[len(topdir):] - if link.startswith('/'): - link = link[1:] - symlinks[target].append(link) - - return symlinks - - -def _cache_symlinks(symlink_data): - with open(SYMLINK_CACHE, 'w') as f: - json.dump(symlink_data, f) - - -def _maintain_symlinks(symlink_type, base_path): - """Switch a real file into a symlink""" - try: - # Try the cache first because going from git checkout to sdist is the - # only time we know that we're going to cache correctly - with open(SYMLINK_CACHE, 'r') as f: - symlink_data = json.load(f) - except (IOError, OSError) as e: - # IOError on py2, OSError on py3. Both have errno - if e.errno == 2: - # SYMLINKS_CACHE doesn't exist. Fallback to trying to create the - # cache now. Will work if we're running directly from a git - # checkout or from an sdist created earlier. - library_symlinks = _find_symlinks('lib', '.py') - library_symlinks.update(_find_symlinks('test/lib')) - - symlink_data = {'script': _find_symlinks('bin'), - 'library': library_symlinks, - } - - # Sanity check that something we know should be a symlink was - # found. We'll take that to mean that the current directory - # structure properly reflects symlinks in the git repo - if 'ansible-playbook' in symlink_data['script']['ansible']: - _cache_symlinks(symlink_data) - else: - raise RuntimeError( - "Pregenerated symlink list was not present and expected " - "symlinks in ./bin were missing or broken. " - "Perhaps this isn't a git checkout?" - ) - else: - raise - symlinks = symlink_data[symlink_type] - - for source in symlinks: - for dest in symlinks[source]: - dest_path = os.path.join(base_path, dest) - if not os.path.islink(dest_path): - try: - os.unlink(dest_path) - except OSError as e: - if e.errno == 2: - # File does not exist which is all we wanted - pass - os.symlink(source, dest_path) - - -class BuildPyCommand(BuildPy): - def run(self): - BuildPy.run(self) - _maintain_symlinks('library', self.build_lib) - - -class BuildScriptsCommand(BuildScripts): - def run(self): - BuildScripts.run(self) - _maintain_symlinks('script', self.build_dir) - - -class InstallLibCommand(InstallLib): - def run(self): - InstallLib.run(self) - _maintain_symlinks('library', self.install_dir) - - -class InstallScriptsCommand(InstallScripts): - def run(self): - InstallScripts.run(self) - _maintain_symlinks('script', self.install_dir) - - -class SDistCommand(SDist): - def run(self): - # have to generate the cache of symlinks for release as sdist is the - # only command that has access to symlinks from the git repo - library_symlinks = _find_symlinks('lib', '.py') - library_symlinks.update(_find_symlinks('test/lib')) - - symlinks = {'script': _find_symlinks('bin'), - 'library': library_symlinks, - } - _cache_symlinks(symlinks) - - SDist.run(self) - - # Print warnings at the end because no one will see warnings before all the normal status - # output - if os.environ.get('_ANSIBLE_SDIST_FROM_MAKEFILE', False) != '1': - warnings.warn('When setup.py sdist is run from outside of the Makefile,' - ' the generated tarball may be incomplete. Use `make snapshot`' - ' to create a tarball from an arbitrary checkout or use' - ' `cd packaging/release && make release version=[..]` for official builds.', - RuntimeWarning) - - -def read_file(file_name): - """Read file and return its contents.""" - with open(file_name, 'r') as f: - return f.read() - - -def read_requirements(file_name): - """Read requirements file as a list.""" - reqs = read_file(file_name).splitlines() - if not reqs: - raise RuntimeError( - "Unable to read requirements from the %s file" - "That indicates this copy of the source code is incomplete." - % file_name - ) - return reqs - - -def get_dynamic_setup_params(): - """Add dynamically calculated setup params to static ones.""" - return { - # Retrieve the long description from the README - 'long_description': read_file('README.rst'), - 'install_requires': read_requirements('requirements.txt'), - } - - -here = os.path.abspath(os.path.dirname(__file__)) -__version__, __author__ = find_package_info(here, 'lib', 'ansible', 'release.py') -static_setup_params = dict( - # Use the distutils SDist so that symlinks are not expanded - # Use a custom Build for the same reason - cmdclass={ - 'build_py': BuildPyCommand, - 'build_scripts': BuildScriptsCommand, - 'install_lib': InstallLibCommand, - 'install_scripts': InstallScriptsCommand, - 'sdist': SDistCommand, - }, - name='ansible-core', - version=__version__, - description='Radically simple IT automation', - author=__author__, - author_email='info@ansible.com', - url='https://ansible.com/', - project_urls={ - 'Bug Tracker': 'https://github.com/ansible/ansible/issues', - 'CI: Azure Pipelines': 'https://dev.azure.com/ansible/ansible/', - 'Code of Conduct': 'https://docs.ansible.com/ansible/latest/community/code_of_conduct.html', - 'Documentation': 'https://docs.ansible.com/ansible/', - 'Mailing lists': 'https://docs.ansible.com/ansible/latest/community/communication.html#mailing-list-information', - 'Source Code': 'https://github.com/ansible/ansible', - }, - license='GPLv3+', - # Ansible will also make use of a system copy of python-six and - # python-selectors2 if installed but use a Bundled copy if it's not. - python_requires='>=3.8', +setup( + install_requires=install_requires, package_dir={'': 'lib', 'ansible_test': 'test/lib/ansible_test'}, packages=find_packages('lib') + find_packages('test/lib'), - include_package_data=True, - classifiers=[ - 'Development Status :: 5 - Production/Stable', - 'Environment :: Console', - 'Intended Audience :: Developers', - 'Intended Audience :: Information Technology', - 'Intended Audience :: System Administrators', - 'License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)', - 'Natural Language :: English', - 'Operating System :: POSIX', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python :: 3.9', - 'Programming Language :: Python :: 3.10', - 'Topic :: System :: Installation/Setup', - 'Topic :: System :: Systems Administration', - 'Topic :: Utilities', - ], - scripts=[ - 'bin/ansible', - 'bin/ansible-playbook', - 'bin/ansible-pull', - 'bin/ansible-doc', - 'bin/ansible-galaxy', - 'bin/ansible-console', - 'bin/ansible-connection', - 'bin/ansible-vault', - 'bin/ansible-config', - 'bin/ansible-inventory', - 'bin/ansible-test', - ], - data_files=[], - # Installing as zip files would break due to references to __file__ - zip_safe=False + entry_points={ + 'console_scripts': [ + 'ansible=ansible.cli.adhoc:main', + 'ansible-config=ansible.cli.config:main', + 'ansible-console=ansible.cli.console:main', + 'ansible-doc=ansible.cli.doc:main', + 'ansible-galaxy=ansible.cli.galaxy:main', + 'ansible-inventory=ansible.cli.inventory:main', + 'ansible-playbook=ansible.cli.playbook:main', + 'ansible-pull=ansible.cli.pull:main', + 'ansible-vault=ansible.cli.vault:main', + 'ansible-connection=ansible.cli.scripts.ansible_connection_cli_stub:main', + ], + }, ) - - -def main(): - """Invoke installation process using setuptools.""" - setup_params = dict(static_setup_params, **get_dynamic_setup_params()) - ignore_warning_regex = ( - r"Unknown distribution option: '(project_urls|python_requires)'" - ) - warnings.filterwarnings( - 'ignore', - message=ignore_warning_regex, - category=UserWarning, - module='distutils.dist', - ) - setup(**setup_params) - warnings.resetwarnings() - - -if __name__ == '__main__': - main() diff --git a/test/integration/targets/connection_local/aliases b/test/integration/targets/connection_local/aliases index b59832142f2..0ca054fafcb 100644 --- a/test/integration/targets/connection_local/aliases +++ b/test/integration/targets/connection_local/aliases @@ -1 +1,2 @@ shippable/posix/group3 +needs/target/connection diff --git a/test/integration/targets/connection_local/runme.sh b/test/integration/targets/connection_local/runme.sh deleted file mode 120000 index 70aa5dbdba4..00000000000 --- a/test/integration/targets/connection_local/runme.sh +++ /dev/null @@ -1 +0,0 @@ -../connection_posix/test.sh \ No newline at end of file diff --git a/test/integration/targets/connection_local/runme.sh b/test/integration/targets/connection_local/runme.sh new file mode 100755 index 00000000000..a2c32adf06b --- /dev/null +++ b/test/integration/targets/connection_local/runme.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +set -eux + +group=local + +cd ../connection + +INVENTORY="../connection_${group}/test_connection.inventory" ./test.sh \ + -e target_hosts="${group}" \ + -e action_prefix= \ + -e local_tmp=/tmp/ansible-local \ + -e remote_tmp=/tmp/ansible-remote \ + "$@" diff --git a/test/integration/targets/connection_paramiko_ssh/aliases b/test/integration/targets/connection_paramiko_ssh/aliases index fd5b08a4166..aa7fd949b17 100644 --- a/test/integration/targets/connection_paramiko_ssh/aliases +++ b/test/integration/targets/connection_paramiko_ssh/aliases @@ -1,4 +1,5 @@ needs/ssh shippable/posix/group3 needs/target/setup_paramiko +needs/target/connection destructive # potentially installs/uninstalls OS packages via setup_paramiko diff --git a/test/integration/targets/connection_paramiko_ssh/test.sh b/test/integration/targets/connection_paramiko_ssh/test.sh deleted file mode 120000 index 70aa5dbdba4..00000000000 --- a/test/integration/targets/connection_paramiko_ssh/test.sh +++ /dev/null @@ -1 +0,0 @@ -../connection_posix/test.sh \ No newline at end of file diff --git a/test/integration/targets/connection_paramiko_ssh/test.sh b/test/integration/targets/connection_paramiko_ssh/test.sh new file mode 100755 index 00000000000..de1ae673423 --- /dev/null +++ b/test/integration/targets/connection_paramiko_ssh/test.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +set -eux + +group=paramiko_ssh + +cd ../connection + +INVENTORY="../connection_${group}/test_connection.inventory" ./test.sh \ + -e target_hosts="${group}" \ + -e action_prefix= \ + -e local_tmp=/tmp/ansible-local \ + -e remote_tmp=/tmp/ansible-remote \ + "$@" diff --git a/test/integration/targets/connection_posix/aliases b/test/integration/targets/connection_posix/aliases deleted file mode 100644 index f5e09799b1f..00000000000 --- a/test/integration/targets/connection_posix/aliases +++ /dev/null @@ -1,2 +0,0 @@ -needs/target/connection -hidden diff --git a/test/integration/targets/connection_posix/test.sh b/test/integration/targets/connection_posix/test.sh deleted file mode 100755 index d3976ff30b2..00000000000 --- a/test/integration/targets/connection_posix/test.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env bash - -set -eux - -# Connection tests for POSIX platforms use this script by linking to it from the appropriate 'connection_' target dir. -# The name of the inventory group to test is extracted from the directory name following the 'connection_' prefix. - -group=$(python -c \ - "from os import path; print(path.basename(path.abspath(path.dirname('$0'))).replace('connection_', ''))") - -cd ../connection - -INVENTORY="../connection_${group}/test_connection.inventory" ./test.sh \ - -e target_hosts="${group}" \ - -e action_prefix= \ - -e local_tmp=/tmp/ansible-local \ - -e remote_tmp=/tmp/ansible-remote \ - "$@" diff --git a/test/integration/targets/connection_ssh/aliases b/test/integration/targets/connection_ssh/aliases index 50fb8eb888a..baa04acb206 100644 --- a/test/integration/targets/connection_ssh/aliases +++ b/test/integration/targets/connection_ssh/aliases @@ -1,2 +1,3 @@ needs/ssh shippable/posix/group1 +needs/target/connection diff --git a/test/integration/targets/connection_ssh/posix.sh b/test/integration/targets/connection_ssh/posix.sh deleted file mode 120000 index 70aa5dbdba4..00000000000 --- a/test/integration/targets/connection_ssh/posix.sh +++ /dev/null @@ -1 +0,0 @@ -../connection_posix/test.sh \ No newline at end of file diff --git a/test/integration/targets/connection_ssh/posix.sh b/test/integration/targets/connection_ssh/posix.sh new file mode 100755 index 00000000000..8f036fbbda0 --- /dev/null +++ b/test/integration/targets/connection_ssh/posix.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +set -eux + +group=ssh + +cd ../connection + +INVENTORY="../connection_${group}/test_connection.inventory" ./test.sh \ + -e target_hosts="${group}" \ + -e action_prefix= \ + -e local_tmp=/tmp/ansible-local \ + -e remote_tmp=/tmp/ansible-remote \ + "$@" diff --git a/test/integration/targets/copy/files/subdir/subdir1/bar.txt b/test/integration/targets/copy/files/subdir/subdir1/bar.txt deleted file mode 120000 index 315e865d2bf..00000000000 --- a/test/integration/targets/copy/files/subdir/subdir1/bar.txt +++ /dev/null @@ -1 +0,0 @@ -../bar.txt \ No newline at end of file diff --git a/test/integration/targets/copy/files/subdir/subdir1/empty.txt b/test/integration/targets/copy/files/subdir/subdir1/empty.txt new file mode 100644 index 00000000000..e69de29bb2d diff --git a/test/integration/targets/copy/tasks/main.yml b/test/integration/targets/copy/tasks/main.yml index bef182b82b5..88d7c4fde4e 100644 --- a/test/integration/targets/copy/tasks/main.yml +++ b/test/integration/targets/copy/tasks/main.yml @@ -16,6 +16,7 @@ invalid2: ../invalid out_of_tree_circle: /tmp/ansible-test-link-dir/out_of_tree_circle subdir3: ../subdir2/subdir3 + bar.txt: ../bar.txt - file: path={{local_temp_dir}} state=directory name: ensure temp dir exists diff --git a/test/integration/targets/entry_points/aliases b/test/integration/targets/entry_points/aliases new file mode 100644 index 00000000000..45c1d4ee561 --- /dev/null +++ b/test/integration/targets/entry_points/aliases @@ -0,0 +1,2 @@ +context/controller +shippable/posix/group5 diff --git a/test/integration/targets/entry_points/runme.sh b/test/integration/targets/entry_points/runme.sh new file mode 100755 index 00000000000..63f1c0dfec5 --- /dev/null +++ b/test/integration/targets/entry_points/runme.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +set -eu +source virtualenv.sh +set +x +unset PYTHONPATH + +base_dir="$(dirname "$(dirname "$(dirname "$(dirname "${OUTPUT_DIR}")")")")" +bin_dir="$(dirname "$(command -v pip)")" + +# deps are already installed, using --no-deps to avoid re-installing them +pip install "${base_dir}" --disable-pip-version-check --no-deps +# --use-feature=in-tree-build not available on all platforms + +for bin in "${bin_dir}/ansible"*; do + name="$(basename "${bin}")" + + entry_point="${name//ansible-/}" + entry_point="${entry_point//ansible/adhoc}" + + echo "=== ${name} (${entry_point})=${bin} ===" + + if [ "${name}" == "ansible-test" ]; then + "${bin}" --help | tee /dev/stderr | grep -Eo "^usage:\ ansible-test\ .*" + python -m ansible "${entry_point}" --help | tee /dev/stderr | grep -Eo "^usage:\ ansible-test\ .*" + else + "${bin}" --version | tee /dev/stderr | grep -Eo "(^${name}\ \[core\ .*|executable location = ${bin}$)" + python -m ansible "${entry_point}" --version | tee /dev/stderr | grep -Eo "(^${name}\ \[core\ .*|executable location = ${bin}$)" + fi +done diff --git a/test/integration/targets/plugin_loader/normal/library/_symlink.py b/test/integration/targets/plugin_loader/normal/library/_symlink.py deleted file mode 120000 index c4142e74d43..00000000000 --- a/test/integration/targets/plugin_loader/normal/library/_symlink.py +++ /dev/null @@ -1 +0,0 @@ -_underscore.py \ No newline at end of file diff --git a/test/integration/targets/plugin_loader/runme.sh b/test/integration/targets/plugin_loader/runme.sh index 2a1bdedaca3..8ce7803a78f 100755 --- a/test/integration/targets/plugin_loader/runme.sh +++ b/test/integration/targets/plugin_loader/runme.sh @@ -2,6 +2,15 @@ set -ux +cleanup() { + unlink normal/library/_symlink.py +} + +pushd normal/library +ln -s _underscore.py _symlink.py +popd + +trap 'cleanup' EXIT # check normal execution for myplay in normal/*.yml diff --git a/test/lib/ansible_test/_internal/__init__.py b/test/lib/ansible_test/_internal/__init__.py index e604a2b35a8..c98f7567829 100644 --- a/test/lib/ansible_test/_internal/__init__.py +++ b/test/lib/ansible_test/_internal/__init__.py @@ -47,11 +47,11 @@ from .provisioning import ( ) -def main(): +def main(cli_args=None): # type: (t.Optional[t.List[str]]) -> None """Main program function.""" try: os.chdir(data_context().content.root) - args = parse_args() + args = parse_args(cli_args) config = args.config(args) # type: CommonConfig display.verbosity = config.verbosity display.truncate = config.truncate diff --git a/test/lib/ansible_test/_internal/cli/__init__.py b/test/lib/ansible_test/_internal/cli/__init__.py index 21c45b6e328..344fe693e3f 100644 --- a/test/lib/ansible_test/_internal/cli/__init__.py +++ b/test/lib/ansible_test/_internal/cli/__init__.py @@ -20,7 +20,7 @@ from .compat import ( ) -def parse_args(): # type: () -> argparse.Namespace +def parse_args(argv=None): # type: (t.Optional[t.List[str]]) -> argparse.Namespace """Parse command line arguments.""" completer = CompositeActionCompletionFinder() @@ -29,7 +29,7 @@ def parse_args(): # type: () -> argparse.Namespace else: epilog = 'Install the "argcomplete" python package to enable tab completion.' - parser = argparse.ArgumentParser(epilog=epilog) + parser = argparse.ArgumentParser(prog='ansible-test', epilog=epilog) do_commands(parser, completer) @@ -38,7 +38,10 @@ def parse_args(): # type: () -> argparse.Namespace always_complete_options=False, ) - argv = sys.argv[1:] + if argv is None: + argv = sys.argv[1:] + else: + argv = argv[1:] args = parser.parse_args(argv) if args.explain and not args.verbosity: diff --git a/test/lib/ansible_test/_internal/constants.py b/test/lib/ansible_test/_internal/constants.py index cac7240872e..846678516e2 100644 --- a/test/lib/ansible_test/_internal/constants.py +++ b/test/lib/ansible_test/_internal/constants.py @@ -35,15 +35,15 @@ SECCOMP_CHOICES = [ # It is necessary for payload creation to reconstruct the bin directory when running ansible-test from an installed version of ansible. # It is also used to construct the injector directory at runtime. ANSIBLE_BIN_SYMLINK_MAP = { - 'ansible': '../lib/ansible/cli/scripts/ansible_cli_stub.py', - 'ansible-config': 'ansible', + 'ansible': '../lib/ansible/cli/adhoc.py', + 'ansible-config': '../lib/ansible/cli/config.py', 'ansible-connection': '../lib/ansible/cli/scripts/ansible_connection_cli_stub.py', - 'ansible-console': 'ansible', - 'ansible-doc': 'ansible', - 'ansible-galaxy': 'ansible', - 'ansible-inventory': 'ansible', - 'ansible-playbook': 'ansible', - 'ansible-pull': 'ansible', + 'ansible-console': '../lib/ansible/cli/console.py', + 'ansible-doc': '../lib/ansible/cli/doc.py', + 'ansible-galaxy': '../lib/ansible/cli/galaxy.py', + 'ansible-inventory': '../lib/ansible/cli/inventory.py', + 'ansible-playbook': '../lib/ansible/cli/playbook.py', + 'ansible-pull': '../lib/ansible/cli/pull.py', 'ansible-test': '../test/lib/ansible_test/_util/target/cli/ansible_test_cli_stub.py', - 'ansible-vault': 'ansible', + 'ansible-vault': '../lib/ansible/cli/vault.py', } diff --git a/test/lib/ansible_test/_util/controller/sanity/code-smell/shebang.py b/test/lib/ansible_test/_util/controller/sanity/code-smell/shebang.py index 401af1aee68..bbbb713a85d 100644 --- a/test/lib/ansible_test/_util/controller/sanity/code-smell/shebang.py +++ b/test/lib/ansible_test/_util/controller/sanity/code-smell/shebang.py @@ -70,6 +70,10 @@ def main(): is_module = True elif path == 'test/lib/ansible_test/_util/target/cli/ansible_test_cli_stub.py': pass # ansible-test entry point must be executable and have a shebang + elif re.search(r'^lib/ansible/cli/[^/]+\.py', path): + pass # cli entry points must be executable and have a shebang + elif path.startswith('examples/'): + continue # examples trigger some false positives due to location elif path.startswith('lib/') or path.startswith('test/lib/'): if executable: print('%s:%d:%d: should not be executable' % (path, 0, 0)) diff --git a/test/lib/ansible_test/_util/target/cli/ansible_test_cli_stub.py b/test/lib/ansible_test/_util/target/cli/ansible_test_cli_stub.py index dc31095a813..a2fe851e7b0 100755 --- a/test/lib/ansible_test/_util/target/cli/ansible_test_cli_stub.py +++ b/test/lib/ansible_test/_util/target/cli/ansible_test_cli_stub.py @@ -11,7 +11,7 @@ import os import sys -def main(): +def main(args=None): # type: (t.Optional[t.List[str]]) -> None """Main program entry point.""" ansible_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) source_root = os.path.join(ansible_root, 'test', 'lib') @@ -30,7 +30,7 @@ def main(): # noinspection PyProtectedMember from ansible_test._internal import main as cli_main - cli_main() + cli_main(args) def version_to_str(version): diff --git a/test/sanity/code-smell/package-data.py b/test/sanity/code-smell/package-data.py index 921cb1971fa..9c64ac6fd27 100644 --- a/test/sanity/code-smell/package-data.py +++ b/test/sanity/code-smell/package-data.py @@ -29,6 +29,7 @@ def assemble_files_to_ship(complete_file_list): 'hacking/tests/*', 'hacking/ticket_stubs/*', 'test/sanity/code-smell/botmeta.*', + 'test/sanity/code-smell/release-names.*', 'test/utils/*', 'test/utils/*/*', 'test/utils/*/*/*', @@ -53,8 +54,9 @@ def assemble_files_to_ship(complete_file_list): 'hacking/report.py', 'hacking/return_skeleton_generator.py', 'hacking/test-module', - 'hacking/test-module.py', 'test/support/README.md', + 'test/lib/ansible_test/_internal/commands/sanity/bin_symlinks.py', + 'test/lib/ansible_test/_internal/commands/sanity/integration_aliases.py', '.cherry_picker.toml', '.mailmap', # Generated as part of a build step @@ -74,22 +76,27 @@ def assemble_files_to_ship(complete_file_list): 'hacking/env-setup', 'hacking/env-setup.fish', 'MANIFEST', + 'setup.cfg', + # docs for test files not included in sdist + 'docs/docsite/rst/dev_guide/testing/sanity/bin-symlinks.rst', + 'docs/docsite/rst/dev_guide/testing/sanity/botmeta.rst', + 'docs/docsite/rst/dev_guide/testing/sanity/integration-aliases.rst', + 'docs/docsite/rst/dev_guide/testing/sanity/release-names.rst', )) # These files are generated and then intentionally added to the sdist # Manpages + ignore_script = ('ansible-connection', 'ansible-test') manpages = ['docs/man/man1/ansible.1'] for dirname, dummy, files in os.walk('bin'): for filename in files: - path = os.path.join(dirname, filename) - if os.path.islink(path): - if os.readlink(path) == 'ansible': - manpages.append('docs/man/man1/%s.1' % filename) + if filename in ignore_script: + continue + manpages.append('docs/man/man1/%s.1' % filename) # Misc misc_generated_files = [ - 'SYMLINK_CACHE.json', 'PKG-INFO', ] @@ -110,7 +117,11 @@ def assemble_files_to_install(complete_file_list): """ This looks for all of the files which should show up in an installation of ansible """ - ignore_patterns = tuple() + ignore_patterns = ( + # Tests excluded from sdist + 'test/lib/ansible_test/_internal/commands/sanity/bin_symlinks.py', + 'test/lib/ansible_test/_internal/commands/sanity/integration_aliases.py', + ) pkg_data_files = [] for path in complete_file_list: @@ -256,12 +267,19 @@ def check_sdist_files_are_wanted(sdist_dir, to_ship_files): dirname = '' for filename in files: + if filename == 'setup.cfg': + continue + path = os.path.join(dirname, filename) if path not in to_ship_files: + if fnmatch.fnmatch(path, 'changelogs/CHANGELOG-v2.[0-9]*.rst'): # changelog files are expected continue + if fnmatch.fnmatch(path, 'lib/ansible_core.egg-info/*'): + continue + # FIXME: ansible-test doesn't pass the paths of symlinks to us so we aren't # checking those if not os.path.islink(os.path.join(sdist_dir, path)): @@ -282,7 +300,7 @@ def check_installed_contains_expected(install_dir, to_install_files): EGG_RE = re.compile('ansible[^/]+\\.egg-info/(PKG-INFO|SOURCES.txt|' - 'dependency_links.txt|not-zip-safe|requires.txt|top_level.txt)$') + 'dependency_links.txt|not-zip-safe|requires.txt|top_level.txt|entry_points.txt)$') def check_installed_files_are_wanted(install_dir, to_install_files): diff --git a/test/sanity/ignore.txt b/test/sanity/ignore.txt index 07e2e15b20d..b4459b4b78d 100644 --- a/test/sanity/ignore.txt +++ b/test/sanity/ignore.txt @@ -1,14 +1,8 @@ docs/docsite/rst/dev_guide/testing/sanity/no-smart-quotes.rst no-smart-quotes docs/docsite/rst/locales/ja/LC_MESSAGES/dev_guide.po no-smart-quotes # Translation of the no-smart-quotes rule -examples/play.yml shebang examples/scripts/ConfigureRemotingForAnsible.ps1 pslint:PSCustomUseLiteralPath -examples/scripts/my_test_facts.py shebang # example module but not in a normal module location -examples/scripts/my_test_info.py shebang # example module but not in a normal module location -examples/scripts/my_test.py shebang # example module but not in a normal module location examples/scripts/upgrade_to_ps3.ps1 pslint:PSCustomUseLiteralPath examples/scripts/upgrade_to_ps3.ps1 pslint:PSUseApprovedVerbs -lib/ansible/cli/console.py pylint:disallowed-name -lib/ansible/cli/scripts/ansible_cli_stub.py shebang lib/ansible/cli/scripts/ansible_connection_cli_stub.py shebang lib/ansible/config/base.yml no-unwanted-files lib/ansible/executor/playbook_executor.py pylint:disallowed-name