Implement a framework for having common code for release scripts (#55893)

* Implement a framework for having common code for release scripts

* Release scripts will go through hacking/build-ansible.  build-ansible is
  a pluggable script which will set a directory that has common code for
  non-enduser scripts.  It will then invoke the plugin which implements
  that subcommand.  Uses straight.plugin for loading each sub-command.

* We're going to add tools which are needed to test ansible (the changelog
  generation, for instance) so we need to include the pieces relevant to
  that in the tarball.

* Add straight.plugin to the sanity test requirements for the same
  reason

* Skip compile test just for build-ansible plugins which won't be run as
  part of sanity tests.

MANIFEST.in

@@ -25,3 +25,5 @@ include changelogs/CHANGELOG*.rst
 include contrib/README.md
 recursive-include contrib/inventory *
 exclude test/sanity/code-smell/botmeta.*
+recursive-include hacking/build_library *.py
+include hacking/build-ansible

hacking/build-ansible

@@ -0,0 +1,76 @@
+#!/usr/bin/env python3
+# coding: utf-8
+# PYTHON_ARGCOMPLETE_OK
+# Copyright: (c) 2019, Ansible Project
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+# Make coding more python3-ish
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+
+import argparse
+import os.path
+import sys
+
+from straight.plugin import load
+
+try:
+    import argcomplete
+except ImportError:
+    argcomplete = None
+
+
+def set_sys_path(this_script=__file__):
+    """Add path to the common librarydirectory to :attr:`sys.path`"""
+    hacking_dir = os.path.dirname(this_script)
+    libdir = os.path.abspath(os.path.join(hacking_dir, 'build_library'))
+
+    if libdir not in sys.path:
+        sys.path.insert(0, libdir)
+
+
+set_sys_path()
+
+from build_ansible import commands
+
+
+def create_arg_parser(program_name):
+    """
+    Creates a command line argument parser
+
+    :arg program_name: The name of the script.  Used in help texts
+    """
+    parser = argparse.ArgumentParser(prog=program_name,
+                                     description="Implements utilities to build Ansible")
+    return parser
+
+
+def main():
+    """
+    Main entrypoint of the script
+
+    "It all starts here"
+    """
+    subcommands = load('build_ansible.command_plugins', subclasses=commands.Command)
+
+    arg_parser = create_arg_parser(os.path.basename(sys.argv[0]))
+    subparsers = arg_parser.add_subparsers(title='Subcommands', dest='command',
+                                           help='for help use build-ansible SUBCOMMANDS -h')
+    subcommands.pipe('init_parser', subparsers.add_parser)
+
+    if argcomplete:
+        argcomplete.autocomplete(arg_parser)
+
+    args = arg_parser.parse_args(sys.argv[1:])
+
+    for subcommand in subcommands:
+        if subcommand.name == args.command:
+            sys.exit(subcommand.main(args))
+
+    print('Error: Select a subcommand')
+    arg_parser.print_usage()
+
+
+if __name__ == '__main__':
+    main()

hacking/build_library/build_ansible/command_plugins/porting_guide.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 # coding: utf-8
 # Copyright: (c) 2019, Ansible Project
 # GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
@@ -9,10 +8,14 @@ __metaclass__ = type
 
 
 import argparse
+import os.path
 import sys
 
 from jinja2 import Environment, DictLoader
 
+# Pylint doesn't understand Python3 namespace modules.
+from ..commands import Command  # pylint: disable=relative-beyond-top-level
+
 
 PORTING_GUIDE_TEMPLATE = """
 .. _porting_{{ ver }}_guide:
@@ -106,16 +109,6 @@ JINJA_ENV = Environment(
 )
 
 
-def parse_args(args):
-    parser = argparse.ArgumentParser(description="Generate a fresh porting guide template")
-    parser.add_argument("--version", dest="version", type=str, required=True, action='store',
-                        help="Version of Ansible to write the porting guide for")
-
-    args = parser.parse_args(args)
-
-    return args
-
-
 def generate_porting_guide(version):
     template = JINJA_ENV.get_template('porting_guide')
 
@@ -133,13 +126,17 @@ def write_guide(version, guide_content):
         out_file.write(guide_content)
 
 
-def main():
+class PortingGuideCommand(Command):
-    args = parse_args(sys.argv[1:])
+    name = 'porting-guide'
-
-    guide_content = generate_porting_guide(args.version)
-
-    write_guide(args.version, guide_content)
 
+    @classmethod
+    def init_parser(cls, add_parser):
+        parser = add_parser(cls.name, description="Generate a fresh porting guide template")
+        parser.add_argument("--version", dest="version", type=str, required=True, action='store',
+                            help="Version of Ansible to write the porting guide for")
 
-if __name__ == '__main__':
+    @staticmethod
-    main()
+    def main(args):
+        guide_content = generate_porting_guide(args.version)
+        write_guide(args.version, guide_content)
+        return 0

hacking/build_library/build_ansible/command_plugins/release_announcement.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 # coding: utf-8
 # Copyright: (c) 2019, Ansible Project
 # GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
@@ -12,6 +11,7 @@ import argparse
 import asyncio
 import datetime
 import hashlib
+import os.path
 import sys
 from collections import UserString
 from distutils.version import LooseVersion
@@ -19,6 +19,10 @@ from distutils.version import LooseVersion
 import aiohttp
 from jinja2 import Environment, DictLoader
 
+# Pylint doesn't understand Python3 namespace modules.
+from ..commands import Command  # pylint: disable=relative-beyond-top-level
+
+
 # pylint: disable=
 VERSION_FRAGMENT = """
 {%- if versions | length > 1 %}
@@ -31,7 +35,7 @@ VERSION_FRAGMENT = """
 """
 
 LONG_TEMPLATE = """
-{% set plural = True if versions | length == 1 else False %}
+{% set plural = False if versions | length == 1 else True %}
 {% set latest_ver = (versions | sort(attribute='ver_obj'))[-1] %}
 
 To: ansible-devel@googlegroups.com, ansible-project@googlegroups.com, ansible-announce@googlegroups.com
@@ -66,7 +70,7 @@ What's new in {{ version_str }}
 {{ '-' * (14 + version_str | length) }}
 
 {% filter wordwrap %}
-{% if plural %}This release is a{% else %}These releases are{% endif %} maintenance release{% if plural %}s{% endif %} containing numerous bugfixes. The full {% if versions | length <= 1 %} changelog is{% else %} changelogs are{% endif %} at:
+{% if plural %}These releases are{% else %}This release is a{% endif %} maintenance release{% if plural %}s{% endif %} containing numerous bugfixes. The full {% if plural %} changelogs are{% else %} changelog is{% endif %} at:
 {% endfilter %}
 
 
@@ -116,15 +120,16 @@ Thanks!
 # proper wrapping to occur
 
 SHORT_TEMPLATE = """
+{% set plural = False if versions | length == 1 else True %}
 @ansible
 {{ version_str }}
-{% if versions | length > 1 %}
+{% if plural %}
   have
 {% else %}
   has
 {% endif %}
 been released! Get
-{% if versions | length > 1 %}
+{% if plural %}
 them
 {% else %}
 it
@@ -152,19 +157,7 @@ class VersionStr(UserString):
         self.ver_obj = LooseVersion(string)
 
 
-def parse_args(args):
+def transform_args(args):
-    parser = argparse.ArgumentParser(description="Generate email and twitter announcements"
-                                                 " from template")
-    parser.add_argument("--version", dest="versions", type=str, required=True, action='append',
-                        help="Versions of Ansible to announce")
-    parser.add_argument("--name", type=str, required=True, help="Real name to use on emails")
-    parser.add_argument("--email-out", type=str, default="-",
-                        help="Filename to place the email announcement into")
-    parser.add_argument("--twitter-out", type=str, default="-",
-                        help="Filename to place the twitter announcement into")
-
-    args = parser.parse_args(args)
-
     # Make it possible to sort versions in the jinja2 templates
     new_versions = []
     for version in args.versions:
@@ -285,15 +278,29 @@ def write_message(filename, message):
         sys.stdout.write(message)
 
 
-def main():
+class ReleaseAnnouncementCommand(Command):
-    args = parse_args(sys.argv[1:])
+    name = 'release-announcement'
+
+    @classmethod
+    def init_parser(cls, add_parser):
+        parser = add_parser(cls.name,
+                            description="Generate email and twitter announcements from template")
 
-    twitter_message = generate_short_message(args.versions)
+        parser.add_argument("--version", dest="versions", type=str, required=True, action='append',
-    email_message = generate_long_message(args.versions, args.name)
+                            help="Versions of Ansible to announce")
+        parser.add_argument("--name", type=str, required=True, help="Real name to use on emails")
+        parser.add_argument("--email-out", type=str, default="-",
+                            help="Filename to place the email announcement into")
+        parser.add_argument("--twitter-out", type=str, default="-",
+                            help="Filename to place the twitter announcement into")
 
-    write_message(args.twitter_out, twitter_message)
+    @staticmethod
-    write_message(args.email_out, email_message)
+    def main(args):
+        args = transform_args(args)
 
+        twitter_message = generate_short_message(args.versions)
+        email_message = generate_long_message(args.versions, args.name)
 
-if __name__ == '__main__':
+        write_message(args.twitter_out, twitter_message)
-    main()
+        write_message(args.email_out, email_message)
+        return 0

hacking/build_library/build_ansible/commands.py

@@ -0,0 +1,50 @@
+# coding: utf-8
+# Copyright: (c) 2019, Ansible Project
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+# Make coding more python3-ish
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+
+from abc import ABCMeta, abstractmethod, abstractproperty
+
+
+class Command:
+    """
+    Subcommands of :program:`build-ansible`.
+
+    This defines an interface that all subcommands must conform to.  :program:`build-ansible` will
+    require that these things are present in order to proceed.
+    """
+    @staticmethod
+    @abstractproperty
+    def name():
+        """Name of the command.  The same as the string is invoked with"""
+
+    @staticmethod
+    @abstractmethod
+    def init_parser(add_parser):
+        """
+        Initialize and register an argparse ArgumentParser
+
+        :arg add_parser: function which creates an ArgumentParser for the main program.
+
+        Implementations should first create an ArgumentParser using `add_parser` and then populate
+        it with the command line arguments that are needed.
+
+        .. seealso:
+            `add_parser` information in the :py:meth:`ArgumentParser.add_subparsers` documentation.
+        """
+
+    @staticmethod
+    @abstractmethod
+    def main(arguments):
+        """
+        Run the command
+
+        :arg arguments: The **parsed** command line args
+
+        This is the Command's entrypoint.  The command line args are already parsed but from here
+        on, the command can do its work.
+        """

test/runner/requirements/sanity.txt

@@ -7,6 +7,7 @@ pylint ; python_version >= '3.5' # pylint 2.0.0 and later require python 3+
 pytest
 rstcheck ; python_version >= '2.7' # rstcheck requires python 2.7+
 sphinx
+straight.plugin  # needed for hacking/build-ansible which will host changelog generation
 virtualenv
 voluptuous
 yamllint

test/sanity/code-smell/shebang.py

@@ -39,8 +39,7 @@ def main():
         'test/utils/shippable/timing.py',
         'test/integration/targets/old_style_modules_posix/library/helloworld.sh',
         # The following are Python 3.6+.  Only run by release engineers
-        'hacking/release-announcement.py',
+        'hacking/build-ansible',
-        'hacking/porting-guide.py',
     ])
 
     # see https://unicode.org/faq/utf_bom.html#bom1

test/sanity/compile/python2.6-skip.txt

@@ -1,3 +1,3 @@
 # The following are only run by release engineers who can be asked to have newer Python3 on their systems
-hacking/release-announcement.py
+hacking/build_library/build_ansible/command_plugins/porting_guide.py
-hacking/porting-guide.py
+hacking/build_library/build_ansible/command_plugins/release_announcement.py

test/sanity/compile/python2.7-skip.txt

@@ -1,3 +1,3 @@
 # The following are only run by release engineers who can be asked to have newer Python3 on their systems
-hacking/release-announcement.py
+hacking/build_library/build_ansible/command_plugins/porting_guide.py
-hacking/porting-guide.py
+hacking/build_library/build_ansible/command_plugins/release_announcement.py

test/sanity/compile/python3.5-skip.txt

@@ -1,3 +1,3 @@
 # The following are only run by release engineers who can be asked to have newer Python3 on their systems
-hacking/release-announcement.py
+hacking/build_library/build_ansible/command_plugins/porting_guide.py
-hacking/porting-guide.py
+hacking/build_library/build_ansible/command_plugins/release_announcement.py