Add a script to update the intersphinx inventory (#66646)

* Add a script to update the intersphinx inventory files

* We're about to add intersphinx inventories for separate ansible docs
  so we need an easy way to update them.  Also, we should be updating
  these cache files for other upstreams occassionally as well.  With a
  script, we can add updating them to a release process.
* Now that we don't know what the version of the cache is, change the
  filenames to not contain versions.

* Update the intersphinx cache files with the latest upstream versions

Results of running:

hacking/build-ansible.py update-intersphinx-cache -o docs/docsite -c docs/docsite/rst/conf.py

* Add a comment to the configuration file which says how to structure the intersphinx mapping and why.

* Update docs/docsite/rst/conf.py

Co-Authored-By: Sandra McCann <samccann@redhat.com>

Co-authored-by: Sandra McCann <samccann@redhat.com>

docs/docsite/rst/conf.py

@@ -263,6 +263,15 @@ latex_documents = [
 
 autoclass_content = 'both'
 
-intersphinx_mapping = {'python': ('https://docs.python.org/2/', (None, '../python2-2.7.13.inv')),
-                       'python3': ('https://docs.python.org/3/', (None, '../python3-3.6.2.inv')),
-                       'jinja2': ('http://jinja.pocoo.org/docs/', (None, '../jinja2-2.9.7.inv'))}
+# Note:  Our strategy for intersphinx mappings is to have the upstream build location as the
+# canonical source and then cached copies of the mapping stored locally in case someone is building
+# when disconnected from the internet.  We then have a script to update the cached copies.
+#
+# Because of that, each entry in this mapping should have this format:
+#   name: ('http://UPSTREAM_URL', (None, 'path/to/local/cache.inv'))
+#
+# The update script depends on this format so deviating from this (for instance, adding a third
+# location for the mappning to live) will confuse it.
+intersphinx_mapping = {'python': ('https://docs.python.org/2/', (None, '../python2.inv')),
+                       'python3': ('https://docs.python.org/3/', (None, '../python3.inv')),
+                       'jinja2': ('http://jinja.pocoo.org/docs/', (None, '../jinja.inv'))}

hacking/build_library/build_ansible/command_plugins/update_intersphinx.py

@@ -0,0 +1,101 @@
+# -*- coding: utf-8 -*-
+# (c) 2020, Ansible Project
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+# Make coding more python3-ish
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+
+import argparse
+import importlib
+import os
+import pathlib
+import time
+import urllib.parse
+
+from collections import defaultdict
+
+from ansible.module_utils.common.collections import is_iterable
+from ansible.module_utils.urls import Request
+
+# Pylint doesn't understand Python3 namespace modules.
+from ..commands import Command  # pylint: disable=relative-beyond-top-level
+from .. import errors  # pylint: disable=relative-beyond-top-level
+
+
+EXAMPLE_CONF = """
+A proper intersphinx_mapping entry should look like:
+    intersphinx_mapping = {
+        'python3': ('https://docs.python.org/3', (None, 'python3.inv'))
+    }
+
+See the intersphinx docs for more info:
+    https://www.sphinx-doc.org/en/master/usage/extensions/intersphinx.html#confval-intersphinx_mapping
+"""
+
+
+class UpdateIntersphinxCache(Command):
+    name = 'update-intersphinx-cache'
+
+    @classmethod
+    def init_parser(cls, add_parser):
+        parser = add_parser(cls.name, description='Update cached intersphinx mappings.  This'
+                            ' updates the cached intersphinx mappings for docs to reference'
+                            ' documentation from other projects.')
+        parser.add_argument('-o', '--output-dir', action='store',
+                            help='Path to directory the cached objects.inv files are stored in')
+        parser.add_argument('-c', '--conf-file', action='store',
+                            help='Path to a sphinx config file to retrieve intersphinx config from')
+
+    @staticmethod
+    def main(args):
+        # Retrieve the intersphinx information from the sphinx config file
+        conf_dir = pathlib.Path(args.conf_file).parent
+
+        conf_module_spec = importlib.util.spec_from_file_location('sphinxconf', args.conf_file)
+        conf_module = importlib.util.module_from_spec(conf_module_spec)
+        conf_module_spec.loader.exec_module(conf_module)
+        intersphinx_mapping = conf_module.intersphinx_mapping
+
+        for intersphinx_name, inventory in intersphinx_mapping.items():
+            if not is_iterable(inventory) or len(inventory) != 2:
+                print('WARNING: The intersphinx entry for {0} must be'
+                      ' a two-tuple.\n{1}'.format(intersphinx_name, EXAMPLE_CONF))
+                continue
+
+            url = cache_file = None
+            for inv_source in inventory:
+                if isinstance(inv_source, str) and url is None:
+                    url = inv_source
+                elif is_iterable(inv_source) and cache_file is None:
+                    if len(inv_source) != 2:
+                        print('WARNING: The fallback entry for {0} should be a tuple of (None,'
+                              ' filename).\n{1}'.format(intersphinx_name, EXAMPLE_CONF))
+                        continue
+                    cache_file = inv_source[1]
+                else:
+                    print('WARNING: The configuration for {0} should be a tuple of one url and one'
+                          ' tuple for a fallback filename.\n{1}'.format(intersphinx_name,
+                                                                        EXAMPLE_CONF))
+                    continue
+
+            if url is None or cache_file is None:
+                print('WARNING: Could not figure out the url or fallback'
+                      ' filename for {0}.\n{1}'.format(intersphinx_name, EXAMPLE_CONF))
+                continue
+
+            url = urllib.parse.urljoin(url, 'objects.inv')
+            # Resolve any relative cache files to be relative to the conf file
+            cache_file = conf_dir / cache_file
+
+            # Retrieve the inventory and cache it
+            # The jinja CDN seems to be blocking the default urllib User-Agent
+            requestor = Request(headers={'User-Agent': 'Definitely Not Python ;-)'})
+            with requestor.open('GET', url) as source_file:
+                with open(cache_file, 'wb') as f:
+                    f.write(source_file.read())
+
+        print('Download of new cache files complete.  Remember to git commit -a the changes')
+
+        return 0

test/sanity/ignore.txt

@@ -129,6 +129,8 @@ hacking/build_library/build_ansible/command_plugins/porting_guide.py compile-3.5
 hacking/build_library/build_ansible/command_plugins/release_announcement.py compile-2.6!skip # release process only, 3.6+ required
 hacking/build_library/build_ansible/command_plugins/release_announcement.py compile-2.7!skip # release process only, 3.6+ required
 hacking/build_library/build_ansible/command_plugins/release_announcement.py compile-3.5!skip # release process only, 3.6+ required
+hacking/build_library/build_ansible/command_plugins/update_intersphinx.py compile-2.6!skip # release process and docs build only, 3.5+ required
+hacking/build_library/build_ansible/command_plugins/update_intersphinx.py compile-2.7!skip # release process and docs build only, 3.5+ required
 hacking/fix_test_syntax.py future-import-boilerplate
 hacking/fix_test_syntax.py metaclass-boilerplate
 hacking/get_library.py future-import-boilerplate