From 310b0a2521a158e4791216a77071af371ee8acbf Mon Sep 17 00:00:00 2001 From: Dag Wieers Date: Wed, 12 Sep 2018 22:50:36 +0200 Subject: [PATCH] Docs: Make docsite rebuilds smarter/faster (#45062) * Make the following scripts idempotent so that we only have to rebuild changed docs, not all docs: * plugin_formatter * generate_man * dump_keywords.py * dump_config.py * testing_formatter.sh --- docs/bin/dump_config.py | 6 +++-- docs/bin/dump_keywords.py | 5 ++-- docs/bin/generate_man.py | 7 ++---- docs/bin/plugin_formatter.py | 5 ++-- docs/bin/testing_formatter.sh | 10 ++++++-- docs/docsite/Makefile | 2 +- lib/ansible/utils/_build_helpers.py | 38 +++++++++++++++++++++++++++++ lib/ansible/utils/plugin_docs.py | 22 +++-------------- 8 files changed, 62 insertions(+), 33 deletions(-) create mode 100644 lib/ansible/utils/_build_helpers.py diff --git a/docs/bin/dump_config.py b/docs/bin/dump_config.py index ffb37a48dc6..f1ff55bcf56 100755 --- a/docs/bin/dump_config.py +++ b/docs/bin/dump_config.py @@ -6,6 +6,8 @@ import sys import yaml from jinja2 import Environment, FileSystemLoader +from ansible.module_utils._text import to_bytes +from ansible.utils._build_helpers import update_file_if_different DEFAULT_TEMPLATE_FILE = 'config.rst.j2' @@ -62,8 +64,8 @@ def main(args): output_name = os.path.join(output_dir, template_file.replace('.j2', '')) temp_vars = {'config_options': config_options} - with open(output_name, 'wb') as f: - f.write(template.render(temp_vars).encode('utf-8')) + data = to_bytes(template.render(temp_vars)) + update_file_if_different(output_name, data) return 0 diff --git a/docs/bin/dump_keywords.py b/docs/bin/dump_keywords.py index 3b1a3eaf565..30056a6fc8c 100755 --- a/docs/bin/dump_keywords.py +++ b/docs/bin/dump_keywords.py @@ -8,10 +8,12 @@ import jinja2 import yaml from jinja2 import Environment, FileSystemLoader +from ansible.module_utils._text import to_bytes from ansible.playbook import Play from ansible.playbook.block import Block from ansible.playbook.role import Role from ansible.playbook.task import Task +from ansible.utils._build_helpers import update_file_if_different template_file = 'playbooks_keywords.rst.j2' oblist = {} @@ -79,5 +81,4 @@ if LooseVersion(jinja2.__version__) < LooseVersion('2.10'): # jinja2 < 2.10's indent filter indents blank lines. Cleanup keyword_page = re.sub(' +\n', '\n', keyword_page) -with open(outputname, 'w') as f: - f.write(keyword_page) +update_file_if_different(outputname, to_bytes(keyword_page)) diff --git a/docs/bin/generate_man.py b/docs/bin/generate_man.py index b2d5bfd4107..938f3435dc4 100755 --- a/docs/bin/generate_man.py +++ b/docs/bin/generate_man.py @@ -2,12 +2,12 @@ import optparse import os -import pprint import sys from jinja2 import Environment, FileSystemLoader from ansible.module_utils._text import to_bytes +from ansible.utils._build_helpers import update_file_if_different def generate_parser(): @@ -274,7 +274,4 @@ if __name__ == '__main__': manpage = template.render(tvars) filename = os.path.join(output_dir, doc_name_formats[output_format] % tvars['cli_name']) - - with open(filename, 'wb') as f: - f.write(to_bytes(manpage)) - print("Wrote doc to %s" % filename) + update_file_if_different(filename, to_bytes(manpage)) diff --git a/docs/bin/plugin_formatter.py b/docs/bin/plugin_formatter.py index 248b1723b48..3f627bcf835 100755 --- a/docs/bin/plugin_formatter.py +++ b/docs/bin/plugin_formatter.py @@ -56,6 +56,7 @@ from ansible.module_utils.parsing.convert_bool import boolean from ansible.plugins.loader import fragment_loader from ansible.utils import plugin_docs from ansible.utils.display import Display +from ansible.utils._build_helpers import update_file_if_different ##################################################################################### @@ -183,8 +184,8 @@ def write_data(text, output_dir, outputname, module=None): os.makedirs(output_dir) fname = os.path.join(output_dir, outputname) fname = fname.replace(".py", "") - with open(fname, 'wb') as f: - f.write(to_bytes(text)) + + update_file_if_different(fname, to_bytes(text)) else: print(text) diff --git a/docs/bin/testing_formatter.sh b/docs/bin/testing_formatter.sh index 0867f9a54ef..d88d67de03d 100755 --- a/docs/bin/testing_formatter.sh +++ b/docs/bin/testing_formatter.sh @@ -1,6 +1,8 @@ #!/bin/bash -eu -cat <<- EOF > ../docsite/rst/dev_guide/testing/sanity/index.rst +FILENAME=../docsite/rst/dev_guide/testing/sanity/index.rst + +cat <<- EOF >$FILENAME.new Sanity Tests ============ @@ -12,5 +14,9 @@ This list is also available using \`\`ansible-test sanity --list-tests\`\`. $(for test in $(../../test/runner/ansible-test sanity --list-tests); do echo " ${test}"; done) - EOF + +# Put file into place if it has changed +if [ "$(sha1sum <$FILENAME)" != "$(sha1sum <$FILENAME.new)" ]; then + mv -f $FILENAME.new $FILENAME +fi diff --git a/docs/docsite/Makefile b/docs/docsite/Makefile index 52d18e504ba..1f4931d3435 100644 --- a/docs/docsite/Makefile +++ b/docs/docsite/Makefile @@ -35,7 +35,7 @@ endif all: docs -docs: clean htmldocs +docs: htmldocs generate_rst: staticmin config cli keywords modules plugins testing diff --git a/lib/ansible/utils/_build_helpers.py b/lib/ansible/utils/_build_helpers.py new file mode 100644 index 00000000000..0a107e36ede --- /dev/null +++ b/lib/ansible/utils/_build_helpers.py @@ -0,0 +1,38 @@ +# Copyright: (c) 2018, Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +""" +This file contains common code for building ansible. If you want to use code from here at runtime, +it needs to be moved out of this file and the implementation looked over to figure out whether API +should be changed before being made public. +""" + +import os.path + + +def update_file_if_different(filename, b_data): + ''' + Replace file content only if content is different. + + This preserves timestamps in case the file content has not changed. It performs multiple + operations on the file so it is not atomic and may be slower than simply writing to the file. + + :arg filename: The filename to write to + :b_data: Byte string containing the data to write to the file + ''' + try: + with open(filename, 'rb') as f: + b_data_old = f.read() + except IOError as e: + if e.errno != 2: + raise + # File did not exist, set b_data_old to a sentinel value so that + # b_data gets written to the filename + b_data_old = None + + if b_data_old != b_data: + with open(filename, 'wb') as f: + f.write(b_data) diff --git a/lib/ansible/utils/plugin_docs.py b/lib/ansible/utils/plugin_docs.py index 2e6d7d19ab9..dc98ac73b3d 100644 --- a/lib/ansible/utils/plugin_docs.py +++ b/lib/ansible/utils/plugin_docs.py @@ -1,22 +1,6 @@ -# (c) 2012, Jan-Piet Mens -# -# This file is part of Ansible -# -# Ansible is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Ansible is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Ansible. If not, see . -# - -# Make coding more python3-ish +# Copyright: (c) 2012, Jan-Piet Mens +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + from __future__ import (absolute_import, division, print_function) __metaclass__ = type