From 1f6c4d5bb20318f10095ea8990f19c44bc9619b1 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 8 Jan 2016 00:40:03 +0000 Subject: [PATCH] Speed up gendoc.py by only running build.py once --- scripts/gendoc.py | 76 ++++++++++++++++++++++++--------------------- templating/build.py | 39 ++++++++++++----------- 2 files changed, 61 insertions(+), 54 deletions(-) diff --git a/scripts/gendoc.py b/scripts/gendoc.py index 88ca6bdb..dbf84fc8 100755 --- a/scripts/gendoc.py +++ b/scripts/gendoc.py @@ -151,6 +151,7 @@ def get_rst(file_info, title_level, title_styles, spec_dir, adjust_titles): def build_spec(target, out_filename): + log("Building templated file %s" % out_filename) with open(out_filename, "wb") as outfile: for file_info in target["files"]: section = get_rst( @@ -174,6 +175,7 @@ This function replaces these relative titles with actual title styles from the array in targets.yaml. """ def fix_relative_titles(target, filename, out_filename): + log("Fix relative titles, %s -> %s" % (filename, out_filename)) title_styles = target["title_styles"] relative_title_chars = [ target["relative_title_styles"]["subtitle"], @@ -226,6 +228,7 @@ def fix_relative_titles(target, filename, out_filename): def rst2html(i, o): + log("rst2html %s -> %s" % (i, o)) with open(i, "r") as in_file: with open(o, "w") as out_file: publish_file( @@ -239,6 +242,8 @@ def rst2html(i, o): def addAnchors(path): + log("add anchors %s" % path) + with open(path, "r") as f: lines = f.readlines() @@ -250,34 +255,27 @@ def addAnchors(path): f.write(line + "\n") -def run_through_template(input, set_verbose, substitutions): - tmpfile = './tmp/output' - try: - with open(tmpfile, 'w') as out: - args = [ - 'python', 'build.py', - "-i", "matrix_templates", - "-o", "../scripts/tmp", - "../scripts/"+input - ] - for k, v in substitutions.items(): - args.append("--substitution=%s=%s" % (k, v)) - - if set_verbose: - args.insert(2, "-v") - log("EXEC: %s" % " ".join(args)) - log(" ==== build.py output ==== ") - print subprocess.check_output( - args, - stderr=out, - cwd="../templating" - ) - except subprocess.CalledProcessError as e: - print e.output - with open(tmpfile, 'r') as f: - sys.stderr.write(f.read() + "\n") - raise +def run_through_template(input_files, set_verbose, substitutions): + args = [ + 'python', 'build.py', + "-o", "../scripts/tmp", + "-i", "matrix_templates", + ] + for k, v in substitutions.items(): + args.append("--substitution=%s=%s" % (k, v)) + + if set_verbose: + args.insert(2, "-v") + + args.extend("../scripts/"+f for f in input_files) + + log("EXEC: %s" % " ".join(args)) + log(" ==== build.py output ==== ") + subprocess.check_call( + args, + cwd="../templating" + ) def get_build_targets(targets_listing): with open(targets_listing, "r") as targ_file: @@ -401,16 +399,27 @@ def main(requested_target_name, keep_intermediates, substitutions): targets = [requested_target_name] if requested_target_name == "all": - targets = get_build_targets("../specification/targets.yaml") + targets = get_build_targets("../specification/targets.yaml") + ["howtos"] + + templated_files = [] + for target_name in targets: + templated_file = "tmp/templated_%s.rst" % (target_name,) + + if target_name == "howtos": + shutil.copy("../supporting-docs/howtos/client-server.rst", templated_file) + else: + target = get_build_target("../specification/targets.yaml", target_name) + build_spec(target=target, out_filename=templated_file) + templated_files.append(templated_file) + + # we do all the templating at once, because it's slow + run_through_template(templated_files, VERBOSE, substitutions) for target_name in targets: templated_file = "tmp/templated_%s.rst" % (target_name,) rst_file = "tmp/spec_%s.rst" % (target_name,) html_file = "gen/%s.html" % (target_name,) - target = get_build_target("../specification/targets.yaml", target_name) - build_spec(target=target, out_filename=templated_file) - run_through_template(templated_file, VERBOSE, substitutions) fix_relative_titles( target=target, filename=templated_file, out_filename=rst_file, @@ -418,11 +427,6 @@ def main(requested_target_name, keep_intermediates, substitutions): rst2html(rst_file, html_file) addAnchors(html_file) - if requested_target_name == "all": - shutil.copy("../supporting-docs/howtos/client-server.rst", "tmp/howto.rst") - run_through_template("tmp/howto.rst", False, substitutions) # too spammy to mark -v on this - rst2html("tmp/howto.rst", "gen/howtos.html") - if not keep_intermediates: cleanup_env() diff --git a/templating/build.py b/templating/build.py index f08c2201..6f49f543 100755 --- a/templating/build.py +++ b/templating/build.py @@ -57,7 +57,7 @@ def check_unaccessed(name, store): log("Found %s unused %s keys." % (len(unaccessed_keys), name)) log(unaccessed_keys) -def main(input_module, file_stream=None, out_dir=None, verbose=False, substitutions={}): +def main(input_module, files=None, out_dir=None, verbose=False, substitutions={}): if out_dir and not os.path.exists(out_dir): os.makedirs(out_dir) @@ -138,7 +138,7 @@ def main(input_module, file_stream=None, out_dir=None, verbose=False, substituti sections = in_mod.exports["sections"](env, units, debug=verbose).get_sections() # print out valid section keys if no file supplied - if not file_stream: + if not files: print "\nValid template variables:" for key in sections.keys(): sec_text = "" if (len(sections[key]) > 75) else ( @@ -152,8 +152,18 @@ def main(input_module, file_stream=None, out_dir=None, verbose=False, substituti return # check the input files and substitute in sections where required - log("Parsing input template: %s" % file_stream.name) - temp_str = file_stream.read().decode("utf-8") + for filename in (files): + output_filename = os.path.join(out_dir, os.path.basename(filename)) + process_file(env, sections, filename, output_filename) + + check_unaccessed("units", units) + +def process_file(env, sections, filename, output_filename): + log("Parsing input template: %s" % filename) + + with open(filename, 'r') as file_stream: + temp_str = file_stream.read().decode("utf-8") + # do sanity checking on the template to make sure they aren't reffing things # which will never be replaced with a section. ast = env.parse(temp_str) @@ -166,7 +176,6 @@ def main(input_module, file_stream=None, out_dir=None, verbose=False, substituti ) # process the template temp = Template(temp_str) - log("Creating output for: %s" % file_stream.name) output = create_from_template(temp, sections) # Do these substitutions outside of the ordinary templating system because @@ -174,12 +183,11 @@ def main(input_module, file_stream=None, out_dir=None, verbose=False, substituti # generate the templates, not just the top-level sections. for old, new in substitutions.items(): output = output.replace(old, new) - with open( - os.path.join(out_dir, os.path.basename(file_stream.name)), "w" - ) as f: + + with open(output_filename, "w") as f: f.write(output.encode("utf-8")) - log("Output file for: %s" % file_stream.name) - check_unaccessed("units", units) + log("Output file for: %s" % output_filename) + def log(line): print "batesian: %s" % line @@ -191,8 +199,8 @@ if __name__ == '__main__': "list of possible template variables, add --show-template-vars." ) parser.add_argument( - "file", nargs="?", type=FileType('r'), - help="The input file to process. This will be passed through Jinja "+ + "files", nargs="+", + help="The input files to process. These will be passed through Jinja "+ "then output under the same name to the output directory." ) parser.add_argument( @@ -232,11 +240,6 @@ if __name__ == '__main__': main(args.input, verbose=args.verbose) sys.exit(0) - if not args.file: - log("No file supplied.") - parser.print_help() - sys.exit(1) - substitutions = {} for substitution in args.substitution: parts = substitution.split("=", 1) @@ -245,6 +248,6 @@ if __name__ == '__main__': substitutions[parts[0]] = parts[1] main( - args.input, file_stream=args.file, out_dir=args.out_directory, + args.input, files=args.files, out_dir=args.out_directory, substitutions=substitutions, verbose=args.verbose )