Merge pull request #259 from matrix-org/rav/doc_speedups

Speed up gendoc.py by only running build.py once

scripts/gendoc.py

@@ -151,6 +151,7 @@ def get_rst(file_info, title_level, title_styles, spec_dir, adjust_titles):
 
 
 def build_spec(target, out_filename):
+    log("Building templated file %s" % out_filename)
     with open(out_filename, "wb") as outfile:
         for file_info in target["files"]:
             section = get_rst(
@@ -174,6 +175,7 @@ This function replaces these relative titles with actual title styles from the
 array in targets.yaml.
 """
 def fix_relative_titles(target, filename, out_filename):
+    log("Fix relative titles, %s -> %s" % (filename, out_filename))
     title_styles = target["title_styles"]
     relative_title_chars = [
         target["relative_title_styles"]["subtitle"],
@@ -226,6 +228,7 @@ def fix_relative_titles(target, filename, out_filename):
 
 
 def rst2html(i, o):
+    log("rst2html %s -> %s" % (i, o))
     with open(i, "r") as in_file:
         with open(o, "w") as out_file:
             publish_file(
@@ -239,6 +242,8 @@ def rst2html(i, o):
 
 
 def addAnchors(path):
+    log("add anchors %s" % path)
+
     with open(path, "r") as f:
         lines = f.readlines()
 
@@ -250,34 +255,27 @@ def addAnchors(path):
             f.write(line + "\n")
 
 
-def run_through_template(input, set_verbose, substitutions):
+def run_through_template(input_files, set_verbose, substitutions):
-    tmpfile = './tmp/output'
+    args = [
-    try:
+        'python', 'build.py',
-        with open(tmpfile, 'w') as out:
+        "-o", "../scripts/tmp",
-            args = [
+        "-i", "matrix_templates",
-                'python', 'build.py',
+    ]
-                "-i", "matrix_templates",
-                "-o", "../scripts/tmp",
-                "../scripts/"+input
-            ]
-            for k, v in substitutions.items():
-                args.append("--substitution=%s=%s" % (k, v))
-
-            if set_verbose:
-                args.insert(2, "-v")
-            log("EXEC: %s" % " ".join(args))
-            log(" ==== build.py output ==== ")
-            print subprocess.check_output(
-                args,
-                stderr=out,
-                cwd="../templating"
-            )
-    except subprocess.CalledProcessError as e:
-        print e.output
-        with open(tmpfile, 'r') as f:
-            sys.stderr.write(f.read() + "\n")
-        raise
 
+    for k, v in substitutions.items():
+        args.append("--substitution=%s=%s" % (k, v))
+
+    if set_verbose:
+        args.insert(2, "-v")
+
+    args.extend("../scripts/"+f for f in input_files)
+
+    log("EXEC: %s" % " ".join(args))
+    log(" ==== build.py output ==== ")
+    subprocess.check_call(
+        args,
+        cwd="../templating"
+    )
 
 def get_build_targets(targets_listing):
     with open(targets_listing, "r") as targ_file:
@@ -401,16 +399,27 @@ def main(requested_target_name, keep_intermediates, substitutions):
 
     targets = [requested_target_name]
     if requested_target_name == "all":
-        targets = get_build_targets("../specification/targets.yaml")
+        targets = get_build_targets("../specification/targets.yaml") + ["howtos"]
+
+    templated_files = []
+    for target_name in targets:
+        templated_file = "tmp/templated_%s.rst" % (target_name,)
+
+        if target_name == "howtos":
+            shutil.copy("../supporting-docs/howtos/client-server.rst", templated_file)
+        else:
+            target = get_build_target("../specification/targets.yaml", target_name)
+            build_spec(target=target, out_filename=templated_file)
+        templated_files.append(templated_file)
+
+    # we do all the templating at once, because it's slow
+    run_through_template(templated_files, VERBOSE, substitutions)
 
     for target_name in targets:
         templated_file = "tmp/templated_%s.rst" % (target_name,)
         rst_file = "tmp/spec_%s.rst" % (target_name,)
         html_file = "gen/%s.html" % (target_name,)
 
-        target = get_build_target("../specification/targets.yaml", target_name)
-        build_spec(target=target, out_filename=templated_file)
-        run_through_template(templated_file, VERBOSE, substitutions)
         fix_relative_titles(
             target=target, filename=templated_file,
             out_filename=rst_file,
@@ -418,11 +427,6 @@ def main(requested_target_name, keep_intermediates, substitutions):
         rst2html(rst_file, html_file)
         addAnchors(html_file)
 
-    if requested_target_name == "all":
-        shutil.copy("../supporting-docs/howtos/client-server.rst", "tmp/howto.rst")
-        run_through_template("tmp/howto.rst", False, substitutions)  # too spammy to mark -v on this
-        rst2html("tmp/howto.rst", "gen/howtos.html")
-
     if not keep_intermediates:
         cleanup_env()
 

templating/build.py

@@ -57,7 +57,7 @@ def check_unaccessed(name, store):
         log("Found %s unused %s keys." % (len(unaccessed_keys), name))
         log(unaccessed_keys)
 
-def main(input_module, file_stream=None, out_dir=None, verbose=False, substitutions={}):
+def main(input_module, files=None, out_dir=None, verbose=False, substitutions={}):
     if out_dir and not os.path.exists(out_dir):
         os.makedirs(out_dir)
 
@@ -138,7 +138,7 @@ def main(input_module, file_stream=None, out_dir=None, verbose=False, substituti
     sections = in_mod.exports["sections"](env, units, debug=verbose).get_sections()
 
     # print out valid section keys if no file supplied
-    if not file_stream:
+    if not files:
         print "\nValid template variables:"
         for key in sections.keys():
             sec_text = "" if (len(sections[key]) > 75) else (
@@ -152,8 +152,19 @@ def main(input_module, file_stream=None, out_dir=None, verbose=False, substituti
         return
 
     # check the input files and substitute in sections where required
-    log("Parsing input template: %s" % file_stream.name)
+    for input_filename in files:
-    temp_str = file_stream.read().decode("utf-8")
+        output_filename = os.path.join(out_dir, 
+                                       os.path.basename(input_filename))
+        process_file(env, sections, input_filename, output_filename)
+
+    check_unaccessed("units", units)
+
+def process_file(env, sections, filename, output_filename):
+    log("Parsing input template: %s" % filename)
+
+    with open(filename, "r") as file_stream:
+        temp_str = file_stream.read().decode("utf-8")
+
     # do sanity checking on the template to make sure they aren't reffing things
     # which will never be replaced with a section.
     ast = env.parse(temp_str)
@@ -166,7 +177,6 @@ def main(input_module, file_stream=None, out_dir=None, verbose=False, substituti
         )
     # process the template
     temp = Template(temp_str)
-    log("Creating output for: %s" % file_stream.name)
     output = create_from_template(temp, sections)
 
     # Do these substitutions outside of the ordinary templating system because
@@ -174,12 +184,11 @@ def main(input_module, file_stream=None, out_dir=None, verbose=False, substituti
     # generate the templates, not just the top-level sections.
     for old, new in substitutions.items():
         output = output.replace(old, new)
-    with open(
+
-            os.path.join(out_dir, os.path.basename(file_stream.name)), "w"
+    with open(output_filename, "w") as f:
-            ) as f:
         f.write(output.encode("utf-8"))
-    log("Output file for: %s" % file_stream.name)
+    log("Output file for: %s" % output_filename)
-    check_unaccessed("units", units)
+
 
 def log(line):
     print "batesian: %s" % line
@@ -191,8 +200,8 @@ if __name__ == '__main__':
         "list of possible template variables, add --show-template-vars."
     )
     parser.add_argument(
-        "file", nargs="?", type=FileType('r'),
+        "files", nargs="+",
-        help="The input file to process. This will be passed through Jinja "+
+        help="The input files to process. These will be passed through Jinja "+
         "then output under the same name to the output directory."
     )
     parser.add_argument(
@@ -232,11 +241,6 @@ if __name__ == '__main__':
         main(args.input, verbose=args.verbose)
         sys.exit(0)
 
-    if not args.file:
-        log("No file supplied.")
-        parser.print_help()
-        sys.exit(1)
-
     substitutions = {}
     for substitution in args.substitution:
         parts = substitution.split("=", 1)
@@ -245,6 +249,6 @@ if __name__ == '__main__':
         substitutions[parts[0]] = parts[1]
 
     main(
-        args.input, file_stream=args.file, out_dir=args.out_directory,
+        args.input, files=args.files, out_dir=args.out_directory,
         substitutions=substitutions, verbose=args.verbose
     )