Merge pull request #259 from matrix-org/rav/doc_speedups

Speed up gendoc.py by only running build.py once
pull/977/head
Richard van der Hoff 9 years ago
commit 18570126b5

@ -151,6 +151,7 @@ def get_rst(file_info, title_level, title_styles, spec_dir, adjust_titles):
def build_spec(target, out_filename): def build_spec(target, out_filename):
log("Building templated file %s" % out_filename)
with open(out_filename, "wb") as outfile: with open(out_filename, "wb") as outfile:
for file_info in target["files"]: for file_info in target["files"]:
section = get_rst( section = get_rst(
@ -174,6 +175,7 @@ This function replaces these relative titles with actual title styles from the
array in targets.yaml. array in targets.yaml.
""" """
def fix_relative_titles(target, filename, out_filename): def fix_relative_titles(target, filename, out_filename):
log("Fix relative titles, %s -> %s" % (filename, out_filename))
title_styles = target["title_styles"] title_styles = target["title_styles"]
relative_title_chars = [ relative_title_chars = [
target["relative_title_styles"]["subtitle"], target["relative_title_styles"]["subtitle"],
@ -226,6 +228,7 @@ def fix_relative_titles(target, filename, out_filename):
def rst2html(i, o): def rst2html(i, o):
log("rst2html %s -> %s" % (i, o))
with open(i, "r") as in_file: with open(i, "r") as in_file:
with open(o, "w") as out_file: with open(o, "w") as out_file:
publish_file( publish_file(
@ -239,6 +242,8 @@ def rst2html(i, o):
def addAnchors(path): def addAnchors(path):
log("add anchors %s" % path)
with open(path, "r") as f: with open(path, "r") as f:
lines = f.readlines() lines = f.readlines()
@ -250,34 +255,27 @@ def addAnchors(path):
f.write(line + "\n") f.write(line + "\n")
def run_through_template(input, set_verbose, substitutions): def run_through_template(input_files, set_verbose, substitutions):
tmpfile = './tmp/output'
try:
with open(tmpfile, 'w') as out:
args = [ args = [
'python', 'build.py', 'python', 'build.py',
"-i", "matrix_templates",
"-o", "../scripts/tmp", "-o", "../scripts/tmp",
"../scripts/"+input "-i", "matrix_templates",
] ]
for k, v in substitutions.items(): for k, v in substitutions.items():
args.append("--substitution=%s=%s" % (k, v)) args.append("--substitution=%s=%s" % (k, v))
if set_verbose: if set_verbose:
args.insert(2, "-v") args.insert(2, "-v")
args.extend("../scripts/"+f for f in input_files)
log("EXEC: %s" % " ".join(args)) log("EXEC: %s" % " ".join(args))
log(" ==== build.py output ==== ") log(" ==== build.py output ==== ")
print subprocess.check_output( subprocess.check_call(
args, args,
stderr=out,
cwd="../templating" cwd="../templating"
) )
except subprocess.CalledProcessError as e:
print e.output
with open(tmpfile, 'r') as f:
sys.stderr.write(f.read() + "\n")
raise
def get_build_targets(targets_listing): def get_build_targets(targets_listing):
with open(targets_listing, "r") as targ_file: with open(targets_listing, "r") as targ_file:
@ -401,16 +399,27 @@ def main(requested_target_name, keep_intermediates, substitutions):
targets = [requested_target_name] targets = [requested_target_name]
if requested_target_name == "all": if requested_target_name == "all":
targets = get_build_targets("../specification/targets.yaml") targets = get_build_targets("../specification/targets.yaml") + ["howtos"]
templated_files = []
for target_name in targets: for target_name in targets:
templated_file = "tmp/templated_%s.rst" % (target_name,) templated_file = "tmp/templated_%s.rst" % (target_name,)
rst_file = "tmp/spec_%s.rst" % (target_name,)
html_file = "gen/%s.html" % (target_name,)
if target_name == "howtos":
shutil.copy("../supporting-docs/howtos/client-server.rst", templated_file)
else:
target = get_build_target("../specification/targets.yaml", target_name) target = get_build_target("../specification/targets.yaml", target_name)
build_spec(target=target, out_filename=templated_file) build_spec(target=target, out_filename=templated_file)
run_through_template(templated_file, VERBOSE, substitutions) templated_files.append(templated_file)
# we do all the templating at once, because it's slow
run_through_template(templated_files, VERBOSE, substitutions)
for target_name in targets:
templated_file = "tmp/templated_%s.rst" % (target_name,)
rst_file = "tmp/spec_%s.rst" % (target_name,)
html_file = "gen/%s.html" % (target_name,)
fix_relative_titles( fix_relative_titles(
target=target, filename=templated_file, target=target, filename=templated_file,
out_filename=rst_file, out_filename=rst_file,
@ -418,11 +427,6 @@ def main(requested_target_name, keep_intermediates, substitutions):
rst2html(rst_file, html_file) rst2html(rst_file, html_file)
addAnchors(html_file) addAnchors(html_file)
if requested_target_name == "all":
shutil.copy("../supporting-docs/howtos/client-server.rst", "tmp/howto.rst")
run_through_template("tmp/howto.rst", False, substitutions) # too spammy to mark -v on this
rst2html("tmp/howto.rst", "gen/howtos.html")
if not keep_intermediates: if not keep_intermediates:
cleanup_env() cleanup_env()

@ -57,7 +57,7 @@ def check_unaccessed(name, store):
log("Found %s unused %s keys." % (len(unaccessed_keys), name)) log("Found %s unused %s keys." % (len(unaccessed_keys), name))
log(unaccessed_keys) log(unaccessed_keys)
def main(input_module, file_stream=None, out_dir=None, verbose=False, substitutions={}): def main(input_module, files=None, out_dir=None, verbose=False, substitutions={}):
if out_dir and not os.path.exists(out_dir): if out_dir and not os.path.exists(out_dir):
os.makedirs(out_dir) os.makedirs(out_dir)
@ -138,7 +138,7 @@ def main(input_module, file_stream=None, out_dir=None, verbose=False, substituti
sections = in_mod.exports["sections"](env, units, debug=verbose).get_sections() sections = in_mod.exports["sections"](env, units, debug=verbose).get_sections()
# print out valid section keys if no file supplied # print out valid section keys if no file supplied
if not file_stream: if not files:
print "\nValid template variables:" print "\nValid template variables:"
for key in sections.keys(): for key in sections.keys():
sec_text = "" if (len(sections[key]) > 75) else ( sec_text = "" if (len(sections[key]) > 75) else (
@ -152,8 +152,19 @@ def main(input_module, file_stream=None, out_dir=None, verbose=False, substituti
return return
# check the input files and substitute in sections where required # check the input files and substitute in sections where required
log("Parsing input template: %s" % file_stream.name) for input_filename in files:
output_filename = os.path.join(out_dir,
os.path.basename(input_filename))
process_file(env, sections, input_filename, output_filename)
check_unaccessed("units", units)
def process_file(env, sections, filename, output_filename):
log("Parsing input template: %s" % filename)
with open(filename, "r") as file_stream:
temp_str = file_stream.read().decode("utf-8") temp_str = file_stream.read().decode("utf-8")
# do sanity checking on the template to make sure they aren't reffing things # do sanity checking on the template to make sure they aren't reffing things
# which will never be replaced with a section. # which will never be replaced with a section.
ast = env.parse(temp_str) ast = env.parse(temp_str)
@ -166,7 +177,6 @@ def main(input_module, file_stream=None, out_dir=None, verbose=False, substituti
) )
# process the template # process the template
temp = Template(temp_str) temp = Template(temp_str)
log("Creating output for: %s" % file_stream.name)
output = create_from_template(temp, sections) output = create_from_template(temp, sections)
# Do these substitutions outside of the ordinary templating system because # Do these substitutions outside of the ordinary templating system because
@ -174,12 +184,11 @@ def main(input_module, file_stream=None, out_dir=None, verbose=False, substituti
# generate the templates, not just the top-level sections. # generate the templates, not just the top-level sections.
for old, new in substitutions.items(): for old, new in substitutions.items():
output = output.replace(old, new) output = output.replace(old, new)
with open(
os.path.join(out_dir, os.path.basename(file_stream.name)), "w" with open(output_filename, "w") as f:
) as f:
f.write(output.encode("utf-8")) f.write(output.encode("utf-8"))
log("Output file for: %s" % file_stream.name) log("Output file for: %s" % output_filename)
check_unaccessed("units", units)
def log(line): def log(line):
print "batesian: %s" % line print "batesian: %s" % line
@ -191,8 +200,8 @@ if __name__ == '__main__':
"list of possible template variables, add --show-template-vars." "list of possible template variables, add --show-template-vars."
) )
parser.add_argument( parser.add_argument(
"file", nargs="?", type=FileType('r'), "files", nargs="+",
help="The input file to process. This will be passed through Jinja "+ help="The input files to process. These will be passed through Jinja "+
"then output under the same name to the output directory." "then output under the same name to the output directory."
) )
parser.add_argument( parser.add_argument(
@ -232,11 +241,6 @@ if __name__ == '__main__':
main(args.input, verbose=args.verbose) main(args.input, verbose=args.verbose)
sys.exit(0) sys.exit(0)
if not args.file:
log("No file supplied.")
parser.print_help()
sys.exit(1)
substitutions = {} substitutions = {}
for substitution in args.substitution: for substitution in args.substitution:
parts = substitution.split("=", 1) parts = substitution.split("=", 1)
@ -245,6 +249,6 @@ if __name__ == '__main__':
substitutions[parts[0]] = parts[1] substitutions[parts[0]] = parts[1]
main( main(
args.input, file_stream=args.file, out_dir=args.out_directory, args.input, files=args.files, out_dir=args.out_directory,
substitutions=substitutions, verbose=args.verbose substitutions=substitutions, verbose=args.verbose
) )

Loading…
Cancel
Save