Merge pull request #259 from matrix-org/rav/doc_speedups

Speed up gendoc.py by only running build.py once
pull/977/head
Richard van der Hoff 9 years ago
commit 18570126b5

@ -151,6 +151,7 @@ def get_rst(file_info, title_level, title_styles, spec_dir, adjust_titles):
def build_spec(target, out_filename):
log("Building templated file %s" % out_filename)
with open(out_filename, "wb") as outfile:
for file_info in target["files"]:
section = get_rst(
@ -174,6 +175,7 @@ This function replaces these relative titles with actual title styles from the
array in targets.yaml.
"""
def fix_relative_titles(target, filename, out_filename):
log("Fix relative titles, %s -> %s" % (filename, out_filename))
title_styles = target["title_styles"]
relative_title_chars = [
target["relative_title_styles"]["subtitle"],
@ -226,6 +228,7 @@ def fix_relative_titles(target, filename, out_filename):
def rst2html(i, o):
log("rst2html %s -> %s" % (i, o))
with open(i, "r") as in_file:
with open(o, "w") as out_file:
publish_file(
@ -239,6 +242,8 @@ def rst2html(i, o):
def addAnchors(path):
log("add anchors %s" % path)
with open(path, "r") as f:
lines = f.readlines()
@ -250,34 +255,27 @@ def addAnchors(path):
f.write(line + "\n")
def run_through_template(input, set_verbose, substitutions):
tmpfile = './tmp/output'
try:
with open(tmpfile, 'w') as out:
def run_through_template(input_files, set_verbose, substitutions):
args = [
'python', 'build.py',
"-i", "matrix_templates",
"-o", "../scripts/tmp",
"../scripts/"+input
"-i", "matrix_templates",
]
for k, v in substitutions.items():
args.append("--substitution=%s=%s" % (k, v))
if set_verbose:
args.insert(2, "-v")
args.extend("../scripts/"+f for f in input_files)
log("EXEC: %s" % " ".join(args))
log(" ==== build.py output ==== ")
print subprocess.check_output(
subprocess.check_call(
args,
stderr=out,
cwd="../templating"
)
except subprocess.CalledProcessError as e:
print e.output
with open(tmpfile, 'r') as f:
sys.stderr.write(f.read() + "\n")
raise
def get_build_targets(targets_listing):
with open(targets_listing, "r") as targ_file:
@ -401,16 +399,27 @@ def main(requested_target_name, keep_intermediates, substitutions):
targets = [requested_target_name]
if requested_target_name == "all":
targets = get_build_targets("../specification/targets.yaml")
targets = get_build_targets("../specification/targets.yaml") + ["howtos"]
templated_files = []
for target_name in targets:
templated_file = "tmp/templated_%s.rst" % (target_name,)
rst_file = "tmp/spec_%s.rst" % (target_name,)
html_file = "gen/%s.html" % (target_name,)
if target_name == "howtos":
shutil.copy("../supporting-docs/howtos/client-server.rst", templated_file)
else:
target = get_build_target("../specification/targets.yaml", target_name)
build_spec(target=target, out_filename=templated_file)
run_through_template(templated_file, VERBOSE, substitutions)
templated_files.append(templated_file)
# we do all the templating at once, because it's slow
run_through_template(templated_files, VERBOSE, substitutions)
for target_name in targets:
templated_file = "tmp/templated_%s.rst" % (target_name,)
rst_file = "tmp/spec_%s.rst" % (target_name,)
html_file = "gen/%s.html" % (target_name,)
fix_relative_titles(
target=target, filename=templated_file,
out_filename=rst_file,
@ -418,11 +427,6 @@ def main(requested_target_name, keep_intermediates, substitutions):
rst2html(rst_file, html_file)
addAnchors(html_file)
if requested_target_name == "all":
shutil.copy("../supporting-docs/howtos/client-server.rst", "tmp/howto.rst")
run_through_template("tmp/howto.rst", False, substitutions) # too spammy to mark -v on this
rst2html("tmp/howto.rst", "gen/howtos.html")
if not keep_intermediates:
cleanup_env()

@ -57,7 +57,7 @@ def check_unaccessed(name, store):
log("Found %s unused %s keys." % (len(unaccessed_keys), name))
log(unaccessed_keys)
def main(input_module, file_stream=None, out_dir=None, verbose=False, substitutions={}):
def main(input_module, files=None, out_dir=None, verbose=False, substitutions={}):
if out_dir and not os.path.exists(out_dir):
os.makedirs(out_dir)
@ -138,7 +138,7 @@ def main(input_module, file_stream=None, out_dir=None, verbose=False, substituti
sections = in_mod.exports["sections"](env, units, debug=verbose).get_sections()
# print out valid section keys if no file supplied
if not file_stream:
if not files:
print "\nValid template variables:"
for key in sections.keys():
sec_text = "" if (len(sections[key]) > 75) else (
@ -152,8 +152,19 @@ def main(input_module, file_stream=None, out_dir=None, verbose=False, substituti
return
# check the input files and substitute in sections where required
log("Parsing input template: %s" % file_stream.name)
for input_filename in files:
output_filename = os.path.join(out_dir,
os.path.basename(input_filename))
process_file(env, sections, input_filename, output_filename)
check_unaccessed("units", units)
def process_file(env, sections, filename, output_filename):
log("Parsing input template: %s" % filename)
with open(filename, "r") as file_stream:
temp_str = file_stream.read().decode("utf-8")
# do sanity checking on the template to make sure they aren't reffing things
# which will never be replaced with a section.
ast = env.parse(temp_str)
@ -166,7 +177,6 @@ def main(input_module, file_stream=None, out_dir=None, verbose=False, substituti
)
# process the template
temp = Template(temp_str)
log("Creating output for: %s" % file_stream.name)
output = create_from_template(temp, sections)
# Do these substitutions outside of the ordinary templating system because
@ -174,12 +184,11 @@ def main(input_module, file_stream=None, out_dir=None, verbose=False, substituti
# generate the templates, not just the top-level sections.
for old, new in substitutions.items():
output = output.replace(old, new)
with open(
os.path.join(out_dir, os.path.basename(file_stream.name)), "w"
) as f:
with open(output_filename, "w") as f:
f.write(output.encode("utf-8"))
log("Output file for: %s" % file_stream.name)
check_unaccessed("units", units)
log("Output file for: %s" % output_filename)
def log(line):
print "batesian: %s" % line
@ -191,8 +200,8 @@ if __name__ == '__main__':
"list of possible template variables, add --show-template-vars."
)
parser.add_argument(
"file", nargs="?", type=FileType('r'),
help="The input file to process. This will be passed through Jinja "+
"files", nargs="+",
help="The input files to process. These will be passed through Jinja "+
"then output under the same name to the output directory."
)
parser.add_argument(
@ -232,11 +241,6 @@ if __name__ == '__main__':
main(args.input, verbose=args.verbose)
sys.exit(0)
if not args.file:
log("No file supplied.")
parser.print_help()
sys.exit(1)
substitutions = {}
for substitution in args.substitution:
parts = substitution.split("=", 1)
@ -245,6 +249,6 @@ if __name__ == '__main__':
substitutions[parts[0]] = parts[1]
main(
args.input, file_stream=args.file, out_dir=args.out_directory,
args.input, files=args.files, out_dir=args.out_directory,
substitutions=substitutions, verbose=args.verbose
)

Loading…
Cancel
Save