Implement relative title styles

Templates don't know at what level they will be inserted. Previously, we
hard-coded the title style which is not compatible with the build target
system. Define a set of styles which will be replaced by the gendoc script
when it encounters them:
 '<' : Make this title a sub-heading
 '/' : Make this title a heading at the same level
 '>' : Make this title a super-heading

The build target system is now basically complete and functioning.
convert-to-modules
Kegan Dougal 9 years ago
parent 067363c629
commit f71763b0d3

@ -17,57 +17,133 @@ stylesheets = {
}
def _list_get(l, index, default=None):
try:
return l[index]
except IndexError:
return default
"""
Read a RST file and replace titles with a different title level if required.
Args:
filename: The name of the file being read (for debugging)
file_stream: The open file stream to read from.
title_level: The integer which determines the offset to *start* from.
title_styles: An array of characters detailing the right title styles to use
e.g. ["=", "-", "~", "+"]
Returns:
string: The file contents with titles adjusted.
Example:
Assume title_styles = ["=", "-", "~", "+"], title_level = 1, and the file
when read line-by-line encounters the titles "===", "---", "---", "===", "---".
This function will bump every title encountered down a sub-heading e.g.
"=" to "-" and "-" to "~" because title_level = 1, so the output would be
"---", "~~~", "~~~", "---", "~~~". There is no bumping "up" a title level.
"""
def load_with_adjusted_titles(filename, file_stream, title_level, title_styles):
rst_lines = []
title_chars = "".join(title_styles)
title_regex = re.compile("^[" + re.escape(title_chars) + "]+$")
title_regex = re.compile("^[" + re.escape(title_chars) + "]{3,}$")
curr_title_level = title_level
prev_line_title_level = 0 # We expect the file to start with '=' titles
file_offset = None
prev_non_title_line = None
for i, line in enumerate(file_stream, 1):
if title_regex.match(line):
line_title_level = title_styles.index(line[0])
# Allowed to go 1 deeper or any number shallower
if curr_title_level - line_title_level < -1:
raise Exception(
("File '%s' line '%s' has a title " +
"style '%s' which doesn't match one of the " +
"allowed title styles of %s because the " +
"title level before this line was '%s'") %
(filename, (i + 1), line[0], title_styles,
title_styles[curr_title_level])
)
curr_title_level = line_title_level
# ignore anything which isn't a title (e.g. '===============')
if not title_regex.match(line):
rst_lines.append(line)
else:
prev_non_title_line = line
continue
# The title underline must match at a minimum the length of the title
if len(prev_non_title_line) > len(line):
rst_lines.append(line)
prev_non_title_line = line
continue
line_title_style = line[0]
line_title_level = title_styles.index(line_title_style)
# Not all files will start with "===" and we should be flexible enough
# to allow that. The first title we encounter sets the "file offset"
# which is added to the title_level desired.
if file_offset is None:
file_offset = line_title_level
if file_offset != 0:
print (" WARNING: %s starts with a title style of '%s' but '%s' " +
"is preferable.") % (filename, line_title_style, title_styles[0])
# Sanity checks: Make sure that this file is obeying the title levels
# specified and bail if it isn't.
# The file is allowed to go 1 deeper or any number shallower
if prev_line_title_level - line_title_level < -1:
raise Exception(
("File '%s' line '%s' has a title " +
"style '%s' which doesn't match one of the " +
"allowed title styles of %s because the " +
"title level before this line was '%s'") %
(filename, (i + 1), line_title_style, title_styles,
title_styles[prev_line_title_level])
)
prev_line_title_level = line_title_level
adjusted_level = (
title_level + line_title_level - file_offset
)
# Sanity check: Make sure we can bump down the title and we aren't at the
# lowest level already
if adjusted_level >= len(title_styles):
raise Exception(
("Files '%s' line '%s' has a sub-title level too low and it " +
"cannot be adjusted to fit. You can add another level to the " +
"'title_styles' key in targets.yaml to fix this.") %
(filename, (i + 1))
)
if adjusted_level == line_title_level:
# no changes required
rst_lines.append(line)
continue
# Adjusting line levels
# print (
# "File: %s Adjusting %s to %s because file_offset=%s title_offset=%s" %
# (filename, line_title_style,
# title_styles[adjusted_level],
# file_offset, title_level)
# )
rst_lines.append(line.replace(
line_title_style,
title_styles[adjusted_level]
))
return "".join(rst_lines)
def get_rst(file_info, title_level, title_styles, spec_dir):
def get_rst(file_info, title_level, title_styles, spec_dir, adjust_titles):
# string are file paths to RST blobs
if isinstance(file_info, basestring):
print "%s %s" % (">" * (1 + title_level), file_info)
with open(spec_dir + file_info, "r") as f:
return load_with_adjusted_titles(file_info, f, title_level, title_styles)
rst = None
if adjust_titles:
rst = load_with_adjusted_titles(
file_info, f, title_level, title_styles
)
else:
rst = f.read()
if rst[-2:] != "\n\n":
raise Exception(
("File %s should end with TWO new-line characters to ensure " +
"file concatenation works correctly.") % (file_info,)
)
return rst
# dicts look like {0: filepath, 1: filepath} where the key is the title level
elif isinstance(file_info, dict):
levels = sorted(file_info.keys())
rst = []
for l in levels:
rst.append(get_rst(file_info[l], l, title_styles, spec_dir))
rst.append(get_rst(file_info[l], l, title_styles, spec_dir, adjust_titles))
return "".join(rst)
# lists are multiple file paths e.g. [filepath, filepath]
elif isinstance(file_info, list):
rst = []
for f in file_info:
rst.append(get_rst(f, title_level, title_styles, spec_dir))
rst.append(get_rst(f, title_level, title_styles, spec_dir, adjust_titles))
return "".join(rst)
raise Exception(
"The following 'file' entry in this target isn't a string, list or dict. " +
@ -82,11 +158,74 @@ def build_spec(target, out_filename):
file_info=file_info,
title_level=0,
title_styles=target["title_styles"],
spec_dir="../specification/"
spec_dir="../specification/",
adjust_titles=True
)
outfile.write(section)
"""
Replaces relative title styles with actual title styles.
The templating system has no idea what the right title style is when it produces
RST because it depends on the build target. As a result, it uses relative title
styles defined in targets.yaml to say "down a level, up a level, same level".
This function replaces these relative titles with actual title styles from the
array in targets.yaml.
"""
def fix_relative_titles(target, filename, out_filename):
title_styles = target["title_styles"] # ["=", "-", "~", "+"]
relative_title_chars = [ # ["<", "/", ">"]
target["relative_title_styles"]["subtitle"],
target["relative_title_styles"]["sametitle"],
target["relative_title_styles"]["supertitle"]
]
relative_title_matcher = re.compile(
"^[" + re.escape("".join(relative_title_chars)) + "]{3,}$"
)
title_matcher = re.compile(
"^[" + re.escape("".join(title_styles)) + "]{3,}$"
)
current_title_style = None
with open(filename, "r") as infile:
with open(out_filename, "w") as outfile:
for line in infile.readlines():
if not relative_title_matcher.match(line):
if title_matcher.match(line):
current_title_style = line[0]
outfile.write(line)
continue
line_char = line[0]
replacement_char = None
current_title_level = title_styles.index(current_title_style)
if line_char == target["relative_title_styles"]["subtitle"]:
if (current_title_level + 1) == len(title_styles):
raise Exception(
"Encountered sub-title line style but we can't go " +
"any lower."
)
replacement_char = title_styles[current_title_level + 1]
elif line_char == target["relative_title_styles"]["sametitle"]:
replacement_char = title_styles[current_title_level]
elif line_char == target["relative_title_styles"]["supertitle"]:
if (current_title_level - 1) < 0:
raise Exception(
"Encountered super-title line style but we can't go " +
"any higher."
)
replacement_char = title_styles[current_title_level - 1]
else:
raise Exception(
"Unknown relative line char %s" % (line_char,)
)
outfile.write(
line.replace(line_char, replacement_char)
)
def rst2html(i, o):
with open(i, "r") as in_file:
with open(o, "w") as out_file:
@ -123,11 +262,13 @@ def run_through_template(input):
def get_build_target(targets_listing, target_name):
build_target = {
"title_styles": [],
"relative_title_styles": {},
"files": []
}
with open(targets_listing, "r") as targ_file:
all_targets = yaml.load(targ_file.read())
build_target["title_styles"] = all_targets["title_styles"]
build_target["relative_title_styles"] = all_targets["relative_title_styles"]
target = all_targets["targets"].get(target_name)
if not target:
raise Exception(
@ -138,17 +279,30 @@ def get_build_target(targets_listing, target_name):
raise Exception(
"Found target but 'files' key is not a list."
)
def get_group(group_id):
group_name = group_id[len("group:"):]
group = all_targets.get("groups", {}).get(group_name)
if not group:
raise Exception(
"Tried to find group '" + group_name + "' but it " +
"doesn't exist."
)
return group
resolved_files = []
for f in target["files"]:
group = None
if isinstance(f, basestring) and f.startswith("group:"):
# copy across the group of files specified
group_name = f[len("group:"):]
group = all_targets.get("groups", {}).get(group_name)
if not group:
raise Exception(
"Tried to find group '" + group_name + "' but it " +
"doesn't exist."
)
group = get_group(f)
elif isinstance(f, dict):
for (k, v) in f.iteritems():
if isinstance(v, basestring) and v.startswith("group:"):
f[k] = get_group(v)
resolved_files.append(f)
continue
if group:
if isinstance(group, list):
resolved_files.extend(group)
else:
@ -178,8 +332,12 @@ def main(target_name):
prepare_env()
print "Building spec [target=%s]" % target_name
target = get_build_target("../specification/targets.yaml", target_name)
build_spec(target=target, out_filename="tmp/full_spec.rst")
run_through_template("tmp/full_spec.rst")
build_spec(target=target, out_filename="tmp/templated_spec.rst")
run_through_template("tmp/templated_spec.rst")
fix_relative_titles(
target=target, filename="tmp/templated_spec.rst",
out_filename="tmp/full_spec.rst"
)
shutil.copy("../supporting-docs/howtos/client-server.rst", "tmp/howto.rst")
run_through_template("tmp/howto.rst")
rst2html("tmp/full_spec.rst", "gen/specification.html")

@ -0,0 +1,5 @@
Feature Profiles
================
Feature profiles blurb goes here.

@ -0,0 +1,5 @@
Modules
=======
Modules intro here.

@ -400,3 +400,4 @@ in their content to provide a way for Matrix clients to link into the 'native'
client from which the event originated. For instance, this could contain the
message-ID for emails/nntp posts, or a link to a blog comment when gatewaying
blog comment traffic in & out of matrix

@ -92,7 +92,7 @@ server by querying other servers.
.. _Perspectives Project: http://perspectives-project.org/
Publishing Keys
_______________
^^^^^^^^^^^^^^^
Home servers publish the allowed TLS fingerprints and signing keys in a JSON
object at ``/_matrix/key/v2/server/{key_id}``. The response contains a list of
@ -178,7 +178,7 @@ events sent by that server can still be checked.
}
Querying Keys Through Another Server
____________________________________
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Servers may offer a query API ``_matrix/key/v2/query/`` for getting the keys
for another server. This API can be used to GET at list of JSON objects for a

@ -0,0 +1,5 @@
Modules
=======
Modules blurb goes here.

@ -2,26 +2,37 @@ targets:
main: # arbitrary name to identify this build target
files: # the sort order of files to cat
- 00_00_intro.rst
- 00_01_feature_profiles.rst
- 00_02a_events.rst
- 00_02b_event_signing.rst
- { 1: 00_01_feature_profiles.rst }
- { 1: 00_02a_events.rst }
- { 1: 00_02b_event_signing.rst }
- 01_00_client_server_api.rst
- 02_00_modules.rst
- "group:modules" # reference a group of files
- { 1: "group:modules" } # reference a group of files
- 03_00_application_service_api.rst
- 04_00_server_server_api.rst
- 05_00_identity_servers.rst
- 06_00_appendices.rst
groups: # reusable blobs of files when prefixed with 'group:'
modules:
0: modules/00_modules_intro.rst
1:
- modules/01_00_voip_events.rst
- modules/02_00_typing_notifications.rst
- modules/03_00_receipts.rst
- modules/04_00_content_repo.rst
- modules/05_00_end_to_end_encryption.rst
- modules/06_00_history_visibility.rst
- 1: modules/07_00_push_overview.rst # Mark a nested file dependency
2: [modules/07_01_push_cs_api.rst , modules/07_02_push_push_gw_api.rst]
title_styles: ["=", "-", "~", "+"]
- modules/00_modules_intro.rst
- modules/01_00_voip_events.rst
- modules/02_00_typing_notifications.rst
- modules/03_00_receipts.rst
- modules/04_00_content_repo.rst
- modules/05_00_end_to_end_encryption.rst
- modules/06_00_history_visibility.rst
- modules/07_00_push_overview.rst
- { 2: [modules/07_01_push_cs_api.rst , modules/07_02_push_push_gw_api.rst] }
title_styles: ["=", "-", "~", "+", "^"]
# The templating system doesn't know the right title style to use when generating
# RST. These symbols are 'relative' to say "make a sub-title" (-1), "make a title
# at the same level (0)", or "make a title one above (+1)". The gendoc script
# will inspect this file and replace these relative styles with actual title
# styles. The templating system will also inspect this file to know which symbols
# to inject.
relative_title_styles:
subtitle: "<"
sametitle: "/"
supertitle: ">"

@ -23,10 +23,13 @@ class MatrixSections(Sections):
spec_meta = self.units.get("spec_meta")
return spec_meta["changelog"]
def _render_events(self, filterFn, sortFn, title_kind="~"):
def _render_events(self, filterFn, sortFn):
template = self.env.get_template("events.tmpl")
examples = self.units.get("event_examples")
schemas = self.units.get("event_schemas")
subtitle_title_char = self.units.get("spec_targets")[
"relative_title_styles"
]["subtitle"]
sections = []
for event_name in sortFn(schemas):
if not filterFn(event_name):
@ -34,14 +37,16 @@ class MatrixSections(Sections):
sections.append(template.render(
example=examples[event_name],
event=schemas[event_name],
title_kind=title_kind
title_kind=subtitle_title_char
))
return "\n\n".join(sections)
def _render_http_api_group(self, group, sortFnOrPathList=None,
title_kind="-"):
def _render_http_api_group(self, group, sortFnOrPathList=None):
template = self.env.get_template("http-api.tmpl")
http_api = self.units.get("swagger_apis")[group]["__meta"]
subtitle_title_char = self.units.get("spec_targets")[
"relative_title_styles"
]["subtitle"]
sections = []
endpoints = []
if sortFnOrPathList:
@ -67,15 +72,14 @@ class MatrixSections(Sections):
for endpoint in endpoints:
sections.append(template.render(
endpoint=endpoint,
title_kind=title_kind
title_kind=subtitle_title_char
))
return "\n\n".join(sections)
def render_profile_http_api(self):
return self._render_http_api_group(
"profile",
sortFnOrPathList=["displayname", "avatar_url"],
title_kind="~"
sortFnOrPathList=["displayname", "avatar_url"]
)
def render_sync_http_api(self):
@ -86,20 +90,17 @@ class MatrixSections(Sections):
def render_presence_http_api(self):
return self._render_http_api_group(
"presence",
sortFnOrPathList=["status"],
title_kind="~"
sortFnOrPathList=["status"]
)
def render_membership_http_api(self):
return self._render_http_api_group(
"membership",
title_kind="~"
"membership"
)
def render_login_http_api(self):
return self._render_http_api_group(
"login",
title_kind="~"
"login"
)
def render_room_events(self):
@ -114,6 +115,9 @@ class MatrixSections(Sections):
template = self.env.get_template("msgtypes.tmpl")
examples = self.units.get("event_examples")
schemas = self.units.get("event_schemas")
subtitle_title_char = self.units.get("spec_targets")[
"relative_title_styles"
]["subtitle"]
sections = []
msgtype_order = [
"m.room.message#m.text", "m.room.message#m.emote",
@ -129,7 +133,8 @@ class MatrixSections(Sections):
continue
sections.append(template.render(
example=examples[event_name],
event=schemas[event_name]
event=schemas[event_name],
title_kind=subtitle_title_char
))
return "\n\n".join(sections)
@ -150,7 +155,7 @@ class MatrixSections(Sections):
def render_presence_events(self):
def filterFn(eventType):
return eventType.startswith("m.presence")
return self._render_events(filterFn, sorted, title_kind="+")
return self._render_events(filterFn, sorted)
def _render_ce_type(self, type):
template = self.env.get_template("common-event-fields.tmpl")

@ -1,5 +1,5 @@
``{{event.msgtype}}``
{{(4 + event.msgtype | length) * '+'}}
{{(4 + event.msgtype | length) * title_kind}}
{{event.desc | wrap(80)}}
{% for table in event.content_fields -%}
{{"``"+table.title+"``" if table.title else "" }}

@ -13,6 +13,7 @@ V1_EVENT_EXAMPLES = "../event-schemas/examples/v1"
V1_EVENT_SCHEMA = "../event-schemas/schema/v1"
CORE_EVENT_SCHEMA = "../event-schemas/schema/v1/core-event-schema"
CHANGELOG = "../CHANGELOG.rst"
TARGETS = "../specification/targets.yaml"
ROOM_EVENT = "core-event-schema/room_event.json"
STATE_EVENT = "core-event-schema/state_event.json"
@ -466,6 +467,12 @@ class MatrixUnits(Units):
"changelog": "".join(changelog_lines)
}
def load_spec_targets(self):
with open(TARGETS, "r") as f:
return yaml.load(f.read())
def load_git_version(self):
null = open(os.devnull, 'w')
cwd = os.path.dirname(os.path.abspath(__file__))

Loading…
Cancel
Save