From 54ee861b5faf85bea35361cdbb89f28bae15711b Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 4 Feb 2019 14:47:20 -0700 Subject: [PATCH 1/4] Fix changelog generation for non-default versions Currently if you generate a changelog for r0.1.1 of an API, you'd get "No significant changes" which is wrong. You should get a real changelog for the version. This is now handled by generating a "preferred" changelog which acts as the default for version variables in the RST. Using a specific version's changelog is still supported for the rare cases where that is desired. --- scripts/templating/matrix_templates/sections.py | 2 +- scripts/templating/matrix_templates/units.py | 14 +++++++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/scripts/templating/matrix_templates/sections.py b/scripts/templating/matrix_templates/sections.py index af497674..4451d21d 100644 --- a/scripts/templating/matrix_templates/sections.py +++ b/scripts/templating/matrix_templates/sections.py @@ -41,7 +41,7 @@ class MatrixSections(Sections): version_var = "%s_%s" % (spec_var, version) logger.info("Rendering changelog for %s" % version_var) rendered[version_var] = changelog - if version == "unstable": + if version == "preferred": rendered[spec_var] = changelog return rendered diff --git a/scripts/templating/matrix_templates/units.py b/scripts/templating/matrix_templates/units.py index 721501ff..fd261838 100644 --- a/scripts/templating/matrix_templates/units.py +++ b/scripts/templating/matrix_templates/units.py @@ -903,9 +903,17 @@ class MatrixUnits(Units): return schema - def load_changelogs(self): + def load_changelogs(self, substitutions): changelogs = {} + preferred_versions = { + "server_server": substitutions.get("%SERVER_RELEASE_LABEL%", "unstable"), + "client_server": substitutions.get("%CLIENT_RELEASE_LABEL%", "unstable"), + "identity_service": substitutions.get("%IDENTITY_RELEASE_LABEL%", "unstable"), + "push_gateway": substitutions.get("%PUSH_GATEWAY_RELEASE_LABEL%", "unstable"), + "application_service": substitutions.get("%APPSERVICE_RELEASE_LABEL%", "unstable"), + } + # Changelog generation is a bit complicated. We rely on towncrier to # generate the unstable/current changelog, but otherwise use the RST # edition to record historical changelogs. This is done by prepending @@ -1007,6 +1015,10 @@ class MatrixUnits(Units): title_part = keyword_versions[title_part] changelog = "".join(changelog_lines) changelogs[name][title_part.replace("^[a-zA-Z0-9]", "_").lower()] = changelog + preferred_changelog = changelogs[name]["unstable"] + if name in preferred_versions: + preferred_changelog = changelogs[name][preferred_versions[name]] + changelogs[name]["preferred"] = preferred_changelog return changelogs From 76946a8a7c6ba2bffb26ef91993e7b993bba03a9 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 6 Feb 2019 22:02:21 -0700 Subject: [PATCH 2/4] Simplify changelog generation We don'e need `{{server_server_changelog_r0.1.0}}` (for example), so don't go through the hassle of generating it. Instead, we'll generate the changelog for the requested versions of each API and put that in place. In the future, we may wish to consider bringing back more complicated variables when/if we start generating released versions of the spec on the fly rather than manually. --- .../templating/matrix_templates/sections.py | 9 +- scripts/templating/matrix_templates/units.py | 181 +++++++++--------- 2 files changed, 92 insertions(+), 98 deletions(-) diff --git a/scripts/templating/matrix_templates/sections.py b/scripts/templating/matrix_templates/sections.py index 4451d21d..5961aa24 100644 --- a/scripts/templating/matrix_templates/sections.py +++ b/scripts/templating/matrix_templates/sections.py @@ -34,15 +34,10 @@ class MatrixSections(Sections): def render_changelogs(self): rendered = {} changelogs = self.units.get("changelogs") - for spec, versioned in changelogs.items(): + for spec, changelog_text in changelogs.items(): spec_var = "%s_changelog" % spec logger.info("Rendering changelog for spec: %s" % spec) - for version, changelog in versioned.items(): - version_var = "%s_%s" % (spec_var, version) - logger.info("Rendering changelog for %s" % version_var) - rendered[version_var] = changelog - if version == "preferred": - rendered[spec_var] = changelog + rendered[spec_var] = changelog_text return rendered def _render_events(self, filterFn, sortFn): diff --git a/scripts/templating/matrix_templates/units.py b/scripts/templating/matrix_templates/units.py index fd261838..0e3546cb 100644 --- a/scripts/templating/matrix_templates/units.py +++ b/scripts/templating/matrix_templates/units.py @@ -904,9 +904,20 @@ class MatrixUnits(Units): return schema def load_changelogs(self, substitutions): + """Loads the changelog unit for later rendering in a section. + + Args: + substitutions: dict of variable name to value. Provided by the gendoc script. + + Returns: + A dict of API name ("client_server", for example) to changelog. + """ changelogs = {} - preferred_versions = { + # The APIs and versions we'll prepare changelogs for. We use the substitutions + # to ensure that we pick up the right version for generated documentation. This + # defaults to "unstable" as a version for incremental generated documentation (CI). + prepare_versions = { "server_server": substitutions.get("%SERVER_RELEASE_LABEL%", "unstable"), "client_server": substitutions.get("%CLIENT_RELEASE_LABEL%", "unstable"), "identity_service": substitutions.get("%IDENTITY_RELEASE_LABEL%", "unstable"), @@ -914,112 +925,100 @@ class MatrixUnits(Units): "application_service": substitutions.get("%APPSERVICE_RELEASE_LABEL%", "unstable"), } - # Changelog generation is a bit complicated. We rely on towncrier to - # generate the unstable/current changelog, but otherwise use the RST - # edition to record historical changelogs. This is done by prepending - # the towncrier output to the RST in memory, then parsing the RST by - # hand. We parse the entire changelog to create a changelog for each - # version which may be of use in some APIs. - - # Map specific headers to specific keys that'll be used eventually - # in variables. Things not listed here will get lowercased and formatted - # such that characters not [a-z0-9] will be replaced with an underscore. - keyword_versions = { - "Unreleased Changes": "unstable" - } - - # Only generate changelogs for things that have an RST document - for f in os.listdir(CHANGELOG_DIR): - if not f.endswith(".rst"): - continue - path = os.path.join(CHANGELOG_DIR, f) - name = f[:-4] # take off ".rst" - - # If there's a directory with the same name, we'll try to generate - # a towncrier changelog and prepend it to the general changelog. - tc_path = os.path.join(CHANGELOG_DIR, name) - tc_lines = [] - if os.path.isdir(tc_path): - logger.info("Generating towncrier changelog for: %s" % name) - p = subprocess.Popen( - ['towncrier', '--version', 'Unreleased Changes', '--name', name, '--draft'], - cwd=tc_path, - stderr=subprocess.PIPE, - stdout=subprocess.PIPE, - ) - stdout, stderr = p.communicate() - if p.returncode != 0: - # Something broke - dump as much information as we can - logger.error("Towncrier exited with code %s" % p.returncode) - logger.error(stdout.decode('UTF-8')) - logger.error(stderr.decode('UTF-8')) - raw_log = "" - else: - raw_log = stdout.decode('UTF-8') - - # This is a bit of a hack, but it does mean that the log at least gets *something* - # to tell us it broke - if not raw_log.startswith("Unreleased Changes"): - logger.error("Towncrier appears to have failed to generate a changelog") - logger.error(raw_log) - raw_log = "" - tc_lines = raw_log.splitlines() + # Changelogs are split into two places: towncrier for the unstable changelog and + # the RST file for historical versions. If the prepare_versions dict above has + # a version other than "unstable" specified for an API, we'll use the historical + # changelog and otherwise generate the towncrier log in-memory. - title_part = None + for api_name, target_version in prepare_versions.items(): + logger.info("Generating changelog for %s at %s" % (api_name, target_version,)) changelog_lines = [] - with open(path, "r", encoding="utf-8") as f: - lines = f.readlines() + if target_version == 'unstable': + # generate towncrier log + tc_path = os.path.join(CHANGELOG_DIR, api_name) + if os.path.isdir(tc_path): + logger.info("Generating towncrier changelog for: %s" % api_name) + p = subprocess.Popen( + ['towncrier', '--version', 'unstable', '--name', api_name, '--draft'], + cwd=tc_path, + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + ) + stdout, stderr = p.communicate() + if p.returncode != 0: + # Something broke - dump as much information as we can + logger.error("Towncrier exited with code %s" % p.returncode) + logger.error(stdout.decode('UTF-8')) + logger.error(stderr.decode('UTF-8')) + raw_log = "" + else: + raw_log = stdout.decode('UTF-8') + + # This is a bit of a hack, but it does mean that the log at least gets *something* + # to tell us it broke + if not raw_log.startswith("unstable"): + logger.error("Towncrier appears to have failed to generate a changelog") + logger.error(raw_log) + raw_log = "" + changelog_lines = raw_log.splitlines() + else: + # read in the existing RST changelog + logger.info("Reading changelog RST for %s" % api_name) + rst_path = os.path.join(CHANGELOG_DIR, "%s.rst" % api_name) + with open(rst_path, 'r', encoding="utf-8") as f: + changelog_lines = f.readlines() + + # Parse the changelog lines to find the header we're looking for and therefore + # the changelog body. prev_line = None - for line in (tc_lines + lines): + title_part = None + changelog_body_lines = [] + have_changelog = False + for line in changelog_lines: if prev_line is None: prev_line = line continue if not title_part: - # find the title underline (at least 3 =) + # Titles we care about are underlined with at least 3 equal signs if re.match("^[=]{3,}$", line.strip()): - title_part = prev_line + logger.info("Found header %s" % prev_line) + title_part = prev_line.strip() continue prev_line = line - else: # have title, get body (stop on next title or EOF) + else: + # we have a title, start parsing the body if re.match("^[=]{3,}$", line.strip()): - # we hit another title, so pop the last line of - # the changelog and record the changelog - new_title = changelog_lines.pop() - if name not in changelogs: - changelogs[name] = {} - if title_part in keyword_versions: - title_part = keyword_versions[title_part] - title_part = title_part.strip().replace("^[a-zA-Z0-9]", "_").lower() - changelog = "".join(changelog_lines) - changelogs[name][title_part] = changelog - - # reset for the next version - changelog_lines = [] - title_part = new_title.strip() + # we hit another title. prev_line will be the new section's header. + # do a check to see if the section we just read is the one we want - if + # it is, use that changelog and move on. If it isn't, keep reading. + if title_part == target_version: + changelogs[api_name] = "".join(changelog_body_lines) + have_changelog = True + break + # not the section we want - start the next section + title_part = changelog_body_lines.pop().strip() + changelog_body_lines = [] continue - # Don't generate subheadings (we'll keep the title though) if re.match("^[-]{3,}$", line.strip()): - continue - if line.strip().startswith(".. version: "): - # The changelog is directing us to use a different title - # for the changelog. - title_part = line.strip()[len(".. version: "):] + # the last line is a subheading - drop this line because it's the underline + # and that causes problems with rendering. We'll keep the header text though. continue if line.strip().startswith(".. "): - continue # skip comments - changelog_lines.append(" " + line + '\n') - if len(changelog_lines) > 0 and title_part is not None: - if name not in changelogs: - changelogs[name] = {} - if title_part in keyword_versions: - title_part = keyword_versions[title_part] - changelog = "".join(changelog_lines) - changelogs[name][title_part.replace("^[a-zA-Z0-9]", "_").lower()] = changelog - preferred_changelog = changelogs[name]["unstable"] - if name in preferred_versions: - preferred_changelog = changelogs[name][preferred_versions[name]] - changelogs[name]["preferred"] = preferred_changelog + # skip comments + continue + # if we made it this far, append the line to the changelog body. We indent it so + # that it renders correctly in the section. We also add newlines so that there's + # intentionally blank lines that make rst2html happy. + changelog_body_lines.append(" " + line + '\n') + # do some quick checks to see if the last read section is our changelog + if not have_changelog: + logger.info("No changelog - testing %s == %s" % (target_version, title_part,)) + if title_part == target_version and len(changelog_body_lines) > 0: + changelogs[api_name] = "".join(changelog_body_lines) + else: + raise ValueError("No changelog for %s at %s" % (api_name, target_version,)) + # return our `dict[api_name] => changelog` as the last step. return changelogs def load_unstable_warnings(self, substitutions): From 375104127cc25fc5cb3d46456e4b7d08623f490c Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 6 Feb 2019 22:03:16 -0700 Subject: [PATCH 3/4] Fix spec release process to match new changelog stuff Also while we're here, make it accurate. Fixes https://github.com/matrix-org/matrix-doc/issues/1858 --- meta/releasing_a_spec.md | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/meta/releasing_a_spec.md b/meta/releasing_a_spec.md index 2d9cd34e..1d7aaea7 100644 --- a/meta/releasing_a_spec.md +++ b/meta/releasing_a_spec.md @@ -18,16 +18,18 @@ The remainder of the process is as follows: 1. Activate your Python 3 virtual environment. 1. Having checked out the new release branch, navigate your way over to `./changelogs`. 1. Follow the release instructions provided in the README.md located there. -1. Update the changelog section of the specification you're releasing to make a reference - to the new version. 1. Update any version/link references across all specifications. -1. Ensure the `targets.yml` file lists the version correctly. -1. Commit the changes and PR them to master. -1. Tag the release with the format `client_server/r0.4.0`. -1. Add the changes to the matrix-org/matrix.org repository (for historic tracking). +1. Generate the specification using `./scripts/gendoc.py -c r0.4.0`, specifying all the + API versions at the time of generation. +1. PR the changes to the matrix-org/matrix.org repository (for historic tracking). * This is done by making a PR to the `unstyled_docs/spec` folder for the version and specification you're releasing. * Don't forget to symlink the new release as `latest`. + * For the client-server API, don't forget to generate the swagger JSON by using + `./scripts/dump-swagger.py -c r0.4.0`. This will also need symlinking to `latest`. +1. Commit the changes and PR them to master. **Wait for review from the spec core team.** + * Link to your matrix-org/matrix.org so both can be reviewed at the same time. +1. Tag the release with the format `client_server/r0.4.0`. 1. Perform a release on GitHub to tag the release. 1. Yell from the mountaintop to the world about the new release. From b1689a3036878cbf152e2e4de45f1ac28a08d5ee Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Sun, 10 Feb 2019 18:03:17 -0700 Subject: [PATCH 4/4] Misc improvements --- meta/releasing_a_spec.md | 4 +- scripts/templating/matrix_templates/units.py | 120 +++++++++---------- 2 files changed, 56 insertions(+), 68 deletions(-) diff --git a/meta/releasing_a_spec.md b/meta/releasing_a_spec.md index 1d7aaea7..f186c4be 100644 --- a/meta/releasing_a_spec.md +++ b/meta/releasing_a_spec.md @@ -19,8 +19,8 @@ The remainder of the process is as follows: 1. Having checked out the new release branch, navigate your way over to `./changelogs`. 1. Follow the release instructions provided in the README.md located there. 1. Update any version/link references across all specifications. -1. Generate the specification using `./scripts/gendoc.py -c r0.4.0`, specifying all the - API versions at the time of generation. +1. Generate the specification using `./scripts/gendoc.py`, specifying all the + API versions at the time of generation. For example: `./scripts/gendoc.py -c r0.4.0 -s r0.1.0 -i r0.1.0 #etc` 1. PR the changes to the matrix-org/matrix.org repository (for historic tracking). * This is done by making a PR to the `unstyled_docs/spec` folder for the version and specification you're releasing. diff --git a/scripts/templating/matrix_templates/units.py b/scripts/templating/matrix_templates/units.py index a061c693..c1755119 100644 --- a/scripts/templating/matrix_templates/units.py +++ b/scripts/templating/matrix_templates/units.py @@ -935,92 +935,80 @@ class MatrixUnits(Units): changelog_lines = [] if target_version == 'unstable': # generate towncrier log - tc_path = os.path.join(CHANGELOG_DIR, api_name) - if os.path.isdir(tc_path): - logger.info("Generating towncrier changelog for: %s" % api_name) - p = subprocess.Popen( - ['towncrier', '--version', 'unstable', '--name', api_name, '--draft'], - cwd=tc_path, - stderr=subprocess.PIPE, - stdout=subprocess.PIPE, - ) - stdout, stderr = p.communicate() - if p.returncode != 0: - # Something broke - dump as much information as we can - logger.error("Towncrier exited with code %s" % p.returncode) - logger.error(stdout.decode('UTF-8')) - logger.error(stderr.decode('UTF-8')) - raw_log = "" - else: - raw_log = stdout.decode('UTF-8') - - # This is a bit of a hack, but it does mean that the log at least gets *something* - # to tell us it broke - if not raw_log.startswith("unstable"): - logger.error("Towncrier appears to have failed to generate a changelog") - logger.error(raw_log) - raw_log = "" - changelog_lines = raw_log.splitlines() + changelog_lines = self._read_towncrier_changelog(api_name) else: # read in the existing RST changelog - logger.info("Reading changelog RST for %s" % api_name) - rst_path = os.path.join(CHANGELOG_DIR, "%s.rst" % api_name) - with open(rst_path, 'r', encoding="utf-8") as f: - changelog_lines = f.readlines() + changelog_lines = self._read_rst_changelog(api_name) # Parse the changelog lines to find the header we're looking for and therefore # the changelog body. prev_line = None title_part = None changelog_body_lines = [] - have_changelog = False for line in changelog_lines: if prev_line is None: prev_line = line continue - if not title_part: - # Titles we care about are underlined with at least 3 equal signs - if re.match("^[=]{3,}$", line.strip()): - logger.info("Found header %s" % prev_line) - title_part = prev_line.strip() - continue - prev_line = line - else: - # we have a title, start parsing the body - if re.match("^[=]{3,}$", line.strip()): - # we hit another title. prev_line will be the new section's header. - # do a check to see if the section we just read is the one we want - if - # it is, use that changelog and move on. If it isn't, keep reading. - if title_part == target_version: - changelogs[api_name] = "".join(changelog_body_lines) - have_changelog = True - break - # not the section we want - start the next section - title_part = changelog_body_lines.pop().strip() - changelog_body_lines = [] - continue - if re.match("^[-]{3,}$", line.strip()): - # the last line is a subheading - drop this line because it's the underline - # and that causes problems with rendering. We'll keep the header text though. - continue - if line.strip().startswith(".. "): - # skip comments - continue + if re.match("^[=]{3,}$", line.strip()): + # the last line was a header - use that as our new title_part + title_part = prev_line.strip() + continue + if re.match("^[-]{3,}$", line.strip()): + # the last line is a subheading - drop this line because it's the underline + # and that causes problems with rendering. We'll keep the header text though. + continue + if line.strip().startswith(".. "): + # skip comments + continue + if title_part == target_version: # if we made it this far, append the line to the changelog body. We indent it so # that it renders correctly in the section. We also add newlines so that there's # intentionally blank lines that make rst2html happy. changelog_body_lines.append(" " + line + '\n') - # do some quick checks to see if the last read section is our changelog - if not have_changelog: - logger.info("No changelog - testing %s == %s" % (target_version, title_part,)) - if title_part == target_version and len(changelog_body_lines) > 0: - changelogs[api_name] = "".join(changelog_body_lines) - else: - raise ValueError("No changelog for %s at %s" % (api_name, target_version,)) + + if len(changelog_body_lines) > 0: + changelogs[api_name] = "".join(changelog_body_lines) + else: + raise ValueError("No changelog for %s at %s" % (api_name, target_version,)) # return our `dict[api_name] => changelog` as the last step. return changelogs + def _read_towncrier_changelog(self, api_name): + tc_path = os.path.join(CHANGELOG_DIR, api_name) + if os.path.isdir(tc_path): + logger.info("Generating towncrier changelog for: %s" % api_name) + p = subprocess.Popen( + ['towncrier', '--version', 'unstable', '--name', api_name, '--draft'], + cwd=tc_path, + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + ) + stdout, stderr = p.communicate() + if p.returncode != 0: + # Something broke - dump as much information as we can + logger.error("Towncrier exited with code %s" % p.returncode) + logger.error(stdout.decode('UTF-8')) + logger.error(stderr.decode('UTF-8')) + raw_log = "" + else: + raw_log = stdout.decode('UTF-8') + + # This is a bit of a hack, but it does mean that the log at least gets *something* + # to tell us it broke + if not raw_log.startswith("unstable"): + logger.error("Towncrier appears to have failed to generate a changelog") + logger.error(raw_log) + raw_log = "" + return raw_log.splitlines() + return [] + + def _read_rst_changelog(self, api_name): + logger.info("Reading changelog RST for %s" % api_name) + rst_path = os.path.join(CHANGELOG_DIR, "%s.rst" % api_name) + with open(rst_path, 'r', encoding="utf-8") as f: + return f.readlines() + def load_unstable_warnings(self, substitutions): warning = """ .. WARNING::