diff --git a/meta/releasing_a_spec.md b/meta/releasing_a_spec.md index 2d9cd34ec..f186c4be1 100644 --- a/meta/releasing_a_spec.md +++ b/meta/releasing_a_spec.md @@ -18,16 +18,18 @@ The remainder of the process is as follows: 1. Activate your Python 3 virtual environment. 1. Having checked out the new release branch, navigate your way over to `./changelogs`. 1. Follow the release instructions provided in the README.md located there. -1. Update the changelog section of the specification you're releasing to make a reference - to the new version. 1. Update any version/link references across all specifications. -1. Ensure the `targets.yml` file lists the version correctly. -1. Commit the changes and PR them to master. -1. Tag the release with the format `client_server/r0.4.0`. -1. Add the changes to the matrix-org/matrix.org repository (for historic tracking). +1. Generate the specification using `./scripts/gendoc.py`, specifying all the + API versions at the time of generation. For example: `./scripts/gendoc.py -c r0.4.0 -s r0.1.0 -i r0.1.0 #etc` +1. PR the changes to the matrix-org/matrix.org repository (for historic tracking). * This is done by making a PR to the `unstyled_docs/spec` folder for the version and specification you're releasing. * Don't forget to symlink the new release as `latest`. + * For the client-server API, don't forget to generate the swagger JSON by using + `./scripts/dump-swagger.py -c r0.4.0`. This will also need symlinking to `latest`. +1. Commit the changes and PR them to master. **Wait for review from the spec core team.** + * Link to your matrix-org/matrix.org so both can be reviewed at the same time. +1. Tag the release with the format `client_server/r0.4.0`. 1. Perform a release on GitHub to tag the release. 1. Yell from the mountaintop to the world about the new release. diff --git a/scripts/templating/matrix_templates/sections.py b/scripts/templating/matrix_templates/sections.py index af4976744..5961aa246 100644 --- a/scripts/templating/matrix_templates/sections.py +++ b/scripts/templating/matrix_templates/sections.py @@ -34,15 +34,10 @@ class MatrixSections(Sections): def render_changelogs(self): rendered = {} changelogs = self.units.get("changelogs") - for spec, versioned in changelogs.items(): + for spec, changelog_text in changelogs.items(): spec_var = "%s_changelog" % spec logger.info("Rendering changelog for spec: %s" % spec) - for version, changelog in versioned.items(): - version_var = "%s_%s" % (spec_var, version) - logger.info("Rendering changelog for %s" % version_var) - rendered[version_var] = changelog - if version == "unstable": - rendered[spec_var] = changelog + rendered[spec_var] = changelog_text return rendered def _render_events(self, filterFn, sortFn): diff --git a/scripts/templating/matrix_templates/units.py b/scripts/templating/matrix_templates/units.py index f697dbdf0..c1755119e 100644 --- a/scripts/templating/matrix_templates/units.py +++ b/scripts/templating/matrix_templates/units.py @@ -903,113 +903,112 @@ class MatrixUnits(Units): return schema - def load_changelogs(self): + def load_changelogs(self, substitutions): + """Loads the changelog unit for later rendering in a section. + + Args: + substitutions: dict of variable name to value. Provided by the gendoc script. + + Returns: + A dict of API name ("client_server", for example) to changelog. + """ changelogs = {} - # Changelog generation is a bit complicated. We rely on towncrier to - # generate the unstable/current changelog, but otherwise use the RST - # edition to record historical changelogs. This is done by prepending - # the towncrier output to the RST in memory, then parsing the RST by - # hand. We parse the entire changelog to create a changelog for each - # version which may be of use in some APIs. - - # Map specific headers to specific keys that'll be used eventually - # in variables. Things not listed here will get lowercased and formatted - # such that characters not [a-z0-9] will be replaced with an underscore. - keyword_versions = { - "Unreleased Changes": "unstable" + # The APIs and versions we'll prepare changelogs for. We use the substitutions + # to ensure that we pick up the right version for generated documentation. This + # defaults to "unstable" as a version for incremental generated documentation (CI). + prepare_versions = { + "server_server": substitutions.get("%SERVER_RELEASE_LABEL%", "unstable"), + "client_server": substitutions.get("%CLIENT_RELEASE_LABEL%", "unstable"), + "identity_service": substitutions.get("%IDENTITY_RELEASE_LABEL%", "unstable"), + "push_gateway": substitutions.get("%PUSH_GATEWAY_RELEASE_LABEL%", "unstable"), + "application_service": substitutions.get("%APPSERVICE_RELEASE_LABEL%", "unstable"), } - # Only generate changelogs for things that have an RST document - for f in os.listdir(CHANGELOG_DIR): - if not f.endswith(".rst"): - continue - path = os.path.join(CHANGELOG_DIR, f) - name = f[:-4] # take off ".rst" - - # If there's a directory with the same name, we'll try to generate - # a towncrier changelog and prepend it to the general changelog. - tc_path = os.path.join(CHANGELOG_DIR, name) - tc_lines = [] - if os.path.isdir(tc_path): - logger.info("Generating towncrier changelog for: %s" % name) - p = subprocess.Popen( - ['towncrier', '--version', 'Unreleased Changes', '--name', name, '--draft'], - cwd=tc_path, - stderr=subprocess.PIPE, - stdout=subprocess.PIPE, - ) - stdout, stderr = p.communicate() - if p.returncode != 0: - # Something broke - dump as much information as we can - logger.error("Towncrier exited with code %s" % p.returncode) - logger.error(stdout.decode('UTF-8')) - logger.error(stderr.decode('UTF-8')) - raw_log = "" - else: - raw_log = stdout.decode('UTF-8') - - # This is a bit of a hack, but it does mean that the log at least gets *something* - # to tell us it broke - if not raw_log.startswith("Unreleased Changes"): - logger.error("Towncrier appears to have failed to generate a changelog") - logger.error(raw_log) - raw_log = "" - tc_lines = raw_log.splitlines() + # Changelogs are split into two places: towncrier for the unstable changelog and + # the RST file for historical versions. If the prepare_versions dict above has + # a version other than "unstable" specified for an API, we'll use the historical + # changelog and otherwise generate the towncrier log in-memory. - title_part = None + for api_name, target_version in prepare_versions.items(): + logger.info("Generating changelog for %s at %s" % (api_name, target_version,)) changelog_lines = [] - with open(path, "r", encoding="utf-8") as f: - lines = f.readlines() + if target_version == 'unstable': + # generate towncrier log + changelog_lines = self._read_towncrier_changelog(api_name) + else: + # read in the existing RST changelog + changelog_lines = self._read_rst_changelog(api_name) + + # Parse the changelog lines to find the header we're looking for and therefore + # the changelog body. prev_line = None - for line in (tc_lines + lines): + title_part = None + changelog_body_lines = [] + for line in changelog_lines: if prev_line is None: prev_line = line continue - if not title_part: - # find the title underline (at least 3 =) - if re.match("^[=]{3,}$", line.strip()): - title_part = prev_line - continue - prev_line = line - else: # have title, get body (stop on next title or EOF) - if re.match("^[=]{3,}$", line.strip()): - # we hit another title, so pop the last line of - # the changelog and record the changelog - new_title = changelog_lines.pop() - if name not in changelogs: - changelogs[name] = {} - if title_part in keyword_versions: - title_part = keyword_versions[title_part] - title_part = title_part.strip().replace("^[a-zA-Z0-9]", "_").lower() - changelog = "".join(changelog_lines) - changelogs[name][title_part] = changelog - - # reset for the next version - changelog_lines = [] - title_part = new_title.strip() - continue - # Don't generate subheadings (we'll keep the title though) - if re.match("^[-]{3,}$", line.strip()): - continue - if line.strip().startswith(".. version: "): - # The changelog is directing us to use a different title - # for the changelog. - title_part = line.strip()[len(".. version: "):] - continue - if line.strip().startswith(".. "): - continue # skip comments - changelog_lines.append(" " + line + '\n') - if len(changelog_lines) > 0 and title_part is not None: - if name not in changelogs: - changelogs[name] = {} - if title_part in keyword_versions: - title_part = keyword_versions[title_part] - changelog = "".join(changelog_lines) - changelogs[name][title_part.replace("^[a-zA-Z0-9]", "_").lower()] = changelog + if re.match("^[=]{3,}$", line.strip()): + # the last line was a header - use that as our new title_part + title_part = prev_line.strip() + continue + if re.match("^[-]{3,}$", line.strip()): + # the last line is a subheading - drop this line because it's the underline + # and that causes problems with rendering. We'll keep the header text though. + continue + if line.strip().startswith(".. "): + # skip comments + continue + if title_part == target_version: + # if we made it this far, append the line to the changelog body. We indent it so + # that it renders correctly in the section. We also add newlines so that there's + # intentionally blank lines that make rst2html happy. + changelog_body_lines.append(" " + line + '\n') + + if len(changelog_body_lines) > 0: + changelogs[api_name] = "".join(changelog_body_lines) + else: + raise ValueError("No changelog for %s at %s" % (api_name, target_version,)) + # return our `dict[api_name] => changelog` as the last step. return changelogs + def _read_towncrier_changelog(self, api_name): + tc_path = os.path.join(CHANGELOG_DIR, api_name) + if os.path.isdir(tc_path): + logger.info("Generating towncrier changelog for: %s" % api_name) + p = subprocess.Popen( + ['towncrier', '--version', 'unstable', '--name', api_name, '--draft'], + cwd=tc_path, + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + ) + stdout, stderr = p.communicate() + if p.returncode != 0: + # Something broke - dump as much information as we can + logger.error("Towncrier exited with code %s" % p.returncode) + logger.error(stdout.decode('UTF-8')) + logger.error(stderr.decode('UTF-8')) + raw_log = "" + else: + raw_log = stdout.decode('UTF-8') + + # This is a bit of a hack, but it does mean that the log at least gets *something* + # to tell us it broke + if not raw_log.startswith("unstable"): + logger.error("Towncrier appears to have failed to generate a changelog") + logger.error(raw_log) + raw_log = "" + return raw_log.splitlines() + return [] + + def _read_rst_changelog(self, api_name): + logger.info("Reading changelog RST for %s" % api_name) + rst_path = os.path.join(CHANGELOG_DIR, "%s.rst" % api_name) + with open(rst_path, 'r', encoding="utf-8") as f: + return f.readlines() + def load_unstable_warnings(self, substitutions): warning = """ .. WARNING::