From e8674688e607c4a8a47d0030dc3c2167335916a7 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 18 Oct 2021 10:09:35 -0600 Subject: [PATCH] Release process: changelog generation and docs (#3446) * Remove extra pyprojects and update changelog docs * Add script for rendering the changelog * Add docs for how to release the spec * Move legacy changelogs out of the way --- .gitignore | 1 + changelogs/README.md | 59 ++-------------- changelogs/application_service/pyproject.toml | 35 ---------- changelogs/client_server/pyproject.toml | 35 ---------- changelogs/header.md | 16 +++++ changelogs/identity_service/pyproject.toml | 35 ---------- .../{ => legacy}/application_service.rst | 0 changelogs/{ => legacy}/client_server.rst | 0 changelogs/{ => legacy}/identity_service.rst | 0 changelogs/{ => legacy}/push_gateway.rst | 0 changelogs/{ => legacy}/server_server.rst | 0 changelogs/push_gateway/pyproject.toml | 35 ---------- changelogs/pyproject.toml | 39 +++++++++++ changelogs/server_server/pyproject.toml | 35 ---------- content/changelog.md | 2 + layouts/partials/changelogs/.gitkeep | 0 .../changelog/changelog-changes.html | 4 +- .../changelog/changelog-rendered.html | 2 + meta/releasing.md | 67 +++++++++++++++++++ meta/releasing_a_spec.md | 52 -------------- scripts/generate-changelog.sh | 35 ++++++++++ scripts/requirements.txt | 2 +- 22 files changed, 172 insertions(+), 282 deletions(-) delete mode 100644 changelogs/application_service/pyproject.toml delete mode 100644 changelogs/client_server/pyproject.toml create mode 100644 changelogs/header.md delete mode 100644 changelogs/identity_service/pyproject.toml rename changelogs/{ => legacy}/application_service.rst (100%) rename changelogs/{ => legacy}/client_server.rst (100%) rename changelogs/{ => legacy}/identity_service.rst (100%) rename changelogs/{ => legacy}/push_gateway.rst (100%) rename changelogs/{ => legacy}/server_server.rst (100%) delete mode 100644 changelogs/push_gateway/pyproject.toml create mode 100644 changelogs/pyproject.toml delete mode 100644 changelogs/server_server/pyproject.toml create mode 100644 layouts/partials/changelogs/.gitkeep create mode 100644 layouts/shortcodes/changelog/changelog-rendered.html create mode 100644 meta/releasing.md delete mode 100644 meta/releasing_a_spec.md create mode 100644 scripts/generate-changelog.sh diff --git a/.gitignore b/.gitignore index 6e5a25c7..d80f9ea7 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,4 @@ _rendered.rst /.vscode/ /.idea/ /spec/ +changelogs/rendered.* diff --git a/changelogs/README.md b/changelogs/README.md index 5a5b6271..2e9bdf6e 100644 --- a/changelogs/README.md +++ b/changelogs/README.md @@ -1,60 +1,15 @@ - - # Changelogs [Towncrier](https://github.com/hawkowl/towncrier) is used to manage the changelog and keep it up to date. Because of this, updating a changelog is really easy. -## How to update a changelog when releasing an API - -1. Ensure you're in your Python 3 virtual environment -2. `cd` your way to the API you're releasing (eg: `cd changelogs/client_server`) -3. Run `towncrier --version "r0.4.0" --name "client-server" --yes` substituting the - variables as approprite. Note that `--name` is required although the value is ignored. -4. Commit the changes and finish the release process. - -## How to prepare a changelog for a new API - -For this example, we're going to pretend that the `server_server` API doesn't exist. - -1. Create the file `changelogs/server_server.rst` -2. Create the folder `changelogs/server_server` -3. In the new folder, create a `pyproject.toml` file with these contents: - ```toml - [tool.towncrier] - filename = "../server_server.rst" - directory = "newsfragments" - issue_format = "`#{issue} `_" - title_format = "{version}" - - [[tool.towncrier.type]] - directory = "breaking" - name = "Breaking Changes" - showcontent = true - - [[tool.towncrier.type]] - directory = "deprecation" - name = "Deprecations" - showcontent = true - - [[tool.towncrier.type]] - directory = "new" - name = "New Endpoints" - showcontent = true +## Generating the changelog - [[tool.towncrier.type]] - directory = "removal" - name = "Removed Endpoints" - showcontent = true +Please see the [release docs](../meta/releasing.md) for more information. - [[tool.towncrier.type]] - directory = "feature" - name = "Backwards Compatible Changes" - showcontent = true +## Creating a new changelog - [[tool.towncrier.type]] - directory = "clarification" - name = "Spec Clarifications" - showcontent = true - ``` -4. Create a `.gitignore` in `changelogs/server_server/newsfragments` with the contents `!.gitignore` +There are a few places you'll have to update: +* `/layouts/shortcodes/changelog/changelog-changes.html` to account for the new changelog. +* `/scripts/generate-changelog.sh` to render the changelog for releases. +* Supporting documentation such as the contributing guidelines. diff --git a/changelogs/application_service/pyproject.toml b/changelogs/application_service/pyproject.toml deleted file mode 100644 index 278def78..00000000 --- a/changelogs/application_service/pyproject.toml +++ /dev/null @@ -1,35 +0,0 @@ -[tool.towncrier] - filename = "../application_service.rst" - directory = "newsfragments" - issue_format = "`#{issue} `_" - title_format = "{version}" - - [[tool.towncrier.type]] - directory = "breaking" - name = "Breaking Changes" - showcontent = true - - [[tool.towncrier.type]] - directory = "deprecation" - name = "Deprecations" - showcontent = true - - [[tool.towncrier.type]] - directory = "new" - name = "New Endpoints" - showcontent = true - - [[tool.towncrier.type]] - directory = "removal" - name = "Removed Endpoints" - showcontent = true - - [[tool.towncrier.type]] - directory = "feature" - name = "Backwards Compatible Changes" - showcontent = true - - [[tool.towncrier.type]] - directory = "clarification" - name = "Spec Clarifications" - showcontent = true diff --git a/changelogs/client_server/pyproject.toml b/changelogs/client_server/pyproject.toml deleted file mode 100644 index eb9e7b4e..00000000 --- a/changelogs/client_server/pyproject.toml +++ /dev/null @@ -1,35 +0,0 @@ -[tool.towncrier] - filename = "../client_server.rst" - directory = "newsfragments" - issue_format = "`#{issue} `_" - title_format = "{version}" - - [[tool.towncrier.type]] - directory = "breaking" - name = "Breaking Changes" - showcontent = true - - [[tool.towncrier.type]] - directory = "deprecation" - name = "Deprecations" - showcontent = true - - [[tool.towncrier.type]] - directory = "new" - name = "New Endpoints" - showcontent = true - - [[tool.towncrier.type]] - directory = "removal" - name = "Removed Endpoints" - showcontent = true - - [[tool.towncrier.type]] - directory = "feature" - name = "Backwards Compatible Changes" - showcontent = true - - [[tool.towncrier.type]] - directory = "clarification" - name = "Spec Clarifications" - showcontent = true diff --git a/changelogs/header.md b/changelogs/header.md new file mode 100644 index 00000000..2b98c8a1 --- /dev/null +++ b/changelogs/header.md @@ -0,0 +1,16 @@ + + +## VERSION + + + + +
Git commithttps://github.com/matrix-org/matrix-doc/tree/VERSION
Release dateDATE
+ + diff --git a/changelogs/identity_service/pyproject.toml b/changelogs/identity_service/pyproject.toml deleted file mode 100644 index 2e50b9df..00000000 --- a/changelogs/identity_service/pyproject.toml +++ /dev/null @@ -1,35 +0,0 @@ -[tool.towncrier] - filename = "../identity_service.rst" - directory = "newsfragments" - issue_format = "`#{issue} `_" - title_format = "{version}" - - [[tool.towncrier.type]] - directory = "breaking" - name = "Breaking Changes" - showcontent = true - - [[tool.towncrier.type]] - directory = "deprecation" - name = "Deprecations" - showcontent = true - - [[tool.towncrier.type]] - directory = "new" - name = "New Endpoints" - showcontent = true - - [[tool.towncrier.type]] - directory = "removal" - name = "Removed Endpoints" - showcontent = true - - [[tool.towncrier.type]] - directory = "feature" - name = "Backwards Compatible Changes" - showcontent = true - - [[tool.towncrier.type]] - directory = "clarification" - name = "Spec Clarifications" - showcontent = true diff --git a/changelogs/application_service.rst b/changelogs/legacy/application_service.rst similarity index 100% rename from changelogs/application_service.rst rename to changelogs/legacy/application_service.rst diff --git a/changelogs/client_server.rst b/changelogs/legacy/client_server.rst similarity index 100% rename from changelogs/client_server.rst rename to changelogs/legacy/client_server.rst diff --git a/changelogs/identity_service.rst b/changelogs/legacy/identity_service.rst similarity index 100% rename from changelogs/identity_service.rst rename to changelogs/legacy/identity_service.rst diff --git a/changelogs/push_gateway.rst b/changelogs/legacy/push_gateway.rst similarity index 100% rename from changelogs/push_gateway.rst rename to changelogs/legacy/push_gateway.rst diff --git a/changelogs/server_server.rst b/changelogs/legacy/server_server.rst similarity index 100% rename from changelogs/server_server.rst rename to changelogs/legacy/server_server.rst diff --git a/changelogs/push_gateway/pyproject.toml b/changelogs/push_gateway/pyproject.toml deleted file mode 100644 index 9f2595c9..00000000 --- a/changelogs/push_gateway/pyproject.toml +++ /dev/null @@ -1,35 +0,0 @@ -[tool.towncrier] - filename = "../push_gateway.rst" - directory = "newsfragments" - issue_format = "`#{issue} `_" - title_format = "{version}" - - [[tool.towncrier.type]] - directory = "breaking" - name = "Breaking Changes" - showcontent = true - - [[tool.towncrier.type]] - directory = "deprecation" - name = "Deprecations" - showcontent = true - - [[tool.towncrier.type]] - directory = "new" - name = "New Endpoints" - showcontent = true - - [[tool.towncrier.type]] - directory = "removal" - name = "Removed Endpoints" - showcontent = true - - [[tool.towncrier.type]] - directory = "feature" - name = "Backwards Compatible Changes" - showcontent = true - - [[tool.towncrier.type]] - directory = "clarification" - name = "Spec Clarifications" - showcontent = true diff --git a/changelogs/pyproject.toml b/changelogs/pyproject.toml new file mode 100644 index 00000000..f114a178 --- /dev/null +++ b/changelogs/pyproject.toml @@ -0,0 +1,39 @@ +[tool.towncrier] + version = "unused" + filename = "../rendered.md" + issue_format = "[#{issue}](https://github.com/matrix-org/matrix-doc/issues/{issue})" + title_format = "### {name}" # Matches rendered spec, even if awkward + underlines = " " # 3 spaces intentionally to hide RST headings + + # Note: The names below have the tag built-in so the rendered spec *and* the generated + # changelog can benefit from sane headings. + + [[tool.towncrier.type]] + directory = "breaking" + name = "Breaking Changes" + showcontent = true + + [[tool.towncrier.type]] + directory = "deprecation" + name = "Deprecations" + showcontent = true + + [[tool.towncrier.type]] + directory = "new" + name = "New Endpoints" + showcontent = true + + [[tool.towncrier.type]] + directory = "removal" + name = "Removed Endpoints" + showcontent = true + + [[tool.towncrier.type]] + directory = "feature" + name = "Backwards Compatible Changes" + showcontent = true + + [[tool.towncrier.type]] + directory = "clarification" + name = "Spec Clarifications" + showcontent = true diff --git a/changelogs/server_server/pyproject.toml b/changelogs/server_server/pyproject.toml deleted file mode 100644 index 6a9dca1d..00000000 --- a/changelogs/server_server/pyproject.toml +++ /dev/null @@ -1,35 +0,0 @@ -[tool.towncrier] - filename = "../server_server.rst" - directory = "newsfragments" - issue_format = "`#{issue} `_" - title_format = "{version}" - - [[tool.towncrier.type]] - directory = "breaking" - name = "Breaking Changes" - showcontent = true - - [[tool.towncrier.type]] - directory = "deprecation" - name = "Deprecations" - showcontent = true - - [[tool.towncrier.type]] - directory = "new" - name = "New Endpoints" - showcontent = true - - [[tool.towncrier.type]] - directory = "removal" - name = "Removed Endpoints" - showcontent = true - - [[tool.towncrier.type]] - directory = "feature" - name = "Backwards Compatible Changes" - showcontent = true - - [[tool.towncrier.type]] - directory = "clarification" - name = "Spec Clarifications" - showcontent = true diff --git a/content/changelog.md b/content/changelog.md index 41b0987e..9cd39ee6 100644 --- a/content/changelog.md +++ b/content/changelog.md @@ -8,6 +8,8 @@ weight: 1000 {{% changelog/changelog-changes %}} + +

Historical versions

Before version 1.1, versioning was applied at the level of individual API specifications. This section includes links to these versions of the APIs. diff --git a/layouts/partials/changelogs/.gitkeep b/layouts/partials/changelogs/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/layouts/shortcodes/changelog/changelog-changes.html b/layouts/shortcodes/changelog/changelog-changes.html index 11a59858..edd46bce 100644 --- a/layouts/shortcodes/changelog/changelog-changes.html +++ b/layouts/shortcodes/changelog/changelog-changes.html @@ -40,7 +40,7 @@ {{ define "partials/render-api-changes" }}

{{ .title }}

{{ $api_path := .path }} - {{ $config_file := path.Join $api_path "pyproject.toml" }} + {{ $config_file := path.Join $api_path ".." "pyproject.toml" }} {{ $config := readFile $config_file | transform.Unmarshal }} {{ $news_path := path.Join $api_path "newsfragments" }} {{ partial "render-newsfragments" (dict "config" $config "news_path" $news_path )}} @@ -74,7 +74,7 @@ {{ range $config.tool.towncrier.type }} {{ $changes_of_type := (index $types .directory) }} {{ if $changes_of_type }} -
  • {{.name}} +
  • {{.name | safeHTML}}

      {{ range $changes_of_type }}
    • {{ .ticket }}: {{ .description | markdownify }}
      • diff --git a/layouts/shortcodes/changelog/changelog-rendered.html b/layouts/shortcodes/changelog/changelog-rendered.html new file mode 100644 index 00000000..bbfd6e53 --- /dev/null +++ b/layouts/shortcodes/changelog/changelog-rendered.html @@ -0,0 +1,2 @@ +{{ $partial := .Params.p }} +{{ partial $partial . }} diff --git a/meta/releasing.md b/meta/releasing.md new file mode 100644 index 00000000..e1480e39 --- /dev/null +++ b/meta/releasing.md @@ -0,0 +1,67 @@ +# How to release the specification + +The whole specification is now released as a single unit/artifact. This document is +the process for releasing the specification and a description of how the (public) +machinery works. + +## Prerequisites / preparation + +First, can we even release the spec? This stage is mostly preperation work needed +to ensure a consistent and reliable specification. + +1. Ensure `main` is committed with all the spec changes you expect to be there. +2. Review the changelog to look for typos, wording inconsistencies, or lines which + can be merged. For example, "Fix typos" and "Fix spelling" can be condensed to + "Fix various typos throughout the specification". +3. Do a quick skim to ensure changelogs reference the MSCs which brought the changes + in. They should be linked to the GitHub MSC PR (not the markdown document). + +## The release + +Assuming the preparation work is complete, all that remains is the actual specification +release. This is done directly on `main`, though local branching for safety is also +welcome. + +1. Update the `params.version` section of `config.toml` to use the following template: + ```toml + [params.version] + status = "stable" + current_version_url = "https://spec.matrix.org/latest" + + # This will be the spec version you're releasing. If that's v1.2, then `major = "1"` + # and `minor = "2"` + major = "1" + minor = "2" + + # Today's date. Please use the format implied here for consistency. + release_date = "October 01, 2021" + ``` +2. Commit the changes. +3. Tag `main` with the spec release with a format of `v1.2` (if releasing Matrix 1.2). +4. Push `main` and the tag. +5. GitHub Actions will run its build steps. Wait until these are successful. If fixes + need to be made to repair the pipeline or spec build, delete and re-tag the release. +6. Generate the changelog. This is done *after* the tagging to ensure the rendered + changelog makes sense. + 1. Activate your python virtual environment. + 2. Run `./scripts/generate-changelog.sh v1.2 "October 01, 2021"` (using the correct + version number and same `release_date` format from the hugo config). + 3. Commit the result. +7. Create a new release on GitHub from the newly created tag. + * The title should be just "v1.2" (for example). + * The description should be a copy/paste of the changelog. The generated changelog + will be at `content/partials/changelogs/v1.2.md` - copy/paste verbatim. + * Upload the artifacts of the GitHub Actions build for the release to the GitHub + release as artifacts themselves. This should be the tarball that got deployed + to spec.matrix.org. +8. Commit a reversion to `params.version` of `config.toml` on `main`: + ```toml + [params.version] + status = "unstable" + current_version_url = "https://spec.matrix.org/latest" + # major = "1" + # minor = "2" + # release_date = "October 01, 2021" + ``` +9. Push pending commits and ensure the unstable spec updates accordingly from the + GitHub Actions pipeline. diff --git a/meta/releasing_a_spec.md b/meta/releasing_a_spec.md deleted file mode 100644 index daec3662..00000000 --- a/meta/releasing_a_spec.md +++ /dev/null @@ -1,52 +0,0 @@ -# How to release a specification - -There are several specifications that belong to matrix, such as the client-server -specification, server-server specification, and identity service specification. Each -of these gets released independently of each other with their own version numbers. - -Once a specification is ready for release, a branch should be created to track the -changes in and to hold potential future hotfixes. This should be the name of the -specification (as it appears in the directory structure of this project) followed -by "release-" and the release version. For example, if the Client-Server Specification -was getting an r0.4.0 release, the branch name would be `client_server/release-r0.4.0`. - -*Note*: Historical releases prior to this process may or may not have an appropriate -release branch. Releases after this document came into place will have an appropriate -branch. - -The remainder of the process is as follows: -1. Activate your Python 3 virtual environment. -1. Having checked out the new release branch, navigate your way over to `./changelogs`. -1. Follow the release instructions provided in the README.md located there. -1. Update any version/link references across all specifications. -1. From translate.riot.im, push repository changes and merge the subsequent PR. -1. Run `./scripts/i18n.py` to ensure all translatable files are up to date. -1. Generate the specification using `./scripts/gendoc.py`, specifying all the - API versions at the time of generation. For example: `./scripts/gendoc.py -c r0.4.0 -s r0.1.0 -i r0.1.0 #etc` -1. PR the changes to the matrix-org/matrix.org repository (for historic tracking). - * This is done by making a PR to the `unstyled_docs/spec` folder for the version and - specification you're releasing. - * Don't forget to symlink the new release as `latest`. - * For the client-server API, don't forget to generate the swagger JSON by using - `./scripts/dump-swagger.py -c r0.4.0`. This will also need symlinking to `latest`. -1. Commit the changes and PR them to master. **Wait for review from the spec core team.** - * Link to your matrix-org/matrix.org so both can be reviewed at the same time. -1. Tag the release with the format `client_server/r0.4.0`. -1. Perform a release on GitHub to tag the release. -1. Yell from the mountaintop to the world about the new release. - -### Creating a release for a brand-new specification - -Some specifications may not have ever had a release, and therefore need a bit more work -to become ready. - -1. Activate your Python 3 virtual environment. -1. Having checked out the new release branch, navigate your way over to `./changelogs`. -1. Follow the "new changelog" instructions provided in the README.md located there. -1. Open the specification RST file and make some changes: - * Using a released specification as a template, update the changelog section. - * Use the appropriate changelog variable in the RST. -1. Create/define the appropriate variables in `gendoc.py`. -1. Update `targets.yml`. -1. Update any version/link references across all specifications. -1. Follow the regular release process. diff --git a/scripts/generate-changelog.sh b/scripts/generate-changelog.sh new file mode 100644 index 00000000..6e2c2f11 --- /dev/null +++ b/scripts/generate-changelog.sh @@ -0,0 +1,35 @@ +# /bin/bash + +# Usage: ./generate.sh v1.2 "April 01, 2021" + +set -e + +MAGIC_STRING="" + +cd changelogs + +# Pre-cleanup just in case it wasn't done on the last run +rm -f rendered.* + +# Reversed order so the push gateway ends up on the bottom +towncrier --name "Push Gateway API" --dir "./push_gateway" --config "./pyproject.toml" --yes +towncrier --name "Identity Service API" --dir "./identity_service" --config "./pyproject.toml" --yes +towncrier --name "Application Service API" --dir "./application_service" --config "./pyproject.toml" --yes +towncrier --name "Server-Server API" --dir "./server_server" --config "./pyproject.toml" --yes +towncrier --name "Client-Server API" --dir "./client_server" --config "./pyproject.toml" --yes + +# Prepare the header +cp header.md rendered.header.md +sed -i "s/VERSION/$1/g" rendered.header.md +sed -i "s/DATE/$2/g" rendered.header.md +cat rendered.header.md rendered.md > rendered.final.md + +# Remove trailing whitespace (such as our intentionally blank RST headings) +sed -i "s/[ ]*$//" rendered.final.md + +# Put the changelog in place +mv rendered.final.md ../layouts/partials/changelogs/$1.md +sed -i "s/$MAGIC_STRING/$MAGIC_STRING\n{{% changelog\\/changelog-rendered p=\"changelogs\\/$1.md\" %}}/" ../content/changelog.md + +# Cleanup +rm -v rendered.* diff --git a/scripts/requirements.txt b/scripts/requirements.txt index 2af156cc..9d9f1167 100644 --- a/scripts/requirements.txt +++ b/scripts/requirements.txt @@ -7,4 +7,4 @@ jsonschema >= 2.6.0, < 3.0.0 PyYAML >= 3.12 requests >= 2.18.4 -towncrier == 18.6.0 +towncrier == 21.9.0rc1