From 08fde5f25730915724bedc3e26b134f1f64d5b37 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com>
Date: Tue, 15 Nov 2022 23:26:55 +0000
Subject: [PATCH] Generate unstable changelogs using towncrier (#1340)

Replace the current stack of hugo templates with a towncrier invocation. The main advantage of this is that it means that the "Changes since last release" section is consistent with the changelogs for the actual releases.

This also changes the release process so that the changelog is generated before tagging, which means that the thing tagged v1.5 is actually the v1.5 spec.

Fixes #908.
---
 .github/workflows/main.yml                    | 31 ++++++-
 .../internal/newsfragments/1340.clarification |  1 +
 content/{changelog.md => changelog/index.md}  |  8 +-
 .../changelogs => content/changelog}/v1.1.md  |  3 +
 .../changelogs => content/changelog}/v1.2.md  |  3 +
 .../changelogs => content/changelog}/v1.3.md  |  3 +
 .../changelogs => content/changelog}/v1.4.md  |  3 +
 layouts/partials/changelogs/.gitkeep          |  0
 .../changelog/changelog-changes.html          | 90 -------------------
 .../changelog/changelog-rendered.html         |  2 -
 layouts/shortcodes/changelog/changelogs.html  | 10 +++
 meta/releasing.md                             | 16 ++--
 scripts/generate-changelog.sh                 | 42 +++++----
 13 files changed, 86 insertions(+), 126 deletions(-)
 create mode 100644 changelogs/internal/newsfragments/1340.clarification
 rename content/{changelog.md => changelog/index.md} (85%)
 rename {layouts/partials/changelogs => content/changelog}/v1.1.md (99%)
 rename {layouts/partials/changelogs => content/changelog}/v1.2.md (99%)
 rename {layouts/partials/changelogs => content/changelog}/v1.3.md (99%)
 rename {layouts/partials/changelogs => content/changelog}/v1.4.md (99%)
 delete mode 100644 layouts/partials/changelogs/.gitkeep
 delete mode 100644 layouts/shortcodes/changelog/changelog-changes.html
 delete mode 100644 layouts/shortcodes/changelog/changelog-rendered.html
 create mode 100644 layouts/shortcodes/changelog/changelogs.html
 mode change 100644 => 100755 scripts/generate-changelog.sh

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index ce54f976..a77191d6 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -99,10 +99,32 @@ jobs:
           name: openapi-artifact
           path: openapi.tar.gz
 
+  generate-changelog:
+    name: "📢 Run towncrier for changelog"
+    # skip for builds of git tags
+    if: "!startsWith(github.ref, 'refs/tags/')"
+    runs-on: ubuntu-latest
+    steps:
+      - name: "📥 Source checkout"
+        uses: actions/checkout@v2
+      - name: "➕ Setup Python"
+        uses: actions/setup-python@v4
+      - name: "➕ Install towncrier"
+        run: "pip install 'towncrier'"
+      - name: "Generate changelog"
+        run: ./scripts/generate-changelog.sh vUNSTABLE
+      - name: "📤 Artifact upload"
+        uses: actions/upload-artifact@v2
+        with:
+          name: changelog-artifact
+          path: content/changelog/vUNSTABLE.md
+
   build-spec:
     name: "📖 Build the spec"
     runs-on: ubuntu-latest
-    needs: [calculate-baseurl, build-openapi]
+    needs: [calculate-baseurl, build-openapi, generate-changelog]
+    # run even if generate-changelog was skipped
+    if: ${{ always() }}
     steps:
       - name: "➕ Setup Node"
         uses: actions/setup-node@v2
@@ -119,9 +141,14 @@ jobs:
         run: |
           npm i
           npm run get-proposals
+      - name: "📥 Download generated changelog"
+        if: "needs.generate-changelog.result == 'success'"
+        uses: actions/download-artifact@v3
+        with:
+          name: changelog-artifact
+          path: content/changelog
       - name: "⚙️ hugo"
         run: hugo --baseURL "${{ needs.calculate-baseurl.outputs.baseURL }}" -d "spec"
-
       # We manually unpack the spec OpenAPI definition JSON to the website tree
       # to make it available to the world in a canonical place:
       # https://spec.matrix.org/latest/client-server-api/api.json
diff --git a/changelogs/internal/newsfragments/1340.clarification b/changelogs/internal/newsfragments/1340.clarification
new file mode 100644
index 00000000..a3180440
--- /dev/null
+++ b/changelogs/internal/newsfragments/1340.clarification
@@ -0,0 +1 @@
+Generate the unstable changelogs with towncrier, for consistency.
diff --git a/content/changelog.md b/content/changelog/index.md
similarity index 85%
rename from content/changelog.md
rename to content/changelog/index.md
index a84514b6..560cb43b 100644
--- a/content/changelog.md
+++ b/content/changelog/index.md
@@ -6,13 +6,7 @@ weight: 1000
 
 {{% changelog/changelog-description %}}
 
-{{% changelog/changelog-changes %}}
-
-<!-- DO NOT REMOVE OR CHANGE - Release script puts next release here -->
-{{% changelog/changelog-rendered p="changelogs/v1.4.md" %}}
-{{% changelog/changelog-rendered p="changelogs/v1.3.md" %}}
-{{% changelog/changelog-rendered p="changelogs/v1.2.md" %}}
-{{% changelog/changelog-rendered p="changelogs/v1.1.md" %}}
+{{% changelog/changelogs %}}
 
 <h2 id="historical-versions" class="no-numbers">Historical versions</h2>
 
diff --git a/layouts/partials/changelogs/v1.1.md b/content/changelog/v1.1.md
similarity index 99%
rename from layouts/partials/changelogs/v1.1.md
rename to content/changelog/v1.1.md
index b8e0832d..8cf43878 100644
--- a/layouts/partials/changelogs/v1.1.md
+++ b/content/changelog/v1.1.md
@@ -1,3 +1,6 @@
+---
+date: 2021-11-09T00:00:00+0000
+---
 <!--
 This is a header file for the generated changelog.
 
diff --git a/layouts/partials/changelogs/v1.2.md b/content/changelog/v1.2.md
similarity index 99%
rename from layouts/partials/changelogs/v1.2.md
rename to content/changelog/v1.2.md
index bce9ea66..22deb1ac 100644
--- a/layouts/partials/changelogs/v1.2.md
+++ b/content/changelog/v1.2.md
@@ -1,3 +1,6 @@
+---
+date: 2022-02-02T00:00:00+0000
+---
 <!--
 This is a header file for the generated changelog.
 
diff --git a/layouts/partials/changelogs/v1.3.md b/content/changelog/v1.3.md
similarity index 99%
rename from layouts/partials/changelogs/v1.3.md
rename to content/changelog/v1.3.md
index eeae95f4..bc05b1cc 100644
--- a/layouts/partials/changelogs/v1.3.md
+++ b/content/changelog/v1.3.md
@@ -1,3 +1,6 @@
+---
+date: 2022-06-15T00:00:00+0100
+---
 <!--
 This is a header file for the generated changelog.
 
diff --git a/layouts/partials/changelogs/v1.4.md b/content/changelog/v1.4.md
similarity index 99%
rename from layouts/partials/changelogs/v1.4.md
rename to content/changelog/v1.4.md
index 2e741e06..7d22c00e 100644
--- a/layouts/partials/changelogs/v1.4.md
+++ b/content/changelog/v1.4.md
@@ -1,3 +1,6 @@
+---
+date: 2022-09-29T00:00:00+0100
+---
 <!--
 This is a header file for the generated changelog.
 
diff --git a/layouts/partials/changelogs/.gitkeep b/layouts/partials/changelogs/.gitkeep
deleted file mode 100644
index e69de29b..00000000
diff --git a/layouts/shortcodes/changelog/changelog-changes.html b/layouts/shortcodes/changelog/changelog-changes.html
deleted file mode 100644
index 99efce08..00000000
--- a/layouts/shortcodes/changelog/changelog-changes.html
+++ /dev/null
@@ -1,90 +0,0 @@
-{{/*
-
-  This template is used to render the set of changes in the changelog page.
-
-  It expects to find a directory "changelogs" containing a subdirectory for
-  each of the 5 APIs in the specification. Inside each of these directories
-  it expects to find newsfragments describing changes to that API.
-
-  If the `version.status` setting in config.toml is anything other than
-  "unstable", then it also expects to find additional settings under
-  `version` in config.toml:
-  - `major`: the major version number of the release
-  - `minor`: the minor version number of the release
-  - `release_date`: the date of the release
-
-  The release tag is calculated as `v<major>.<minor>`; for example `v1.5`.
-
-  It then renders this into a table displayed before the list of changes.
-
-*/}}
-
-{{ $path := path.Join "changelogs" }}
-{{ $status := .Site.Params.version.status }}
-{{ $release_tag := delimit (slice "v" .Site.Params.version.major "." .Site.Params.version.minor) "" }}
-
-{{ if ne $status "unstable" }}
-<table class="release-info">
-<tr><th>Git commit</th><td><a href="https://github.com/matrix-org/matrix-spec/tree/{{ $release_tag }}">https://github.com/matrix-org/matrix-spec/tree/{{ $release_tag }}</a></td>
-<tr><th>Release date</th><td>{{ .Site.Params.version.release_date }}</td>
-</table>
-{{ end }}
-
-<h2 id=api-changes>Changes since last release</h2>
-{{ partial "render-api-changes" (dict "title" "Client-Server API" "id" "client-server-api" "path" (path.Join $path "client_server")) }}
-{{ partial "render-api-changes" (dict "title" "Server-Server API" "id" "server-server-api" "path" (path.Join $path "server_server")) }}
-{{ partial "render-api-changes" (dict "title" "Application Service API" "id" "application-service-api" "path" (path.Join $path "application_service")) }}
-{{ partial "render-api-changes" (dict "title" "Identity Service API" "id" "identity-service-api" "path" (path.Join $path "identity_service")) }}
-{{ partial "render-api-changes" (dict "title" "Push Gateway API" "id" "push-gateway-api" "path" (path.Join $path "push_gateway")) }}
-{{ partial "render-api-changes" (dict "title" "Room Versions" "id" "room-versions" "path" (path.Join $path "room_versions")) }}
-{{ partial "render-api-changes" (dict "title" "Appendices" "id" "appendices" "path" (path.Join $path "appendices")) }}
-{{ partial "render-api-changes" (dict "title" "Internal Changes/Tooling" "id" "internal" "path" (path.Join $path "internal")) }}
-
-{{ define "partials/render-api-changes" }}
-<h3 id="{{.id}}">{{ .title }}</h3>
-    {{ $api_path := .path }}
-    {{ $config_file := path.Join $api_path ".." "pyproject.toml" }}
-    {{ $config := readFile $config_file | transform.Unmarshal }}
-    {{ $news_path := path.Join $api_path "newsfragments" }}
-    {{ partial "render-newsfragments"  (dict "config" $config "news_path" $news_path )}}
-{{ end }}
-
-{{ define "partials/render-newsfragments" }}
-{{ $config := .config }}
-{{ $news_path := .news_path }}
-
-{{ $types := dict }}
-{{ range $config.tool.towncrier.type }}
-    {{ $types = merge $types (dict .directory (slice)) }}
-{{ end }}
-
-{{ range (readDir $news_path) }}
-
-    {{ $pieces := split .Name "." }}
-
-    {{ $ticket := index $pieces 0 }}
-    {{ $description := readFile (path.Join $news_path .Name ) }}
-    {{ $change_info := (dict "ticket" $ticket "description" $description )}}
-
-    {{ $type := index $pieces 1 }}
-    {{ $instances := index $types $type }}
-    {{ $instances = $instances | append $change_info }}
-    {{ $types = merge $types (dict $type $instances) }}
-
-{{ end }}
-
-<ul>
-{{ range $config.tool.towncrier.type }}
-    {{ $changes_of_type := (index $types .directory) }}
-    {{ if $changes_of_type }}
-<li>{{.name | safeHTML}}
-<p><ul>
-        {{ range $changes_of_type }}
-<li><a href="https://github.com/matrix-org/matrix-spec/issues/{{.ticket}}"><strong>{{ .ticket }}: </strong></a>{{ .description | markdownify }}</li>
-        {{ end }}
-</ul></p>
-</li>
-    {{ end }}
-{{ end }}
-</ul>
-{{ end }}
diff --git a/layouts/shortcodes/changelog/changelog-rendered.html b/layouts/shortcodes/changelog/changelog-rendered.html
deleted file mode 100644
index bbfd6e53..00000000
--- a/layouts/shortcodes/changelog/changelog-rendered.html
+++ /dev/null
@@ -1,2 +0,0 @@
-{{ $partial := .Params.p }}
-{{ partial $partial . }}
diff --git a/layouts/shortcodes/changelog/changelogs.html b/layouts/shortcodes/changelog/changelogs.html
new file mode 100644
index 00000000..78b5932f
--- /dev/null
+++ b/layouts/shortcodes/changelog/changelogs.html
@@ -0,0 +1,10 @@
+{{/*
+   This template is used to render all of the changelog sections under
+   "content/changelogs"
+*/}}
+
+{{ with .Page.Resources.Match "*.md" }}
+{{ range ((sort . "Params.date" "desc")) }}
+{{ .Content }}
+{{ end }}
+{{ end }}
diff --git a/meta/releasing.md b/meta/releasing.md
index 3e01dc86..7483c357 100644
--- a/meta/releasing.md
+++ b/meta/releasing.md
@@ -37,21 +37,21 @@ release.
    release_date = "October 01, 2021"
    ```
 3. Commit the changes.
-4. Tag the branch with the spec release with a format of `v1.2` (if releasing Matrix 1.2).
-5. Push the release branch and the tag.
-6. GitHub Actions will run its build steps. Wait until these are successful. If fixes
-   need to be made to repair the pipeline or spec build, delete and re-tag the release.
-7. Check out and fast-forward `main` to the release branch.
-8. Generate the changelog. This is done *after* the tagging to ensure the rendered
-   changelog makes sense.
+4. Generate the changelog.
    1. Activate your python virtual environment.
    2. Run `./scripts/generate-changelog.sh v1.2 "October 01, 2021"` (using the correct
       version number and same `release_date` format from the hugo config).
    3. Commit the result.
+5. Tag the branch with the spec release with a format of `v1.2` (if releasing Matrix 1.2).
+6. Push the release branch and the tag.
+7. GitHub Actions will run its build steps. Wait until these are successful. If fixes
+   need to be made to repair the pipeline or spec build, delete and re-tag the release.
+   You may need to fix up the changelog file by hand in this case.
+8. Check out and fast-forward `main` to the release branch.
 9. Create a new release on GitHub from the newly created tag.
    * The title should be just "v1.2" (for example).
    * The description should be a copy/paste of the changelog. The generated changelog
-     will be at `content/partials/changelogs/v1.2.md` - copy/paste verbatim.
+     will be at `content/changelog/v1.2.md` - copy/paste verbatim.
    * Upload the artifacts of the GitHub Actions build for the release to the GitHub
      release as artifacts themselves. This should be the tarball that will be deployed
      to spec.matrix.org.
diff --git a/scripts/generate-changelog.sh b/scripts/generate-changelog.sh
old mode 100644
new mode 100755
index f66e5358..73e59940
--- a/scripts/generate-changelog.sh
+++ b/scripts/generate-changelog.sh
@@ -1,15 +1,17 @@
 # /bin/bash
 
-# Usage: ./generate.sh v1.2 "April 01, 2021"
+# Usage: ./scripts/generate-changelog.sh v1.2 "April 01, 2021"
+# or: ./scripts/generate-changelog.sh vUNSTABLE
 
 set -e
 
-MAGIC_STRING="<!-- DO NOT REMOVE OR CHANGE - Release script puts next release here -->"
+VERSION="$1"
+DATE="$2"
 
-cd changelogs
+cd `dirname $0`/../changelogs
 
 # Pre-cleanup just in case it wasn't done on the last run
-rm -f rendered.*
+rm -f rendered.md
 
 # Reversed order so that room versions ends up on the bottom
 towncrier --name "Internal Changes/Tooling" --dir "./internal" --config "./pyproject.toml" --yes
@@ -21,18 +23,24 @@ towncrier --name "Application Service API" --dir "./application_service" --confi
 towncrier --name "Server-Server API" --dir "./server_server" --config "./pyproject.toml" --yes
 towncrier --name "Client-Server API" --dir "./client_server" --config "./pyproject.toml" --yes
 
-# Prepare the header
-cp header.md rendered.header.md
-sed -i "s/VERSION/$1/g" rendered.header.md
-sed -i "s/DATE/$2/g" rendered.header.md
-cat rendered.header.md rendered.md > rendered.final.md
-
-# Remove trailing whitespace (such as our intentionally blank RST headings)
-sed -i "s/[ ]*$//" rendered.final.md
-
-# Put the changelog in place
-mv rendered.final.md ../layouts/partials/changelogs/$1.md
-sed -i "s/$MAGIC_STRING/$MAGIC_STRING\n{{% changelog\\/changelog-rendered p=\"changelogs\\/$1.md\" %}}/" ../content/changelog.md
+{
+    # Prepare the header
+    # We include the generation date in the front matter so that we can use it
+    # to sort the changelogs at build time.
+    cat <<EOF
+---
+date: $(date -Iseconds)
+---
+EOF
+    if [ "$VERSION" = "vUNSTABLE" ]; then
+        echo "## Changes since last release"
+    else
+        sed -e "s/VERSION/$1/g" -e "s/DATE/$2/g" header.md
+    fi
+
+    # Remove trailing whitespace (such as our intentionally blank RST headings)
+    sed -e "s/[ ]*$//" rendered.md
+} > ../content/changelog/$VERSION.md
 
 # Cleanup
-rm -v rendered.*
+rm -v rendered.md