Update CircleCI and Buildkite configs to build the new spec (#3017)

This PR attempts to update the CI of matrix-doc to build [the new spec redesign](https://github.com/matrix-org/matrix-doc/issues/2906). It does so by additionally building the new spec in parallel to the old. The plan is to continue to host the old spec at https://matrix.org/docs/spec, while the new spec will be at https://spec.matrix.org. Eventually we will retire the old version of the spec, and have the old URL redirect to the new one.

In detail, this PR:
* Adds a new step to CircleCI to build the new spec with `hugo`. This step uses alpine, grabs some dependencies, and then builds the HTML.
  * We needed to hand some specific options to hugo for CircleCI in order to continue allowing CircleCI to host temporary builds of the spec after each CI run. This required changing some assumptions related to relative paths.
  * CircleCI's artifacts hosting is also quite limited. Specifically it will not automatically resolve `/some/path` to `/some/path/index.html`, which our hugo theme relied on. Fixes were implemented for this, but we may want to consider switching away from CircleCI artifacts as a host, and using something like [netlify](https://www.netlify.com/) instead.
* Modifies the existing Buildkite pipeline step to build both the new spec in a separate step. It additionally modifies the old spec to be built with alpine. (Separate out into another PR)
  * We'd like to separate out the deployment of matrix.org from the new spec. Therefore a new step, with a separate artifact build (`spec.tar.gz`). We will eventually remove the old step and the matrix.org build trigger.
* Modifies `pyproject.toml` to update the config of [giles](https://github.com/OpenAstronomy/baldrick/blob/master/baldrick/plugins/circleci_artifacts.py), which is what creates the "docs", "swagger" links in the CI steps for matrix-docs PRs.
  * A new step was added for the new spec. The old spec was renamed to "legacy".

.buildkite/pipeline.yaml

@@ -1,5 +1,5 @@
 steps:
-  - label: ":books: Build spec"
+  - label: ":books: Build the legacy spec"
     command:
       # Install the python dependencies necessary to build the spec
       - python3 -m venv env && . env/bin/activate
@@ -16,3 +16,23 @@ steps:
     trigger: "matrix-dot-org"
     async: true
     branches: "master"
+
+  - label: ":books: Build the spec"
+    command:
+      # Install package dependencies
+      - apk add nodejs npm git hugo
+      # Install the node dependencies necessary to build the spec
+      - npm i
+      # Pull all git submodules, required for the hugo theme
+      - git submodule update --init --recursive
+      # Pull current proposal information
+      - npm run get-proposals
+      # Build the spec, will build to './spec'
+      - hugo -d "spec"
+      # Compress the result and make it available as an artifact
+      - tar -czf spec.tar.gz spec
+    artifact_paths:
+      - spec.tar.gz
+    plugins:
+      - docker#v3.7.0:
+          image: alpine

.circleci/config.yml

@@ -1,9 +1,20 @@
-gendoc: &gendoc
+genlegacydoc: &genlegacydoc
-  name: Generate the docs
+  name: Generate the legacy docs
   command: |
     source /env/bin/activate
     scripts/gendoc.py
 
+gendoc: &gendoc
+  name: Generate the docs
+  # Note: Node dependencies are required for the hugo build.
+  # Note: We use a custom config file for circleci due to some specifics with hosting the
+  #       site using CircleCI's artifacts platform. See config-circleci.toml for details.
+  command: |
+    apk add nodejs npm git hugo
+    npm i
+    cat config-circleci.toml config.toml > hugo-config.toml
+    hugo --config hugo-config.toml --baseURL "/${CIRCLE_NODE_INDEX}/public"
+
 genswagger: &genswagger
   name: Validate sources and generate swagger json
   command: |
@@ -63,17 +74,32 @@ jobs:
     steps:
       - checkout
       - run: *checkexamples
-  build-docs:
+  build-legacy-docs:
     docker:
       - image: uhoreg/matrix-doc-build
     steps:
       - checkout
-      - run: *gendoc
+      - run: *genlegacydoc
       - store_artifacts:
           path: scripts/gen
       - run:
-          name: "Doc build is available at:"
+          name: "Legacy doc build is available at:"
           command: DOCS_URL="${CIRCLE_BUILD_URL}/artifacts/${CIRCLE_NODE_INDEX}/${CIRCLE_WORKING_DIRECTORY/#\~/$HOME}/scripts/gen/index.html"; echo $DOCS_URL
+  build-docs:
+    docker:
+      - image: alpine
+    steps:
+      # Note: We install git in the image so we can pull git submodules. The hugo theme in use
+      # is a git submodule, which has its own submodules, and all need to be loaded.
+      - run: apk add git
+      - checkout
+      - run: git submodule update --init --recursive
+      - run: *gendoc
+      - store_artifacts:
+          path: public
+      - run:
+          name: "Doc build is available at:"
+          command: DOCS_URL="${CIRCLE_BUILD_URL}/artifacts/${CIRCLE_NODE_INDEX}/public/index.html"; echo $DOCS_URL
   build-swagger:
     docker:
       - image: uhoreg/matrix-doc-build
@@ -104,6 +130,7 @@ workflows:
 
   build-spec:
     jobs:
+      - build-legacy-docs
       - build-docs
       - build-swagger
       - check-docs

.gitignore

@@ -11,6 +11,8 @@
 /scripts/swagger
 /scripts/tmp
 /templating/out
+/hugo-config.toml
+/public
 *.pyc
 *.swp
 _rendered.rst

assets-hugo/scss/custom.scss

@@ -409,7 +409,7 @@ Make padding symmetrical (this selector is used in the default styles to apply p
 }
 
 .pageinfo-unstable {
-  background-image: url('/icons/unstable.png');
+  background-image: url('../icons/unstable.png');
   background-position: left 1rem center;
   background-repeat: no-repeat;
   padding-left: 100px;

config-circleci.toml

@@ -0,0 +1,5 @@
+# When serving the documentation via CircleCI, we need to use '/path/index.html' URLs instead
+# of hugo's default of prettier '/path/' URLs. This is because CircleCI's artifacts webserver
+# does not resolve '/path/' to '/path/index.html' like webservers often do.
+# CircleCI discussion: https://discuss.circleci.com/t/circle-artifacts-com-not-using-index-html/320/3
+uglyURLs = true

pyproject.toml

@@ -1,7 +1,11 @@
 [ tool.gilesbot ]
 
-  [ tool.gilesbot.circleci_artifacts.docs ]
+  [ tool.gilesbot.circleci_artifacts.legacydocs ]
     url = "gen/index.html"
+    message = "Click details to preview the legacy HTML documentation."
+
+  [ tool.gilesbot.circleci_artifacts.docs ]
+    url = "public/index.html"
     message = "Click details to preview the HTML documentation."
 
   [ tool.gilesbot.circleci_artifacts.swagger ]

scripts/generate-matrix-org-assets

@@ -11,7 +11,7 @@ mkdir -p assets
 # generate specification/proposals.rst
 ./scripts/proposals.py
 
-# generate the spec docs
+# generate the legacy spec docs
 ./scripts/gendoc.py -d assets/spec
 
 # and the swagger