From a17023962ca6c084a03514c58342e52a1a72ee16 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Mon, 1 Mar 2021 10:04:06 +0000 Subject: [PATCH] Update CircleCI and Buildkite configs to build the new spec (#3017) This PR attempts to update the CI of matrix-doc to build [the new spec redesign](https://github.com/matrix-org/matrix-doc/issues/2906). It does so by additionally building the new spec in parallel to the old. The plan is to continue to host the old spec at https://matrix.org/docs/spec, while the new spec will be at https://spec.matrix.org. Eventually we will retire the old version of the spec, and have the old URL redirect to the new one. In detail, this PR: * Adds a new step to CircleCI to build the new spec with `hugo`. This step uses alpine, grabs some dependencies, and then builds the HTML. * We needed to hand some specific options to hugo for CircleCI in order to continue allowing CircleCI to host temporary builds of the spec after each CI run. This required changing some assumptions related to relative paths. * CircleCI's artifacts hosting is also quite limited. Specifically it will not automatically resolve `/some/path` to `/some/path/index.html`, which our hugo theme relied on. Fixes were implemented for this, but we may want to consider switching away from CircleCI artifacts as a host, and using something like [netlify](https://www.netlify.com/) instead. * Modifies the existing Buildkite pipeline step to build both the new spec in a separate step. It additionally modifies the old spec to be built with alpine. (Separate out into another PR) * We'd like to separate out the deployment of matrix.org from the new spec. Therefore a new step, with a separate artifact build (`spec.tar.gz`). We will eventually remove the old step and the matrix.org build trigger. * Modifies `pyproject.toml` to update the config of [giles](https://github.com/OpenAstronomy/baldrick/blob/master/baldrick/plugins/circleci_artifacts.py), which is what creates the "docs", "swagger" links in the CI steps for matrix-docs PRs. * A new step was added for the new spec. The old spec was renamed to "legacy". --- .buildkite/pipeline.yaml | 22 +++++++++++++++++- .circleci/config.yml | 37 ++++++++++++++++++++++++++---- .gitignore | 2 ++ assets-hugo/scss/custom.scss | 2 +- config-circleci.toml | 5 ++++ pyproject.toml | 6 ++++- scripts/generate-matrix-org-assets | 2 +- 7 files changed, 67 insertions(+), 9 deletions(-) create mode 100644 config-circleci.toml diff --git a/.buildkite/pipeline.yaml b/.buildkite/pipeline.yaml index fd8c58a8..1835bc96 100644 --- a/.buildkite/pipeline.yaml +++ b/.buildkite/pipeline.yaml @@ -1,5 +1,5 @@ steps: - - label: ":books: Build spec" + - label: ":books: Build the legacy spec" command: # Install the python dependencies necessary to build the spec - python3 -m venv env && . env/bin/activate @@ -16,3 +16,23 @@ steps: trigger: "matrix-dot-org" async: true branches: "master" + + - label: ":books: Build the spec" + command: + # Install package dependencies + - apk add nodejs npm git hugo + # Install the node dependencies necessary to build the spec + - npm i + # Pull all git submodules, required for the hugo theme + - git submodule update --init --recursive + # Pull current proposal information + - npm run get-proposals + # Build the spec, will build to './spec' + - hugo -d "spec" + # Compress the result and make it available as an artifact + - tar -czf spec.tar.gz spec + artifact_paths: + - spec.tar.gz + plugins: + - docker#v3.7.0: + image: alpine \ No newline at end of file diff --git a/.circleci/config.yml b/.circleci/config.yml index b629a018..09c5a6fc 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,9 +1,20 @@ -gendoc: &gendoc - name: Generate the docs +genlegacydoc: &genlegacydoc + name: Generate the legacy docs command: | source /env/bin/activate scripts/gendoc.py +gendoc: &gendoc + name: Generate the docs + # Note: Node dependencies are required for the hugo build. + # Note: We use a custom config file for circleci due to some specifics with hosting the + # site using CircleCI's artifacts platform. See config-circleci.toml for details. + command: | + apk add nodejs npm git hugo + npm i + cat config-circleci.toml config.toml > hugo-config.toml + hugo --config hugo-config.toml --baseURL "/${CIRCLE_NODE_INDEX}/public" + genswagger: &genswagger name: Validate sources and generate swagger json command: | @@ -63,17 +74,32 @@ jobs: steps: - checkout - run: *checkexamples - build-docs: + build-legacy-docs: docker: - image: uhoreg/matrix-doc-build steps: - checkout - - run: *gendoc + - run: *genlegacydoc - store_artifacts: path: scripts/gen - run: - name: "Doc build is available at:" + name: "Legacy doc build is available at:" command: DOCS_URL="${CIRCLE_BUILD_URL}/artifacts/${CIRCLE_NODE_INDEX}/${CIRCLE_WORKING_DIRECTORY/#\~/$HOME}/scripts/gen/index.html"; echo $DOCS_URL + build-docs: + docker: + - image: alpine + steps: + # Note: We install git in the image so we can pull git submodules. The hugo theme in use + # is a git submodule, which has its own submodules, and all need to be loaded. + - run: apk add git + - checkout + - run: git submodule update --init --recursive + - run: *gendoc + - store_artifacts: + path: public + - run: + name: "Doc build is available at:" + command: DOCS_URL="${CIRCLE_BUILD_URL}/artifacts/${CIRCLE_NODE_INDEX}/public/index.html"; echo $DOCS_URL build-swagger: docker: - image: uhoreg/matrix-doc-build @@ -104,6 +130,7 @@ workflows: build-spec: jobs: + - build-legacy-docs - build-docs - build-swagger - check-docs diff --git a/.gitignore b/.gitignore index 4b1c7721..5245c82b 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,8 @@ /scripts/swagger /scripts/tmp /templating/out +/hugo-config.toml +/public *.pyc *.swp _rendered.rst diff --git a/assets-hugo/scss/custom.scss b/assets-hugo/scss/custom.scss index 24abdc6e..3116762e 100644 --- a/assets-hugo/scss/custom.scss +++ b/assets-hugo/scss/custom.scss @@ -409,7 +409,7 @@ Make padding symmetrical (this selector is used in the default styles to apply p } .pageinfo-unstable { - background-image: url('/icons/unstable.png'); + background-image: url('../icons/unstable.png'); background-position: left 1rem center; background-repeat: no-repeat; padding-left: 100px; diff --git a/config-circleci.toml b/config-circleci.toml new file mode 100644 index 00000000..6b448a7f --- /dev/null +++ b/config-circleci.toml @@ -0,0 +1,5 @@ +# When serving the documentation via CircleCI, we need to use '/path/index.html' URLs instead +# of hugo's default of prettier '/path/' URLs. This is because CircleCI's artifacts webserver +# does not resolve '/path/' to '/path/index.html' like webservers often do. +# CircleCI discussion: https://discuss.circleci.com/t/circle-artifacts-com-not-using-index-html/320/3 +uglyURLs = true diff --git a/pyproject.toml b/pyproject.toml index 060a44fc..17a9e62d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,11 @@ [ tool.gilesbot ] - [ tool.gilesbot.circleci_artifacts.docs ] + [ tool.gilesbot.circleci_artifacts.legacydocs ] url = "gen/index.html" + message = "Click details to preview the legacy HTML documentation." + + [ tool.gilesbot.circleci_artifacts.docs ] + url = "public/index.html" message = "Click details to preview the HTML documentation." [ tool.gilesbot.circleci_artifacts.swagger ] diff --git a/scripts/generate-matrix-org-assets b/scripts/generate-matrix-org-assets index ed08f81d..3aff31f5 100755 --- a/scripts/generate-matrix-org-assets +++ b/scripts/generate-matrix-org-assets @@ -11,7 +11,7 @@ mkdir -p assets # generate specification/proposals.rst ./scripts/proposals.py -# generate the spec docs +# generate the legacy spec docs ./scripts/gendoc.py -d assets/spec # and the swagger