From 643cdd19c807710ce0f84e92c0f8a197c05a65b9 Mon Sep 17 00:00:00 2001 From: Will Date: Mon, 8 Feb 2021 15:39:16 -0800 Subject: [PATCH] Support rendering of proposal tables --- .gitignore | 1 + content/proposals.md | 40 ++++- layouts/shortcodes/proposal-tables.html | 81 +++++++++ package-lock.json | 6 + package.json | 2 + scripts/proposals.js | 219 ++++++++++++++++++++++++ 6 files changed, 348 insertions(+), 1 deletion(-) create mode 100644 layouts/shortcodes/proposal-tables.html create mode 100644 scripts/proposals.js diff --git a/.gitignore b/.gitignore index 35f067d8..4b1c7721 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ /api/node_modules /assets /assets.tar.gz +/data/msc /env* /node_modules /resources diff --git a/content/proposals.md b/content/proposals.md index 985716f7..28cb670c 100644 --- a/content/proposals.md +++ b/content/proposals.md @@ -338,7 +338,7 @@ request trackers. | Spec PR In Review | [spec-pr-in-review](https://github.com/matrix-org/matrix-doc/issues?q=label%3Aproposal+label%3Aspec-pr-in-review+) | The spec PR has been written, and is currently under review | | Spec PR Merged | [merged](https://github.com/matrix-org/matrix-doc/issues?q=label%3Aproposal+label%3Amerged) | A proposal with a sufficient working implementation and whose Spec PR has been merged! | | Postponed | [proposal-postponed](https://github.com/matrix-org/matrix-doc/issues?q=label%3Aproposal+label%3Aproposal-postponed+) | A proposal that is temporarily blocked or a feature that may not be useful currently but perhaps sometime in the future | -| Abandoned | [proposal-closed](https://github.com/matrix-org/matrix-doc/issues?q=label%3Aproposal+label%3Aabandoned) | A proposal where the author/shepherd is not responsive | +| Abandoned | [abandoned](https://github.com/matrix-org/matrix-doc/issues?q=label%3Aproposal+label%3Aabandoned) | A proposal where the author/shepherd is not responsive | | Obsolete | [obsolete](https://github.com/matrix-org/matrix-doc/issues?q=label%3Aproposal+label%3Aobsolete+) | A proposal which has been made obsolete by another proposal or decision elsewhere. | ## Categories @@ -479,3 +479,41 @@ In summary: way that proposes new stable endpoints. Typically this is solved by a small table at the bottom mapping the various values from stable to unstable. + +## Proposal Tracking + +This is a living document generated from the list of proposals on the +issue and pull request trackers of the +[matrix-doc](https://github.com/matrix-org/matrix-doc) repo. + +We use labels and some metadata in MSC PR descriptions to generate this +page. Labels are assigned by the Spec Core Team whilst triaging the +proposals based on those which exist in the +[matrix-doc](https://github.com/matrix-org/matrix-doc) repo already. + +It is worth mentioning that a previous version of the MSC process used a +mixture of GitHub issues and PRs, leading to some MSC numbers deriving +from GitHub issue IDs instead. A useful feature of GitHub is that it +does automatically resolve to an issue, if an issue ID is placed in a +pull URL. This means that + will correctly +resolve to the desired MSC, whether it started as an issue or a PR. + +Other metadata: + +- The MSC number is taken from the GitHub Pull Request ID. This is + carried for the lifetime of the proposal. These IDs do not necessary + represent a chronological order. +- The GitHub PR title will act as the MSC's title. +- Please link to the spec PR (if any) by adding a "PRs: \#1234" line + in the issue description. +- The creation date is taken from the GitHub PR, but can be overridden + by adding a "Date: yyyy-mm-dd" line in the PR description. +- Updated Date is taken from GitHub. +- Author is the creator of the MSC PR, but can be overridden by adding + a "Author: @username" line in the body of the issue description. + Please make sure @username is a GitHub user (include the @!) +- A shepherd can be assigned by adding a "Shepherd: @username" line in + the issue description. Again, make sure this is a real GitHub user. + +{{% proposal-tables %}} diff --git a/layouts/shortcodes/proposal-tables.html b/layouts/shortcodes/proposal-tables.html new file mode 100644 index 00000000..94fbdae7 --- /dev/null +++ b/layouts/shortcodes/proposal-tables.html @@ -0,0 +1,81 @@ +{{/* + + This template is used to render tables of MSC proposals. + + It expects there to be a "proposals.json" under /data/msc. + It expects "proposals.json" to contain an array of objects, + one for each MSC state. Each object contains: + * `title`: human-readable title for the state, like "Proposal In Review" + * `label`: the GitHub label used for the state, like "proposal-in-review" + * `proposals`: an array of objects, each of which represents an MSC and contains: + * `number`: GitHub issue number + * `url`: GitHub URL for this issue + * `title`: Issue title + * `created_at`: issue creation date + * `updated_at`: issue last-updated date + * `authors`: array of GitHub user names representing authors of this MSC + * `shepherd`: GitHub user name representing the shepherd of this MSC, or null + * `documentation`: Links to further documentation referenced in the GitHub issue + + This data is scraped from GitHub using the /scripts/proposals.js Node script. + The script is run in CI: so typically if you run a local server the data will + be missing and no tables will be generated. If you do want to see the tables locally, + you can run the script locally: + + npm install + npm run get-proposals + + If this template does find the data, it renders one table for each MSC state, + containing a row for each MSC in that state. + +*/}} + +{{ $states := .Site.Data.msc.proposals }} + +{{ range $states }} +

{{ .title }}

+ {{ if .proposals }} + + + + + + + + + + + + + + + {{ range .proposals }} + + {{ $index := 0 }} + {{ $docs_links_list := slice }} + {{ range .documentation }} + {{ $index = add $index 1 }} + {{ $docs_link := printf "%d" . $index }} + {{ $docs_links_list = $docs_links_list | append $docs_link }} + {{ end }} + {{ $docs_links := delimit $docs_links_list ", " }} + + {{ $authors_list := apply .authors "printf" "@%s" "." "." }} + {{ $authors := delimit $authors_list ", " }} + + + + + + + + + + + {{ end }} + +
MSCTitleCreated atUpdated atDocsAuthorShepherd
{{ .number }}{{ .title }}{{ .created_at }}{{ .updated_at }}{{ with $docs_links }}{{ $docs_links }}{{ end }}{{ $authors }}{{ with .shepherd }}@{{ . }}{{ end }}
+ {{ else }} +

No MSCs are currently in this state.

+ {{ end }} +{{ end }} diff --git a/package-lock.json b/package-lock.json index ac740160..da521dae 100644 --- a/package-lock.json +++ b/package-lock.json @@ -569,6 +569,12 @@ "picomatch": "^2.0.5" } }, + "node-fetch": { + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz", + "integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==", + "dev": true + }, "node-releases": { "version": "1.1.60", "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-1.1.60.tgz", diff --git a/package.json b/package.json index fdf39899..5a4e3142 100644 --- a/package.json +++ b/package.json @@ -4,6 +4,7 @@ "description": "Hugo theme for the Matrix specification.", "main": "none.js", "scripts": { + "get-proposals": "node ./scripts/proposals.js", "test": "echo \"Error: no test specified\" && exit 1" }, "repository": { @@ -20,6 +21,7 @@ "dependencies": {}, "devDependencies": { "autoprefixer": "^9.8.6", + "node-fetch": "^2.6.1", "postcss-cli": "^7.1.2" } } diff --git a/scripts/proposals.js b/scripts/proposals.js new file mode 100644 index 00000000..af78ac91 --- /dev/null +++ b/scripts/proposals.js @@ -0,0 +1,219 @@ +"use strict"; + +/** + * This Node script fetches MSC proposals and stores them in /data/msc, + * so they can be used by a Hugo template to render summary tables of them + * in the specification. + * + * In detail, it: + * - fetches all GitHub issues from matrix-doc that have the `proposal` label + * - groups them by their state in the MSC process + * - does some light massaging of them so it's easier for the Hugo template to work with them + * - store them at /data/msc + */ + +// built-in modules +const path = require('path'); +const fs = require('fs'); + +// third-party modules +const fetch = require('node-fetch'); + +// We will write proposals into the /data/msc directory +const outputDir = path.join(__dirname, "../data/msc"); + +/** + * This defines the different proposal lifecycle states. + * Each state has: + * - `label`: a GitHub label used to identify issues in this state + * - `title`: used for things like headings in renderings of the proposals + */ +const states = [ + { + label: "proposal-in-review", + title: "Proposal In Review" + }, + { + label: "proposed-final-comment-period", + title: "Proposed Final Comment Period" + }, + { + label: "final-comment-period", + title: "Final Comment Period" + }, + { + label: "finished-final-comment-period", + title: "Finished Final Comment Period" + }, + { + label: "spec-pr-missing", + title: "Spec PR Missing" + }, + { + label: "spec-pr-in-review", + title: "Spec PR In Review" + }, + { + label: "merged", + title: "Merged" + }, + { + label: "proposal-postoned", + title: "Postponed" + }, + { + label: "abandoned", + title: "Abandoned" + }, + { + label: "obsolete", + title: "Obsolete" + } +]; + +let issues = []; + +/** + * Fetch all the MSC proposals from GitHub. + * + * GitHub only lets us fetch 100 items at a time, and it gives us a `link` + * response header containing the URL for the next batch. + * So we will keep fetching until the response doesn't contain the "next" link. + */ +async function getIssues() { + + /** + * A pretty ugly function to get us the "next" link in the header if there + * was one, or `null` otherwise. + */ + function getNextLink(header) { + const links = header.split(","); + for (const link of links) { + const linkPieces = link.split(";"); + if (linkPieces[1] == ` rel=\"next\"`) { + const next = linkPieces[0].trim(); + return next.substring(1, next.length-1); + } + } + return null; + } + + let pageLink = "https://api.github.com/repos/matrix-org/matrix-doc/issues?state=all&labels=proposal&per_page=100"; + while (pageLink) { + const response = await fetch(pageLink); + const issuesForPage = await response.json(); + issues = issues.concat(issuesForPage); + const linkHeader = response.headers.get("link"); + pageLink = getNextLink(linkHeader); + } +} + +getIssues().then(processIssues); + +/** + * Rather than just store the complete issue, we'll extract + * only the pieces we need. + * We'll also do some transformation of the issues, jsut because + * it's easier to do in JS than in the template. + */ +function getProposalFromIssue(issue) { + + /** + * A helper function to fetch the contents of special + * directives in the issue body. + * Looks for a directive in the format: + * `^${directiveName}: (.+?)$`, returning the matched + * part or null if the directive wasn't found. + */ + function getDirective(directiveName, issue) { + const re = new RegExp(`^${directiveName}: (.+?)$`, "m"); + const found = issue.body.match(re); + return found? found[1]: null; + } + + function getDocumentation(issue) { + const found = getDirective("Documentation", issue); + if (found) { + return found.split(",").map(a => a.trim()); + } else { + return []; + } + } + + function getAuthors(issue) { + const found = getDirective("Author", issue); + if (found) { + return found.split(",").map(a => a.trim().substr(1)); + } else { + return [`${issue.user.login}`]; + } + } + + function getShepherd(issue) { + const found = getDirective("Shepherd", issue); + if (found) { + return found.substr(1); + } else { + return null; + } + } + + return { + number: issue.number, + url: issue.html_url, + title: issue.title, + created_at: issue.created_at.substr(0, 10), + updated_at: issue.updated_at.substr(0, 10), + authors: getAuthors(issue), + shepherd: getShepherd(issue), + documentation: getDocumentation(issue) + } +} + +/** + * Returns the intersection of two arrays. + */ +function intersection(array1, array2) { + return array1.filter(i => array2.includes(i)); +} + +/** + * Given all the GitHub issues with a "proposal" label: + * - group issues by the state they are in, and for each group: + * - extract the bits we need from each issue + * - write the result under /data/msc + */ +function processIssues() { + if (!fs.existsSync(outputDir)){ + fs.mkdirSync(outputDir); + } + const output = []; + // make a group of "work in progress" proposals, + // which are identified by not having any of the state labels + const stateLabels = states.map(s => s.label); + const worksInProgress = issues.filter(issue => { + const labelsForIssue = issue.labels.map(l => l.name); + return intersection(labelsForIssue, stateLabels).length === 0; + }); + output.push({ + title: "Work In Progress", + label: "work-in-progress", + proposals: worksInProgress.map(issue => getProposalFromIssue(issue)) + }); + // for each defined state + for (const state of states) { + // get the set of issues for that state + const issuesForState = issues.filter(msc => { + return msc.labels.some(l => l.name === state.label); + }); + // store it in /data + output.push({ + title: state.title, + label: state.label, + proposals: issuesForState.map(issue => getProposalFromIssue(issue)) + }); + } + const outputData = JSON.stringify(output, null, '\t'); + const outputFile = path.join(outputDir, `proposals.json`); + fs.writeFileSync(outputFile, outputData); +}