Cut out legacy build scripts
Travis Ralston
4 years ago
committed by
Richard van der Hoff
parent
726be04841
commit
8d73a17f5b
@ -1,3 +0,0 @@
|
||||
continuserv proactively re-generates the spec on filesystem changes, and serves
|
||||
it over HTTP. For notes on using it, see [the main
|
||||
readme](../../README.rst#continuserv).
|
||||
@ -1,15 +0,0 @@
|
||||
<head>
|
||||
<script>
|
||||
window.onload = function() {
|
||||
var url = new URL(window.location);
|
||||
url.pathname += "api-docs.json";
|
||||
var newLoc = "http://petstore.swagger.io/?url=" + encodeURIComponent(url);
|
||||
document.getElementById("apidocs").href = newLoc;
|
||||
};
|
||||
</script>
|
||||
</head>
|
||||
<body><ul>
|
||||
<li><a id="apidocs">api docs</a></li>
|
||||
<li><a href="index.html">spec</a></li>
|
||||
</ul>
|
||||
</body>
|
||||
@ -1,274 +0,0 @@
|
||||
// continuserv proactively re-generates the spec on filesystem changes, and serves it over HTTP.
|
||||
// It will always serve the most recent version of the spec, and may block an HTTP request until regeneration is finished.
|
||||
// It does not currently pre-empt stale generations, but will block until they are complete.
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
fsnotify "gopkg.in/fsnotify/fsnotify.v1"
|
||||
)
|
||||
|
||||
var (
|
||||
port = flag.Int("port", 8000, "Port on which to serve HTTP")
|
||||
|
||||
mu sync.Mutex // Prevent multiple updates in parallel.
|
||||
toServe atomic.Value // Always contains a bytesOrErr. May be stale unless wg is zero.
|
||||
|
||||
wgMu sync.Mutex // Prevent multiple calls to wg.Wait() or wg.Add(positive number) in parallel.
|
||||
wg sync.WaitGroup // Indicates how many updates are pending.
|
||||
)
|
||||
|
||||
func main() {
|
||||
flag.Parse()
|
||||
|
||||
w, err := fsnotify.NewWatcher()
|
||||
if err != nil {
|
||||
log.Fatalf("Error making watcher: %v", err)
|
||||
}
|
||||
|
||||
dir, err := os.Getwd()
|
||||
if err != nil {
|
||||
log.Fatalf("Error getting wd: %v", err)
|
||||
}
|
||||
for ; !exists(path.Join(dir, ".git")); dir = path.Dir(dir) {
|
||||
if dir == "/" {
|
||||
log.Fatalf("Could not find git root")
|
||||
}
|
||||
}
|
||||
|
||||
walker := makeWalker(dir, w)
|
||||
paths := []string{"api", "changelogs", "event-schemas", "scripts",
|
||||
"specification", "schemas", "data-definitions"}
|
||||
|
||||
for _, p := range paths {
|
||||
filepath.Walk(path.Join(dir, p), walker)
|
||||
}
|
||||
|
||||
wg.Add(1)
|
||||
populateOnce(dir)
|
||||
|
||||
ch := make(chan struct{}, 100) // Buffered to ensure we can multiple-increment wg for pending writes
|
||||
go doPopulate(ch, dir)
|
||||
|
||||
go watchFS(ch, w)
|
||||
fmt.Printf("Listening on port %d\n", *port)
|
||||
http.HandleFunc("/", serve)
|
||||
log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", *port), nil))
|
||||
|
||||
}
|
||||
|
||||
func watchFS(ch chan struct{}, w *fsnotify.Watcher) {
|
||||
for {
|
||||
select {
|
||||
case e := <-w.Events:
|
||||
if filter(e) {
|
||||
fmt.Printf("Noticed change to %s, re-generating spec\n", e.Name)
|
||||
ch <- struct{}{}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func makeWalker(base string, w *fsnotify.Watcher) filepath.WalkFunc {
|
||||
return func(path string, i os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
log.Fatalf("Error walking: %v", err)
|
||||
}
|
||||
if !i.IsDir() {
|
||||
// we set watches on directories, not files
|
||||
return nil
|
||||
}
|
||||
|
||||
rel, err := filepath.Rel(base, path)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to get relative path of %s: %v", path, err)
|
||||
}
|
||||
|
||||
// Normalize slashes
|
||||
rel = filepath.ToSlash(rel)
|
||||
|
||||
// skip a few things that we know don't form part of the spec
|
||||
if rel == "api/node_modules" ||
|
||||
rel == "scripts/gen" ||
|
||||
rel == "scripts/tmp" {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
|
||||
// log.Printf("Adding watch on %s", path)
|
||||
if err := w.Add(path); err != nil {
|
||||
log.Fatalf("Failed to add watch on %s: %v", path, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// Return true if event should trigger re-population
|
||||
func filter(e fsnotify.Event) bool {
|
||||
// vim is *really* noisy about how it writes files
|
||||
if e.Op != fsnotify.Write {
|
||||
return false
|
||||
}
|
||||
|
||||
_, fname := filepath.Split(e.Name)
|
||||
|
||||
// Avoid some temp files that vim or emacs writes
|
||||
if strings.HasSuffix(e.Name, "~") || strings.HasSuffix(e.Name, ".swp") || strings.HasPrefix(fname, ".") ||
|
||||
(strings.HasPrefix(fname, "#") && strings.HasSuffix(fname, "#")) {
|
||||
return false
|
||||
}
|
||||
|
||||
// Forcefully ignore directories we don't care about (Windows, at least, tries to notify about some directories)
|
||||
filePath := filepath.ToSlash(e.Name) // normalize slashes
|
||||
if strings.Contains(filePath, "/scripts/tmp") ||
|
||||
strings.Contains(filePath, "/scripts/gen") ||
|
||||
strings.Contains(filePath, "/api/node_modules") {
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
func serve(w http.ResponseWriter, req *http.Request) {
|
||||
wgMu.Lock()
|
||||
wg.Wait()
|
||||
wgMu.Unlock()
|
||||
|
||||
m := toServe.Load().(bytesOrErr)
|
||||
if m.err != nil {
|
||||
w.Header().Set("Content-Type", "text/plain")
|
||||
w.Write([]byte(m.err.Error()))
|
||||
return
|
||||
}
|
||||
|
||||
ok := true
|
||||
var b []byte
|
||||
|
||||
file := req.URL.Path
|
||||
if file[0] == '/' {
|
||||
file = file[1:]
|
||||
}
|
||||
b, ok = m.bytes[filepath.FromSlash(file)] // de-normalize slashes
|
||||
|
||||
if ok && file == "api-docs.json" {
|
||||
w.Header().Set("Access-Control-Allow-Origin", "*")
|
||||
}
|
||||
|
||||
if ok {
|
||||
w.Header().Set("Content-Type", "text/html")
|
||||
w.Write([]byte(b))
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "text/plain")
|
||||
w.WriteHeader(404)
|
||||
w.Write([]byte("Not found"))
|
||||
}
|
||||
|
||||
func generate(dir string) (map[string][]byte, error) {
|
||||
cmd := exec.Command("python", "gendoc.py")
|
||||
cmd.Dir = path.Join(dir, "scripts")
|
||||
var b bytes.Buffer
|
||||
cmd.Stderr = &b
|
||||
err := cmd.Run()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error generating spec: %v\nOutput from gendoc:\n%v", err, b.String())
|
||||
}
|
||||
|
||||
// cheekily dump the swagger docs into the gen directory so that it is
|
||||
// easy to serve
|
||||
cmd = exec.Command("python", "dump-swagger.py", "-o", "gen/api-docs.json")
|
||||
cmd.Dir = path.Join(dir, "scripts")
|
||||
cmd.Stderr = &b
|
||||
if err := cmd.Run(); err != nil {
|
||||
return nil, fmt.Errorf("error generating api docs: %v\nOutput from dump-swagger:\n%v", err, b.String())
|
||||
}
|
||||
|
||||
files := make(map[string][]byte)
|
||||
base := path.Join(dir, "scripts", "gen")
|
||||
walker := func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if info.IsDir() {
|
||||
return nil
|
||||
}
|
||||
|
||||
rel, err := filepath.Rel(base, path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Failed to get relative path of %s: %v", path, err)
|
||||
}
|
||||
|
||||
bytes, err := ioutil.ReadFile(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
files[rel] = bytes
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := filepath.Walk(base, walker); err != nil {
|
||||
return nil, fmt.Errorf("error reading spec: %v", err)
|
||||
}
|
||||
|
||||
// load the special index
|
||||
indexpath := path.Join(dir, "scripts", "continuserv", "index.html")
|
||||
bytes, err := ioutil.ReadFile(indexpath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error reading index: %v", err)
|
||||
}
|
||||
files[""] = bytes
|
||||
|
||||
return files, nil
|
||||
}
|
||||
|
||||
func populateOnce(dir string) {
|
||||
defer wg.Done()
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
|
||||
files, err := generate(dir)
|
||||
toServe.Store(bytesOrErr{files, err})
|
||||
}
|
||||
|
||||
func doPopulate(ch chan struct{}, dir string) {
|
||||
var pending int
|
||||
for {
|
||||
select {
|
||||
case <-ch:
|
||||
if pending == 0 {
|
||||
wgMu.Lock()
|
||||
wg.Add(1)
|
||||
wgMu.Unlock()
|
||||
}
|
||||
pending++
|
||||
case <-time.After(10 * time.Millisecond):
|
||||
if pending > 0 {
|
||||
pending = 0
|
||||
populateOnce(dir)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func exists(path string) bool {
|
||||
_, err := os.Stat(path)
|
||||
return !os.IsNotExist(err)
|
||||
}
|
||||
|
||||
type bytesOrErr struct {
|
||||
bytes map[string][]byte // filename -> contents
|
||||
err error
|
||||
}
|
||||
@ -1,6 +0,0 @@
|
||||
with import <nixpkgs> {};
|
||||
|
||||
(python.buildEnv.override {
|
||||
extraLibs = with pythonPackages;
|
||||
[ docutils pyyaml jinja2 pygments ];
|
||||
}).env
|
||||
@ -1,561 +0,0 @@
|
||||
#! /usr/bin/env python
|
||||
|
||||
# Copyright 2016 OpenMarket Ltd
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from argparse import ArgumentParser
|
||||
from docutils.core import publish_file
|
||||
import copy
|
||||
import fileinput
|
||||
import glob
|
||||
import os
|
||||
import os.path
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import yaml
|
||||
|
||||
script_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
docs_dir = os.path.dirname(script_dir)
|
||||
spec_dir = os.path.join(docs_dir, "specification")
|
||||
tmp_dir = os.path.join(script_dir, "tmp")
|
||||
changelog_dir = os.path.join(docs_dir, "changelogs")
|
||||
|
||||
VERBOSE = False
|
||||
|
||||
"""
|
||||
Read a RST file and replace titles with a different title level if required.
|
||||
Args:
|
||||
filename: The name of the file being read (for debugging)
|
||||
file_stream: The open file stream to read from.
|
||||
title_level: The integer which determines the offset to *start* from.
|
||||
title_styles: An array of characters detailing the right title styles to use
|
||||
e.g. ["=", "-", "~", "+"]
|
||||
Returns:
|
||||
string: The file contents with titles adjusted.
|
||||
Example:
|
||||
Assume title_styles = ["=", "-", "~", "+"], title_level = 1, and the file
|
||||
when read line-by-line encounters the titles "===", "---", "---", "===", "---".
|
||||
This function will bump every title encountered down a sub-heading e.g.
|
||||
"=" to "-" and "-" to "~" because title_level = 1, so the output would be
|
||||
"---", "~~~", "~~~", "---", "~~~". There is no bumping "up" a title level.
|
||||
"""
|
||||
def load_with_adjusted_titles(filename, file_stream, title_level, title_styles):
|
||||
rst_lines = []
|
||||
|
||||
prev_line_title_level = 0 # We expect the file to start with '=' titles
|
||||
file_offset = None
|
||||
prev_non_title_line = None
|
||||
for i, line in enumerate(file_stream):
|
||||
if (prev_non_title_line is None
|
||||
or not is_title_line(prev_non_title_line, line, title_styles)
|
||||
):
|
||||
rst_lines.append(line)
|
||||
prev_non_title_line = line
|
||||
continue
|
||||
|
||||
line_title_style = line[0]
|
||||
line_title_level = title_styles.index(line_title_style)
|
||||
|
||||
# Not all files will start with "===" and we should be flexible enough
|
||||
# to allow that. The first title we encounter sets the "file offset"
|
||||
# which is added to the title_level desired.
|
||||
if file_offset is None:
|
||||
file_offset = line_title_level
|
||||
if file_offset != 0:
|
||||
logv((" WARNING: %s starts with a title style of '%s' but '%s' " +
|
||||
"is preferable.") % (filename, line_title_style, title_styles[0]))
|
||||
|
||||
# Sanity checks: Make sure that this file is obeying the title levels
|
||||
# specified and bail if it isn't.
|
||||
# The file is allowed to go 1 deeper or any number shallower
|
||||
if prev_line_title_level - line_title_level < -1:
|
||||
raise Exception(
|
||||
("File '%s' line '%s' has a title " +
|
||||
"style '%s' which doesn't match one of the " +
|
||||
"allowed title styles of %s because the " +
|
||||
"title level before this line was '%s'") %
|
||||
(filename, (i + 1), line_title_style, title_styles,
|
||||
title_styles[prev_line_title_level])
|
||||
)
|
||||
prev_line_title_level = line_title_level
|
||||
|
||||
adjusted_level = (
|
||||
title_level + line_title_level - file_offset
|
||||
)
|
||||
|
||||
# Sanity check: Make sure we can bump down the title and we aren't at the
|
||||
# lowest level already
|
||||
if adjusted_level >= len(title_styles):
|
||||
raise Exception(
|
||||
("Files '%s' line '%s' has a sub-title level too low and it " +
|
||||
"cannot be adjusted to fit. You can add another level to the " +
|
||||
"'title_styles' key in targets.yaml to fix this.") %
|
||||
(filename, (i + 1))
|
||||
)
|
||||
|
||||
if adjusted_level == line_title_level:
|
||||
# no changes required
|
||||
rst_lines.append(line)
|
||||
continue
|
||||
|
||||
# Adjusting line levels
|
||||
logv(
|
||||
"File: %s Adjusting %s to %s because file_offset=%s title_offset=%s" %
|
||||
(filename, line_title_style, title_styles[adjusted_level],
|
||||
file_offset, title_level)
|
||||
)
|
||||
rst_lines.append(line.replace(
|
||||
line_title_style,
|
||||
title_styles[adjusted_level]
|
||||
))
|
||||
|
||||
return "".join(rst_lines)
|
||||
|
||||
|
||||
def is_title_line(prev_line, line, title_styles):
|
||||
# The title underline must match at a minimum the length of the title
|
||||
if len(prev_line) > len(line):
|
||||
return False
|
||||
|
||||
line = line.rstrip()
|
||||
|
||||
# must be at least 3 chars long
|
||||
if len(line) < 3:
|
||||
return False
|
||||
|
||||
# must start with a title char
|
||||
title_char = line[0]
|
||||
if title_char not in title_styles:
|
||||
return False
|
||||
|
||||
# all characters must be the same
|
||||
for char in line[1:]:
|
||||
if char != title_char:
|
||||
return False
|
||||
|
||||
# looks like a title line
|
||||
return True
|
||||
|
||||
|
||||
def get_rst(file_info, title_level, title_styles, spec_dir, adjust_titles):
|
||||
# string are file paths to RST blobs
|
||||
if isinstance(file_info, str):
|
||||
log("%s %s" % (">" * (1 + title_level), file_info))
|
||||
with open(os.path.join(spec_dir, file_info), "r", encoding="utf-8") as f:
|
||||
rst = None
|
||||
if adjust_titles:
|
||||
rst = load_with_adjusted_titles(
|
||||
file_info, f, title_level, title_styles
|
||||
)
|
||||
else:
|
||||
rst = f.read()
|
||||
|
||||
rst += "\n\n"
|
||||
return rst
|
||||
# dicts look like {0: filepath, 1: filepath} where the key is the title level
|
||||
elif isinstance(file_info, dict):
|
||||
levels = sorted(file_info.keys())
|
||||
rst = []
|
||||
for l in levels:
|
||||
rst.append(get_rst(file_info[l], l, title_styles, spec_dir, adjust_titles))
|
||||
return "".join(rst)
|
||||
# lists are multiple file paths e.g. [filepath, filepath]
|
||||
elif isinstance(file_info, list):
|
||||
rst = []
|
||||
for f in file_info:
|
||||
rst.append(get_rst(f, title_level, title_styles, spec_dir, adjust_titles))
|
||||
return "".join(rst)
|
||||
raise Exception(
|
||||
"The following 'file' entry in this target isn't a string, list or dict. " +
|
||||
"It really really should be. Entry: %s" % (file_info,)
|
||||
)
|
||||
|
||||
|
||||
def build_spec(target, out_filename):
|
||||
log("Building templated file %s" % out_filename)
|
||||
with open(out_filename, "w", encoding="utf-8") as outfile:
|
||||
for file_info in target["files"]:
|
||||
section = get_rst(
|
||||
file_info=file_info,
|
||||
title_level=0,
|
||||
title_styles=target["title_styles"],
|
||||
spec_dir=spec_dir,
|
||||
adjust_titles=True
|
||||
)
|
||||
outfile.write(section)
|
||||
|
||||
|
||||
"""
|
||||
Replaces relative title styles with actual title styles.
|
||||
|
||||
The templating system has no idea what the right title style is when it produces
|
||||
RST because it depends on the build target. As a result, it uses relative title
|
||||
styles defined in targets.yaml to say "down a level, up a level, same level".
|
||||
|
||||
This function replaces these relative titles with actual title styles from the
|
||||
array in targets.yaml.
|
||||
"""
|
||||
def fix_relative_titles(target, filename, out_filename):
|
||||
log("Fix relative titles, %s -> %s" % (filename, out_filename))
|
||||
title_styles = target["title_styles"]
|
||||
relative_title_chars = [
|
||||
target["relative_title_styles"]["subtitle"],
|
||||
target["relative_title_styles"]["sametitle"],
|
||||
target["relative_title_styles"]["supertitle"]
|
||||
]
|
||||
relative_title_matcher = re.compile(
|
||||
"^[" + re.escape("".join(relative_title_chars)) + "]{3,}$"
|
||||
)
|
||||
title_matcher = re.compile(
|
||||
"^[" + re.escape("".join(title_styles)) + "]{3,}$"
|
||||
)
|
||||
current_title_style = None
|
||||
with open(filename, "r", encoding="utf-8") as infile:
|
||||
with open(out_filename, "w", encoding="utf-8") as outfile:
|
||||
for line in infile.readlines():
|
||||
if not relative_title_matcher.match(line):
|
||||
if title_matcher.match(line):
|
||||
current_title_style = line[0]
|
||||
outfile.write(line)
|
||||
continue
|
||||
line_char = line[0]
|
||||
replacement_char = None
|
||||
current_title_level = title_styles.index(current_title_style)
|
||||
if line_char == target["relative_title_styles"]["subtitle"]:
|
||||
if (current_title_level + 1) == len(title_styles):
|
||||
raise Exception(
|
||||
"Encountered sub-title line style but we can't go " +
|
||||
"any lower."
|
||||
)
|
||||
replacement_char = title_styles[current_title_level + 1]
|
||||
elif line_char == target["relative_title_styles"]["sametitle"]:
|
||||
replacement_char = title_styles[current_title_level]
|
||||
elif line_char == target["relative_title_styles"]["supertitle"]:
|
||||
if (current_title_level - 1) < 0:
|
||||
raise Exception(
|
||||
"Encountered super-title line style but we can't go " +
|
||||
"any higher."
|
||||
)
|
||||
replacement_char = title_styles[current_title_level - 1]
|
||||
else:
|
||||
raise Exception(
|
||||
"Unknown relative line char %s" % (line_char,)
|
||||
)
|
||||
|
||||
outfile.write(
|
||||
line.replace(line_char, replacement_char)
|
||||
)
|
||||
|
||||
|
||||
|
||||
def rst2html(i, o, stylesheets):
|
||||
log("rst2html %s -> %s" % (i, o))
|
||||
with open(i, "r", encoding="utf-8") as in_file:
|
||||
with open(o, "w", encoding="utf-8") as out_file:
|
||||
publish_file(
|
||||
source=in_file,
|
||||
destination=out_file,
|
||||
reader_name="standalone",
|
||||
parser_name="restructuredtext",
|
||||
writer_name="html",
|
||||
settings_overrides={
|
||||
"stylesheet_path": stylesheets,
|
||||
"syntax_highlight": "short",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def addAnchors(path):
|
||||
log("add anchors %s" % path)
|
||||
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
lines = f.readlines()
|
||||
|
||||
replacement = r'<p><a class="anchor" id="\2"></a></p>\n\1'
|
||||
with open(path, "w", encoding="utf-8") as f:
|
||||
for line in lines:
|
||||
line = re.sub(r'(<h\d id="#?(.*?)">)', replacement, line.rstrip())
|
||||
line = re.sub(r'(<div class="section" id="(.*?)">)', replacement, line.rstrip())
|
||||
f.write(line + "\n")
|
||||
|
||||
|
||||
def run_through_template(input_files, set_verbose, substitutions):
|
||||
args = [
|
||||
'python', script_dir+'/templating/build.py',
|
||||
"-o", tmp_dir,
|
||||
"-i", "matrix_templates",
|
||||
]
|
||||
|
||||
for k, v in substitutions.items():
|
||||
args.append("--substitution=%s=%s" % (k, v))
|
||||
|
||||
if set_verbose:
|
||||
args.insert(2, "-v")
|
||||
|
||||
args.extend(input_files)
|
||||
|
||||
log("EXEC: %s" % " ".join(args))
|
||||
log(" ==== build.py output ==== ")
|
||||
subprocess.check_call(args)
|
||||
|
||||
"""
|
||||
Extract and resolve groups for the given target in the given targets listing.
|
||||
Args:
|
||||
all_targets (dict): The parsed YAML file containing a list of targets
|
||||
target_name (str): The name of the target to extract from the listings.
|
||||
Returns:
|
||||
dict: Containing "filees" (a list of file paths), "relative_title_styles"
|
||||
(a dict of relative style keyword to title character) and "title_styles"
|
||||
(a list of characters which represent the global title style to follow,
|
||||
with the top section title first, the second section second, and so on.)
|
||||
"""
|
||||
def get_build_target(all_targets, target_name):
|
||||
build_target = {
|
||||
"title_styles": [],
|
||||
"relative_title_styles": {},
|
||||
"files": []
|
||||
}
|
||||
|
||||
build_target["title_styles"] = all_targets["title_styles"]
|
||||
build_target["relative_title_styles"] = all_targets["relative_title_styles"]
|
||||
target = all_targets["targets"].get(target_name)
|
||||
if not target:
|
||||
raise Exception(
|
||||
"No target by the name '" + target_name + "' exists in '" +
|
||||
targets_listing + "'."
|
||||
)
|
||||
if not isinstance(target.get("files"), list):
|
||||
raise Exception(
|
||||
"Found target but 'files' key is not a list."
|
||||
)
|
||||
|
||||
def get_group(group_id, depth):
|
||||
group_name = group_id[len("group:"):]
|
||||
group = all_targets.get("groups", {}).get(group_name)
|
||||
if not group:
|
||||
raise Exception(
|
||||
"Tried to find group '%s' but it doesn't exist." % group_name
|
||||
)
|
||||
if not isinstance(group, list):
|
||||
raise Exception(
|
||||
"Expected group '%s' to be a list but it isn't." % group_name
|
||||
)
|
||||
# deep copy so changes to depths don't contaminate multiple uses of this group
|
||||
group = copy.deepcopy(group)
|
||||
# swap relative depths for absolute ones
|
||||
for i, entry in enumerate(group):
|
||||
if isinstance(entry, dict):
|
||||
group[i] = {
|
||||
(rel_depth + depth): v for (rel_depth, v) in entry.items()
|
||||
}
|
||||
return group
|
||||
|
||||
resolved_files = []
|
||||
for file_entry in target["files"]:
|
||||
# file_entry is a group id
|
||||
if isinstance(file_entry, str) and file_entry.startswith("group:"):
|
||||
group = get_group(file_entry, 0)
|
||||
# The group may be resolved to a list of file entries, in which case
|
||||
# we want to extend the array to insert each of them rather than
|
||||
# insert the entire list as a single element (which is what append does)
|
||||
if isinstance(group, list):
|
||||
resolved_files.extend(group)
|
||||
else:
|
||||
resolved_files.append(group)
|
||||
# file_entry is a dict which has more file entries as values
|
||||
elif isinstance(file_entry, dict):
|
||||
resolved_entry = {}
|
||||
for (depth, entry) in file_entry.items():
|
||||
if not isinstance(entry, str):
|
||||
raise Exception(
|
||||
"Double-nested depths are not supported. Entry: %s" % (file_entry,)
|
||||
)
|
||||
if entry.startswith("group:"):
|
||||
resolved_entry[depth] = get_group(entry, depth)
|
||||
else:
|
||||
# map across without editing (e.g. normal file path)
|
||||
resolved_entry[depth] = entry
|
||||
resolved_files.append(resolved_entry)
|
||||
continue
|
||||
# file_entry is just a plain ol' file path
|
||||
else:
|
||||
resolved_files.append(file_entry)
|
||||
build_target["files"] = resolved_files
|
||||
return build_target
|
||||
|
||||
def log(line):
|
||||
print("gendoc: %s" % line)
|
||||
|
||||
def logv(line):
|
||||
if VERBOSE:
|
||||
print("gendoc:V: %s" % line)
|
||||
|
||||
|
||||
def cleanup_env():
|
||||
shutil.rmtree(tmp_dir)
|
||||
|
||||
|
||||
def mkdirp(d) :
|
||||
if not os.path.exists(d):
|
||||
os.makedirs(d)
|
||||
|
||||
|
||||
def main(targets, dest_dir, keep_intermediates, substitutions):
|
||||
try:
|
||||
mkdirp(dest_dir)
|
||||
except Exception as e:
|
||||
log("Error creating destination directory '%s': %s" % (dest_dir, str(e)))
|
||||
return 1
|
||||
try:
|
||||
mkdirp(tmp_dir)
|
||||
except Exception as e:
|
||||
log("Error creating temporary directory '%s': %s" % (tmp_dir, str(e)))
|
||||
return 1
|
||||
|
||||
with open(os.path.join(spec_dir, "targets.yaml"), "r") as targ_file:
|
||||
target_defs = yaml.load(targ_file.read())
|
||||
|
||||
if targets == ["all"]:
|
||||
targets = target_defs["targets"].keys()
|
||||
|
||||
log("Building spec [targets=%s]" % targets)
|
||||
|
||||
templated_files = {} # map from target name to templated file
|
||||
|
||||
for target_name in targets:
|
||||
templated_file = os.path.join(tmp_dir, "templated_%s.rst" % (target_name,))
|
||||
|
||||
target = get_build_target(target_defs, target_name)
|
||||
build_spec(target=target, out_filename=templated_file)
|
||||
templated_files[target_name] = templated_file
|
||||
|
||||
# we do all the templating at once, because it's slow
|
||||
run_through_template(templated_files.values(), VERBOSE, substitutions)
|
||||
|
||||
stylesheets = glob.glob(os.path.join(script_dir, "css", "*.css"))
|
||||
|
||||
for target_name, templated_file in templated_files.items():
|
||||
target = target_defs["targets"].get(target_name)
|
||||
version_label = None
|
||||
if target:
|
||||
version_label = target.get("version_label")
|
||||
if version_label:
|
||||
for old, new in substitutions.items():
|
||||
version_label = version_label.replace(old, new)
|
||||
|
||||
rst_file = os.path.join(tmp_dir, "spec_%s.rst" % (target_name,))
|
||||
if version_label:
|
||||
d = os.path.join(dest_dir, target_name.split('@')[0])
|
||||
if not os.path.exists(d):
|
||||
os.mkdir(d)
|
||||
html_file = os.path.join(d, "%s.html" % version_label)
|
||||
else:
|
||||
html_file = os.path.join(dest_dir, "%s.html" % (target_name, ))
|
||||
|
||||
fix_relative_titles(
|
||||
target=target_defs, filename=templated_file,
|
||||
out_filename=rst_file,
|
||||
)
|
||||
rst2html(rst_file, html_file, stylesheets=stylesheets)
|
||||
addAnchors(html_file)
|
||||
|
||||
if not keep_intermediates:
|
||||
cleanup_env()
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
def list_targets():
|
||||
with open(os.path.join(spec_dir, "targets.yaml"), "r") as targ_file:
|
||||
target_defs = yaml.load(targ_file.read())
|
||||
targets = target_defs["targets"].keys()
|
||||
print("\n".join(targets))
|
||||
|
||||
|
||||
def extract_major(s):
|
||||
major_version = s
|
||||
match = re.match("^(r\d+)(\.\d+)*$", s)
|
||||
if match:
|
||||
major_version = match.group(1)
|
||||
return major_version
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = ArgumentParser(
|
||||
"gendoc.py - Generate the Matrix specification as HTML."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--nodelete", "-n", action="store_true",
|
||||
help="Do not delete intermediate files. They will be found in scripts/tmp/"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--target", "-t", action="append",
|
||||
help="Specify the build target to build from specification/targets.yaml. " +
|
||||
"The value 'all' will build all of the targets therein."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--verbose", "-v", action="store_true",
|
||||
help="Turn on verbose mode."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--client_release", "-c", action="store", default="unstable",
|
||||
help="The client-server release tag to generate, e.g. r1.2"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--server_release", "-s", action="store", default="unstable",
|
||||
help="The server-server release tag to generate, e.g. r1.2"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--appservice_release", "-a", action="store", default="unstable",
|
||||
help="The appservice release tag to generate, e.g. r1.2"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--push_gateway_release", "-p", action="store", default="unstable",
|
||||
help="The push gateway release tag to generate, e.g. r1.2"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--identity_release", "-i", action="store", default="unstable",
|
||||
help="The identity service release tag to generate, e.g. r1.2"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--list_targets", action="store_true",
|
||||
help="Do not update the specification. Instead print a list of targets.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--dest", "-d", default=os.path.join(script_dir, "gen"),
|
||||
help="Set destination directory (default: scripts/gen)",
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
VERBOSE = args.verbose
|
||||
|
||||
if args.list_targets:
|
||||
list_targets()
|
||||
exit(0)
|
||||
|
||||
substitutions = {
|
||||
"%CLIENT_RELEASE_LABEL%": args.client_release,
|
||||
# we hardcode the major versions. This ends up in the example
|
||||
# API URLs. When we have released a new major version, we'll
|
||||
# have to bump them.
|
||||
"%CLIENT_MAJOR_VERSION%": "r0",
|
||||
"%SERVER_RELEASE_LABEL%": args.server_release,
|
||||
"%APPSERVICE_RELEASE_LABEL%": args.appservice_release,
|
||||
"%IDENTITY_RELEASE_LABEL%": args.identity_release,
|
||||
"%PUSH_GATEWAY_RELEASE_LABEL%": args.push_gateway_release,
|
||||
}
|
||||
|
||||
exit (main(args.target or ["all"], args.dest, args.nodelete, substitutions))
|
||||
@ -1,218 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
#
|
||||
# proposals.py: generate an RST file (proposals.rst) from queries to github.com/matrix.org/matrix-doc/issues.
|
||||
|
||||
import requests
|
||||
import re
|
||||
from datetime import datetime
|
||||
|
||||
# a list of the labels we care about
|
||||
LABELS_LIST=[
|
||||
'proposal-in-review',
|
||||
'proposed-final-comment-period',
|
||||
'final-comment-period',
|
||||
'finished-final-comment-period',
|
||||
'spec-pr-missing',
|
||||
'spec-pr-in-review',
|
||||
'merged',
|
||||
'proposal-postponed',
|
||||
'abandoned',
|
||||
'obsolete',
|
||||
]
|
||||
|
||||
|
||||
authors = set()
|
||||
prs = set()
|
||||
|
||||
def getpage(url):
|
||||
"""Request the given URL, and extract the pagecount from the response headers
|
||||
|
||||
Args:
|
||||
url (str): URL to fetch
|
||||
|
||||
Returns:
|
||||
Tuple[int, list]: number of pages, and the list of items on this page
|
||||
"""
|
||||
resp = requests.get(url)
|
||||
|
||||
pagecount = 1
|
||||
for link in resp.links.values():
|
||||
if link['rel'] == 'last':
|
||||
# we extract the pagecount from the `page` param of the last url
|
||||
# in the response, eg
|
||||
# 'https://api.github.com/repositories/24998719/issues?state=all&labels=proposal&page=10'
|
||||
pagecount = int(re.search('page=(\d+)', link['url']).group(1))
|
||||
|
||||
val = resp.json()
|
||||
if not isinstance(val, list):
|
||||
print(val) # Just dump the raw (likely error) response to the log
|
||||
raise Exception("Error calling %s" % url)
|
||||
return (pagecount, val)
|
||||
|
||||
def getbylabel(label):
|
||||
"""Fetch all the issues with a given label
|
||||
|
||||
Args:
|
||||
label (str): label to fetch
|
||||
|
||||
Returns:
|
||||
Iterator[dict]: an iterator over the issue list.
|
||||
"""
|
||||
urlbase = 'https://api.github.com/repos/matrix-org/matrix-doc/issues?state=all&labels=' + label + '&page='
|
||||
page = 1
|
||||
while True:
|
||||
(pagecount, results) = getpage(urlbase + str(page))
|
||||
for i in results:
|
||||
yield i
|
||||
page += 1
|
||||
if page > pagecount:
|
||||
return
|
||||
|
||||
def print_issue_list(text_file, label, issues):
|
||||
text_file.write(label + "\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\n")
|
||||
|
||||
if (len(issues) == 0):
|
||||
text_file.write("No proposals.\n\n")
|
||||
return
|
||||
|
||||
text_file.write(".. list-table::\n :header-rows: 1\n :widths: auto\n :stub-columns: 1\n\n")
|
||||
text_file.write(" * - MSC\n")
|
||||
text_file.write(" - Proposal Title\n")
|
||||
text_file.write(" - Creation Date\n")
|
||||
text_file.write(" - Update Date\n")
|
||||
text_file.write(" - Documentation\n")
|
||||
text_file.write(" - Author\n")
|
||||
text_file.write(" - Shepherd\n")
|
||||
text_file.write(" - PRs\n")
|
||||
|
||||
for item in issues:
|
||||
# set the created date, find local field, otherwise Github
|
||||
body = str(item['body'])
|
||||
created = re.search('^Date: (.+?)\n', body, flags=re.MULTILINE)
|
||||
if created is not None:
|
||||
created = created.group(1).strip()
|
||||
try:
|
||||
created = datetime.strptime(created, "%d/%m/%Y")
|
||||
created = created.strftime('%Y-%m-%d')
|
||||
except:
|
||||
pass
|
||||
try:
|
||||
created = datetime.strptime(created, "%Y-%m-%d")
|
||||
created = created.strftime('%Y-%m-%d')
|
||||
except:
|
||||
pass
|
||||
else :
|
||||
created = datetime.strptime(item['created_at'], "%Y-%m-%dT%XZ")
|
||||
created = created.strftime('%Y-%m-%d')
|
||||
item['created'] = created
|
||||
|
||||
issues_to_print = sorted(issues, key=lambda issue_sort: issue_sort["created"])
|
||||
|
||||
for item in issues_to_print:
|
||||
# MSC number
|
||||
text_file.write(" * - `MSC" + str(item['number']) + " <" + item['html_url'] + ">`_\n")
|
||||
|
||||
# title from Github issue
|
||||
text_file.write(" - " + item['title'] + "\n")
|
||||
|
||||
# created date
|
||||
text_file.write(" - " + item['created'] + "\n")
|
||||
|
||||
# last updated, purely Github
|
||||
updated = datetime.strptime(item['updated_at'], "%Y-%m-%dT%XZ")
|
||||
text_file.write(" - " + updated.strftime('%Y-%m-%d') + "\n")
|
||||
|
||||
# list of document links (urls comma-separated)
|
||||
maindoc = re.search('^Documentation: (.+?)$', str(item['body']), flags=re.MULTILINE)
|
||||
if maindoc is not None:
|
||||
maindoc = maindoc.group(1)
|
||||
doc_list_formatted = ["`" + str(item['number']) + "-" + str(i) + " <" + x.strip() + ">`_" for i, x in enumerate(maindoc.split(','),1)]
|
||||
text_file.write(" - " + ', '.join(doc_list_formatted))
|
||||
else:
|
||||
text_file.write(" - ")
|
||||
text_file.write("\n")
|
||||
|
||||
# author list, if missing just use Github issue creator
|
||||
author = re.search('^Author: (.+?)$', str(item['body']), flags=re.MULTILINE)
|
||||
if author is not None:
|
||||
author_list_formatted = set()
|
||||
author_list = author.group(1)
|
||||
for a in author_list.split(","):
|
||||
authors.add(a.strip())
|
||||
author_list_formatted.add("`" + str(a.strip()) + "`_")
|
||||
text_file.write(" - " + ', '.join(author_list_formatted))
|
||||
else:
|
||||
author = "@" + item['user']['login']
|
||||
authors.add(author)
|
||||
text_file.write(" - `" + str(author) + "`_")
|
||||
text_file.write("\n")
|
||||
|
||||
# shepherd (currently only one)
|
||||
shepherd = re.search('Shepherd: (.+?)\n', str(item['body']))
|
||||
if shepherd is not None:
|
||||
authors.add(shepherd.group(1).strip())
|
||||
shepherd = "`" + shepherd.group(1).strip() + "`_"
|
||||
text_file.write(" - " + str(shepherd) + "\n")
|
||||
|
||||
# PRs
|
||||
try:
|
||||
pr_list = re.search('PRs: (.+?)$', str(item['body']))
|
||||
if pr_list is not None:
|
||||
pr_list_formatted = set()
|
||||
pr_list = pr_list.group(1)
|
||||
for p in pr_list.split(","):
|
||||
if re.match(r"#\d", p.strip()):
|
||||
prs.add(p.strip())
|
||||
pr_list_formatted.add("`PR" + str(p.strip()) + "`_")
|
||||
elif re.match(r"https://github.com/matrix-org/matrix-doc/pulls/\d", p.strip()):
|
||||
pr = "#" + p.strip().replace('https://github.com/matrix-org/matrix-doc/pulls/', '')
|
||||
prs.add(pr)
|
||||
pr_list_formatted.add("`PR" + str(pr) + "`_")
|
||||
else:
|
||||
raise RuntimeWarning
|
||||
text_file.write(" - " + ', '.join(pr_list_formatted))
|
||||
text_file.write("\n")
|
||||
else:
|
||||
text_file.write(" - \n")
|
||||
except:
|
||||
print("exception parsing PRs for MSC" + str(item['number']))
|
||||
text_file.write(" - \n")
|
||||
|
||||
text_file.write("\n\n\n")
|
||||
|
||||
|
||||
# first get all of the issues, filtering by label
|
||||
issues = {n: [] for n in LABELS_LIST}
|
||||
# use the magic 'None' key for a proposal in progress
|
||||
issues[None] = []
|
||||
|
||||
for prop in getbylabel('proposal'):
|
||||
print("%s: %s" % (prop['number'], [l['name'] for l in prop['labels']]))
|
||||
found_label = False
|
||||
for label in prop['labels']:
|
||||
label_name = label['name']
|
||||
if label_name in issues:
|
||||
issues[label_name].append(prop)
|
||||
found_label = True
|
||||
|
||||
# if it doesn't have any other label, assume it's work-in-progress
|
||||
if not found_label:
|
||||
issues[None].append(prop)
|
||||
|
||||
text_file = open("specification/proposals.rst", "w")
|
||||
|
||||
text_file.write("Tables of Tracked Proposals\n---------------------------\n\n")
|
||||
|
||||
print_issue_list(text_file, "<work-in-progress>", issues[None])
|
||||
for label in LABELS_LIST:
|
||||
print_issue_list(text_file, label, issues[label])
|
||||
|
||||
text_file.write("\n")
|
||||
|
||||
for author in authors:
|
||||
text_file.write("\n.. _" + author + ": https://github.com/" + author[1:])
|
||||
|
||||
for pr in prs:
|
||||
text_file.write("\n.. _PR" + pr + ": https://github.com/matrix-org/matrix-doc/pull/" + pr.replace('#', ''))
|
||||
|
||||
text_file.close()
|
||||
@ -1,38 +0,0 @@
|
||||
#! /bin/bash
|
||||
|
||||
set -ex
|
||||
|
||||
cd `dirname $0`/..
|
||||
|
||||
virtualenv -p python3 env
|
||||
. env/bin/activate
|
||||
|
||||
# Print out the python versions for debugging purposes
|
||||
python --version
|
||||
pip --version
|
||||
|
||||
# Install python dependencies
|
||||
pip install -r scripts/requirements.txt
|
||||
|
||||
# Install node dependencies
|
||||
npm install --prefix=scripts
|
||||
|
||||
# do sanity checks on the examples and swagger
|
||||
scripts/check-event-schema-examples.py
|
||||
scripts/check-swagger-sources.py
|
||||
node scripts/validator.js --schema "data/api/client-server"
|
||||
|
||||
: ${GOPATH:=${WORKSPACE}/.gopath}
|
||||
mkdir -p "${GOPATH}"
|
||||
export GOPATH
|
||||
go get github.com/hashicorp/golang-lru
|
||||
go get gopkg.in/fsnotify/fsnotify.v1
|
||||
|
||||
# make sure that the scripts build
|
||||
(cd scripts/continuserv && go build)
|
||||
(cd scripts/speculator && go build)
|
||||
|
||||
# build the spec for matrix.org.
|
||||
# (we don't actually use it on travis, but it's still useful to check we
|
||||
# can build it. On Buildkite, this is then used to deploy to matrix.org).
|
||||
./scripts/generate-matrix-org-assets
|
||||
Loading…
Reference in New Issue