Merge branch 'master' into spec-feature-profiles

Conflicts:
	templating/build.py
pull/977/head
Kegan Dougal 9 years ago
commit 11c586c560

@ -29,6 +29,33 @@ except ImportError as e:
raise raise
def check_parameter(filepath, request, parameter):
schema = parameter.get("schema")
example = None
try:
example_json = schema.get('example')
if example_json:
example = json.loads(example_json)
except Exception as e:
raise ValueError("Error parsing JSON example request for %r" % (
request
), e)
fileurl = "file://" + os.path.abspath(filepath)
if example and schema:
try:
print ("Checking request schema for: %r %r" % (
filepath, request
))
# Setting the 'id' tells jsonschema where the file is so that it
# can correctly resolve relative $ref references in the schema
schema['id'] = fileurl
jsonschema.validate(example, schema)
except Exception as e:
raise ValueError("Error validating JSON schema for %r %r" % (
request, code
), e)
def check_response(filepath, request, code, response): def check_response(filepath, request, code, response):
example = None example = None
try: try:
@ -43,7 +70,9 @@ def check_response(filepath, request, code, response):
fileurl = "file://" + os.path.abspath(filepath) fileurl = "file://" + os.path.abspath(filepath)
if example and schema: if example and schema:
try: try:
print ("Checking schema for: %r %r %r" % (filepath, request, code)) print ("Checking response schema for: %r %r %r" % (
filepath, request, code
))
# Setting the 'id' tells jsonschema where the file is so that it # Setting the 'id' tells jsonschema where the file is so that it
# can correctly resolve relative $ref references in the schema # can correctly resolve relative $ref references in the schema
schema['id'] = fileurl schema['id'] = fileurl
@ -59,8 +88,13 @@ def check_swagger_file(filepath):
swagger = yaml.load(f) swagger = yaml.load(f)
for path, path_api in swagger.get('paths', {}).items(): for path, path_api in swagger.get('paths', {}).items():
for method, request_api in path_api.items(): for method, request_api in path_api.items():
request = "%s %s" % (method.upper(), path) request = "%s %s" % (method.upper(), path)
for parameter in request_api.get('parameters', ()):
if parameter['in'] == 'body':
check_parameter(filepath, request, parameter)
try: try:
responses = request_api['responses'] responses = request_api['responses']
except KeyError: except KeyError:

@ -29,7 +29,6 @@ paths:
parameters: parameters:
- in: body - in: body
name: body name: body
required: true
schema: schema:
type: object type: object
example: |- example: |-
@ -63,7 +62,19 @@ paths:
description: The fully-qualified Matrix ID that has been registered. description: The fully-qualified Matrix ID that has been registered.
access_token: access_token:
type: string type: string
description: An access token for the account. This access token can then be used to authorize other requests. description: |-
An access token for the account.
This access token can then be used to authorize other requests.
The access token may expire at some point, and if so, it SHOULD come with a ``refresh_token``.
There is no specific error message to indicate that a request has failed because
an access token has expired; instead, if a client has reason to believe its
access token is valid, and it receives an auth error, they should attempt to
refresh for a new token on failure, and retry the request with the new token.
refresh_token:
type: string
# TODO: Work out how to linkify /tokenrefresh
description: |-
(optional) A ``refresh_token`` may be exchanged for a new ``access_token`` using the /tokenrefresh API endpoint.
home_server: home_server:
type: string type: string
description: The hostname of the Home Server on which the account has been registered. description: The hostname of the Home Server on which the account has been registered.
@ -77,3 +88,60 @@ paths:
description: This request was rate-limited. description: This request was rate-limited.
schema: schema:
"$ref": "definitions/error.yaml" "$ref": "definitions/error.yaml"
"/tokenrefresh":
post:
summary: Exchanges a refresh token for an access token.
description: |-
Exchanges a refresh token for a new access token.
This is intended to be used if the access token has expired.
security:
- accessToken: []
parameters:
- in: body
name: body
schema:
type: object
example: |-
{
"refresh_token": "a1b2c3"
}
properties:
refresh_token:
type: string
description: The refresh token which was issued by the server.
required: ["refresh_token"]
responses:
200:
description: |-
The refresh token was accepted, and a new access token has been issued.
The passed refresh token is no longer valid and cannot be used.
A new refresh token will have been returned unless some policy does
not allow the user to continue to renew their session.
examples:
application/json: |-
{
"access_token": "bearwithme123",
"refresh_token": "exchangewithme987"
}
schema:
type: object
properties:
access_token:
type: string
description: |-
An access token for the account.
This access token can then be used to authorize other requests.
The access token may expire at some point, and if so, it SHOULD come with a ``refresh_token``.
refresh_token:
type: string
description: (optional) A ``refresh_token`` may be exchanged for a new ``access_token`` using the TODO Linkify /tokenrefresh API endpoint.
403:
description: |-
The exchange attempt failed. For example, the refresh token may have already been used.
examples:
application/json: |-
{"errcode": "M_FORBIDDEN"}
429:
description: This request was rate-limited.
schema:
"$ref": "definitions/error.yaml"

@ -69,7 +69,6 @@ paths:
"/rooms/{roomId}/invite": "/rooms/{roomId}/invite":
post: post:
summary: Invite a user to participate in a particular room. summary: Invite a user to participate in a particular room.
# It's a crying shame that I don't know how to force line breaks.
description: |- description: |-
This API invites a user to participate in a particular room. This API invites a user to participate in a particular room.
They do not start participating in the room until they actually join the They do not start participating in the room until they actually join the

@ -0,0 +1,34 @@
Macaroon Caveats
================
`Macaroons`_ are issued by Matrix servers as authorization tokens. Macaroons may be restricted by adding caveats to them.
.. _Macaroons: http://theory.stanford.edu/~ataly/Papers/macaroons.pdf
Caveats can only be used for reducing the scope of a token, never for increasing it. Servers are required to reject any macroon with a caveat that they do not understand.
Some caveats are specified in this specification, and must be understood by all servers. The use of non-standard caveats is allowed.
All caveats must take the form:
`key` `operator` `value`
where `key` is a non-empty string drawn from the character set [A-Za-z0-9_]
`operator` is a non-empty string which does not contain whitespace
`value` is a non-empty string
And these are joined by single space characters.
Specified caveats:
+-------------+--------------------------------------------------+------------------------------------------------------------------------------------------------+
| Caveat name | Description | Legal Values |
+-------------+--------------------------------------------------+------------------------------------------------------------------------------------------------+
| gen | Generation of the macaroon caveat spec. | 1 |
| user_id | ID of the user for which this macaroon is valid. | Pure equality check. Operator must be =. |
| type | The purpose of this macaroon. | access - used to authorize any action except token refresh |
| refresh - only used to authorize a token refresh |
| time | Time before/after which this macaroon is valid. | A POSIX timestamp in milliseconds (in UTC). |
| Operator < means the macaroon is valid before the timestamp, as interpreted by the server. |
| Operator > means the macaroon is valid after the timestamp, as interpreted by the server. |
| Operator == means the macaroon is valid at exactly the timestamp, as interpreted by the server.|
| Note that exact equality of time is largely meaningless. |
+-------------+--------------------------------------------------+------------------------------------------------------------------------------------------------+

@ -4,6 +4,7 @@ It serves the following HTTP endpoints:
- / lists open pull requests - / lists open pull requests
- /spec/123 which renders the spec as html at pull request 123. - /spec/123 which renders the spec as html at pull request 123.
- /diff/rst/123 which gives a diff of the spec's rst at pull request 123. - /diff/rst/123 which gives a diff of the spec's rst at pull request 123.
- /diff/html/123 which gives a diff of the spec's HTML at pull request 123.
To run it, you must install the `go` tool, and run: To run it, you must install the `go` tool, and run:
`go run main.go` `go run main.go`

@ -0,0 +1,564 @@
#!/usr/bin/perl
#
# htmldiff - present a diff marked version of two html documents
#
# Copyright (c) 1998-2006 MACS, Inc.
#
# Copyright (c) 2007 SiSco, Inc.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
#
# See http://www.themacs.com for more information.
#
# usage: htmldiff [[-c] [-l] [-o] oldversion newversion [output]]
#
# -c - disable metahtml comment processing
# -o - disable outputting of old text
# -l - use navindex to create sequence of diffs
# oldversion - the previous version of the document
# newversion - the newer version of the document
# output - a filename to place the output in. If omitted, the output goes to
# standard output.
#
# if invoked with no options or arguments, operates as a CGI script. It then
# takes the following parameters:
#
# oldfile - the URL of the original file
# newfile - the URL of the new file
# mhtml - a flag to indicate whether it should be aware of MetaHTML comments.
#
# requires GNU diff utility
# also requires the perl modules Getopt::Std
#
# NOTE: The markup created by htmldiff may not validate against the HTML 4.0
# DTD. This is because the algorithm is realtively simple, and there are
# places in the markup content model where the span element is not allowed.
# Htmldiff is NOT aware of these places.
#
# $Source: /u/sources/public/2009/htmldiff/htmldiff.pl,v $
# $Revision: 1.1 $
#
# $Log: htmldiff.pl,v $
# Revision 1.1 2014/01/06 08:04:51 dom
# added copy of htmldiff perl script since aptest.com repo no longer available
#
# Revision 1.5 2008/03/05 13:23:16 ahby
# Fixed a problem with leading whitespace before markup.
#
# Revision 1.4 2007/12/13 13:09:16 ahby
# Updated copyright and license.
#
# Revision 1.3 2007/12/13 12:53:34 ahby
# Changed use of span to ins and del
#
# Revision 1.2 2002/02/13 16:27:23 ahby
# Changed processing model.
# Improved handling of old text and changed styles.
#
# Revision 1.1 2000/07/12 12:20:04 ahby
# Updated to remove empty spans - this fixes validation problems under
# strict.
#
# Revision 1.11 1999/12/08 19:46:45 ahby
# Fixed validation errors introduced by placing markup where it didn't
# belong.
#
# Revision 1.10 1999/10/18 13:42:58 ahby
# Added -o to the usage message.
#
# Revision 1.9 1999/05/04 12:29:11 ahby
# Added an option to turn off the display of old text.
#
# Revision 1.8 1999/04/09 14:37:27 ahby
# Fixed a perl syntax error.
#
# Revision 1.7 1999/04/09 14:35:49 ahby
# Added reference to MACS homepage.
#
# Revision 1.6 1999/04/09 14:35:09 ahby
# Added comment about validity of generated markup.
#
# Revision 1.5 1999/02/22 22:17:54 ahby
# Changed to use stylesheets.
# Changed to rely upon span.
# Changed to work around content model problems.
#
# Revision 1.4 1999/02/08 02:32:22 ahby
# Added a copyright statement.
#
# Revision 1.3 1999/02/08 02:30:40 ahby
# Added header processing.
#
# Revision 1.2 1998/12/10 17:31:31 ahby
# Fixed to escape less-thans in change blocks and to not permit change
# markup within specific elements (like TITLE).
#
# Revision 1.1 1998/11/26 00:09:22 ahby
# Initial revision
#
#
use Getopt::Std;
sub usage {
print STDERR "htmldiff [-c] [-o] oldversion newversion [output]\n";
exit;
}
sub url_encode {
my $str = shift;
$str =~ s/([\x00-\x1f\x7F-\xFF])/
sprintf ('%%%02x', ord ($1))/eg;
return $str;
}
# markit - diff-mark the streams
#
# markit(file1, file2)
#
# markit relies upon GNUdiff to mark up the text.
#
# The markup is encoded using special control sequences:
#
# a block wrapped in control-a is deleted text
# a block wrapped in control-b is old text
# a block wrapped in control-c is new text
#
# The main processing loop attempts to wrap the text blocks in appropriate
# SPANs based upon the type of text that it is.
#
# When the loop encounters a < in the text, it stops the span. Then it outputs
# the element that is defined, then it restarts the span.
sub markit {
my $retval = "";
my($file1) = shift;
my($file2) = shift;
# my $old="<span class=\\\"diff-old-a\\\">deleted text: </span>%c'\012'%c'\001'%c'\012'%<%c'\012'%c'\001'%c'\012'";
my $old="%c'\012'%c'\001'%c'\012'%<%c'\012'%c'\001'%c'\012'";
my $new="%c'\012'%c'\003'%c'\012'%>%c'\012'%c'\003'%c'\012'";
my $unchanged="%=";
my $changed="%c'\012'%c'\001'%c'\012'%<%c'\012'%c'\001'%c'\012'%c'\004'%c'\012'%>%c'\012'%c'\004'%c'\012'";
if ($opt_o) {
$old = "";
$changed = "%c'\012'%c'\004'%c'\012'%>%c'\012'%c'\004'%c'\012'";
}
# my $old="%c'\002'<font color=\\\"purple\\\" size=\\\"-2\\\">deleted text:</font><s>%c'\012'%c'\001'%c'\012'%<%c'\012'%c'\001'%c'\012'</s>%c'\012'%c'\002'";
# my $new="%c'\002'<font color=\\\"purple\\\"><u>%c'\012'%c'\002'%>%c'\002'</u></font>%c'\002'%c'\012'";
# my $unchanged="%=";
# my $changed="%c'\002'<s>%c'\012'%c'\001'%c'\012'%<%c'\012'%c'\001'%c'\012'</s><font color=\\\"purple\\\"><u>%c'\002'%c'\012'%>%c'\012'%c'\002'</u></font>%c'\002'%c'\012'";
my @span;
$span[0]="</span>";
$span[1]="<del class=\"diff-old\">";
$span[2]="<del class=\"diff-old\">";
$span[3]="<ins class=\"diff-new\">";
$span[4]="<ins class=\"diff-chg\">";
my @diffEnd ;
$diffEnd[1] = '</del>';
$diffEnd[2] = '</del>';
$diffEnd[3] = '</ins>';
$diffEnd[4] = '</ins>';
my $diffcounter = 0;
open(FILE, qq(diff -d --old-group-format="$old" --new-group-format="$new" --changed-group-format="$changed" --unchanged-group-format="$unchanged" $file1 $file2 |)) || die("Diff failed: $!");
# system (qq(diff --old-group-format="$old" --new-group-format="$new" --changed-group-format="$changed" --unchanged-group-format="$unchanged" $file1 $file2 > /tmp/output));
my $state = 0;
my $inblock = 0;
my $temp = "";
my $lineCount = 0;
# strategy:
#
# process the output of diff...
#
# a link with control A-D means the start/end of the corresponding ordinal
# state (1-4). Resting state is state 0.
#
# While in a state, accumulate the contents for that state. When exiting the
# state, determine if it is appropriate to emit the contents with markup or
# not (basically, if the accumulated buffer contains only empty lines or lines
# with markup, then we don't want to emit the wrappers. We don't need them.
#
# Note that if there is markup in the "old" block, that markup is silently
# removed. It isn't really that interesting, and it messes up the output
# something fierce.
while (<FILE>) {
my $anchor = $opt_l ? qq[<a tabindex="$diffcounter">] : "" ;
my $anchorEnd = $opt_l ? q[</a>] : "" ;
$lineCount ++;
if ($state == 0) { # if we are resting and we find a marker,
# then we must be entering a block
if (m/^([\001-\004])/) {
$state = ord($1);
$_ = "";
}
# if (m/^\001/) {
# $state = 1;
# s/^/$span[1]/;
# } elsif (m/^\002/) {
# $state = 2;
# s/^/$span[2]/;
# } elsif (m/^\003/) {
# $state = 3;
# s/^/$span[3]/;
# } elsif (m/^\004/) {
# $state = 4;
# s/^/$span[4]/;
# }
} else {
# if we are in "old" state, remove markup
if (($state == 1) || ($state == 2)) {
s/\<.*\>//; # get rid of any old markup
s/\</&lt;/g; # escape any remaining STAG or ETAGs
s/\>/&gt;/g;
}
# if we found another marker, we must be exiting the state
if (m/^([\001-\004])/) {
if ($temp ne "") {
$_ = $span[$state] . $anchor . $temp . $anchorEnd . $diffEnd[$state] . "\n";
$temp = "";
} else {
$_ = "" ;
}
$state = 0;
} elsif (m/^\s*\</) { # otherwise, is this line markup?
# if it is markup AND we haven't seen anything else yet,
# then we will emit the markup
if ($temp eq "") {
$retval .= $_;
$_ = "";
} else { # we wrap it with the state switches and hold it
s/^/$anchorEnd$diffEnd[$state]/;
s/$/$span[$state]$anchor/;
$temp .= $_;
$_ = "";
}
} else {
if (m/.+/) {
$temp .= $_;
$_ = "";
}
}
}
s/\001//g;
s/\002//g;
s/\003//g;
s/\004//g;
if ($_ !~ m/^$/) {
$retval .= $_;
}
$diffcounter++;
}
close FILE;
$retval =~ s/$span[1]\n+$diffEnd[1]//g;
$retval =~ s/$span[2]\n+$diffEnd[2]//g;
$retval =~ s/$span[3]\n+$diffEnd[3]//g;
$retval =~ s/$span[4]\n+$diffEnd[4]//g;
$retval =~ s/$span[1]\n*$//g;
$retval =~ s/$span[2]\n*$//g;
$retval =~ s/$span[3]\n*$//g;
$retval =~ s/$span[4]\n*$//g;
return $retval;
}
sub splitit {
my $filename = shift;
my $headertmp = shift;
my $inheader=0;
my $preformatted=0;
my $inelement=0;
my $retval = "";
my $styles = q(<style type='text/css'>
.diff-old-a {
font-size: smaller;
color: red;
}
.diff-new { background-color: yellow; }
.diff-chg { background-color: lime; }
.diff-new:before,
.diff-new:after
{ content: "\2191" }
.diff-chg:before, .diff-chg:after
{ content: "\2195" }
.diff-old { text-decoration: line-through; background-color: #FBB; }
.diff-old:before,
.diff-old:after
{ content: "\2193" }
:focus { border: thin red solid}
</style>
);
if ($opt_t) {
$styles .= q(
<script type="text/javascript">
<!--
function setOldDisplay() {
for ( var s = 0; s < document.styleSheets.length; s++ ) {
var css = document.styleSheets[s];
var mydata ;
try { mydata = css.cssRules ;
if ( ! mydata ) mydata = css.rules;
for ( var r = 0; r < mydata.length; r++ ) {
if ( mydata[r].selectorText == '.diff-old' ) {
mydata[r].style.display = ( mydata[r].style.display == '' ) ? 'none'
: '';
return;
}
}
} catch(e) {} ;
}
}
-->
</script>
);
}
if ($stripheader) {
open(HEADER, ">$headertmp");
}
my $incomment = 0;
my $inhead = 1;
open(FILE, $filename) || die("File $filename cannot be opened: $!");
while (<FILE>) {
if ($inhead == 1) {
if (m/\<\/head/i) {
print HEADER $styles;
}
if (m/\<body/i) {
$inhead = 0;
print HEADER;
if ($opt_t) {
print HEADER q(
<form action=""><input type="button" onclick="setOldDisplay()" value="Show/Hide Old Content" /></form>
);
}
close HEADER;
} else {
print HEADER;
}
} else {
if ($incomment) {
if (m;-->;) {
$incomment = 0;
s/.*-->//;
} else {
next;
}
}
if (m;<!--;) {
while (m;<!--.*-->;) {
s/<!--.*?-->//;
}
if (m;<!--; ) {
$incomment = 1;
s/<!--.*//;
}
}
if (m/\<pre/i) {
$preformatted = 1;
}
if (m/\<\/pre\>/i) {
$preformatted = 0;
}
if ($preformatted) {
$retval .= $_;
} elsif ($mhtmlcomments && /^;;;/) {
$retval .= $_;
} else {
my @list = split(' ');
foreach $element (@list) {
if ($element =~ m/\<H[1-6]/i) {
# $inheader = 1;
}
if ($inheader == 0) {
$element =~ s/</\n</g;
$element =~ s/^\n//;
$element =~ s/>/>\n/g;
$element =~ s/\n$//;
$element =~ s/>\n([.,:!]+)/>$1/g;
}
if ($element =~ m/\<\/H[1-6]\>/i) {
$inheader = 0;
}
$retval .= "$element";
$inelement += ($element =~ s/</&lt;/g);
$inelement -= ($element =~ s/>/&gt;/g);
if ($inelement < 0) {
$inelement = 0;
}
if (($inelement == 0) && ($inheader == 0)) {
$retval .= "\n";
} else {
$retval .= " ";
}
}
undef @list;
}
}
}
$retval .= "\n";
close FILE;
return $retval;
}
$mhtmlcomments = 1;
sub cli {
getopts("clto") || usage();
if ($opt_c) {$mhtmlcomments = 0;}
if (@ARGV < 2) { usage(); }
$file1 = $ARGV[0];
$file2 = $ARGV[1];
$file3 = $ARGV[2];
$tmp = splitit($file1, $headertmp1);
open (FILE, ">$tmp1");
print FILE $tmp;
close FILE;
$tmp = splitit($file2, $headertmp2);
open (FILE, ">$tmp2");
print FILE $tmp;
close FILE;
$output = "";
if ($stripheader) {
open(FILE, $headertmp2);
while (<FILE>) {
$output .= $_;
}
close(FILE);
}
$output .= markit($tmp1, $tmp2);
if ($file3) {
open(FILE, ">$file3");
print FILE $output;
close FILE;
} else {
print $output;
}
}
sub cgi {
# use LWP::UserAgent;
# use CGI;
my $query = new CGI;
my $url1 = $query->param("oldfile");
my $url2 = $query->param("newfile");
my $mhtml = $query->param("mhtml");
my $file1 = "/tmp/htdcgi1.$$";
my $file2 = "/tmp/htdcgi2.$$";
my $ua = new LWP::UserAgent;
$ua->agent("MACS, Inc. HTMLdiff/0.9 " . $ua->agent);
# Create a request
my $req1 = new HTTP::Request GET => $url1;
my $res1 = $ua->request($req1, $file1);
if ($res1->is_error) {
print $res1->error_as_HTML();
print "<p>The URL $url1 could not be found. Please check it and try again.</p>";
return;
}
my $req2 = new HTTP::Request GET => $url2;
my $res2 = $ua->request($req2, $file2);
if ($res2->is_error) {
print $res2->error_as_HTML();
print "<p>The URL $url2 could not be found. Please check it and try again.</p>";
return;
}
$split1 = splitit($file1, $headertmp1);
open (FILE, ">$tmp1");
print FILE $split1;
close FILE;
$split2 = splitit($file2, $headertmp2);
open (FILE, ">$tmp2");
print FILE $split2;
close FILE;
$output = "";
if ($stripheader) {
open(FILE, $headertmp2);
while (<FILE>) {
$output .= $_;
}
close(FILE);
}
$output .= markit($tmp1, $tmp2);
my $base=$res2->base;
if ($base !~ /\/$/) {
$base =~ s/[^\/]*$//;
}
if ( $output !~ /<base/i ) {
$output =~ s/<head>/<head>\n<base href="$base">/i ||
$output =~ s/<html>/<html>\n<base href="$base">/i ;
}
print $query->header(-type=>'text/html',-nph=>1);
print $output;
unlink $file1;
unlink $file2;
}
$tmp1="/tmp/htdtmp1.$$";
$headertmp1="/tmp/htdhtmp1.$$";
$tmp2="/tmp/htdtmp2.$$";
$headertmp2="/tmp/htdhtmp2.$$";
$stripheader = 1;
if (@ARGV == 0) {
cgi(); # if no arguments, we must be operating as a cgi script
} else {
cli(); # if there are arguments, then we are operating as a CLI
}
unlink $tmp1;
unlink $headertmp1;
unlink $tmp2;
unlink $headertmp2;

@ -3,6 +3,7 @@
// - / lists open pull requests // - / lists open pull requests
// - /spec/123 which renders the spec as html at pull request 123. // - /spec/123 which renders the spec as html at pull request 123.
// - /diff/rst/123 which gives a diff of the spec's rst at pull request 123. // - /diff/rst/123 which gives a diff of the spec's rst at pull request 123.
// - /diff/html/123 which gives a diff of the spec's HTML at pull request 123.
// It is currently woefully inefficient, and there is a lot of low hanging fruit for improvement. // It is currently woefully inefficient, and there is a lot of low hanging fruit for improvement.
package main package main
@ -16,6 +17,7 @@ import (
"log" "log"
"math/rand" "math/rand"
"net/http" "net/http"
"net/url"
"os" "os"
"os/exec" "os/exec"
"path" "path"
@ -52,6 +54,10 @@ var (
allowedMembers map[string]bool allowedMembers map[string]bool
) )
func (u *User) IsTrusted() bool {
return allowedMembers[u.Login]
}
const pullsPrefix = "https://api.github.com/repos/matrix-org/matrix-doc/pulls" const pullsPrefix = "https://api.github.com/repos/matrix-org/matrix-doc/pulls"
func gitClone(url string) (string, error) { func gitClone(url string) (string, error) {
@ -74,7 +80,15 @@ func gitCheckout(path, sha string) error {
return nil return nil
} }
func lookupPullRequest(prNumber string) (*PullRequest, error) { func lookupPullRequest(url url.URL, pathPrefix string) (*PullRequest, error) {
if !strings.HasPrefix(url.Path, pathPrefix+"/") {
return nil, fmt.Errorf("invalid path passed: %s expect %s/123", url.Path, pathPrefix)
}
prNumber := url.Path[len(pathPrefix)+1:]
if strings.Contains(prNumber, "/") {
return nil, fmt.Errorf("invalid path passed: %s expect %s/123", url.Path, pathPrefix)
}
resp, err := http.Get(fmt.Sprintf("%s/%s", pullsPrefix, prNumber)) resp, err := http.Get(fmt.Sprintf("%s/%s", pullsPrefix, prNumber))
defer resp.Body.Close() defer resp.Body.Close()
if err != nil { if err != nil {
@ -100,8 +114,8 @@ func generate(dir string) error {
return nil return nil
} }
func writeError(w http.ResponseWriter, err error) { func writeError(w http.ResponseWriter, code int, err error) {
w.WriteHeader(500) w.WriteHeader(code)
io.WriteString(w, fmt.Sprintf("%v\n", err)) io.WriteString(w, fmt.Sprintf("%v\n", err))
} }
@ -122,75 +136,66 @@ func generateAt(repo, sha string) (dst string, err error) {
} }
func serveSpec(w http.ResponseWriter, req *http.Request) { func serveSpec(w http.ResponseWriter, req *http.Request) {
parts := strings.Split(req.URL.Path, "/") pr, err := lookupPullRequest(*req.URL, "/spec")
if len(parts) != 3 {
w.WriteHeader(400)
io.WriteString(w, fmt.Sprintf("Invalid path passed: %v expect /pull/123", req.URL.Path))
return
}
pr, err := lookupPullRequest(parts[2])
if err != nil { if err != nil {
writeError(w, err) writeError(w, 400, err)
return return
} }
// We're going to run whatever Python is specified in the pull request, which // We're going to run whatever Python is specified in the pull request, which
// may do bad things, so only trust people we trust. // may do bad things, so only trust people we trust.
if !allowedMembers[pr.User.Login] { if err := checkAuth(pr); err != nil {
w.WriteHeader(403) writeError(w, 403, err)
io.WriteString(w, fmt.Sprintf("%q is not a trusted pull requester", pr.User.Login))
return return
} }
dst, err := generateAt(pr.Head.Repo.CloneURL, pr.Head.SHA) dst, err := generateAt(pr.Head.Repo.CloneURL, pr.Head.SHA)
defer os.RemoveAll(dst) defer os.RemoveAll(dst)
if err != nil { if err != nil {
writeError(w, err) writeError(w, 500, err)
return return
} }
b, err := ioutil.ReadFile(path.Join(dst, "scripts/gen/specification.html")) b, err := ioutil.ReadFile(path.Join(dst, "scripts/gen/specification.html"))
if err != nil { if err != nil {
writeError(w, fmt.Errorf("Error reading spec: %v", err)) writeError(w, 500, fmt.Errorf("Error reading spec: %v", err))
return return
} }
w.Write(b) w.Write(b)
} }
func serveRstDiff(w http.ResponseWriter, req *http.Request) { func checkAuth(pr *PullRequest) error {
parts := strings.Split(req.URL.Path, "/") if !pr.User.IsTrusted() {
if len(parts) != 4 { return fmt.Errorf("%q is not a trusted pull requester", pr.User.Login)
w.WriteHeader(400)
io.WriteString(w, fmt.Sprintf("Invalid path passed: %v expect /diff/rst/123", req.URL.Path))
return
} }
return nil
}
pr, err := lookupPullRequest(parts[3]) func serveRSTDiff(w http.ResponseWriter, req *http.Request) {
pr, err := lookupPullRequest(*req.URL, "/diff/rst")
if err != nil { if err != nil {
writeError(w, err) writeError(w, 400, err)
return return
} }
// We're going to run whatever Python is specified in the pull request, which // We're going to run whatever Python is specified in the pull request, which
// may do bad things, so only trust people we trust. // may do bad things, so only trust people we trust.
if !allowedMembers[pr.User.Login] { if err := checkAuth(pr); err != nil {
w.WriteHeader(403) writeError(w, 403, err)
io.WriteString(w, fmt.Sprintf("%q is not a trusted pull requester", pr.User.Login))
return return
} }
base, err := generateAt(pr.Base.Repo.CloneURL, pr.Base.SHA) base, err := generateAt(pr.Base.Repo.CloneURL, pr.Base.SHA)
defer os.RemoveAll(base) defer os.RemoveAll(base)
if err != nil { if err != nil {
writeError(w, err) writeError(w, 500, err)
return return
} }
head, err := generateAt(pr.Head.Repo.CloneURL, pr.Head.SHA) head, err := generateAt(pr.Head.Repo.CloneURL, pr.Head.SHA)
defer os.RemoveAll(head) defer os.RemoveAll(head)
if err != nil { if err != nil {
writeError(w, err) writeError(w, 500, err)
return return
} }
@ -198,23 +203,79 @@ func serveRstDiff(w http.ResponseWriter, req *http.Request) {
var diff bytes.Buffer var diff bytes.Buffer
diffCmd.Stdout = &diff diffCmd.Stdout = &diff
if err := ignoreExitCodeOne(diffCmd.Run()); err != nil { if err := ignoreExitCodeOne(diffCmd.Run()); err != nil {
writeError(w, fmt.Errorf("error running diff: %v", err)) writeError(w, 500, fmt.Errorf("error running diff: %v", err))
return return
} }
w.Write(diff.Bytes()) w.Write(diff.Bytes())
} }
func serveHTMLDiff(w http.ResponseWriter, req *http.Request) {
pr, err := lookupPullRequest(*req.URL, "/diff/html")
if err != nil {
writeError(w, 400, err)
return
}
// We're going to run whatever Python is specified in the pull request, which
// may do bad things, so only trust people we trust.
if err := checkAuth(pr); err != nil {
writeError(w, 403, err)
return
}
base, err := generateAt(pr.Base.Repo.CloneURL, pr.Base.SHA)
defer os.RemoveAll(base)
if err != nil {
writeError(w, 500, err)
return
}
head, err := generateAt(pr.Head.Repo.CloneURL, pr.Head.SHA)
defer os.RemoveAll(head)
if err != nil {
writeError(w, 500, err)
return
}
htmlDiffer, err := findHTMLDiffer()
if err != nil {
writeError(w, 500, fmt.Errorf("could not find HTML differ"))
return
}
cmd := exec.Command(htmlDiffer, path.Join(base, "scripts", "gen", "specification.html"), path.Join(head, "scripts", "gen", "specification.html"))
var b bytes.Buffer
cmd.Stdout = &b
if err := cmd.Run(); err != nil {
writeError(w, 500, fmt.Errorf("error running HTML differ: %v", err))
return
}
w.Write(b.Bytes())
}
func findHTMLDiffer() (string, error) {
wd, err := os.Getwd()
if err != nil {
return "", err
}
differ := path.Join(wd, "htmldiff.pl")
if _, err := os.Stat(differ); err == nil {
return differ, nil
}
return "", fmt.Errorf("unable to find htmldiff.pl")
}
func listPulls(w http.ResponseWriter, req *http.Request) { func listPulls(w http.ResponseWriter, req *http.Request) {
resp, err := http.Get(pullsPrefix) resp, err := http.Get(pullsPrefix)
if err != nil { if err != nil {
writeError(w, err) writeError(w, 500, err)
return return
} }
defer resp.Body.Close() defer resp.Body.Close()
dec := json.NewDecoder(resp.Body) dec := json.NewDecoder(resp.Body)
var pulls []PullRequest var pulls []PullRequest
if err := dec.Decode(&pulls); err != nil { if err := dec.Decode(&pulls); err != nil {
writeError(w, err) writeError(w, 500, err)
return return
} }
if len(pulls) == 0 { if len(pulls) == 0 {
@ -223,8 +284,8 @@ func listPulls(w http.ResponseWriter, req *http.Request) {
} }
s := "<body><ul>" s := "<body><ul>"
for _, pull := range pulls { for _, pull := range pulls {
s += fmt.Sprintf(`<li>%d: <a href="%s">%s</a>: <a href="%s">%s</a>: <a href="spec/%d">spec</a> <a href="diff/rst/%d">rst diff</a></li>`, s += fmt.Sprintf(`<li>%d: <a href="%s">%s</a>: <a href="%s">%s</a>: <a href="spec/%d">spec</a> <a href="diff/html/%d">spec diff</a> <a href="diff/rst/%d">rst diff</a></li>`,
pull.Number, pull.User.HTMLURL, pull.User.Login, pull.HTMLURL, pull.Title, pull.Number, pull.Number) pull.Number, pull.User.HTMLURL, pull.User.Login, pull.HTMLURL, pull.Title, pull.Number, pull.Number, pull.Number)
} }
s += "</ul></body>" s += "</ul></body>"
io.WriteString(w, s) io.WriteString(w, s)
@ -257,7 +318,8 @@ func main() {
"NegativeMjark": true, "NegativeMjark": true,
} }
http.HandleFunc("/spec/", serveSpec) http.HandleFunc("/spec/", serveSpec)
http.HandleFunc("/diff/rst/", serveRstDiff) http.HandleFunc("/diff/rst/", serveRSTDiff)
http.HandleFunc("/diff/html/", serveHTMLDiff)
http.HandleFunc("/healthz", serveText("ok")) http.HandleFunc("/healthz", serveText("ok"))
http.HandleFunc("/", listPulls) http.HandleFunc("/", listPulls)
log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", *port), nil)) log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", *port), nil))

@ -38,7 +38,7 @@ Processing
""" """
from batesian import AccessKeyStore from batesian import AccessKeyStore
from jinja2 import Environment, FileSystemLoader, StrictUndefined, Template from jinja2 import Environment, FileSystemLoader, StrictUndefined, Template, meta
from argparse import ArgumentParser, FileType from argparse import ArgumentParser, FileType
import importlib import importlib
import json import json
@ -122,7 +122,19 @@ def main(input_module, file_stream=None, out_dir=None, verbose=False):
# check the input files and substitute in sections where required # check the input files and substitute in sections where required
log("Parsing input template: %s" % file_stream.name) log("Parsing input template: %s" % file_stream.name)
temp = Template(file_stream.read().decode("utf-8")) temp_str = file_stream.read().decode("utf-8")
# do sanity checking on the template to make sure they aren't reffing things
# which will never be replaced with a section.
ast = env.parse(temp_str)
template_vars = meta.find_undeclared_variables(ast)
unused_vars = [var for var in template_vars if var not in sections]
if len(unused_vars) > 0:
raise Exception(
"You have {{ variables }} which are not found in sections: %s" %
(unused_vars,)
)
# process the template
temp = Template(temp_str)
log("Creating output for: %s" % file_stream.name) log("Creating output for: %s" % file_stream.name)
output = create_from_template(temp, sections) output = create_from_template(temp, sections)
with open( with open(

Loading…
Cancel
Save