Merge pull request #1626 from matrix-org/rav/encoding

Set the encoding to UTF-8 on all file I/O
pull/1557/head^2
Richard van der Hoff 6 years ago committed by GitHub
commit f649d04ae1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -154,7 +154,7 @@ def get_rst(file_info, title_level, title_styles, spec_dir, adjust_titles):
# string are file paths to RST blobs # string are file paths to RST blobs
if isinstance(file_info, str): if isinstance(file_info, str):
log("%s %s" % (">" * (1 + title_level), file_info)) log("%s %s" % (">" * (1 + title_level), file_info))
with open(os.path.join(spec_dir, file_info), "r") as f: with open(os.path.join(spec_dir, file_info), "r", encoding="utf-8") as f:
rst = None rst = None
if adjust_titles: if adjust_titles:
rst = load_with_adjusted_titles( rst = load_with_adjusted_titles(
@ -186,7 +186,7 @@ def get_rst(file_info, title_level, title_styles, spec_dir, adjust_titles):
def build_spec(target, out_filename): def build_spec(target, out_filename):
log("Building templated file %s" % out_filename) log("Building templated file %s" % out_filename)
with open(out_filename, "wb") as outfile: with open(out_filename, "w", encoding="utf-8") as outfile:
for file_info in target["files"]: for file_info in target["files"]:
section = get_rst( section = get_rst(
file_info=file_info, file_info=file_info,
@ -195,7 +195,7 @@ def build_spec(target, out_filename):
spec_dir=spec_dir, spec_dir=spec_dir,
adjust_titles=True adjust_titles=True
) )
outfile.write(section.encode('UTF-8')) outfile.write(section)
""" """
@ -223,8 +223,8 @@ def fix_relative_titles(target, filename, out_filename):
"^[" + re.escape("".join(title_styles)) + "]{3,}$" "^[" + re.escape("".join(title_styles)) + "]{3,}$"
) )
current_title_style = None current_title_style = None
with open(filename, "r") as infile: with open(filename, "r", encoding="utf-8") as infile:
with open(out_filename, "w") as outfile: with open(out_filename, "w", encoding="utf-8") as outfile:
for line in infile.readlines(): for line in infile.readlines():
if not relative_title_matcher.match(line): if not relative_title_matcher.match(line):
if title_matcher.match(line): if title_matcher.match(line):
@ -263,8 +263,8 @@ def fix_relative_titles(target, filename, out_filename):
def rst2html(i, o, stylesheets): def rst2html(i, o, stylesheets):
log("rst2html %s -> %s" % (i, o)) log("rst2html %s -> %s" % (i, o))
with open(i, "r") as in_file: with open(i, "r", encoding="utf-8") as in_file:
with open(o, "w") as out_file: with open(o, "w", encoding="utf-8") as out_file:
publish_file( publish_file(
source=in_file, source=in_file,
destination=out_file, destination=out_file,
@ -280,16 +280,15 @@ def rst2html(i, o, stylesheets):
def addAnchors(path): def addAnchors(path):
log("add anchors %s" % path) log("add anchors %s" % path)
with open(path, "rb") as f: with open(path, "r", encoding="utf-8") as f:
lines = f.readlines() lines = f.readlines()
replacement = r'<p><a class="anchor" id="\2"></a></p>\n\1' replacement = r'<p><a class="anchor" id="\2"></a></p>\n\1'
with open(path, "wb") as f: with open(path, "w", encoding="utf-8") as f:
for line in lines: for line in lines:
line = line.decode("UTF-8")
line = re.sub(r'(<h\d id="#?(.*?)">)', replacement, line.rstrip()) line = re.sub(r'(<h\d id="#?(.*?)">)', replacement, line.rstrip())
line = re.sub(r'(<div class="section" id="(.*?)">)', replacement, line.rstrip()) line = re.sub(r'(<div class="section" id="(.*?)">)', replacement, line.rstrip())
f.write((line + "\n").encode('UTF-8')) f.write(line + "\n")
def run_through_template(input_files, set_verbose, substitutions): def run_through_template(input_files, set_verbose, substitutions):

@ -125,7 +125,7 @@ def resolve_references(path, schema):
if '$ref' in schema: if '$ref' in schema:
value = schema['$ref'] value = schema['$ref']
path = os.path.join(os.path.dirname(path), value) path = os.path.join(os.path.dirname(path), value)
with open(path) as f: with open(path, encoding="utf-8") as f:
ref = yaml.load(f, OrderedLoader) ref = yaml.load(f, OrderedLoader)
result = resolve_references(path, ref) result = resolve_references(path, ref)
del schema['$ref'] del schema['$ref']
@ -664,11 +664,11 @@ class MatrixUnits(Units):
continue continue
filepath = os.path.join(path, filename) filepath = os.path.join(path, filename)
logger.info("Reading swagger API: %s" % filepath) logger.info("Reading swagger API: %s" % filepath)
with open(filepath, "r") as f: with open(filepath, "r", encoding="utf-8") as f:
# strip .yaml # strip .yaml
group_name = filename[:-5].replace("-", "_") group_name = filename[:-5].replace("-", "_")
group_name = "%s_%s" % (group_name, suffix) group_name = "%s_%s" % (group_name, suffix)
api = yaml.load(f.read(), OrderedLoader) api = yaml.load(f, OrderedLoader)
api = resolve_references(filepath, api) api = resolve_references(filepath, api)
api["__meta"] = self._load_swagger_meta( api["__meta"] = self._load_swagger_meta(
api, group_name api, group_name
@ -698,11 +698,11 @@ class MatrixUnits(Units):
continue continue
filepath = os.path.join(path, filename) filepath = os.path.join(path, filename)
logger.info("Reading swagger definition: %s" % filepath) logger.info("Reading swagger definition: %s" % filepath)
with open(filepath, "r") as f: with open(filepath, "r", encoding="utf-8") as f:
# strip .yaml # strip .yaml
group_name = re.sub(r"[^a-zA-Z0-9_]", "_", filename[:-5]) group_name = re.sub(r"[^a-zA-Z0-9_]", "_", filename[:-5])
group_name = "%s_%s" % (prefix, group_name) group_name = "%s_%s" % (prefix, group_name)
definition = yaml.load(f.read(), OrderedLoader) definition = yaml.load(f, OrderedLoader)
definition = resolve_references(filepath, definition) definition = resolve_references(filepath, definition)
if 'type' not in definition: if 'type' not in definition:
continue continue
@ -741,7 +741,7 @@ class MatrixUnits(Units):
event_type = filename[:-5] # strip the ".yaml" event_type = filename[:-5] # strip the ".yaml"
logger.info("Reading event schema: %s" % filepath) logger.info("Reading event schema: %s" % filepath)
with open(filepath) as f: with open(filepath, encoding="utf-8") as f:
event_schema = yaml.load(f, OrderedLoader) event_schema = yaml.load(f, OrderedLoader)
schema_info = process_data_type( schema_info = process_data_type(
@ -793,7 +793,7 @@ class MatrixUnits(Units):
filepath = os.path.join(path, filename) filepath = os.path.join(path, filename)
logger.info("Reading event example: %s" % filepath) logger.info("Reading event example: %s" % filepath)
try: try:
with open(filepath, "r") as f: with open(filepath, "r", encoding="utf-8") as f:
example = json.load(f) example = json.load(f)
examples[filename] = examples.get(filename, []) examples[filename] = examples.get(filename, [])
examples[filename].append(example) examples[filename].append(example)
@ -831,7 +831,7 @@ class MatrixUnits(Units):
def read_event_schema(self, filepath): def read_event_schema(self, filepath):
logger.info("Reading %s" % filepath) logger.info("Reading %s" % filepath)
with open(filepath, "r") as f: with open(filepath, "r", encoding="utf-8") as f:
json_schema = yaml.load(f, OrderedLoader) json_schema = yaml.load(f, OrderedLoader)
schema = { schema = {
@ -943,7 +943,7 @@ class MatrixUnits(Units):
title_part = None title_part = None
changelog_lines = [] changelog_lines = []
with open(path, "r") as f: with open(path, "r", encoding="utf-8") as f:
lines = f.readlines() lines = f.readlines()
prev_line = None prev_line = None
for line in (tc_lines + lines): for line in (tc_lines + lines):

Loading…
Cancel
Save