Merge pull request #1626 from matrix-org/rav/encoding

Set the encoding to UTF-8 on all file I/O

scripts/gendoc.py

@@ -154,7 +154,7 @@ def get_rst(file_info, title_level, title_styles, spec_dir, adjust_titles):
     # string are file paths to RST blobs
     if isinstance(file_info, str):
         log("%s %s" % (">" * (1 + title_level), file_info))
-        with open(os.path.join(spec_dir, file_info), "r") as f:
+        with open(os.path.join(spec_dir, file_info), "r", encoding="utf-8") as f:
             rst = None
             if adjust_titles:
                 rst = load_with_adjusted_titles(
@@ -186,7 +186,7 @@ def get_rst(file_info, title_level, title_styles, spec_dir, adjust_titles):
 
 def build_spec(target, out_filename):
     log("Building templated file %s" % out_filename)
-    with open(out_filename, "wb") as outfile:
+    with open(out_filename, "w", encoding="utf-8") as outfile:
         for file_info in target["files"]:
             section = get_rst(
                 file_info=file_info,
@@ -195,7 +195,7 @@ def build_spec(target, out_filename):
                 spec_dir=spec_dir,
                 adjust_titles=True
             )
-            outfile.write(section.encode('UTF-8'))
+            outfile.write(section)
 
 
 """
@@ -223,8 +223,8 @@ def fix_relative_titles(target, filename, out_filename):
         "^[" + re.escape("".join(title_styles)) + "]{3,}$"
     )
     current_title_style = None
-    with open(filename, "r") as infile:
+    with open(filename, "r", encoding="utf-8") as infile:
-        with open(out_filename, "w") as outfile:
+        with open(out_filename, "w", encoding="utf-8") as outfile:
             for line in infile.readlines():
                 if not relative_title_matcher.match(line):
                     if title_matcher.match(line):
@@ -263,8 +263,8 @@ def fix_relative_titles(target, filename, out_filename):
 
 def rst2html(i, o, stylesheets):
     log("rst2html %s -> %s" % (i, o))
-    with open(i, "r") as in_file:
+    with open(i, "r", encoding="utf-8") as in_file:
-        with open(o, "w") as out_file:
+        with open(o, "w", encoding="utf-8") as out_file:
             publish_file(
                 source=in_file,
                 destination=out_file,
@@ -280,16 +280,15 @@ def rst2html(i, o, stylesheets):
 def addAnchors(path):
     log("add anchors %s" % path)
 
-    with open(path, "rb") as f:
+    with open(path, "r", encoding="utf-8") as f:
         lines = f.readlines()
 
     replacement = r'<p><a class="anchor" id="\2"></a></p>\n\1'
-    with open(path, "wb") as f:
+    with open(path, "w", encoding="utf-8") as f:
         for line in lines:
-            line = line.decode("UTF-8")
             line = re.sub(r'(<h\d id="#?(.*?)">)', replacement, line.rstrip())
             line = re.sub(r'(<div class="section" id="(.*?)">)', replacement, line.rstrip())
-            f.write((line + "\n").encode('UTF-8'))
+            f.write(line + "\n")
 
 
 def run_through_template(input_files, set_verbose, substitutions):

scripts/templating/matrix_templates/units.py

@@ -125,7 +125,7 @@ def resolve_references(path, schema):
         if '$ref' in schema:
             value = schema['$ref']
             path = os.path.join(os.path.dirname(path), value)
-            with open(path) as f:
+            with open(path, encoding="utf-8") as f:
                 ref = yaml.load(f, OrderedLoader)
             result = resolve_references(path, ref)
             del schema['$ref']
@@ -664,11 +664,11 @@ class MatrixUnits(Units):
                     continue
                 filepath = os.path.join(path, filename)
                 logger.info("Reading swagger API: %s" % filepath)
-                with open(filepath, "r") as f:
+                with open(filepath, "r", encoding="utf-8") as f:
                     # strip .yaml
                     group_name = filename[:-5].replace("-", "_")
                     group_name = "%s_%s" % (group_name, suffix)
-                    api = yaml.load(f.read(), OrderedLoader)
+                    api = yaml.load(f, OrderedLoader)
                     api = resolve_references(filepath, api)
                     api["__meta"] = self._load_swagger_meta(
                         api, group_name
@@ -698,11 +698,11 @@ class MatrixUnits(Units):
                 continue
             filepath = os.path.join(path, filename)
             logger.info("Reading swagger definition: %s" % filepath)
-            with open(filepath, "r") as f:
+            with open(filepath, "r", encoding="utf-8") as f:
                 # strip .yaml
                 group_name = re.sub(r"[^a-zA-Z0-9_]", "_", filename[:-5])
                 group_name = "%s_%s" % (prefix, group_name)
-                definition = yaml.load(f.read(), OrderedLoader)
+                definition = yaml.load(f, OrderedLoader)
                 definition = resolve_references(filepath, definition)
                 if 'type' not in definition:
                     continue
@@ -741,7 +741,7 @@ class MatrixUnits(Units):
             event_type = filename[:-5]  # strip the ".yaml"
             logger.info("Reading event schema: %s" % filepath)
 
-            with open(filepath) as f:
+            with open(filepath, encoding="utf-8") as f:
                 event_schema = yaml.load(f, OrderedLoader)
 
             schema_info = process_data_type(
@@ -793,7 +793,7 @@ class MatrixUnits(Units):
             filepath = os.path.join(path, filename)
             logger.info("Reading event example: %s" % filepath)
             try:
-                with open(filepath, "r") as f:
+                with open(filepath, "r", encoding="utf-8") as f:
                     example = json.load(f)
                     examples[filename] = examples.get(filename, [])
                     examples[filename].append(example)
@@ -831,7 +831,7 @@ class MatrixUnits(Units):
     def read_event_schema(self, filepath):
         logger.info("Reading %s" % filepath)
 
-        with open(filepath, "r") as f:
+        with open(filepath, "r", encoding="utf-8") as f:
             json_schema = yaml.load(f, OrderedLoader)
 
         schema = {
@@ -943,7 +943,7 @@ class MatrixUnits(Units):
 
             title_part = None
             changelog_lines = []
-            with open(path, "r") as f:
+            with open(path, "r", encoding="utf-8") as f:
                 lines = f.readlines()
             prev_line = None
             for line in (tc_lines + lines):