Misc improvements

meta/releasing_a_spec.md

@@ -19,8 +19,8 @@ The remainder of the process is as follows:
 1. Having checked out the new release branch, navigate your way over to `./changelogs`.
 1. Follow the release instructions provided in the README.md located there.
 1. Update any version/link references across all specifications.
-1. Generate the specification using `./scripts/gendoc.py -c r0.4.0`, specifying all the
+1. Generate the specification using `./scripts/gendoc.py`, specifying all the
-   API versions at the time of generation.
+   API versions at the time of generation. For example: `./scripts/gendoc.py -c r0.4.0 -s r0.1.0 -i r0.1.0 #etc`
 1. PR the changes to the matrix-org/matrix.org repository (for historic tracking).
    * This is done by making a PR to the `unstyled_docs/spec` folder for the version and
      specification you're releasing.

scripts/templating/matrix_templates/units.py

@@ -935,92 +935,80 @@ class MatrixUnits(Units):
             changelog_lines = []
             if target_version == 'unstable':
                 # generate towncrier log
-                tc_path = os.path.join(CHANGELOG_DIR, api_name)
+                changelog_lines = self._read_towncrier_changelog(api_name)
-                if os.path.isdir(tc_path):
-                    logger.info("Generating towncrier changelog for: %s" % api_name)
-                    p = subprocess.Popen(
-                        ['towncrier', '--version', 'unstable', '--name', api_name, '--draft'],
-                        cwd=tc_path,
-                        stderr=subprocess.PIPE,
-                        stdout=subprocess.PIPE,
-                    )
-                    stdout, stderr = p.communicate()
-                    if p.returncode != 0:
-                        # Something broke - dump as much information as we can
-                        logger.error("Towncrier exited with code %s" % p.returncode)
-                        logger.error(stdout.decode('UTF-8'))
-                        logger.error(stderr.decode('UTF-8'))
-                        raw_log = ""
-                    else:
-                        raw_log = stdout.decode('UTF-8')
-
-                        # This is a bit of a hack, but it does mean that the log at least gets *something*
-                        # to tell us it broke
-                        if not raw_log.startswith("unstable"):
-                            logger.error("Towncrier appears to have failed to generate a changelog")
-                            logger.error(raw_log)
-                            raw_log = ""
-                    changelog_lines = raw_log.splitlines()
             else:
                 # read in the existing RST changelog
-                logger.info("Reading changelog RST for %s" % api_name)
+                changelog_lines = self._read_rst_changelog(api_name)
-                rst_path = os.path.join(CHANGELOG_DIR, "%s.rst" % api_name)
-                with open(rst_path, 'r', encoding="utf-8") as f:
-                    changelog_lines = f.readlines()
 
             # Parse the changelog lines to find the header we're looking for and therefore
             # the changelog body.
             prev_line = None
             title_part = None
             changelog_body_lines = []
-            have_changelog = False
             for line in changelog_lines:
                 if prev_line is None:
                     prev_line = line
                     continue
-                if not title_part:
+                if re.match("^[=]{3,}$", line.strip()):
-                    # Titles we care about are underlined with at least 3 equal signs
+                    # the last line was a header - use that as our new title_part
-                    if re.match("^[=]{3,}$", line.strip()):
+                    title_part = prev_line.strip()
-                        logger.info("Found header %s" % prev_line)
+                    continue
-                        title_part = prev_line.strip()
+                if re.match("^[-]{3,}$", line.strip()):
-                        continue
+                    # the last line is a subheading - drop this line because it's the underline
-                    prev_line = line
+                    # and that causes problems with rendering. We'll keep the header text though.
-                else:
+                    continue
-                    # we have a title, start parsing the body
+                if line.strip().startswith(".. "):
-                    if re.match("^[=]{3,}$", line.strip()):
+                    # skip comments
-                        # we hit another title. prev_line will be the new section's header.
+                    continue
-                        # do a check to see if the section we just read is the one we want - if
+                if title_part == target_version:
-                        # it is, use that changelog and move on. If it isn't, keep reading.
-                        if title_part == target_version:
-                            changelogs[api_name] = "".join(changelog_body_lines)
-                            have_changelog = True
-                            break
-                        # not the section we want - start the next section
-                        title_part = changelog_body_lines.pop().strip()
-                        changelog_body_lines = []
-                        continue
-                    if re.match("^[-]{3,}$", line.strip()):
-                        # the last line is a subheading - drop this line because it's the underline
-                        # and that causes problems with rendering. We'll keep the header text though.
-                        continue
-                    if line.strip().startswith(".. "):
-                        # skip comments
-                        continue
                     # if we made it this far, append the line to the changelog body. We indent it so
                     # that it renders correctly in the section. We also add newlines so that there's
                     # intentionally blank lines that make rst2html happy.
                     changelog_body_lines.append("    " + line + '\n')
-            # do some quick checks to see if the last read section is our changelog
+
-            if not have_changelog:
+            if len(changelog_body_lines) > 0:
-                logger.info("No changelog - testing %s == %s" % (target_version, title_part,))
+                changelogs[api_name] = "".join(changelog_body_lines)
-                if title_part == target_version and len(changelog_body_lines) > 0:
+            else:
-                    changelogs[api_name] = "".join(changelog_body_lines)
+                raise ValueError("No changelog for %s at %s" % (api_name, target_version,))
-                else:
-                    raise ValueError("No changelog for %s at %s" % (api_name, target_version,))
 
         # return our `dict[api_name] => changelog` as the last step.
         return changelogs
 
+    def _read_towncrier_changelog(self, api_name):
+        tc_path = os.path.join(CHANGELOG_DIR, api_name)
+        if os.path.isdir(tc_path):
+            logger.info("Generating towncrier changelog for: %s" % api_name)
+            p = subprocess.Popen(
+                ['towncrier', '--version', 'unstable', '--name', api_name, '--draft'],
+                cwd=tc_path,
+                stderr=subprocess.PIPE,
+                stdout=subprocess.PIPE,
+            )
+            stdout, stderr = p.communicate()
+            if p.returncode != 0:
+                # Something broke - dump as much information as we can
+                logger.error("Towncrier exited with code %s" % p.returncode)
+                logger.error(stdout.decode('UTF-8'))
+                logger.error(stderr.decode('UTF-8'))
+                raw_log = ""
+            else:
+                raw_log = stdout.decode('UTF-8')
+
+                # This is a bit of a hack, but it does mean that the log at least gets *something*
+                # to tell us it broke
+                if not raw_log.startswith("unstable"):
+                    logger.error("Towncrier appears to have failed to generate a changelog")
+                    logger.error(raw_log)
+                    raw_log = ""
+            return raw_log.splitlines()
+        return []
+
+    def _read_rst_changelog(self, api_name):
+        logger.info("Reading changelog RST for %s" % api_name)
+        rst_path = os.path.join(CHANGELOG_DIR, "%s.rst" % api_name)
+        with open(rst_path, 'r', encoding="utf-8") as f:
+            return f.readlines()
+
     def load_unstable_warnings(self, substitutions):
         warning = """
 .. WARNING::