From a7d7df145015298627679c9cab67760ba14e1876 Mon Sep 17 00:00:00 2001
From: Matt Clay <matt@mystile.com>
Date: Wed, 9 May 2018 16:47:24 -0700
Subject: [PATCH] Make docs-build sanity test disabled by default.

---
 test/runner/lib/config.py              |  1 +
 test/runner/lib/sanity/__init__.py     | 36 +++++++++++++++++---------
 test/runner/test.py                    |  4 +++
 test/sanity/code-smell/docs-build.json |  1 +
 test/utils/shippable/sanity.sh         |  2 +-
 5 files changed, 31 insertions(+), 13 deletions(-)

diff --git a/test/runner/lib/config.py b/test/runner/lib/config.py
index ba9e7442d22..5da9eb64d19 100644
--- a/test/runner/lib/config.py
+++ b/test/runner/lib/config.py
@@ -139,6 +139,7 @@ class SanityConfig(TestConfig):
         self.test = args.test  # type: list [str]
         self.skip_test = args.skip_test  # type: list [str]
         self.list_tests = args.list_tests  # type: bool
+        self.allow_disabled = args.allow_disabled  # type: bool
 
         if args.base_branch:
             self.base_branch = args.base_branch  # str
diff --git a/test/runner/lib/sanity/__init__.py b/test/runner/lib/sanity/__init__.py
index f903b6eaa6a..57d237cf4ad 100644
--- a/test/runner/lib/sanity/__init__.py
+++ b/test/runner/lib/sanity/__init__.py
@@ -72,6 +72,12 @@ def command_sanity(args):
 
     if args.test:
         tests = [t for t in tests if t.name in args.test]
+    else:
+        disabled = [t.name for t in tests if not t.enabled and not args.allow_disabled]
+        tests = [t for t in tests if t.enabled or args.allow_disabled]
+
+        if disabled:
+            display.warning('Skipping tests disabled by default without --allow-disabled: %s' % ', '.join(sorted(disabled)))
 
     if args.skip_test:
         tests = [t for t in tests if t.name not in args.skip_test]
@@ -203,18 +209,27 @@ class SanityTest(ABC):
 
     def __init__(self, name):
         self.name = name
+        self.enabled = True
 
 
 class SanityCodeSmellTest(SanityTest):
     """Sanity test script."""
     def __init__(self, path):
         name = os.path.splitext(os.path.basename(path))[0]
-        config = os.path.splitext(path)[0] + '.json'
+        config_path = os.path.splitext(path)[0] + '.json'
+
+        super(SanityCodeSmellTest, self).__init__(name)
 
         self.path = path
-        self.config = config if os.path.exists(config) else None
+        self.config_path = config_path if os.path.exists(config_path) else None
+        self.config = None
 
-        super(SanityCodeSmellTest, self).__init__(name)
+        if self.config_path:
+            with open(self.config_path, 'r') as config_fd:
+                self.config = json.load(config_fd)
+
+        if self.config:
+            self.enabled = not self.config.get('disabled')
 
     def test(self, args, targets):
         """
@@ -233,15 +248,12 @@ class SanityCodeSmellTest(SanityTest):
         data = None
 
         if self.config:
-            with open(self.config, 'r') as config_fd:
-                config = json.load(config_fd)
-
-            output = config.get('output')
-            extensions = config.get('extensions')
-            prefixes = config.get('prefixes')
-            files = config.get('files')
-            always = config.get('always')
-            text = config.get('text')
+            output = self.config.get('output')
+            extensions = self.config.get('extensions')
+            prefixes = self.config.get('prefixes')
+            files = self.config.get('files')
+            always = self.config.get('always')
+            text = self.config.get('text')
 
             if output == 'path-line-column-message':
                 pattern = '^(?P<path>[^:]*):(?P<line>[0-9]+):(?P<column>[0-9]+): (?P<message>.*)$'
diff --git a/test/runner/test.py b/test/runner/test.py
index aacc38f1bdd..95bb33f3a12 100755
--- a/test/runner/test.py
+++ b/test/runner/test.py
@@ -357,6 +357,10 @@ def parse_args():
                         choices=[test.name for test in sanity_get_tests()],
                         help='tests to skip').completer = complete_sanity_test
 
+    sanity.add_argument('--allow-disabled',
+                        action='store_true',
+                        help='allow tests to run which are disabled by default')
+
     sanity.add_argument('--list-tests',
                         action='store_true',
                         help='list available tests')
diff --git a/test/sanity/code-smell/docs-build.json b/test/sanity/code-smell/docs-build.json
index 376970e6824..9b608c91f97 100644
--- a/test/sanity/code-smell/docs-build.json
+++ b/test/sanity/code-smell/docs-build.json
@@ -1,4 +1,5 @@
 {
+    "disabled": true,
     "always": true,
     "output": "path-line-column-message"
 }
diff --git a/test/utils/shippable/sanity.sh b/test/utils/shippable/sanity.sh
index d62251b5a5e..e17cea48439 100755
--- a/test/utils/shippable/sanity.sh
+++ b/test/utils/shippable/sanity.sh
@@ -23,4 +23,4 @@ esac
 # shellcheck disable=SC2086
 ansible-test sanity --color -v --junit ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} \
     --docker --docker-keep-git --base-branch "${base_branch}" \
-    "${options[@]}"
+    "${options[@]}" --allow-disabled