Add Shippable request signing to ansible-test. (#69526)

changelogs/fragments/ansible-test-ci-support-shippable-auth.yml

@@ -0,0 +1,2 @@
+minor_changes:
+    - ansible-test - Added support for Ansible Core CI request signing for Shippable.

test/lib/ansible_test/_internal/ci/__init__.py

@@ -3,10 +3,24 @@ from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
 import abc
+import base64
+import json
+import os
+import tempfile
 
 
 from .. import types as t
 
+from ..encoding import (
+    to_bytes,
+    to_text,
+)
+
+from ..io import (
+    read_text_file,
+    write_text_file,
+)
+
 from ..config import (
     CommonConfig,
     TestConfig,
@@ -18,6 +32,7 @@ from ..util import (
     display,
     get_subclasses,
     import_plugins,
+    raw_command,
 )
 
 
@@ -99,3 +114,71 @@ def get_ci_provider():  # type: () -> CIProvider
     get_ci_provider.provider = provider
 
     return provider
+
+
+class AuthHelper(ABC):
+    """Public key based authentication helper for Ansible Core CI."""
+    def sign_request(self, request):  # type: (t.Dict[str, t.Any]) -> None
+        """Sign the given auth request and make the public key available."""
+        payload_bytes = to_bytes(json.dumps(request, sort_keys=True))
+        signature_raw_bytes = self.sign_bytes(payload_bytes)
+        signature = to_text(base64.b64encode(signature_raw_bytes))
+
+        request.update(signature=signature)
+
+    def initialize_private_key(self):  # type: () -> str
+        """
+        Initialize and publish a new key pair (if needed) and return the private key.
+        The private key is cached across ansible-test invocations so it is only generated and published once per CI job.
+        """
+        path = os.path.expanduser('~/.ansible-core-ci-private.key')
+
+        if os.path.exists(to_bytes(path)):
+            private_key_pem = read_text_file(path)
+        else:
+            private_key_pem = self.generate_private_key()
+            write_text_file(path, private_key_pem)
+
+        return private_key_pem
+
+    @abc.abstractmethod
+    def sign_bytes(self, payload_bytes):  # type: (bytes) -> bytes
+        """Sign the given payload and return the signature, initializing a new key pair if required."""
+
+    @abc.abstractmethod
+    def publish_public_key(self, public_key_pem):  # type: (str) -> None
+        """Publish the given public key."""
+
+    @abc.abstractmethod
+    def generate_private_key(self):  # type: () -> str
+        """Generate a new key pair, publishing the public key and returning the private key."""
+
+
+class OpenSSLAuthHelper(AuthHelper, ABC):  # pylint: disable=abstract-method
+    """OpenSSL based public key based authentication helper for Ansible Core CI."""
+    def sign_bytes(self, payload_bytes):  # type: (bytes) -> bytes
+        """Sign the given payload and return the signature, initializing a new key pair if required."""
+        private_key_pem = self.initialize_private_key()
+
+        with tempfile.NamedTemporaryFile() as private_key_file:
+            private_key_file.write(to_bytes(private_key_pem))
+            private_key_file.flush()
+
+            with tempfile.NamedTemporaryFile() as payload_file:
+                payload_file.write(payload_bytes)
+                payload_file.flush()
+
+                with tempfile.NamedTemporaryFile() as signature_file:
+                    raw_command(['openssl', 'dgst', '-sha256', '-sign', private_key_file.name, '-out', signature_file.name, payload_file.name], capture=True)
+                    signature_raw_bytes = signature_file.read()
+
+        return signature_raw_bytes
+
+    def generate_private_key(self):  # type: () -> str
+        """Generate a new key pair, publishing the public key and returning the private key."""
+        private_key_pem = raw_command(['openssl', 'ecparam', '-genkey', '-name', 'secp384r1', '-noout'], capture=True)[0]
+        public_key_pem = raw_command(['openssl', 'ec', '-pubout'], data=private_key_pem, capture=True)[0]
+
+        self.publish_public_key(public_key_pem)
+
+        return private_key_pem

test/lib/ansible_test/_internal/ci/shippable.py

@@ -4,6 +4,7 @@ __metaclass__ = type
 
 import os
 import re
+import time
 
 from .. import types as t
 
@@ -32,6 +33,7 @@ from . import (
     AuthContext,
     ChangeDetectionNotSupported,
     CIProvider,
+    OpenSSLAuthHelper,
 )
 
 
@@ -40,6 +42,9 @@ CODE = 'shippable'
 
 class Shippable(CIProvider):
     """CI provider implementation for Shippable."""
+    def __init__(self):
+        self.auth = ShippableAuthHelper()
+
     @staticmethod
     def is_supported():  # type: () -> bool
         """Return True if this provider is supported in the current running environment."""
@@ -114,6 +119,8 @@ class Shippable(CIProvider):
         except KeyError as ex:
             raise MissingEnvironmentVariable(name=ex.args[0])
 
+        self.auth.sign_request(request)
+
         auth = dict(
             shippable=request,
         )
@@ -184,6 +191,19 @@ class Shippable(CIProvider):
         return last_commit
 
 
+class ShippableAuthHelper(OpenSSLAuthHelper):
+    """
+    Authentication helper for Shippable.
+    Based on OpenSSL since cryptography is not provided by the default Shippable environment.
+    """
+    def publish_public_key(self, public_key_pem):  # type: (str) -> None
+        """Publish the given public key."""
+        # display the public key as a single line to avoid mangling such as when prefixing each line with a timestamp
+        display.info(public_key_pem.replace('\n', ' '))
+        # allow time for logs to become available to reduce repeated API calls
+        time.sleep(3)
+
+
 class ShippableChanges:
     """Change information for Shippable build."""
     def __init__(self, args):  # type: (CommonConfig) -> None

test/units/ansible_test/ci/test_shippable.py

@@ -0,0 +1,31 @@
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+from .util import common_auth_test
+
+
+def test_auth():
+    # noinspection PyProtectedMember
+    from ansible_test._internal.ci.shippable import (
+        ShippableAuthHelper,
+    )
+
+    class TestShippableAuthHelper(ShippableAuthHelper):
+        def __init__(self):
+            self.public_key_pem = None
+            self.private_key_pem = None
+
+        def publish_public_key(self, public_key_pem):
+            # avoid publishing key
+            self.public_key_pem = public_key_pem
+
+        def initialize_private_key(self):
+            # cache in memory instead of on disk
+            if not self.private_key_pem:
+                self.private_key_pem = self.generate_private_key()
+
+            return self.private_key_pem
+
+    auth = TestShippableAuthHelper()
+
+    common_auth_test(auth)

test/units/ansible_test/ci/util.py

@@ -0,0 +1,53 @@
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import base64
+import json
+import re
+
+
+def common_auth_test(auth):
+    private_key_pem = auth.initialize_private_key()
+    public_key_pem = auth.public_key_pem
+
+    extract_pem_key(private_key_pem, private=True)
+    extract_pem_key(public_key_pem, private=False)
+
+    request = dict(hello='World')
+    auth.sign_request(request)
+
+    verify_signature(request, public_key_pem)
+
+
+def extract_pem_key(value, private):
+    assert isinstance(value, type(u''))
+
+    key_type = '(EC )?PRIVATE' if private else 'PUBLIC'
+    pattern = r'^-----BEGIN ' + key_type + r' KEY-----\n(?P<key>.*?)\n-----END ' + key_type + r' KEY-----\n$'
+    match = re.search(pattern, value, flags=re.DOTALL)
+
+    assert match, 'key "%s" does not match pattern "%s"' % (value, pattern)
+
+    base64.b64decode(match.group('key'))  # make sure the key can be decoded
+
+
+def verify_signature(request, public_key_pem):
+    signature = request.pop('signature')
+    payload_bytes = json.dumps(request, sort_keys=True).encode()
+
+    assert isinstance(signature, type(u''))
+
+    from cryptography.hazmat.backends import default_backend
+    from cryptography.hazmat.primitives import hashes
+    from cryptography.hazmat.primitives.asymmetric import ec
+    from cryptography.hazmat.primitives.serialization import load_pem_public_key
+
+    public_key = load_pem_public_key(public_key_pem.encode(), default_backend())
+
+    verifier = public_key.verifier(
+        base64.b64decode(signature.encode()),
+        ec.ECDSA(hashes.SHA256()),
+    )
+
+    verifier.update(payload_bytes)
+    verifier.verify()

test/units/ansible_test/conftest.py

@@ -0,0 +1,14 @@
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+
+import os
+import pytest
+import sys
+
+
+@pytest.fixture(autouse=True, scope='session')
+def ansible_test():
+    """Make ansible_test available on sys.path for unit testing ansible-test."""
+    test_lib = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), 'lib')
+    sys.path.insert(0, test_lib)