ansible/test/runner/lib/core_ci.py

"""Access Ansible Core CI remote services."""

from __future__ import absolute_import, print_function

import json
import os
import traceback
import uuid
import errno
import time

from lib.http import (
    HttpClient,
    HttpResponse,
    HttpError,
)

from lib.util import (
    ApplicationError,
    run_command,
    make_dirs,
    CommonConfig,
    display,
    is_shippable,
)


class AnsibleCoreCI(object):
    """Client for Ansible Core CI services."""
    def __init__(self, args, platform, version, stage='prod', persist=True, name=None):
        """
        :type args: CommonConfig
        :type platform: str
        :type version: str
        :type stage: str
        :type persist: bool
        :type name: str
        """
        self.args = args
        self.platform = platform
        self.version = version
        self.stage = stage
        self.client = HttpClient(args)
        self.connection = None
        self.instance_id = None
        self.name = name if name else '%s-%s' % (self.platform, self.version)

        if self.platform == 'windows':
            self.ssh_key = None
            self.endpoint = 'https://14blg63h2i.execute-api.us-east-1.amazonaws.com'
            self.port = 5986
        elif self.platform == 'freebsd':
            self.ssh_key = SshKey(args)
            self.endpoint = 'https://14blg63h2i.execute-api.us-east-1.amazonaws.com'
            self.port = 22
        elif self.platform == 'osx':
            self.ssh_key = SshKey(args)
            self.endpoint = 'https://osx.testing.ansible.com'
            self.port = None
        else:
            raise ApplicationError('Unsupported platform: %s' % platform)

        self.path = os.path.expanduser('~/.ansible/test/instances/%s-%s' % (self.name, self.stage))

        if persist and self._load():
            try:
                display.info('Checking existing %s/%s instance %s.' % (self.platform, self.version, self.instance_id),
                             verbosity=1)

                self.connection = self.get()

                display.info('Loaded existing %s/%s instance %s.' % (self.platform, self.version, self.instance_id),
                             verbosity=1)
            except HttpError as ex:
                if ex.status != 404:
                    raise

                self._clear()

                display.info('Cleared stale %s/%s instance %s.' % (self.platform, self.version, self.instance_id),
                             verbosity=1)

                self.instance_id = None
        else:
            self.instance_id = None
            self._clear()

        if self.instance_id:
            self.started = True
        else:
            self.started = False
            self.instance_id = str(uuid.uuid4())

            display.info('Initializing new %s/%s instance %s.' % (self.platform, self.version, self.instance_id),
                         verbosity=1)

    def start(self):
        """Start instance."""
        if is_shippable():
            self.start_shippable()
        else:
            self.start_remote()

    def start_remote(self):
        """Start instance for remote development/testing."""
        with open(os.path.expanduser('~/.ansible-core-ci.key'), 'r') as key_fd:
            auth_key = key_fd.read().strip()

        self._start(dict(
            remote=dict(
                key=auth_key,
                nonce=None,
            ),
        ))

    def start_shippable(self):
        """Start instance on Shippable."""
        self._start(dict(
            shippable=dict(
                run_id=os.environ['SHIPPABLE_BUILD_ID'],
                job_number=int(os.environ['SHIPPABLE_JOB_NUMBER']),
            ),
        ))

    def stop(self):
        """Stop instance."""
        if not self.started:
            display.info('Skipping invalid %s/%s instance %s.' % (self.platform, self.version, self.instance_id),
                         verbosity=1)
            return

        response = self.client.delete(self._uri)

        if response.status_code == 404:
            self._clear()
            display.info('Cleared invalid %s/%s instance %s.' % (self.platform, self.version, self.instance_id),
                         verbosity=1)
            return

        if response.status_code == 200:
            self._clear()
            display.info('Stopped running %s/%s instance %s.' % (self.platform, self.version, self.instance_id),
                         verbosity=1)
            return

        raise self._create_http_error(response)

    def get(self):
        """
        Get instance connection information.
        :rtype: InstanceConnection
        """
        if not self.started:
            display.info('Skipping invalid %s/%s instance %s.' % (self.platform, self.version, self.instance_id),
                         verbosity=1)
            return None

        if self.connection and self.connection.running:
            return self.connection

        response = self.client.get(self._uri)

        if response.status_code != 200:
            raise self._create_http_error(response)

        if self.args.explain:
            self.connection = InstanceConnection(
                running=True,
                hostname='cloud.example.com',
                port=self.port or 12345,
                username='username',
                password='password' if self.platform == 'windows' else None,
            )
        else:
            response_json = response.json()

            status = response_json['status']
            con = response_json['connection']

            self.connection = InstanceConnection(
                running=status == 'running',
                hostname=con['hostname'],
                port=int(con.get('port', self.port)),
                username=con['username'],
                password=con.get('password'),
            )

        status = 'running' if self.connection.running else 'starting'

        display.info('Retrieved %s %s/%s instance %s.' % (status, self.platform, self.version, self.instance_id),
                     verbosity=1)

        return self.connection

    def wait(self):
        """Wait for the instance to become ready."""
        for _ in range(1, 90):
            if self.get().running:
                return
            time.sleep(10)

        raise ApplicationError('Timeout waiting for %s/%s instance %s.' %
                               (self.platform, self.version, self.instance_id))

    @property
    def _uri(self):
        return '%s/%s/jobs/%s' % (self.endpoint, self.stage, self.instance_id)

    def _start(self, auth):
        """Start instance."""
        if self.started:
            display.info('Skipping started %s/%s instance %s.' % (self.platform, self.version, self.instance_id),
                         verbosity=1)
            return

        data = dict(
            config=dict(
                platform=self.platform,
                version=self.version,
                public_key=self.ssh_key.pub_contents if self.ssh_key else None,
                query=False,
            )
        )

        data.update(dict(auth=auth))

        headers = {
            'Content-Type': 'application/json',
        }

        response = self.client.put(self._uri, data=json.dumps(data), headers=headers)

        if response.status_code != 200:
            raise self._create_http_error(response)

        self.started = True
        self._save()

        display.info('Started %s/%s instance %s.' % (self.platform, self.version, self.instance_id),
                     verbosity=1)

    def _clear(self):
        """Clear instance information."""
        try:
            self.connection = None
            os.remove(self.path)
        except OSError as ex:
            if ex.errno != errno.ENOENT:
                raise

    def _load(self):
        """Load instance information."""
        try:
            with open(self.path, 'r') as instance_fd:
                self.instance_id = instance_fd.read()
                self.started = True
        except IOError as ex:
            if ex.errno != errno.ENOENT:
                raise
            self.instance_id = None

        return self.instance_id

    def _save(self):
        """Save instance information."""
        if self.args.explain:
            return

        make_dirs(os.path.dirname(self.path))

        with open(self.path, 'w') as instance_fd:
            instance_fd.write(self.instance_id)

    @staticmethod
    def _create_http_error(response):
        """
        :type response: HttpResponse
        :rtype: ApplicationError
        """
        response_json = response.json()
        stack_trace = ''

        if 'message' in response_json:
            message = response_json['message']
        elif 'errorMessage' in response_json:
            message = response_json['errorMessage'].strip()
            if 'stackTrace' in response_json:
                trace = '\n'.join([x.rstrip() for x in traceback.format_list(response_json['stackTrace'])])
                stack_trace = ('\nTraceback (from remote server):\n%s' % trace)
        else:
            message = str(response_json)

        return HttpError(response.status_code, '%s%s' % (message, stack_trace))


class SshKey(object):
    """Container for SSH key used to connect to remote instances."""
    def __init__(self, args):
        """
        :type args: CommonConfig
        """
        tmp = os.path.expanduser('~/.ansible/test/')

        self.key = os.path.join(tmp, 'id_rsa')
        self.pub = os.path.join(tmp, 'id_rsa.pub')

        if not os.path.isfile(self.pub):
            if not args.explain:
                make_dirs(tmp)

            run_command(args, ['ssh-keygen', '-q', '-t', 'rsa', '-N', '', '-f', self.key])

        if args.explain:
            self.pub_contents = None
        else:
            with open(self.pub, 'r') as pub_fd:
                self.pub_contents = pub_fd.read().strip()


class InstanceConnection(object):
    """Container for remote instance status and connection details."""
    def __init__(self, running, hostname, port, username, password):
        """
        :type running: bool
        :type hostname: str
        :type port: int
        :type username: str
        :type password: str | None
        """
        self.running = running
        self.hostname = hostname
        self.port = port
        self.username = username
        self.password = password

    def __str__(self):
        if self.password:
            return '%s:%s [%s:%s]' % (self.hostname, self.port, self.username, self.password)

        return '%s:%s [%s]' % (self.hostname, self.port, self.username)