Update ansible-test remote endpoint handling. (#71413)

* Request ansible-core-ci resources by provider.
* Remove obsolete us-east-2 CI endpoint.
* Add new --remote-endpoint option.
* Add warning for --remote-aws-region option.
* Update service endpoints.
* Allow non-standard remote stages.
* Add changelog fragment.

changelogs/fragments/ansible-test-endpoint-update.yml

@@ -0,0 +1,7 @@
+minor_changes:
+  - ansible-test - Allow custom ``--remote-stage`` options for development and testing.
+  - ansible-test - Update built-in service endpoints for the ``--remote`` option.
+  - ansible-test - Show a warning when the obsolete ``--remote-aws-region`` option is used.
+  - ansible-test - Support custom remote endpoints with the ``--remote-endpoint`` option.
+  - ansible-test - Remove the discontinued ``us-east-2`` choice from the ``--remote-aws-region`` option.
+  - ansible-test - Request remote resources by provider name for all provider types.

test/lib/ansible_test/_internal/cli.py

@@ -947,9 +947,8 @@ def add_environments(parser, isolated_delegation=True):
 
     remote.add_argument('--remote-stage',
                         metavar='STAGE',
-                        help='remote stage to use: %(choices)s',
-                        choices=['prod', 'dev'],
-                        default='prod')
+                        help='remote stage to use: prod, dev',
+                        default='prod').completer = complete_remote_stage
 
     remote.add_argument('--remote-provider',
                         metavar='PROVIDER',
@@ -957,6 +956,11 @@ def add_environments(parser, isolated_delegation=True):
                         choices=['default', 'aws', 'azure', 'parallels', 'ibmvpc', 'ibmps'],
                         default='default')
 
+    remote.add_argument('--remote-endpoint',
+                        metavar='ENDPOINT',
+                        help='remote provisioning endpoint to use (default: auto)',
+                        default=None)
+
     remote.add_argument('--remote-aws-region',
                         metavar='REGION',
                         help='remote aws region to use: %(choices)s (default: auto)',
@@ -1057,6 +1061,16 @@ def add_extra_docker_options(parser, integration=True):
                         help='memory limit for docker in bytes', type=int)
 
 
+# noinspection PyUnusedLocal
+def complete_remote_stage(prefix, parsed_args, **_):  # pylint: disable=unused-argument
+    """
+    :type prefix: unicode
+    :type parsed_args: any
+    :rtype: list[str]
+    """
+    return [stage for stage in ('prod', 'dev') if stage.startswith(prefix)]
+
+
 def complete_target(prefix, parsed_args, **_):
     """
     :type prefix: unicode

test/lib/ansible_test/_internal/config.py

@@ -96,6 +96,7 @@ class EnvironmentConfig(CommonConfig):
 
         self.remote_stage = args.remote_stage  # type: str
         self.remote_provider = args.remote_provider  # type: str
+        self.remote_endpoint = args.remote_endpoint  # type: t.Optional[str]
         self.remote_aws_region = args.remote_aws_region  # type: str
         self.remote_terminate = args.remote_terminate  # type: str
 

test/lib/ansible_test/_internal/core_ci.py

@@ -50,8 +50,7 @@ from .data import (
 )
 
 AWS_ENDPOINTS = {
-    'us-east-1': 'https://14blg63h2i.execute-api.us-east-1.amazonaws.com',
-    'us-east-2': 'https://g5xynwbk96.execute-api.us-east-2.amazonaws.com',
+    'us-east-1': 'https://ansible-core-ci.testing.ansible.com',
 }
 
 
@@ -87,8 +86,6 @@ class AnsibleCoreCI:
         else:
             self.name = '%s-%s' % (self.platform, self.version)
 
-        self.resource = 'jobs'
-
         # Assign each supported platform to one provider.
         # This is used to determine the provider from the platform when no provider is specified.
         providers = dict(
@@ -157,10 +154,8 @@ class AnsibleCoreCI:
         self.path = os.path.expanduser('~/.ansible/test/instances/%s-%s-%s' % (self.name, self.provider, self.stage))
 
         if self.provider in ('aws', 'azure', 'ibmps', 'ibmvpc'):
-            if self.provider != 'aws':
-                self.resource = self.provider
-
             if args.remote_aws_region:
+                display.warning('The --remote-aws-region option is obsolete and will be removed in a future version of ansible-test.')
                 # permit command-line override of region selection
                 region = args.remote_aws_region
                 # use a dedicated CI key when overriding the region selection
@@ -169,7 +164,12 @@ class AnsibleCoreCI:
                 region = 'us-east-1'
 
             self.path = "%s-%s" % (self.path, region)
-            self.endpoints = (AWS_ENDPOINTS[region],)
+
+            if self.args.remote_endpoint:
+                self.endpoints = (self.args.remote_endpoint,)
+            else:
+                self.endpoints = (AWS_ENDPOINTS[region],)
+
             self.ssh_key = SshKey(args)
 
             if self.platform == 'windows':
@@ -183,8 +183,11 @@ class AnsibleCoreCI:
                 # 90 seconds
                 self.retries = 7
         elif self.provider == 'parallels':
-            self.endpoints = self._get_parallels_endpoints()
-            self.max_threshold = 6
+            if self.args.remote_endpoint:
+                self.endpoints = (self.args.remote_endpoint,)
+            else:
+                self.endpoints = self._get_parallels_endpoints()
+                self.max_threshold = 6
 
             self.ssh_key = SshKey(args)
             self.port = None
@@ -236,7 +239,7 @@ class AnsibleCoreCI:
         sleep = 3
 
         for _iteration in range(1, 10):
-            response = client.get('https://s3.amazonaws.com/ansible-ci-files/ansible-test/parallels-endpoints.txt')
+            response = client.get('https://ansible-ci-files.s3.amazonaws.com/ansible-test/parallels-endpoints.txt')
 
             if response.status_code == 200:
                 endpoints = tuple(response.response.splitlines())
@@ -370,7 +373,7 @@ class AnsibleCoreCI:
 
     @property
     def _uri(self):
-        return '%s/%s/%s/%s' % (self.endpoint, self.stage, self.resource, self.instance_id)
+        return '%s/%s/%s/%s' % (self.endpoint, self.stage, self.provider, self.instance_id)
 
     def _start(self, auth):
         """Start instance."""