ansible-test - Update locale logic to match core. (#78389)

Now that core requires UTF-8 filesystem encoding, ansible-test does as well.

Additionally, the `en_US.UTF-8` or `C.UTF-8` encoding must be available.
Previously the `en_US.UTF-8` encoding was requested, but its availability was never verified.
The fallback to `C.UTF-8` maintains UTF-8 encoding while allowing more flexibility in the running environment.

changelogs/fragments/ansible-test-locale.yml

@@ -0,0 +1,16 @@
+major_changes:
+  - ansible-test - At startup the filesystem encoding is checked to verify it is UTF-8.
+                   If not, the process exits with an error reporting the errant encoding.
+  - ansible-test - At startup the locale is configured as ``en_US.UTF-8``, with a fallback to ``C.UTF-8``.
+                   If neither encoding is available the process exits with an error.
+                   If the fallback is used, a warning is displayed.
+                   In previous versions the ``en_US.UTF-8`` locale was always requested.
+                   However, no startup checking was performed to verify the locale was successfully configured.
+breaking_changes:
+  - ansible-test - At startup the filesystem encoding is checked to verify it is UTF-8.
+                   If not, the process exits with an error reporting the errant encoding.
+  - ansible-test - At startup the locale is configured as ``en_US.UTF-8``, with a fallback to ``C.UTF-8``.
+                   If neither encoding is available the process exits with an error.
+                   If the fallback is used, a warning is displayed.
+                   In previous versions the ``en_US.UTF-8`` locale was always requested.
+                   However, no startup checking was performed to verify the locale was successfully configured.

test/lib/ansible_test/_internal/__init__.py

@@ -14,6 +14,7 @@ from .init import (
 from .util import (
     ApplicationError,
     display,
+    report_locale,
 )
 
 from .delegation import (
@@ -59,6 +60,7 @@ def main(cli_args=None):  # type: (t.Optional[t.List[str]]) -> None
         display.color = config.color
         display.fd = sys.stderr if config.display_stderr else sys.stdout
         configure_timeout(config)
+        report_locale()
 
         display.info('RLIMIT_NOFILE: %s' % (CURRENT_RLIMIT_NOFILE,), verbosity=2)
 

test/lib/ansible_test/_internal/delegation.py

@@ -7,6 +7,10 @@ import os
 import tempfile
 import typing as t
 
+from .locale_util import (
+    STANDARD_LOCALE,
+)
+
 from .io import (
     make_dirs,
 )
@@ -256,12 +260,7 @@ def generate_command(
     cmd = [os.path.join(ansible_bin_path, 'ansible-test')]
     cmd = [python.path] + cmd
 
-    # Force the encoding used during delegation.
-    # This is only needed because ansible-test relies on Python's file system encoding.
-    # Environments that do not have the locale configured are thus unable to work with unicode file paths.
-    # Examples include FreeBSD and some Linux containers.
     env_vars = dict(
-        LC_ALL='en_US.UTF-8',
         ANSIBLE_TEST_CONTENT_ROOT=content_root,
     )
 
@@ -276,6 +275,14 @@ def generate_command(
         env_vars.update(
             PYTHONPATH=library_path,
         )
+    else:
+        # When delegating to a host other than the origin, the locale must be explicitly set.
+        # Setting of the locale for the origin host is handled by common_environment().
+        # Not all connections support setting the locale, and for those that do, it isn't guaranteed to work.
+        # This is needed to make sure the delegated environment is configured for UTF-8 before running Python.
+        env_vars.update(
+            LC_ALL=STANDARD_LOCALE,
+        )
 
     # Propagate the TERM environment variable to the remote host when using the shell command.
     if isinstance(args, ShellConfig):

test/lib/ansible_test/_internal/locale_util.py

@@ -0,0 +1,61 @@
+"""Initialize locale settings. This must be imported very early in ansible-test startup."""
+
+from __future__ import annotations
+
+import locale
+import sys
+import typing as t
+
+STANDARD_LOCALE = 'en_US.UTF-8'
+"""
+The standard locale used by ansible-test and its subprocesses and delegated instances.
+"""
+
+FALLBACK_LOCALE = 'C.UTF-8'
+"""
+The fallback locale to use when the standard locale is not available.
+This was added in ansible-core 2.14 to allow testing in environments without the standard locale.
+It was not needed in previous ansible-core releases since they do not verify the locale during startup.
+"""
+
+
+class LocaleError(SystemExit):
+    """Exception to raise when locale related errors occur."""
+    def __init__(self, message: str) -> None:
+        super().__init__(f'ERROR: {message}')
+
+
+def configure_locale() -> t.Tuple[str, t.Optional[str]]:
+    """Configure the locale, returning the selected locale and an optional warning."""
+
+    if (fs_encoding := sys.getfilesystemencoding()).lower() != 'utf-8':
+        raise LocaleError(f'ansible-test requires the filesystem encoding to be UTF-8, but "{fs_encoding}" was detected.')
+
+    candidate_locales = STANDARD_LOCALE, FALLBACK_LOCALE
+
+    errors: dict[str, str] = {}
+    warning: t.Optional[str] = None
+    configured_locale: t.Optional[str] = None
+
+    for candidate_locale in candidate_locales:
+        try:
+            locale.setlocale(locale.LC_ALL, candidate_locale)
+            locale.getlocale()
+        except (locale.Error, ValueError) as ex:
+            errors[candidate_locale] = str(ex)
+        else:
+            configured_locale = candidate_locale
+            break
+
+    if not configured_locale:
+        raise LocaleError('ansible-test could not initialize a supported locale:\n' +
+                          '\n'.join(f'{key}: {value}' for key, value in errors.items()))
+
+    if configured_locale != STANDARD_LOCALE:
+        warning = (f'Using locale "{configured_locale}" instead of "{STANDARD_LOCALE}". '
+                   'Tests which depend on the locale may behave unexpectedly.')
+
+    return configured_locale, warning
+
+
+CONFIGURED_LOCALE, LOCALE_WARNING = configure_locale()

test/lib/ansible_test/_internal/util.py

@@ -32,6 +32,11 @@ try:
 except ImportError:
     TypeGuard = None
 
+from .locale_util import (
+    LOCALE_WARNING,
+    CONFIGURED_LOCALE,
+)
+
 from .encoding import (
     to_bytes,
     to_optional_bytes,
@@ -611,7 +616,7 @@ class OutputThread(ReaderThread):
 def common_environment():
     """Common environment used for executing all programs."""
     env = dict(
-        LC_ALL='en_US.UTF-8',
+        LC_ALL=CONFIGURED_LOCALE,
         PATH=os.environ.get('PATH', os.path.defpath),
     )
 
@@ -651,6 +656,15 @@ def common_environment():
     return env
 
 
+def report_locale() -> None:
+    """Report the configured locale and the locale warning, if applicable."""
+
+    display.info(f'Configured locale: {CONFIGURED_LOCALE}', verbosity=1)
+
+    if LOCALE_WARNING:
+        display.warning(LOCALE_WARNING)
+
+
 def pass_vars(required, optional):  # type: (t.Collection[str], t.Collection[str]) -> t.Dict[str, str]
     """Return a filtered dictionary of environment variables based on the current environment."""
     env = {}