fix get_data on case_insensitive fs (#69955)

* fix get_data on case_insensitive fs

* implement case-sensitive-forcing versions of various os.path methods that just pass through on case-sensitive systems.

* catch broader IOError for py2/py3 compat

* optimization: factor out case-insensitive comparison

* implement case-sensitive open

changelogs/fragments/case_insensitive_fix.yml

@@ -0,0 +1,2 @@
+bugfixes:
+  - collection loader - fix file/module/class confusion issues on case-insensitive filesystems

lib/ansible/executor/module_common.py

@@ -648,7 +648,7 @@ class CollectionModuleInfo(ModuleInfo):
             self._package_name = '.'.join(path.split('/'))
             try:
                 self.get_source()
-            except FileNotFoundError:
+            except IOError:
                 pass
             else:
                 self.path = os.path.join(path, self._mod_name) + '.py'

lib/ansible/utils/collection_loader.py

@@ -6,7 +6,6 @@ __metaclass__ = type
 
 import os
 import os.path
-import pkgutil
 import re
 import sys
 
@@ -14,6 +13,7 @@ from types import ModuleType
 
 from ansible.module_utils._text import to_bytes, to_native, to_text
 from ansible.module_utils.six import iteritems, string_types, with_metaclass
+from ansible.utils.path import cs_open
 from ansible.utils.singleton import Singleton
 
 # HACK: keep Python 2.6 controller tests happy in CI until they're properly split
@@ -269,7 +269,7 @@ class AnsibleCollectionLoader(with_metaclass(Singleton, object)):
         return os.path.join(path, ns_path_add)
 
     def get_data(self, filename):
-        with open(filename, 'rb') as fd:
+        with cs_open(filename, 'rb') as fd:
             return fd.read()
 
 

lib/ansible/utils/path.py

@@ -20,12 +20,17 @@ __metaclass__ = type
 import os
 import shutil
 
-from errno import EEXIST
+from errno import EEXIST, ENOENT
 from ansible.errors import AnsibleError
 from ansible.module_utils._text import to_bytes, to_native, to_text
 
 
-__all__ = ['unfrackpath', 'makedirs_safe']
+__all__ = ['unfrackpath', 'makedirs_safe', 'cs_exists', 'cs_isdir', 'cs_isfile']
+
+if not os.path.exists(__file__):
+    raise Exception('unable to determine filesystem case-sensitivity ({0} does not exist)'.format(__file__))
+
+_is_case_insensitive_fs = os.path.exists(__file__.upper())
 
 
 def unfrackpath(path, follow=True, basedir=None):
@@ -155,3 +160,73 @@ def is_subpath(child, parent):
         pass
 
     return test
+
+
+def _explicit_case_sensitive_exists(path):
+    """
+    Standalone case-sensitive existence check for case-insensitive filesystems. This assumes the parent
+    dir exists and is otherwise accessible by the caller.
+    :param path: a bytes or text path string to check for existence
+    :return: True if the path exists and the paths pass a case-sensitive comparison
+    """
+    parent, leaf = os.path.split(path)
+
+    if not leaf:
+        # root directory or '.', of course it exists
+        return True
+
+    # ensure that the leaf matches, and that the parent dirs match (recursively)
+    return any(p for p in os.listdir(parent) if p == leaf) and cs_isdir(parent)
+
+
+def cs_open(file, *args, **kwargs):
+    """
+    A replacement for open that behaves case-sensitively on case-insensitive filesystems (passes through all args to underlying platform open)
+    :param file: a bytes or text path string to open
+    :return: a file descriptor if the file exists and the path passes a case-sensitive comparison
+    """
+    fd = open(file, *args, **kwargs)
+    try:
+        if _is_case_insensitive_fs and not _explicit_case_sensitive_exists(file):
+            try:
+                extype = FileNotFoundError
+            except NameError:
+                extype = IOError
+            raise extype(ENOENT, os.strerror(ENOENT), file)
+    except Exception:
+        fd.close()
+        raise
+
+    return fd
+
+
+def cs_exists(path):
+    """
+    A replacement for os.path.exists that behaves case-sensitive on case-insensitive filesystems
+    :param path: a bytes or text path string to check for existence
+    :return: True if the path exists and the paths pass a case-sensitive comparison
+    """
+    raw_exists = os.path.exists(path)
+    if not _is_case_insensitive_fs or not raw_exists:
+        return raw_exists
+
+    # we're on a case-insensitive filesystem and the file exists, verify its case matches
+    return _explicit_case_sensitive_exists(path)
+
+
+def cs_isdir(path):
+    """
+    A replacement for os.path.isdir that behaves case-sensitive on case-insensitive filesystems
+    :param path: a bytes or text path string to check if isdir
+    :return: True if the path is a dir (or resolves to one) and the paths pass a case-sensitive comparison
+    """
+    return os.path.isdir(path) and (not _is_case_insensitive_fs or _explicit_case_sensitive_exists(path))
+
+
+def cs_isfile(path):
+    """
+    A replacement for os.path.isfile that behaves case-sensitive on case-insensitive filesystems
+    :param path: a bytes or text path string to check if isfile
+    :return: True if the path is a file (or resolves to one) and the paths pass a case-sensitive comparison
+    """
+    return os.path.isfile(path) and (not _is_case_insensitive_fs or _explicit_case_sensitive_exists(path))

test/units/utils/test_path.py

@@ -0,0 +1,53 @@
+# -*- coding: utf-8 -*-
+# (c) 2020 Ansible Project
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+import os
+import pytest
+from ansible.utils.path import cs_exists, cs_isdir, cs_isfile, cs_open
+
+
+def iter_parent_paths(path):
+    parent = path
+    while True:
+        parent, leaf = os.path.split(parent)
+        if not parent or not leaf:
+            break
+        else:
+            yield parent
+
+
+def test_cs_open():
+    with open(__file__) as fd:
+        with cs_open(__file__) as csfd:
+            assert fd.read() == csfd.read()
+
+    with pytest.raises(IOError):
+        cs_open(__file__.upper())
+
+
+def test_cs_isfile():
+    assert cs_isfile(__file__)
+    for p in iter_parent_paths(__file__):
+        assert not cs_isfile(p)
+    assert not cs_isfile(__file__.upper())
+
+
+def test_cs_isdir():
+    assert not cs_isdir(__file__)
+    for p in iter_parent_paths(__file__):
+        assert cs_isdir(p)
+        if p != p.upper():
+            assert not cs_isdir(p.upper())
+
+
+def test_cs_exists():
+    assert cs_exists(__file__)
+    assert not cs_exists(__file__.upper())
+    for p in iter_parent_paths(__file__):
+        assert cs_exists(p)
+        if p != p.upper():
+            assert not cs_exists(p.upper())