Fix collection loader builtin handling. (#58897)

- Support more import statements:

  from ansible_collections.ansible.builtin.plugins.module_utils import basic
  from ansible_collections.ansible.builtin.plugins.module_utils.basic import AnsibleModule

- Add unit tests for more import statements.
- Raise ImportError instead of returning None if load_module fails.

lib/ansible/utils/collection_loader.py

@@ -110,6 +110,11 @@ class AnsibleCollectionLoader(object):
         sub_collection = fullname.count('.') > 1
 
         synpkg_def = _SYNTHETIC_PACKAGES.get(fullname)
+        synpkg_remainder = ''
+
+        if not synpkg_def:
+            synpkg_def = _SYNTHETIC_PACKAGES.get(parent_pkg_name)
+            synpkg_remainder = '.' + fullname.rpartition('.')[2]
 
         # FIXME: collapse as much of this back to on-demand as possible (maybe stub packages that get replaced when actually loaded?)
         if synpkg_def:
@@ -121,7 +126,7 @@ class AnsibleCollectionLoader(object):
 
                 if not map_package:
                     raise KeyError('invalid synthetic map package definition (no target "map" defined)')
-                mod = import_module(map_package)
+                mod = import_module(map_package + synpkg_remainder)
 
                 sys.modules[fullname] = mod
 
@@ -195,7 +200,7 @@ class AnsibleCollectionLoader(object):
 
         # FIXME: need to handle the "no dirs present" case for at least the root and synthetic internal collections like ansible.builtin
 
-        return None
+        raise ImportError('module {0} not found'.format(fullname))
 
     @staticmethod
     def _extend_path_with_ns(path, ns):

test/integration/targets/collections/collection_root_user/ansible_collections/testns/testcoll/plugins/module_utils/base.py

@@ -1,7 +1,9 @@
-# FIXME: this style (full module import via from) doesn't work yet from collections
-# from ansible_collections.testns.testcoll.plugins.module_utils import secondary
+from ansible_collections.testns.testcoll.plugins.module_utils import secondary
 import ansible_collections.testns.testcoll.plugins.module_utils.secondary
 
 
 def thingtocall():
+    if secondary != ansible_collections.testns.testcoll.plugins.module_utils.secondary:
+        raise Exception()
+
     return "thingtocall in base called " + ansible_collections.testns.testcoll.plugins.module_utils.secondary.thingtocall()

test/units/utils/test_collection_loader.py

@@ -69,6 +69,46 @@ def test_import_from_collection(monkeypatch):
     finally:
         sys.settrace(original_trace_function)
 
+    # make sure 'import ... as ...' works on builtin synthetic collections
+    # the following import is not supported (it tries to find module_utils in ansible.plugins)
+    # import ansible_collections.ansible.builtin.plugins.module_utils as c1
+    import ansible_collections.ansible.builtin.plugins.action as c2
+    import ansible_collections.ansible.builtin.plugins as c3
+    import ansible_collections.ansible.builtin as c4
+    import ansible_collections.ansible as c5
+    import ansible_collections as c6
+
+    # make sure 'import ...' works on builtin synthetic collections
+    import ansible_collections.ansible.builtin.plugins.module_utils
+
+    import ansible_collections.ansible.builtin.plugins.action
+    assert ansible_collections.ansible.builtin.plugins.action == c3.action == c2
+
+    import ansible_collections.ansible.builtin.plugins
+    assert ansible_collections.ansible.builtin.plugins == c4.plugins == c3
+
+    import ansible_collections.ansible.builtin
+    assert ansible_collections.ansible.builtin == c5.builtin == c4
+
+    import ansible_collections.ansible
+    assert ansible_collections.ansible == c6.ansible == c5
+
+    import ansible_collections
+    assert ansible_collections == c6
+
+    # make sure 'from ... import ...' works on builtin synthetic collections
+    from ansible_collections.ansible import builtin
+    from ansible_collections.ansible.builtin import plugins
+    assert builtin.plugins == plugins
+
+    from ansible_collections.ansible.builtin.plugins import action
+    from ansible_collections.ansible.builtin.plugins.action import command
+    assert action.command == command
+
+    from ansible_collections.ansible.builtin.plugins.module_utils import basic
+    from ansible_collections.ansible.builtin.plugins.module_utils.basic import AnsibleModule
+    assert basic.AnsibleModule == AnsibleModule
+
     # make sure relative imports work from collections code
     # these require __package__ to be set correctly
     import ansible_collections.my_namespace.my_collection.plugins.module_utils.my_other_util