ansible-galaxy - fall back to galaxy.yml when listing collections (#68925)

Fallback to galaxy.yml for listing collections that don't have a FILES.json or MANIFEST.json

Co-authored-by: Sam Doran <sdoran@redhat.com>

docs/docsite/rst/user_guide/collections_using.rst

@@ -114,7 +114,7 @@ collections on a host without access to a Galaxy or Automation Hub server.
 Listing collections
 ===================
 
-To list installed collections, run ``ansible-galaxy collection list``. This shows all of the installed collections found in the configured collections search paths. The path where the collections are located are displayed as well as version information. If no version information is available, a ``*`` is displayed for the version number.
+To list installed collections, run ``ansible-galaxy collection list``. This shows all of the installed collections found in the configured collections search paths. It will also show collections under development which contain a galaxy.yml file instead of a MANIFEST.json. The path where the collections are located are displayed as well as version information. If no version information is available, a ``*`` is displayed for the version number.
 
 .. code-block:: shell
 

lib/ansible/cli/galaxy.py

@@ -1235,7 +1235,7 @@ class GalaxyCLI(CLI):
                     continue
 
                 collection_found = True
-                collection = CollectionRequirement.from_path(b_collection_path, False)
+                collection = CollectionRequirement.from_path(b_collection_path, False, fallback_metadata=True)
                 fqcn_width, version_width = _get_collection_widths(collection)
 
                 _display_header(collection_path, 'Collection', 'Version', fqcn_width, version_width)
@@ -1246,7 +1246,7 @@ class GalaxyCLI(CLI):
                 collection_path = validate_collection_path(path)
                 if os.path.isdir(collection_path):
                     display.vvv("Searching {0} for collections".format(collection_path))
-                    collections = find_existing_collections(collection_path)
+                    collections = find_existing_collections(collection_path, fallback_metadata=True)
                 else:
                     # There was no 'ansible_collections/' directory in the path, so there
                     # or no collections here.

lib/ansible/galaxy/collection.py

@@ -224,7 +224,7 @@ class CollectionRequirement:
                         _extract_tar_file(collection_tar, file_name, b_collection_path, b_temp_path,
                                           expected_hash=file_info['chksum_sha256'])
                     else:
-                        os.makedirs(os.path.join(b_collection_path, to_bytes(file_name, errors='surrogate_or_strict')))
+                        os.makedirs(os.path.join(b_collection_path, to_bytes(file_name, errors='surrogate_or_strict')), mode=0o0755)
         except Exception:
             # Ensure we don't leave the dir behind in case of a failure.
             shutil.rmtree(b_collection_path)
@@ -386,7 +386,7 @@ class CollectionRequirement:
                                      metadata=meta, files=files, allow_pre_releases=allow_pre_release)
 
     @staticmethod
-    def from_path(b_path, force, parent=None):
+    def from_path(b_path, force, parent=None, fallback_metadata=False):
         info = {}
         for b_file_name, property_name in CollectionRequirement._FILE_MAPPING:
             b_file_path = os.path.join(b_path, b_file_name)
@@ -399,6 +399,13 @@ class CollectionRequirement:
                 except ValueError:
                     raise AnsibleError("Collection file at '%s' does not contain a valid json string."
                                        % to_native(b_file_path))
+        if not info and fallback_metadata:
+            b_galaxy_path = os.path.join(b_path, b'galaxy.yml')
+            if os.path.exists(b_galaxy_path):
+                collection_meta = _get_galaxy_yml(b_galaxy_path)
+                info['files_file'] = _build_files_manifest(b_path, collection_meta['namespace'], collection_meta['name'],
+                                                           collection_meta['build_ignore'])
+                info['manifest_file'] = _build_manifest(**collection_meta)
 
         allow_pre_release = False
         if 'manifest_file' in info:
@@ -419,8 +426,11 @@ class CollectionRequirement:
 
             dependencies = manifest['dependencies']
         else:
-            display.warning("Collection at '%s' does not have a MANIFEST.json file, cannot detect version."
+            if fallback_metadata:
-                            % to_text(b_path))
+                warning = "Collection at '%s' does not have a galaxy.yml or a MANIFEST.json file, cannot detect version."
+            else:
+                warning = "Collection at '%s' does not have a MANIFEST.json file, cannot detect version."
+            display.warning(warning % to_text(b_path))
             parent_dir, name = os.path.split(to_text(b_path, errors='surrogate_or_strict'))
             namespace = os.path.split(parent_dir)[1]
 
@@ -998,7 +1008,7 @@ def _build_collection_tar(b_collection_path, b_tar_path, collection_manifest, fi
         display.display('Created collection for %s at %s' % (collection_name, to_text(b_tar_path)))
 
 
-def find_existing_collections(path):
+def find_existing_collections(path, fallback_metadata=False):
     collections = []
 
     b_path = to_bytes(path, errors='surrogate_or_strict')
@@ -1010,7 +1020,7 @@ def find_existing_collections(path):
         for b_collection in os.listdir(b_namespace_path):
             b_collection_path = os.path.join(b_namespace_path, b_collection)
             if os.path.isdir(b_collection_path):
-                req = CollectionRequirement.from_path(b_collection_path, False)
+                req = CollectionRequirement.from_path(b_collection_path, False, fallback_metadata=fallback_metadata)
                 display.vvv("Found installed collection %s:%s at '%s'" % (to_text(req), req.latest_version,
                                                                           to_text(b_collection_path)))
                 collections.append(req)

test/integration/targets/ansible-galaxy/runme.sh

@@ -336,6 +336,10 @@ sed -i -e 's#^version:.*#version: 2.5.0#' ansible_test/zoo/galaxy.yml
 ansible-galaxy collection build ansible_test/zoo
 ansible-galaxy collection install ansible_test-zoo-2.5.0.tar.gz -p ./local
 
+# Test listing a collection that contains a galaxy.yml
+ansible-galaxy collection init "ansible_test.development"
+mv ./ansible_test/development "${galaxy_testdir}/local/ansible_collections/ansible_test/"
+
 export ANSIBLE_COLLECTIONS_PATHS=~/.ansible/collections:${galaxy_testdir}/local
 
 f_ansible_galaxy_status \
@@ -343,7 +347,7 @@ f_ansible_galaxy_status \
 
     ansible-galaxy collection list -p ./install | tee out.txt
 
-    [[ $(grep -c ansible_test out.txt) -eq 4 ]]
+    [[ $(grep -c ansible_test out.txt) -eq 5 ]]
 
 f_ansible_galaxy_status \
     "collection list specific collection"
@@ -352,6 +356,14 @@ f_ansible_galaxy_status \
 
     [[ $(grep -c 'ansible_test\.airport' out.txt) -eq 1 ]]
 
+f_ansible_galaxy_status \
+    "collection list specific collection which contains galaxy.yml"
+
+    ansible-galaxy collection list -p ./install ansible_test.development 2>&1 | tee out.txt
+
+    [[ $(grep -c 'ansible_test\.development' out.txt) -eq 1 ]]
+    [[ $(grep -c 'WARNING' out.txt) -eq 0 ]]
+
 f_ansible_galaxy_status \
     "collection list specific collection found in multiple places"