mirror of https://github.com/ansible/ansible.git
You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
1893 lines
76 KiB
Python
1893 lines
76 KiB
Python
# -*- coding: utf-8 -*-
|
|
# Copyright: (c) 2019-2021, Ansible Project
|
|
# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
|
|
"""Installed collections management package."""
|
|
|
|
from __future__ import (absolute_import, division, print_function)
|
|
__metaclass__ = type
|
|
|
|
import errno
|
|
import fnmatch
|
|
import functools
|
|
import json
|
|
import os
|
|
import pathlib
|
|
import queue
|
|
import re
|
|
import shutil
|
|
import stat
|
|
import sys
|
|
import tarfile
|
|
import tempfile
|
|
import textwrap
|
|
import threading
|
|
import time
|
|
import typing as t
|
|
|
|
from collections import namedtuple
|
|
from contextlib import contextmanager
|
|
from dataclasses import dataclass, fields as dc_fields
|
|
from hashlib import sha256
|
|
from io import BytesIO
|
|
from importlib.metadata import distribution
|
|
from itertools import chain
|
|
|
|
try:
|
|
from packaging.requirements import Requirement as PkgReq
|
|
except ImportError:
|
|
class PkgReq: # type: ignore[no-redef]
|
|
pass
|
|
|
|
HAS_PACKAGING = False
|
|
else:
|
|
HAS_PACKAGING = True
|
|
|
|
try:
|
|
from distlib.manifest import Manifest # type: ignore[import]
|
|
from distlib import DistlibException # type: ignore[import]
|
|
except ImportError:
|
|
HAS_DISTLIB = False
|
|
else:
|
|
HAS_DISTLIB = True
|
|
|
|
if t.TYPE_CHECKING:
|
|
from ansible.galaxy.collection.concrete_artifact_manager import (
|
|
ConcreteArtifactsManager,
|
|
)
|
|
|
|
ManifestKeysType = t.Literal[
|
|
'collection_info', 'file_manifest_file', 'format',
|
|
]
|
|
FileMetaKeysType = t.Literal[
|
|
'name',
|
|
'ftype',
|
|
'chksum_type',
|
|
'chksum_sha256',
|
|
'format',
|
|
]
|
|
CollectionInfoKeysType = t.Literal[
|
|
# collection meta:
|
|
'namespace', 'name', 'version',
|
|
'authors', 'readme',
|
|
'tags', 'description',
|
|
'license', 'license_file',
|
|
'dependencies',
|
|
'repository', 'documentation',
|
|
'homepage', 'issues',
|
|
|
|
# files meta:
|
|
FileMetaKeysType,
|
|
]
|
|
ManifestValueType = t.Dict[CollectionInfoKeysType, t.Union[int, str, t.List[str], t.Dict[str, str], None]]
|
|
CollectionManifestType = t.Dict[ManifestKeysType, ManifestValueType]
|
|
FileManifestEntryType = t.Dict[FileMetaKeysType, t.Union[str, int, None]]
|
|
FilesManifestType = t.Dict[t.Literal['files', 'format'], t.Union[t.List[FileManifestEntryType], int]]
|
|
|
|
import ansible.constants as C
|
|
from ansible.compat.importlib_resources import files
|
|
from ansible.errors import AnsibleError
|
|
from ansible.galaxy.api import GalaxyAPI
|
|
from ansible.galaxy.collection.concrete_artifact_manager import (
|
|
_consume_file,
|
|
_download_file,
|
|
_get_json_from_installed_dir,
|
|
_get_meta_from_src_dir,
|
|
_tarfile_extract,
|
|
)
|
|
from ansible.galaxy.collection.galaxy_api_proxy import MultiGalaxyAPIProxy
|
|
from ansible.galaxy.collection.gpg import (
|
|
run_gpg_verify,
|
|
parse_gpg_errors,
|
|
get_signature_from_source,
|
|
GPG_ERROR_MAP,
|
|
)
|
|
try:
|
|
from ansible.galaxy.dependency_resolution import (
|
|
build_collection_dependency_resolver,
|
|
)
|
|
from ansible.galaxy.dependency_resolution.errors import (
|
|
CollectionDependencyResolutionImpossible,
|
|
CollectionDependencyInconsistentCandidate,
|
|
)
|
|
from ansible.galaxy.dependency_resolution.providers import (
|
|
RESOLVELIB_VERSION,
|
|
RESOLVELIB_LOWERBOUND,
|
|
RESOLVELIB_UPPERBOUND,
|
|
)
|
|
except ImportError:
|
|
HAS_RESOLVELIB = False
|
|
else:
|
|
HAS_RESOLVELIB = True
|
|
|
|
from ansible.galaxy.dependency_resolution.dataclasses import (
|
|
Candidate, Requirement, _is_installed_collection_dir,
|
|
)
|
|
from ansible.galaxy.dependency_resolution.versioning import meets_requirements
|
|
from ansible.plugins.loader import get_all_plugin_loaders
|
|
from ansible.module_utils.six import raise_from
|
|
from ansible.module_utils._text import to_bytes, to_native, to_text
|
|
from ansible.module_utils.common.collections import is_sequence
|
|
from ansible.module_utils.common.yaml import yaml_dump
|
|
from ansible.utils.collection_loader import AnsibleCollectionRef
|
|
from ansible.utils.display import Display
|
|
from ansible.utils.hashing import secure_hash, secure_hash_s
|
|
from ansible.utils.sentinel import Sentinel
|
|
|
|
|
|
display = Display()
|
|
|
|
MANIFEST_FORMAT = 1
|
|
MANIFEST_FILENAME = 'MANIFEST.json'
|
|
|
|
ModifiedContent = namedtuple('ModifiedContent', ['filename', 'expected', 'installed'])
|
|
|
|
SIGNATURE_COUNT_RE = r"^(?P<strict>\+)?(?:(?P<count>\d+)|(?P<all>all))$"
|
|
|
|
|
|
@dataclass
|
|
class ManifestControl:
|
|
directives: list[str] = None
|
|
omit_default_directives: bool = False
|
|
|
|
def __post_init__(self):
|
|
# Allow a dict representing this dataclass to be splatted directly.
|
|
# Requires attrs to have a default value, so anything with a default
|
|
# of None is swapped for its, potentially mutable, default
|
|
for field in dc_fields(self):
|
|
if getattr(self, field.name) is None:
|
|
super().__setattr__(field.name, field.type())
|
|
|
|
|
|
class CollectionSignatureError(Exception):
|
|
def __init__(self, reasons=None, stdout=None, rc=None, ignore=False):
|
|
self.reasons = reasons
|
|
self.stdout = stdout
|
|
self.rc = rc
|
|
self.ignore = ignore
|
|
|
|
self._reason_wrapper = None
|
|
|
|
def _report_unexpected(self, collection_name):
|
|
return (
|
|
f"Unexpected error for '{collection_name}': "
|
|
f"GnuPG signature verification failed with the return code {self.rc} and output {self.stdout}"
|
|
)
|
|
|
|
def _report_expected(self, collection_name):
|
|
header = f"Signature verification failed for '{collection_name}' (return code {self.rc}):"
|
|
return header + self._format_reasons()
|
|
|
|
def _format_reasons(self):
|
|
if self._reason_wrapper is None:
|
|
self._reason_wrapper = textwrap.TextWrapper(
|
|
initial_indent=" * ", # 6 chars
|
|
subsequent_indent=" ", # 6 chars
|
|
)
|
|
|
|
wrapped_reasons = [
|
|
'\n'.join(self._reason_wrapper.wrap(reason))
|
|
for reason in self.reasons
|
|
]
|
|
|
|
return '\n' + '\n'.join(wrapped_reasons)
|
|
|
|
def report(self, collection_name):
|
|
if self.reasons:
|
|
return self._report_expected(collection_name)
|
|
|
|
return self._report_unexpected(collection_name)
|
|
|
|
|
|
# FUTURE: expose actual verify result details for a collection on this object, maybe reimplement as dataclass on py3.8+
|
|
class CollectionVerifyResult:
|
|
def __init__(self, collection_name): # type: (str) -> None
|
|
self.collection_name = collection_name # type: str
|
|
self.success = True # type: bool
|
|
|
|
|
|
def verify_local_collection(local_collection, remote_collection, artifacts_manager):
|
|
# type: (Candidate, t.Optional[Candidate], ConcreteArtifactsManager) -> CollectionVerifyResult
|
|
"""Verify integrity of the locally installed collection.
|
|
|
|
:param local_collection: Collection being checked.
|
|
:param remote_collection: Upstream collection (optional, if None, only verify local artifact)
|
|
:param artifacts_manager: Artifacts manager.
|
|
:return: a collection verify result object.
|
|
"""
|
|
result = CollectionVerifyResult(local_collection.fqcn)
|
|
|
|
b_collection_path = to_bytes(local_collection.src, errors='surrogate_or_strict')
|
|
|
|
display.display("Verifying '{coll!s}'.".format(coll=local_collection))
|
|
display.display(
|
|
u"Installed collection found at '{path!s}'".
|
|
format(path=to_text(local_collection.src)),
|
|
)
|
|
|
|
modified_content = [] # type: list[ModifiedContent]
|
|
|
|
verify_local_only = remote_collection is None
|
|
|
|
# partial away the local FS detail so we can just ask generically during validation
|
|
get_json_from_validation_source = functools.partial(_get_json_from_installed_dir, b_collection_path)
|
|
get_hash_from_validation_source = functools.partial(_get_file_hash, b_collection_path)
|
|
|
|
if not verify_local_only:
|
|
# Compare installed version versus requirement version
|
|
if local_collection.ver != remote_collection.ver:
|
|
err = (
|
|
"{local_fqcn!s} has the version '{local_ver!s}' but "
|
|
"is being compared to '{remote_ver!s}'".format(
|
|
local_fqcn=local_collection.fqcn,
|
|
local_ver=local_collection.ver,
|
|
remote_ver=remote_collection.ver,
|
|
)
|
|
)
|
|
display.display(err)
|
|
result.success = False
|
|
return result
|
|
|
|
manifest_file = os.path.join(to_text(b_collection_path, errors='surrogate_or_strict'), MANIFEST_FILENAME)
|
|
signatures = list(local_collection.signatures)
|
|
if verify_local_only and local_collection.source_info is not None:
|
|
signatures = [info["signature"] for info in local_collection.source_info["signatures"]] + signatures
|
|
elif not verify_local_only and remote_collection.signatures:
|
|
signatures = list(remote_collection.signatures) + signatures
|
|
|
|
keyring_configured = artifacts_manager.keyring is not None
|
|
if not keyring_configured and signatures:
|
|
display.warning(
|
|
"The GnuPG keyring used for collection signature "
|
|
"verification was not configured but signatures were "
|
|
"provided by the Galaxy server. "
|
|
"Configure a keyring for ansible-galaxy to verify "
|
|
"the origin of the collection. "
|
|
"Skipping signature verification."
|
|
)
|
|
elif keyring_configured:
|
|
if not verify_file_signatures(
|
|
local_collection.fqcn,
|
|
manifest_file,
|
|
signatures,
|
|
artifacts_manager.keyring,
|
|
artifacts_manager.required_successful_signature_count,
|
|
artifacts_manager.ignore_signature_errors,
|
|
):
|
|
result.success = False
|
|
return result
|
|
display.vvvv(f"GnuPG signature verification succeeded, verifying contents of {local_collection}")
|
|
|
|
if verify_local_only:
|
|
# since we're not downloading this, just seed it with the value from disk
|
|
manifest_hash = get_hash_from_validation_source(MANIFEST_FILENAME)
|
|
elif keyring_configured and remote_collection.signatures:
|
|
manifest_hash = get_hash_from_validation_source(MANIFEST_FILENAME)
|
|
else:
|
|
# fetch remote
|
|
b_temp_tar_path = ( # NOTE: AnsibleError is raised on URLError
|
|
artifacts_manager.get_artifact_path
|
|
if remote_collection.is_concrete_artifact
|
|
else artifacts_manager.get_galaxy_artifact_path
|
|
)(remote_collection)
|
|
|
|
display.vvv(
|
|
u"Remote collection cached as '{path!s}'".format(path=to_text(b_temp_tar_path))
|
|
)
|
|
|
|
# partial away the tarball details so we can just ask generically during validation
|
|
get_json_from_validation_source = functools.partial(_get_json_from_tar_file, b_temp_tar_path)
|
|
get_hash_from_validation_source = functools.partial(_get_tar_file_hash, b_temp_tar_path)
|
|
|
|
# Verify the downloaded manifest hash matches the installed copy before verifying the file manifest
|
|
manifest_hash = get_hash_from_validation_source(MANIFEST_FILENAME)
|
|
_verify_file_hash(b_collection_path, MANIFEST_FILENAME, manifest_hash, modified_content)
|
|
|
|
display.display('MANIFEST.json hash: {manifest_hash}'.format(manifest_hash=manifest_hash))
|
|
|
|
manifest = get_json_from_validation_source(MANIFEST_FILENAME)
|
|
|
|
# Use the manifest to verify the file manifest checksum
|
|
file_manifest_data = manifest['file_manifest_file']
|
|
file_manifest_filename = file_manifest_data['name']
|
|
expected_hash = file_manifest_data['chksum_%s' % file_manifest_data['chksum_type']]
|
|
|
|
# Verify the file manifest before using it to verify individual files
|
|
_verify_file_hash(b_collection_path, file_manifest_filename, expected_hash, modified_content)
|
|
file_manifest = get_json_from_validation_source(file_manifest_filename)
|
|
|
|
collection_dirs = set()
|
|
collection_files = {
|
|
os.path.join(b_collection_path, b'MANIFEST.json'),
|
|
os.path.join(b_collection_path, b'FILES.json'),
|
|
}
|
|
|
|
# Use the file manifest to verify individual file checksums
|
|
for manifest_data in file_manifest['files']:
|
|
name = manifest_data['name']
|
|
|
|
if manifest_data['ftype'] == 'file':
|
|
collection_files.add(
|
|
os.path.join(b_collection_path, to_bytes(name, errors='surrogate_or_strict'))
|
|
)
|
|
expected_hash = manifest_data['chksum_%s' % manifest_data['chksum_type']]
|
|
_verify_file_hash(b_collection_path, name, expected_hash, modified_content)
|
|
|
|
if manifest_data['ftype'] == 'dir':
|
|
collection_dirs.add(
|
|
os.path.join(b_collection_path, to_bytes(name, errors='surrogate_or_strict'))
|
|
)
|
|
|
|
# Find any paths not in the FILES.json
|
|
for root, dirs, files in os.walk(b_collection_path):
|
|
for name in files:
|
|
full_path = os.path.join(root, name)
|
|
path = to_text(full_path[len(b_collection_path) + 1::], errors='surrogate_or_strict')
|
|
|
|
if full_path not in collection_files:
|
|
modified_content.append(
|
|
ModifiedContent(filename=path, expected='the file does not exist', installed='the file exists')
|
|
)
|
|
for name in dirs:
|
|
full_path = os.path.join(root, name)
|
|
path = to_text(full_path[len(b_collection_path) + 1::], errors='surrogate_or_strict')
|
|
|
|
if full_path not in collection_dirs:
|
|
modified_content.append(
|
|
ModifiedContent(filename=path, expected='the directory does not exist', installed='the directory exists')
|
|
)
|
|
|
|
if modified_content:
|
|
result.success = False
|
|
display.display(
|
|
'Collection {fqcn!s} contains modified content '
|
|
'in the following files:'.
|
|
format(fqcn=to_text(local_collection.fqcn)),
|
|
)
|
|
for content_change in modified_content:
|
|
display.display(' %s' % content_change.filename)
|
|
display.v(" Expected: %s\n Found: %s" % (content_change.expected, content_change.installed))
|
|
else:
|
|
what = "are internally consistent with its manifest" if verify_local_only else "match the remote collection"
|
|
display.display(
|
|
"Successfully verified that checksums for '{coll!s}' {what!s}.".
|
|
format(coll=local_collection, what=what),
|
|
)
|
|
|
|
return result
|
|
|
|
|
|
def verify_file_signatures(fqcn, manifest_file, detached_signatures, keyring, required_successful_count, ignore_signature_errors):
|
|
# type: (str, str, list[str], str, str, list[str]) -> bool
|
|
successful = 0
|
|
error_messages = []
|
|
|
|
signature_count_requirements = re.match(SIGNATURE_COUNT_RE, required_successful_count).groupdict()
|
|
|
|
strict = signature_count_requirements['strict'] or False
|
|
require_all = signature_count_requirements['all']
|
|
require_count = signature_count_requirements['count']
|
|
if require_count is not None:
|
|
require_count = int(require_count)
|
|
|
|
for signature in detached_signatures:
|
|
signature = to_text(signature, errors='surrogate_or_strict')
|
|
try:
|
|
verify_file_signature(manifest_file, signature, keyring, ignore_signature_errors)
|
|
except CollectionSignatureError as error:
|
|
if error.ignore:
|
|
# Do not include ignored errors in either the failed or successful count
|
|
continue
|
|
error_messages.append(error.report(fqcn))
|
|
else:
|
|
successful += 1
|
|
|
|
if require_all:
|
|
continue
|
|
|
|
if successful == require_count:
|
|
break
|
|
|
|
if strict and not successful:
|
|
verified = False
|
|
display.display(f"Signature verification failed for '{fqcn}': no successful signatures")
|
|
elif require_all:
|
|
verified = not error_messages
|
|
if not verified:
|
|
display.display(f"Signature verification failed for '{fqcn}': some signatures failed")
|
|
else:
|
|
verified = not detached_signatures or require_count == successful
|
|
if not verified:
|
|
display.display(f"Signature verification failed for '{fqcn}': fewer successful signatures than required")
|
|
|
|
if not verified:
|
|
for msg in error_messages:
|
|
display.vvvv(msg)
|
|
|
|
return verified
|
|
|
|
|
|
def verify_file_signature(manifest_file, detached_signature, keyring, ignore_signature_errors):
|
|
# type: (str, str, str, list[str]) -> None
|
|
"""Run the gpg command and parse any errors. Raises CollectionSignatureError on failure."""
|
|
gpg_result, gpg_verification_rc = run_gpg_verify(manifest_file, detached_signature, keyring, display)
|
|
|
|
if gpg_result:
|
|
errors = parse_gpg_errors(gpg_result)
|
|
try:
|
|
error = next(errors)
|
|
except StopIteration:
|
|
pass
|
|
else:
|
|
reasons = []
|
|
ignored_reasons = 0
|
|
|
|
for error in chain([error], errors):
|
|
# Get error status (dict key) from the class (dict value)
|
|
status_code = list(GPG_ERROR_MAP.keys())[list(GPG_ERROR_MAP.values()).index(error.__class__)]
|
|
if status_code in ignore_signature_errors:
|
|
ignored_reasons += 1
|
|
reasons.append(error.get_gpg_error_description())
|
|
|
|
ignore = len(reasons) == ignored_reasons
|
|
raise CollectionSignatureError(reasons=set(reasons), stdout=gpg_result, rc=gpg_verification_rc, ignore=ignore)
|
|
|
|
if gpg_verification_rc:
|
|
raise CollectionSignatureError(stdout=gpg_result, rc=gpg_verification_rc)
|
|
|
|
# No errors and rc is 0, verify was successful
|
|
return None
|
|
|
|
|
|
def build_collection(u_collection_path, u_output_path, force):
|
|
# type: (str, str, bool) -> str
|
|
"""Creates the Ansible collection artifact in a .tar.gz file.
|
|
|
|
:param u_collection_path: The path to the collection to build. This should be the directory that contains the
|
|
galaxy.yml file.
|
|
:param u_output_path: The path to create the collection build artifact. This should be a directory.
|
|
:param force: Whether to overwrite an existing collection build artifact or fail.
|
|
:return: The path to the collection build artifact.
|
|
"""
|
|
b_collection_path = to_bytes(u_collection_path, errors='surrogate_or_strict')
|
|
try:
|
|
collection_meta = _get_meta_from_src_dir(b_collection_path)
|
|
except LookupError as lookup_err:
|
|
raise_from(AnsibleError(to_native(lookup_err)), lookup_err)
|
|
|
|
collection_manifest = _build_manifest(**collection_meta)
|
|
file_manifest = _build_files_manifest(
|
|
b_collection_path,
|
|
collection_meta['namespace'], # type: ignore[arg-type]
|
|
collection_meta['name'], # type: ignore[arg-type]
|
|
collection_meta['build_ignore'], # type: ignore[arg-type]
|
|
collection_meta['manifest'], # type: ignore[arg-type]
|
|
collection_meta['license_file'], # type: ignore[arg-type]
|
|
)
|
|
|
|
artifact_tarball_file_name = '{ns!s}-{name!s}-{ver!s}.tar.gz'.format(
|
|
name=collection_meta['name'],
|
|
ns=collection_meta['namespace'],
|
|
ver=collection_meta['version'],
|
|
)
|
|
b_collection_output = os.path.join(
|
|
to_bytes(u_output_path),
|
|
to_bytes(artifact_tarball_file_name, errors='surrogate_or_strict'),
|
|
)
|
|
|
|
if os.path.exists(b_collection_output):
|
|
if os.path.isdir(b_collection_output):
|
|
raise AnsibleError("The output collection artifact '%s' already exists, "
|
|
"but is a directory - aborting" % to_native(b_collection_output))
|
|
elif not force:
|
|
raise AnsibleError("The file '%s' already exists. You can use --force to re-create "
|
|
"the collection artifact." % to_native(b_collection_output))
|
|
|
|
collection_output = _build_collection_tar(b_collection_path, b_collection_output, collection_manifest, file_manifest)
|
|
return collection_output
|
|
|
|
|
|
def download_collections(
|
|
collections, # type: t.Iterable[Requirement]
|
|
output_path, # type: str
|
|
apis, # type: t.Iterable[GalaxyAPI]
|
|
no_deps, # type: bool
|
|
allow_pre_release, # type: bool
|
|
artifacts_manager, # type: ConcreteArtifactsManager
|
|
): # type: (...) -> None
|
|
"""Download Ansible collections as their tarball from a Galaxy server to the path specified and creates a requirements
|
|
file of the downloaded requirements to be used for an install.
|
|
|
|
:param collections: The collections to download, should be a list of tuples with (name, requirement, Galaxy Server).
|
|
:param output_path: The path to download the collections to.
|
|
:param apis: A list of GalaxyAPIs to query when search for a collection.
|
|
:param validate_certs: Whether to validate the certificate if downloading a tarball from a non-Galaxy host.
|
|
:param no_deps: Ignore any collection dependencies and only download the base requirements.
|
|
:param allow_pre_release: Do not ignore pre-release versions when selecting the latest.
|
|
"""
|
|
with _display_progress("Process download dependency map"):
|
|
dep_map = _resolve_depenency_map(
|
|
set(collections),
|
|
galaxy_apis=apis,
|
|
preferred_candidates=None,
|
|
concrete_artifacts_manager=artifacts_manager,
|
|
no_deps=no_deps,
|
|
allow_pre_release=allow_pre_release,
|
|
upgrade=False,
|
|
# Avoid overhead getting signatures since they are not currently applicable to downloaded collections
|
|
include_signatures=False,
|
|
offline=False,
|
|
)
|
|
|
|
b_output_path = to_bytes(output_path, errors='surrogate_or_strict')
|
|
|
|
requirements = []
|
|
with _display_progress(
|
|
"Starting collection download process to '{path!s}'".
|
|
format(path=output_path),
|
|
):
|
|
for fqcn, concrete_coll_pin in dep_map.copy().items(): # FIXME: move into the provider
|
|
if concrete_coll_pin.is_virtual:
|
|
display.display(
|
|
'Virtual collection {coll!s} is not downloadable'.
|
|
format(coll=to_text(concrete_coll_pin)),
|
|
)
|
|
continue
|
|
|
|
display.display(
|
|
u"Downloading collection '{coll!s}' to '{path!s}'".
|
|
format(coll=to_text(concrete_coll_pin), path=to_text(b_output_path)),
|
|
)
|
|
|
|
b_src_path = (
|
|
artifacts_manager.get_artifact_path
|
|
if concrete_coll_pin.is_concrete_artifact
|
|
else artifacts_manager.get_galaxy_artifact_path
|
|
)(concrete_coll_pin)
|
|
|
|
b_dest_path = os.path.join(
|
|
b_output_path,
|
|
os.path.basename(b_src_path),
|
|
)
|
|
|
|
if concrete_coll_pin.is_dir:
|
|
b_dest_path = to_bytes(
|
|
build_collection(
|
|
to_text(b_src_path, errors='surrogate_or_strict'),
|
|
to_text(output_path, errors='surrogate_or_strict'),
|
|
force=True,
|
|
),
|
|
errors='surrogate_or_strict',
|
|
)
|
|
else:
|
|
shutil.copy(to_native(b_src_path), to_native(b_dest_path))
|
|
|
|
display.display(
|
|
"Collection '{coll!s}' was downloaded successfully".
|
|
format(coll=concrete_coll_pin),
|
|
)
|
|
requirements.append({
|
|
# FIXME: Consider using a more specific upgraded format
|
|
# FIXME: having FQCN in the name field, with src field
|
|
# FIXME: pointing to the file path, and explicitly set
|
|
# FIXME: type. If version and name are set, it'd
|
|
# FIXME: perform validation against the actual metadata
|
|
# FIXME: in the artifact src points at.
|
|
'name': to_native(os.path.basename(b_dest_path)),
|
|
'version': concrete_coll_pin.ver,
|
|
})
|
|
|
|
requirements_path = os.path.join(output_path, 'requirements.yml')
|
|
b_requirements_path = to_bytes(
|
|
requirements_path, errors='surrogate_or_strict',
|
|
)
|
|
display.display(
|
|
u'Writing requirements.yml file of downloaded collections '
|
|
"to '{path!s}'".format(path=to_text(requirements_path)),
|
|
)
|
|
yaml_bytes = to_bytes(
|
|
yaml_dump({'collections': requirements}),
|
|
errors='surrogate_or_strict',
|
|
)
|
|
with open(b_requirements_path, mode='wb') as req_fd:
|
|
req_fd.write(yaml_bytes)
|
|
|
|
|
|
def publish_collection(collection_path, api, wait, timeout):
|
|
"""Publish an Ansible collection tarball into an Ansible Galaxy server.
|
|
|
|
:param collection_path: The path to the collection tarball to publish.
|
|
:param api: A GalaxyAPI to publish the collection to.
|
|
:param wait: Whether to wait until the import process is complete.
|
|
:param timeout: The time in seconds to wait for the import process to finish, 0 is indefinite.
|
|
"""
|
|
import_uri = api.publish_collection(collection_path)
|
|
|
|
if wait:
|
|
# Galaxy returns a url fragment which differs between v2 and v3. The second to last entry is
|
|
# always the task_id, though.
|
|
# v2: {"task": "https://galaxy-dev.ansible.com/api/v2/collection-imports/35573/"}
|
|
# v3: {"task": "/api/automation-hub/v3/imports/collections/838d1308-a8f4-402c-95cb-7823f3806cd8/"}
|
|
task_id = None
|
|
for path_segment in reversed(import_uri.split('/')):
|
|
if path_segment:
|
|
task_id = path_segment
|
|
break
|
|
|
|
if not task_id:
|
|
raise AnsibleError("Publishing the collection did not return valid task info. Cannot wait for task status. Returned task info: '%s'" % import_uri)
|
|
|
|
with _display_progress(
|
|
"Collection has been published to the Galaxy server "
|
|
"{api.name!s} {api.api_server!s}".format(api=api),
|
|
):
|
|
api.wait_import_task(task_id, timeout)
|
|
display.display("Collection has been successfully published and imported to the Galaxy server %s %s"
|
|
% (api.name, api.api_server))
|
|
else:
|
|
display.display("Collection has been pushed to the Galaxy server %s %s, not waiting until import has "
|
|
"completed due to --no-wait being set. Import task results can be found at %s"
|
|
% (api.name, api.api_server, import_uri))
|
|
|
|
|
|
def install_collections(
|
|
collections, # type: t.Iterable[Requirement]
|
|
output_path, # type: str
|
|
apis, # type: t.Iterable[GalaxyAPI]
|
|
ignore_errors, # type: bool
|
|
no_deps, # type: bool
|
|
force, # type: bool
|
|
force_deps, # type: bool
|
|
upgrade, # type: bool
|
|
allow_pre_release, # type: bool
|
|
artifacts_manager, # type: ConcreteArtifactsManager
|
|
disable_gpg_verify, # type: bool
|
|
offline, # type: bool
|
|
): # type: (...) -> None
|
|
"""Install Ansible collections to the path specified.
|
|
|
|
:param collections: The collections to install.
|
|
:param output_path: The path to install the collections to.
|
|
:param apis: A list of GalaxyAPIs to query when searching for a collection.
|
|
:param validate_certs: Whether to validate the certificates if downloading a tarball.
|
|
:param ignore_errors: Whether to ignore any errors when installing the collection.
|
|
:param no_deps: Ignore any collection dependencies and only install the base requirements.
|
|
:param force: Re-install a collection if it has already been installed.
|
|
:param force_deps: Re-install a collection as well as its dependencies if they have already been installed.
|
|
"""
|
|
existing_collections = {
|
|
Requirement(coll.fqcn, coll.ver, coll.src, coll.type, None)
|
|
for coll in find_existing_collections(output_path, artifacts_manager)
|
|
}
|
|
|
|
unsatisfied_requirements = set(
|
|
chain.from_iterable(
|
|
(
|
|
Requirement.from_dir_path(sub_coll, artifacts_manager)
|
|
for sub_coll in (
|
|
artifacts_manager.
|
|
get_direct_collection_dependencies(install_req).
|
|
keys()
|
|
)
|
|
)
|
|
if install_req.is_subdirs else (install_req, )
|
|
for install_req in collections
|
|
),
|
|
)
|
|
requested_requirements_names = {req.fqcn for req in unsatisfied_requirements}
|
|
|
|
# NOTE: Don't attempt to reevaluate already installed deps
|
|
# NOTE: unless `--force` or `--force-with-deps` is passed
|
|
unsatisfied_requirements -= set() if force or force_deps else {
|
|
req
|
|
for req in unsatisfied_requirements
|
|
for exs in existing_collections
|
|
if req.fqcn == exs.fqcn and meets_requirements(exs.ver, req.ver)
|
|
}
|
|
|
|
if not unsatisfied_requirements and not upgrade:
|
|
display.display(
|
|
'Nothing to do. All requested collections are already '
|
|
'installed. If you want to reinstall them, '
|
|
'consider using `--force`.'
|
|
)
|
|
return
|
|
|
|
# FIXME: This probably needs to be improved to
|
|
# FIXME: properly match differing src/type.
|
|
existing_non_requested_collections = {
|
|
coll for coll in existing_collections
|
|
if coll.fqcn not in requested_requirements_names
|
|
}
|
|
|
|
preferred_requirements = (
|
|
[] if force_deps
|
|
else existing_non_requested_collections if force
|
|
else existing_collections
|
|
)
|
|
preferred_collections = {
|
|
# NOTE: No need to include signatures if the collection is already installed
|
|
Candidate(coll.fqcn, coll.ver, coll.src, coll.type, None)
|
|
for coll in preferred_requirements
|
|
}
|
|
with _display_progress("Process install dependency map"):
|
|
dependency_map = _resolve_depenency_map(
|
|
collections,
|
|
galaxy_apis=apis,
|
|
preferred_candidates=preferred_collections,
|
|
concrete_artifacts_manager=artifacts_manager,
|
|
no_deps=no_deps,
|
|
allow_pre_release=allow_pre_release,
|
|
upgrade=upgrade,
|
|
include_signatures=not disable_gpg_verify,
|
|
offline=offline,
|
|
)
|
|
|
|
keyring_exists = artifacts_manager.keyring is not None
|
|
with _display_progress("Starting collection install process"):
|
|
for fqcn, concrete_coll_pin in dependency_map.items():
|
|
if concrete_coll_pin.is_virtual:
|
|
display.vvvv(
|
|
"'{coll!s}' is virtual, skipping.".
|
|
format(coll=to_text(concrete_coll_pin)),
|
|
)
|
|
continue
|
|
|
|
if concrete_coll_pin in preferred_collections:
|
|
display.display(
|
|
"'{coll!s}' is already installed, skipping.".
|
|
format(coll=to_text(concrete_coll_pin)),
|
|
)
|
|
continue
|
|
|
|
if not disable_gpg_verify and concrete_coll_pin.signatures and not keyring_exists:
|
|
# Duplicate warning msgs are not displayed
|
|
display.warning(
|
|
"The GnuPG keyring used for collection signature "
|
|
"verification was not configured but signatures were "
|
|
"provided by the Galaxy server to verify authenticity. "
|
|
"Configure a keyring for ansible-galaxy to use "
|
|
"or disable signature verification. "
|
|
"Skipping signature verification."
|
|
)
|
|
|
|
try:
|
|
install(concrete_coll_pin, output_path, artifacts_manager)
|
|
except AnsibleError as err:
|
|
if ignore_errors:
|
|
display.warning(
|
|
'Failed to install collection {coll!s} but skipping '
|
|
'due to --ignore-errors being set. Error: {error!s}'.
|
|
format(
|
|
coll=to_text(concrete_coll_pin),
|
|
error=to_text(err),
|
|
)
|
|
)
|
|
else:
|
|
raise
|
|
|
|
|
|
# NOTE: imported in ansible.cli.galaxy
|
|
def validate_collection_name(name): # type: (str) -> str
|
|
"""Validates the collection name as an input from the user or a requirements file fit the requirements.
|
|
|
|
:param name: The input name with optional range specifier split by ':'.
|
|
:return: The input value, required for argparse validation.
|
|
"""
|
|
collection, dummy, dummy = name.partition(':')
|
|
if AnsibleCollectionRef.is_valid_collection_name(collection):
|
|
return name
|
|
|
|
raise AnsibleError("Invalid collection name '%s', "
|
|
"name must be in the format <namespace>.<collection>. \n"
|
|
"Please make sure namespace and collection name contains "
|
|
"characters from [a-zA-Z0-9_] only." % name)
|
|
|
|
|
|
# NOTE: imported in ansible.cli.galaxy
|
|
def validate_collection_path(collection_path): # type: (str) -> str
|
|
"""Ensure a given path ends with 'ansible_collections'
|
|
|
|
:param collection_path: The path that should end in 'ansible_collections'
|
|
:return: collection_path ending in 'ansible_collections' if it does not already.
|
|
"""
|
|
|
|
if os.path.split(collection_path)[1] != 'ansible_collections':
|
|
return os.path.join(collection_path, 'ansible_collections')
|
|
|
|
return collection_path
|
|
|
|
|
|
def verify_collections(
|
|
collections, # type: t.Iterable[Requirement]
|
|
search_paths, # type: t.Iterable[str]
|
|
apis, # type: t.Iterable[GalaxyAPI]
|
|
ignore_errors, # type: bool
|
|
local_verify_only, # type: bool
|
|
artifacts_manager, # type: ConcreteArtifactsManager
|
|
): # type: (...) -> list[CollectionVerifyResult]
|
|
r"""Verify the integrity of locally installed collections.
|
|
|
|
:param collections: The collections to check.
|
|
:param search_paths: Locations for the local collection lookup.
|
|
:param apis: A list of GalaxyAPIs to query when searching for a collection.
|
|
:param ignore_errors: Whether to ignore any errors when verifying the collection.
|
|
:param local_verify_only: When True, skip downloads and only verify local manifests.
|
|
:param artifacts_manager: Artifacts manager.
|
|
:return: list of CollectionVerifyResult objects describing the results of each collection verification
|
|
"""
|
|
results = [] # type: list[CollectionVerifyResult]
|
|
|
|
api_proxy = MultiGalaxyAPIProxy(apis, artifacts_manager)
|
|
|
|
with _display_progress():
|
|
for collection in collections:
|
|
try:
|
|
if collection.is_concrete_artifact:
|
|
raise AnsibleError(
|
|
message="'{coll_type!s}' type is not supported. "
|
|
'The format namespace.name is expected.'.
|
|
format(coll_type=collection.type)
|
|
)
|
|
|
|
# NOTE: Verify local collection exists before
|
|
# NOTE: downloading its source artifact from
|
|
# NOTE: a galaxy server.
|
|
default_err = 'Collection %s is not installed in any of the collection paths.' % collection.fqcn
|
|
for search_path in search_paths:
|
|
b_search_path = to_bytes(
|
|
os.path.join(
|
|
search_path,
|
|
collection.namespace, collection.name,
|
|
),
|
|
errors='surrogate_or_strict',
|
|
)
|
|
if not os.path.isdir(b_search_path):
|
|
continue
|
|
if not _is_installed_collection_dir(b_search_path):
|
|
default_err = (
|
|
"Collection %s does not have a MANIFEST.json. "
|
|
"A MANIFEST.json is expected if the collection has been built "
|
|
"and installed via ansible-galaxy" % collection.fqcn
|
|
)
|
|
continue
|
|
|
|
local_collection = Candidate.from_dir_path(
|
|
b_search_path, artifacts_manager,
|
|
)
|
|
supplemental_signatures = [
|
|
get_signature_from_source(source, display)
|
|
for source in collection.signature_sources or []
|
|
]
|
|
local_collection = Candidate(
|
|
local_collection.fqcn,
|
|
local_collection.ver,
|
|
local_collection.src,
|
|
local_collection.type,
|
|
signatures=frozenset(supplemental_signatures),
|
|
)
|
|
|
|
break
|
|
else:
|
|
raise AnsibleError(message=default_err)
|
|
|
|
if local_verify_only:
|
|
remote_collection = None
|
|
else:
|
|
signatures = api_proxy.get_signatures(local_collection)
|
|
signatures.extend([
|
|
get_signature_from_source(source, display)
|
|
for source in collection.signature_sources or []
|
|
])
|
|
|
|
remote_collection = Candidate(
|
|
collection.fqcn,
|
|
collection.ver if collection.ver != '*'
|
|
else local_collection.ver,
|
|
None, 'galaxy',
|
|
frozenset(signatures),
|
|
)
|
|
|
|
# Download collection on a galaxy server for comparison
|
|
try:
|
|
# NOTE: If there are no signatures, trigger the lookup. If found,
|
|
# NOTE: it'll cache download URL and token in artifact manager.
|
|
# NOTE: If there are no Galaxy server signatures, only user-provided signature URLs,
|
|
# NOTE: those alone validate the MANIFEST.json and the remote collection is not downloaded.
|
|
# NOTE: The remote MANIFEST.json is only used in verification if there are no signatures.
|
|
if not signatures and not collection.signature_sources:
|
|
api_proxy.get_collection_version_metadata(
|
|
remote_collection,
|
|
)
|
|
except AnsibleError as e: # FIXME: does this actually emit any errors?
|
|
# FIXME: extract the actual message and adjust this:
|
|
expected_error_msg = (
|
|
'Failed to find collection {coll.fqcn!s}:{coll.ver!s}'.
|
|
format(coll=collection)
|
|
)
|
|
if e.message == expected_error_msg:
|
|
raise AnsibleError(
|
|
'Failed to find remote collection '
|
|
"'{coll!s}' on any of the galaxy servers".
|
|
format(coll=collection)
|
|
)
|
|
raise
|
|
|
|
result = verify_local_collection(local_collection, remote_collection, artifacts_manager)
|
|
|
|
results.append(result)
|
|
|
|
except AnsibleError as err:
|
|
if ignore_errors:
|
|
display.warning(
|
|
"Failed to verify collection '{coll!s}' but skipping "
|
|
'due to --ignore-errors being set. '
|
|
'Error: {err!s}'.
|
|
format(coll=collection, err=to_text(err)),
|
|
)
|
|
else:
|
|
raise
|
|
|
|
return results
|
|
|
|
|
|
@contextmanager
|
|
def _tempdir():
|
|
b_temp_path = tempfile.mkdtemp(dir=to_bytes(C.DEFAULT_LOCAL_TMP, errors='surrogate_or_strict'))
|
|
try:
|
|
yield b_temp_path
|
|
finally:
|
|
shutil.rmtree(b_temp_path)
|
|
|
|
|
|
@contextmanager
|
|
def _display_progress(msg=None):
|
|
config_display = C.GALAXY_DISPLAY_PROGRESS
|
|
display_wheel = sys.stdout.isatty() if config_display is None else config_display
|
|
|
|
global display
|
|
if msg is not None:
|
|
display.display(msg)
|
|
|
|
if not display_wheel:
|
|
yield
|
|
return
|
|
|
|
def progress(display_queue, actual_display):
|
|
actual_display.debug("Starting display_progress display thread")
|
|
t = threading.current_thread()
|
|
|
|
while True:
|
|
for c in "|/-\\":
|
|
actual_display.display(c + "\b", newline=False)
|
|
time.sleep(0.1)
|
|
|
|
# Display a message from the main thread
|
|
while True:
|
|
try:
|
|
method, args, kwargs = display_queue.get(block=False, timeout=0.1)
|
|
except queue.Empty:
|
|
break
|
|
else:
|
|
func = getattr(actual_display, method)
|
|
func(*args, **kwargs)
|
|
|
|
if getattr(t, "finish", False):
|
|
actual_display.debug("Received end signal for display_progress display thread")
|
|
return
|
|
|
|
class DisplayThread(object):
|
|
|
|
def __init__(self, display_queue):
|
|
self.display_queue = display_queue
|
|
|
|
def __getattr__(self, attr):
|
|
def call_display(*args, **kwargs):
|
|
self.display_queue.put((attr, args, kwargs))
|
|
|
|
return call_display
|
|
|
|
# Temporary override the global display class with our own which add the calls to a queue for the thread to call.
|
|
old_display = display
|
|
try:
|
|
display_queue = queue.Queue()
|
|
display = DisplayThread(display_queue)
|
|
t = threading.Thread(target=progress, args=(display_queue, old_display))
|
|
t.daemon = True
|
|
t.start()
|
|
|
|
try:
|
|
yield
|
|
finally:
|
|
t.finish = True
|
|
t.join()
|
|
except Exception:
|
|
# The exception is re-raised so we can sure the thread is finished and not using the display anymore
|
|
raise
|
|
finally:
|
|
display = old_display
|
|
|
|
|
|
def _verify_file_hash(b_path, filename, expected_hash, error_queue):
|
|
b_file_path = to_bytes(os.path.join(to_text(b_path), filename), errors='surrogate_or_strict')
|
|
|
|
if not os.path.isfile(b_file_path):
|
|
actual_hash = None
|
|
else:
|
|
with open(b_file_path, mode='rb') as file_object:
|
|
actual_hash = _consume_file(file_object)
|
|
|
|
if expected_hash != actual_hash:
|
|
error_queue.append(ModifiedContent(filename=filename, expected=expected_hash, installed=actual_hash))
|
|
|
|
|
|
def _make_manifest():
|
|
return {
|
|
'files': [
|
|
{
|
|
'name': '.',
|
|
'ftype': 'dir',
|
|
'chksum_type': None,
|
|
'chksum_sha256': None,
|
|
'format': MANIFEST_FORMAT,
|
|
},
|
|
],
|
|
'format': MANIFEST_FORMAT,
|
|
}
|
|
|
|
|
|
def _make_entry(name, ftype, chksum_type='sha256', chksum=None):
|
|
return {
|
|
'name': name,
|
|
'ftype': ftype,
|
|
'chksum_type': chksum_type if chksum else None,
|
|
f'chksum_{chksum_type}': chksum,
|
|
'format': MANIFEST_FORMAT
|
|
}
|
|
|
|
|
|
def _build_files_manifest(b_collection_path, namespace, name, ignore_patterns,
|
|
manifest_control, license_file):
|
|
# type: (bytes, str, str, list[str], dict[str, t.Any], t.Optional[str]) -> FilesManifestType
|
|
if ignore_patterns and manifest_control is not Sentinel:
|
|
raise AnsibleError('"build_ignore" and "manifest" are mutually exclusive')
|
|
|
|
if manifest_control is not Sentinel:
|
|
return _build_files_manifest_distlib(
|
|
b_collection_path,
|
|
namespace,
|
|
name,
|
|
manifest_control,
|
|
license_file,
|
|
)
|
|
|
|
return _build_files_manifest_walk(b_collection_path, namespace, name, ignore_patterns)
|
|
|
|
|
|
def _build_files_manifest_distlib(b_collection_path, namespace, name, manifest_control,
|
|
license_file):
|
|
# type: (bytes, str, str, dict[str, t.Any], t.Optional[str]) -> FilesManifestType
|
|
if not HAS_DISTLIB:
|
|
raise AnsibleError('Use of "manifest" requires the python "distlib" library')
|
|
|
|
if manifest_control is None:
|
|
manifest_control = {}
|
|
|
|
try:
|
|
control = ManifestControl(**manifest_control)
|
|
except TypeError as ex:
|
|
raise AnsibleError(f'Invalid "manifest" provided: {ex}')
|
|
|
|
if not is_sequence(control.directives):
|
|
raise AnsibleError(f'"manifest.directives" must be a list, got: {control.directives.__class__.__name__}')
|
|
|
|
if not isinstance(control.omit_default_directives, bool):
|
|
raise AnsibleError(
|
|
'"manifest.omit_default_directives" is expected to be a boolean, got: '
|
|
f'{control.omit_default_directives.__class__.__name__}'
|
|
)
|
|
|
|
if control.omit_default_directives and not control.directives:
|
|
raise AnsibleError(
|
|
'"manifest.omit_default_directives" was set to True, but no directives were defined '
|
|
'in "manifest.directives". This would produce an empty collection artifact.'
|
|
)
|
|
|
|
directives = []
|
|
if control.omit_default_directives:
|
|
directives.extend(control.directives)
|
|
else:
|
|
directives.extend([
|
|
'include meta/*.yml',
|
|
'include *.txt *.md *.rst *.license COPYING LICENSE',
|
|
'recursive-include .reuse **',
|
|
'recursive-include LICENSES **',
|
|
'recursive-include tests **',
|
|
'recursive-include docs **.rst **.yml **.yaml **.json **.j2 **.txt **.license',
|
|
'recursive-include roles **.yml **.yaml **.json **.j2 **.license',
|
|
'recursive-include playbooks **.yml **.yaml **.json **.license',
|
|
'recursive-include changelogs **.yml **.yaml **.license',
|
|
'recursive-include plugins */**.py */**.license',
|
|
])
|
|
|
|
if license_file:
|
|
directives.append(f'include {license_file}')
|
|
|
|
plugins = set(l.package.split('.')[-1] for d, l in get_all_plugin_loaders())
|
|
for plugin in sorted(plugins):
|
|
if plugin in ('modules', 'module_utils'):
|
|
continue
|
|
elif plugin in C.DOCUMENTABLE_PLUGINS:
|
|
directives.append(
|
|
f'recursive-include plugins/{plugin} **.yml **.yaml'
|
|
)
|
|
|
|
directives.extend([
|
|
'recursive-include plugins/modules **.ps1 **.yml **.yaml **.license',
|
|
'recursive-include plugins/module_utils **.ps1 **.psm1 **.cs **.license',
|
|
])
|
|
|
|
directives.extend(control.directives)
|
|
|
|
directives.extend([
|
|
f'exclude galaxy.yml galaxy.yaml MANIFEST.json FILES.json {namespace}-{name}-*.tar.gz',
|
|
'recursive-exclude tests/output **',
|
|
'global-exclude /.* /__pycache__ *.pyc *.pyo *.bak *~ *.swp',
|
|
])
|
|
|
|
display.vvv('Manifest Directives:')
|
|
display.vvv(textwrap.indent('\n'.join(directives), ' '))
|
|
|
|
u_collection_path = to_text(b_collection_path, errors='surrogate_or_strict')
|
|
m = Manifest(u_collection_path)
|
|
for directive in directives:
|
|
try:
|
|
m.process_directive(directive)
|
|
except DistlibException as e:
|
|
raise AnsibleError(f'Invalid manifest directive: {e}')
|
|
except Exception as e:
|
|
raise AnsibleError(f'Unknown error processing manifest directive: {e}')
|
|
|
|
manifest = _make_manifest()
|
|
|
|
for abs_path in m.sorted(wantdirs=True):
|
|
rel_path = os.path.relpath(abs_path, u_collection_path)
|
|
if os.path.isdir(abs_path):
|
|
manifest_entry = _make_entry(rel_path, 'dir')
|
|
else:
|
|
manifest_entry = _make_entry(
|
|
rel_path,
|
|
'file',
|
|
chksum_type='sha256',
|
|
chksum=secure_hash(abs_path, hash_func=sha256)
|
|
)
|
|
|
|
manifest['files'].append(manifest_entry)
|
|
|
|
return manifest
|
|
|
|
|
|
def _build_files_manifest_walk(b_collection_path, namespace, name, ignore_patterns):
|
|
# type: (bytes, str, str, list[str]) -> FilesManifestType
|
|
# We always ignore .pyc and .retry files as well as some well known version control directories. The ignore
|
|
# patterns can be extended by the build_ignore key in galaxy.yml
|
|
b_ignore_patterns = [
|
|
b'MANIFEST.json',
|
|
b'FILES.json',
|
|
b'galaxy.yml',
|
|
b'galaxy.yaml',
|
|
b'.git',
|
|
b'*.pyc',
|
|
b'*.retry',
|
|
b'tests/output', # Ignore ansible-test result output directory.
|
|
to_bytes('{0}-{1}-*.tar.gz'.format(namespace, name)), # Ignores previously built artifacts in the root dir.
|
|
]
|
|
b_ignore_patterns += [to_bytes(p) for p in ignore_patterns]
|
|
b_ignore_dirs = frozenset([b'CVS', b'.bzr', b'.hg', b'.git', b'.svn', b'__pycache__', b'.tox'])
|
|
|
|
manifest = _make_manifest()
|
|
|
|
def _walk(b_path, b_top_level_dir):
|
|
for b_item in os.listdir(b_path):
|
|
b_abs_path = os.path.join(b_path, b_item)
|
|
b_rel_base_dir = b'' if b_path == b_top_level_dir else b_path[len(b_top_level_dir) + 1:]
|
|
b_rel_path = os.path.join(b_rel_base_dir, b_item)
|
|
rel_path = to_text(b_rel_path, errors='surrogate_or_strict')
|
|
|
|
if os.path.isdir(b_abs_path):
|
|
if any(b_item == b_path for b_path in b_ignore_dirs) or \
|
|
any(fnmatch.fnmatch(b_rel_path, b_pattern) for b_pattern in b_ignore_patterns):
|
|
display.vvv("Skipping '%s' for collection build" % to_text(b_abs_path))
|
|
continue
|
|
|
|
if os.path.islink(b_abs_path):
|
|
b_link_target = os.path.realpath(b_abs_path)
|
|
|
|
if not _is_child_path(b_link_target, b_top_level_dir):
|
|
display.warning("Skipping '%s' as it is a symbolic link to a directory outside the collection"
|
|
% to_text(b_abs_path))
|
|
continue
|
|
|
|
manifest['files'].append(_make_entry(rel_path, 'dir'))
|
|
|
|
if not os.path.islink(b_abs_path):
|
|
_walk(b_abs_path, b_top_level_dir)
|
|
else:
|
|
if any(fnmatch.fnmatch(b_rel_path, b_pattern) for b_pattern in b_ignore_patterns):
|
|
display.vvv("Skipping '%s' for collection build" % to_text(b_abs_path))
|
|
continue
|
|
|
|
# Handling of file symlinks occur in _build_collection_tar, the manifest for a symlink is the same for
|
|
# a normal file.
|
|
manifest['files'].append(
|
|
_make_entry(
|
|
rel_path,
|
|
'file',
|
|
chksum_type='sha256',
|
|
chksum=secure_hash(b_abs_path, hash_func=sha256)
|
|
)
|
|
)
|
|
|
|
_walk(b_collection_path, b_collection_path)
|
|
|
|
return manifest
|
|
|
|
|
|
# FIXME: accept a dict produced from `galaxy.yml` instead of separate args
|
|
def _build_manifest(namespace, name, version, authors, readme, tags, description, license_file,
|
|
dependencies, repository, documentation, homepage, issues, **kwargs):
|
|
manifest = {
|
|
'collection_info': {
|
|
'namespace': namespace,
|
|
'name': name,
|
|
'version': version,
|
|
'authors': authors,
|
|
'readme': readme,
|
|
'tags': tags,
|
|
'description': description,
|
|
'license': kwargs['license'],
|
|
'license_file': license_file or None, # Handle galaxy.yml having an empty string (None)
|
|
'dependencies': dependencies,
|
|
'repository': repository,
|
|
'documentation': documentation,
|
|
'homepage': homepage,
|
|
'issues': issues,
|
|
},
|
|
'file_manifest_file': {
|
|
'name': 'FILES.json',
|
|
'ftype': 'file',
|
|
'chksum_type': 'sha256',
|
|
'chksum_sha256': None, # Filled out in _build_collection_tar
|
|
'format': MANIFEST_FORMAT
|
|
},
|
|
'format': MANIFEST_FORMAT,
|
|
}
|
|
|
|
return manifest
|
|
|
|
|
|
def _build_collection_tar(
|
|
b_collection_path, # type: bytes
|
|
b_tar_path, # type: bytes
|
|
collection_manifest, # type: CollectionManifestType
|
|
file_manifest, # type: FilesManifestType
|
|
): # type: (...) -> str
|
|
"""Build a tar.gz collection artifact from the manifest data."""
|
|
files_manifest_json = to_bytes(json.dumps(file_manifest, indent=True), errors='surrogate_or_strict')
|
|
collection_manifest['file_manifest_file']['chksum_sha256'] = secure_hash_s(files_manifest_json, hash_func=sha256)
|
|
collection_manifest_json = to_bytes(json.dumps(collection_manifest, indent=True), errors='surrogate_or_strict')
|
|
|
|
with _tempdir() as b_temp_path:
|
|
b_tar_filepath = os.path.join(b_temp_path, os.path.basename(b_tar_path))
|
|
|
|
with tarfile.open(b_tar_filepath, mode='w:gz') as tar_file:
|
|
# Add the MANIFEST.json and FILES.json file to the archive
|
|
for name, b in [(MANIFEST_FILENAME, collection_manifest_json), ('FILES.json', files_manifest_json)]:
|
|
b_io = BytesIO(b)
|
|
tar_info = tarfile.TarInfo(name)
|
|
tar_info.size = len(b)
|
|
tar_info.mtime = int(time.time())
|
|
tar_info.mode = 0o0644
|
|
tar_file.addfile(tarinfo=tar_info, fileobj=b_io)
|
|
|
|
for file_info in file_manifest['files']: # type: ignore[union-attr]
|
|
if file_info['name'] == '.':
|
|
continue
|
|
|
|
# arcname expects a native string, cannot be bytes
|
|
filename = to_native(file_info['name'], errors='surrogate_or_strict')
|
|
b_src_path = os.path.join(b_collection_path, to_bytes(filename, errors='surrogate_or_strict'))
|
|
|
|
def reset_stat(tarinfo):
|
|
if tarinfo.type != tarfile.SYMTYPE:
|
|
existing_is_exec = tarinfo.mode & stat.S_IXUSR
|
|
tarinfo.mode = 0o0755 if existing_is_exec or tarinfo.isdir() else 0o0644
|
|
tarinfo.uid = tarinfo.gid = 0
|
|
tarinfo.uname = tarinfo.gname = ''
|
|
|
|
return tarinfo
|
|
|
|
if os.path.islink(b_src_path):
|
|
b_link_target = os.path.realpath(b_src_path)
|
|
if _is_child_path(b_link_target, b_collection_path):
|
|
b_rel_path = os.path.relpath(b_link_target, start=os.path.dirname(b_src_path))
|
|
|
|
tar_info = tarfile.TarInfo(filename)
|
|
tar_info.type = tarfile.SYMTYPE
|
|
tar_info.linkname = to_native(b_rel_path, errors='surrogate_or_strict')
|
|
tar_info = reset_stat(tar_info)
|
|
tar_file.addfile(tarinfo=tar_info)
|
|
|
|
continue
|
|
|
|
# Dealing with a normal file, just add it by name.
|
|
tar_file.add(
|
|
to_native(os.path.realpath(b_src_path)),
|
|
arcname=filename,
|
|
recursive=False,
|
|
filter=reset_stat,
|
|
)
|
|
|
|
shutil.copy(to_native(b_tar_filepath), to_native(b_tar_path))
|
|
collection_name = "%s.%s" % (collection_manifest['collection_info']['namespace'],
|
|
collection_manifest['collection_info']['name'])
|
|
tar_path = to_text(b_tar_path)
|
|
display.display(u'Created collection for %s at %s' % (collection_name, tar_path))
|
|
return tar_path
|
|
|
|
|
|
def _build_collection_dir(b_collection_path, b_collection_output, collection_manifest, file_manifest):
|
|
"""Build a collection directory from the manifest data.
|
|
|
|
This should follow the same pattern as _build_collection_tar.
|
|
"""
|
|
os.makedirs(b_collection_output, mode=0o0755)
|
|
|
|
files_manifest_json = to_bytes(json.dumps(file_manifest, indent=True), errors='surrogate_or_strict')
|
|
collection_manifest['file_manifest_file']['chksum_sha256'] = secure_hash_s(files_manifest_json, hash_func=sha256)
|
|
collection_manifest_json = to_bytes(json.dumps(collection_manifest, indent=True), errors='surrogate_or_strict')
|
|
|
|
# Write contents to the files
|
|
for name, b in [(MANIFEST_FILENAME, collection_manifest_json), ('FILES.json', files_manifest_json)]:
|
|
b_path = os.path.join(b_collection_output, to_bytes(name, errors='surrogate_or_strict'))
|
|
with open(b_path, 'wb') as file_obj, BytesIO(b) as b_io:
|
|
shutil.copyfileobj(b_io, file_obj)
|
|
|
|
os.chmod(b_path, 0o0644)
|
|
|
|
base_directories = []
|
|
for file_info in sorted(file_manifest['files'], key=lambda x: x['name']):
|
|
if file_info['name'] == '.':
|
|
continue
|
|
|
|
src_file = os.path.join(b_collection_path, to_bytes(file_info['name'], errors='surrogate_or_strict'))
|
|
dest_file = os.path.join(b_collection_output, to_bytes(file_info['name'], errors='surrogate_or_strict'))
|
|
|
|
existing_is_exec = os.stat(src_file, follow_symlinks=False).st_mode & stat.S_IXUSR
|
|
mode = 0o0755 if existing_is_exec else 0o0644
|
|
|
|
# ensure symlinks to dirs are not translated to empty dirs
|
|
if os.path.isdir(src_file) and not os.path.islink(src_file):
|
|
mode = 0o0755
|
|
base_directories.append(src_file)
|
|
os.mkdir(dest_file, mode)
|
|
else:
|
|
# do not follow symlinks to ensure the original link is used
|
|
shutil.copyfile(src_file, dest_file, follow_symlinks=False)
|
|
|
|
# avoid setting specific permission on symlinks since it does not
|
|
# support avoid following symlinks and will thrown an exception if the
|
|
# symlink target does not exist
|
|
if not os.path.islink(dest_file):
|
|
os.chmod(dest_file, mode)
|
|
|
|
collection_output = to_text(b_collection_output)
|
|
return collection_output
|
|
|
|
|
|
def _normalize_collection_path(path):
|
|
str_path = path.as_posix() if isinstance(path, pathlib.Path) else path
|
|
return pathlib.Path(
|
|
# This is annoying, but GalaxyCLI._resolve_path did it
|
|
os.path.expandvars(str_path)
|
|
).expanduser().absolute()
|
|
|
|
|
|
def find_existing_collections(path_filter, artifacts_manager, namespace_filter=None, collection_filter=None, dedupe=True):
|
|
"""Locate all collections under a given path.
|
|
|
|
:param path: Collection dirs layout search path.
|
|
:param artifacts_manager: Artifacts manager.
|
|
"""
|
|
if files is None:
|
|
raise AnsibleError('importlib_resources is not installed and is required')
|
|
|
|
if path_filter and not is_sequence(path_filter):
|
|
path_filter = [path_filter]
|
|
|
|
paths = set()
|
|
for path in files('ansible_collections').glob('*/*/'):
|
|
path = _normalize_collection_path(path)
|
|
if not path.is_dir():
|
|
continue
|
|
if path_filter:
|
|
for pf in path_filter:
|
|
try:
|
|
path.relative_to(_normalize_collection_path(pf))
|
|
except ValueError:
|
|
continue
|
|
break
|
|
else:
|
|
continue
|
|
paths.add(path)
|
|
|
|
seen = set()
|
|
for path in paths:
|
|
namespace = path.parent.name
|
|
name = path.name
|
|
if namespace_filter and namespace != namespace_filter:
|
|
continue
|
|
if collection_filter and name != collection_filter:
|
|
continue
|
|
|
|
if dedupe:
|
|
try:
|
|
collection_path = files(f'ansible_collections.{namespace}.{name}')
|
|
except ImportError:
|
|
continue
|
|
if collection_path in seen:
|
|
continue
|
|
seen.add(collection_path)
|
|
else:
|
|
collection_path = path
|
|
|
|
b_collection_path = to_bytes(collection_path.as_posix())
|
|
|
|
try:
|
|
req = Candidate.from_dir_path_as_unknown(b_collection_path, artifacts_manager)
|
|
except ValueError as val_err:
|
|
display.warning(f'{val_err}')
|
|
continue
|
|
|
|
display.vvv(
|
|
u"Found installed collection {coll!s} at '{path!s}'".
|
|
format(coll=to_text(req), path=to_text(req.src))
|
|
)
|
|
yield req
|
|
|
|
|
|
def install(collection, path, artifacts_manager): # FIXME: mv to dataclasses?
|
|
# type: (Candidate, str, ConcreteArtifactsManager) -> None
|
|
"""Install a collection under a given path.
|
|
|
|
:param collection: Collection to be installed.
|
|
:param path: Collection dirs layout path.
|
|
:param artifacts_manager: Artifacts manager.
|
|
"""
|
|
b_artifact_path = (
|
|
artifacts_manager.get_artifact_path if collection.is_concrete_artifact
|
|
else artifacts_manager.get_galaxy_artifact_path
|
|
)(collection)
|
|
|
|
collection_path = os.path.join(path, collection.namespace, collection.name)
|
|
b_collection_path = to_bytes(collection_path, errors='surrogate_or_strict')
|
|
display.display(
|
|
u"Installing '{coll!s}' to '{path!s}'".
|
|
format(coll=to_text(collection), path=collection_path),
|
|
)
|
|
|
|
if os.path.exists(b_collection_path):
|
|
shutil.rmtree(b_collection_path)
|
|
|
|
if collection.is_dir:
|
|
install_src(collection, b_artifact_path, b_collection_path, artifacts_manager)
|
|
else:
|
|
install_artifact(
|
|
b_artifact_path,
|
|
b_collection_path,
|
|
artifacts_manager._b_working_directory,
|
|
collection.signatures,
|
|
artifacts_manager.keyring,
|
|
artifacts_manager.required_successful_signature_count,
|
|
artifacts_manager.ignore_signature_errors,
|
|
)
|
|
if (collection.is_online_index_pointer and isinstance(collection.src, GalaxyAPI)):
|
|
write_source_metadata(
|
|
collection,
|
|
b_collection_path,
|
|
artifacts_manager
|
|
)
|
|
|
|
display.display(
|
|
'{coll!s} was installed successfully'.
|
|
format(coll=to_text(collection)),
|
|
)
|
|
|
|
|
|
def write_source_metadata(collection, b_collection_path, artifacts_manager):
|
|
# type: (Candidate, bytes, ConcreteArtifactsManager) -> None
|
|
source_data = artifacts_manager.get_galaxy_artifact_source_info(collection)
|
|
|
|
b_yaml_source_data = to_bytes(yaml_dump(source_data), errors='surrogate_or_strict')
|
|
b_info_dest = collection.construct_galaxy_info_path(b_collection_path)
|
|
b_info_dir = os.path.split(b_info_dest)[0]
|
|
|
|
if os.path.exists(b_info_dir):
|
|
shutil.rmtree(b_info_dir)
|
|
|
|
try:
|
|
os.mkdir(b_info_dir, mode=0o0755)
|
|
with open(b_info_dest, mode='w+b') as fd:
|
|
fd.write(b_yaml_source_data)
|
|
os.chmod(b_info_dest, 0o0644)
|
|
except Exception:
|
|
# Ensure we don't leave the dir behind in case of a failure.
|
|
if os.path.isdir(b_info_dir):
|
|
shutil.rmtree(b_info_dir)
|
|
raise
|
|
|
|
|
|
def verify_artifact_manifest(manifest_file, signatures, keyring, required_signature_count, ignore_signature_errors):
|
|
# type: (str, list[str], str, str, list[str]) -> None
|
|
failed_verify = False
|
|
coll_path_parts = to_text(manifest_file, errors='surrogate_or_strict').split(os.path.sep)
|
|
collection_name = '%s.%s' % (coll_path_parts[-3], coll_path_parts[-2]) # get 'ns' and 'coll' from /path/to/ns/coll/MANIFEST.json
|
|
if not verify_file_signatures(collection_name, manifest_file, signatures, keyring, required_signature_count, ignore_signature_errors):
|
|
raise AnsibleError(f"Not installing {collection_name} because GnuPG signature verification failed.")
|
|
display.vvvv(f"GnuPG signature verification succeeded for {collection_name}")
|
|
|
|
|
|
def install_artifact(b_coll_targz_path, b_collection_path, b_temp_path, signatures, keyring, required_signature_count, ignore_signature_errors):
|
|
"""Install a collection from tarball under a given path.
|
|
|
|
:param b_coll_targz_path: Collection tarball to be installed.
|
|
:param b_collection_path: Collection dirs layout path.
|
|
:param b_temp_path: Temporary dir path.
|
|
:param signatures: frozenset of signatures to verify the MANIFEST.json
|
|
:param keyring: The keyring used during GPG verification
|
|
:param required_signature_count: The number of signatures that must successfully verify the collection
|
|
:param ignore_signature_errors: GPG errors to ignore during signature verification
|
|
"""
|
|
try:
|
|
with tarfile.open(b_coll_targz_path, mode='r') as collection_tar:
|
|
# Verify the signature on the MANIFEST.json before extracting anything else
|
|
_extract_tar_file(collection_tar, MANIFEST_FILENAME, b_collection_path, b_temp_path)
|
|
|
|
if keyring is not None:
|
|
manifest_file = os.path.join(to_text(b_collection_path, errors='surrogate_or_strict'), MANIFEST_FILENAME)
|
|
verify_artifact_manifest(manifest_file, signatures, keyring, required_signature_count, ignore_signature_errors)
|
|
|
|
files_member_obj = collection_tar.getmember('FILES.json')
|
|
with _tarfile_extract(collection_tar, files_member_obj) as (dummy, files_obj):
|
|
files = json.loads(to_text(files_obj.read(), errors='surrogate_or_strict'))
|
|
|
|
_extract_tar_file(collection_tar, 'FILES.json', b_collection_path, b_temp_path)
|
|
|
|
for file_info in files['files']:
|
|
file_name = file_info['name']
|
|
if file_name == '.':
|
|
continue
|
|
|
|
if file_info['ftype'] == 'file':
|
|
_extract_tar_file(collection_tar, file_name, b_collection_path, b_temp_path,
|
|
expected_hash=file_info['chksum_sha256'])
|
|
|
|
else:
|
|
_extract_tar_dir(collection_tar, file_name, b_collection_path)
|
|
|
|
except Exception:
|
|
# Ensure we don't leave the dir behind in case of a failure.
|
|
shutil.rmtree(b_collection_path)
|
|
|
|
b_namespace_path = os.path.dirname(b_collection_path)
|
|
if not os.listdir(b_namespace_path):
|
|
os.rmdir(b_namespace_path)
|
|
|
|
raise
|
|
|
|
|
|
def install_src(collection, b_collection_path, b_collection_output_path, artifacts_manager):
|
|
r"""Install the collection from source control into given dir.
|
|
|
|
Generates the Ansible collection artifact data from a galaxy.yml and
|
|
installs the artifact to a directory.
|
|
This should follow the same pattern as build_collection, but instead
|
|
of creating an artifact, install it.
|
|
|
|
:param collection: Collection to be installed.
|
|
:param b_collection_path: Collection dirs layout path.
|
|
:param b_collection_output_path: The installation directory for the \
|
|
collection artifact.
|
|
:param artifacts_manager: Artifacts manager.
|
|
|
|
:raises AnsibleError: If no collection metadata found.
|
|
"""
|
|
collection_meta = artifacts_manager.get_direct_collection_meta(collection)
|
|
|
|
if 'build_ignore' not in collection_meta: # installed collection, not src
|
|
# FIXME: optimize this? use a different process? copy instead of build?
|
|
collection_meta['build_ignore'] = []
|
|
collection_manifest = _build_manifest(**collection_meta)
|
|
file_manifest = _build_files_manifest(
|
|
b_collection_path,
|
|
collection_meta['namespace'], collection_meta['name'],
|
|
collection_meta['build_ignore'],
|
|
collection_meta['manifest'],
|
|
collection_meta['license_file'],
|
|
)
|
|
|
|
collection_output_path = _build_collection_dir(
|
|
b_collection_path, b_collection_output_path,
|
|
collection_manifest, file_manifest,
|
|
)
|
|
|
|
display.display(
|
|
'Created collection for {coll!s} at {path!s}'.
|
|
format(coll=collection, path=collection_output_path)
|
|
)
|
|
|
|
|
|
def _extract_tar_dir(tar, dirname, b_dest):
|
|
""" Extracts a directory from a collection tar. """
|
|
member_names = [to_native(dirname, errors='surrogate_or_strict')]
|
|
|
|
# Create list of members with and without trailing separator
|
|
if not member_names[-1].endswith(os.path.sep):
|
|
member_names.append(member_names[-1] + os.path.sep)
|
|
|
|
# Try all of the member names and stop on the first one that are able to successfully get
|
|
for member in member_names:
|
|
try:
|
|
tar_member = tar.getmember(member)
|
|
except KeyError:
|
|
continue
|
|
break
|
|
else:
|
|
# If we still can't find the member, raise a nice error.
|
|
raise AnsibleError("Unable to extract '%s' from collection" % to_native(member, errors='surrogate_or_strict'))
|
|
|
|
b_dir_path = os.path.join(b_dest, to_bytes(dirname, errors='surrogate_or_strict'))
|
|
|
|
b_parent_path = os.path.dirname(b_dir_path)
|
|
try:
|
|
os.makedirs(b_parent_path, mode=0o0755)
|
|
except OSError as e:
|
|
if e.errno != errno.EEXIST:
|
|
raise
|
|
|
|
if tar_member.type == tarfile.SYMTYPE:
|
|
b_link_path = to_bytes(tar_member.linkname, errors='surrogate_or_strict')
|
|
if not _is_child_path(b_link_path, b_dest, link_name=b_dir_path):
|
|
raise AnsibleError("Cannot extract symlink '%s' in collection: path points to location outside of "
|
|
"collection '%s'" % (to_native(dirname), b_link_path))
|
|
|
|
os.symlink(b_link_path, b_dir_path)
|
|
|
|
else:
|
|
if not os.path.isdir(b_dir_path):
|
|
os.mkdir(b_dir_path, 0o0755)
|
|
|
|
|
|
def _extract_tar_file(tar, filename, b_dest, b_temp_path, expected_hash=None):
|
|
""" Extracts a file from a collection tar. """
|
|
with _get_tar_file_member(tar, filename) as (tar_member, tar_obj):
|
|
if tar_member.type == tarfile.SYMTYPE:
|
|
actual_hash = _consume_file(tar_obj)
|
|
|
|
else:
|
|
with tempfile.NamedTemporaryFile(dir=b_temp_path, delete=False) as tmpfile_obj:
|
|
actual_hash = _consume_file(tar_obj, tmpfile_obj)
|
|
|
|
if expected_hash and actual_hash != expected_hash:
|
|
raise AnsibleError("Checksum mismatch for '%s' inside collection at '%s'"
|
|
% (to_native(filename, errors='surrogate_or_strict'), to_native(tar.name)))
|
|
|
|
b_dest_filepath = os.path.abspath(os.path.join(b_dest, to_bytes(filename, errors='surrogate_or_strict')))
|
|
b_parent_dir = os.path.dirname(b_dest_filepath)
|
|
if not _is_child_path(b_parent_dir, b_dest):
|
|
raise AnsibleError("Cannot extract tar entry '%s' as it will be placed outside the collection directory"
|
|
% to_native(filename, errors='surrogate_or_strict'))
|
|
|
|
if not os.path.exists(b_parent_dir):
|
|
# Seems like Galaxy does not validate if all file entries have a corresponding dir ftype entry. This check
|
|
# makes sure we create the parent directory even if it wasn't set in the metadata.
|
|
os.makedirs(b_parent_dir, mode=0o0755)
|
|
|
|
if tar_member.type == tarfile.SYMTYPE:
|
|
b_link_path = to_bytes(tar_member.linkname, errors='surrogate_or_strict')
|
|
if not _is_child_path(b_link_path, b_dest, link_name=b_dest_filepath):
|
|
raise AnsibleError("Cannot extract symlink '%s' in collection: path points to location outside of "
|
|
"collection '%s'" % (to_native(filename), b_link_path))
|
|
|
|
os.symlink(b_link_path, b_dest_filepath)
|
|
|
|
else:
|
|
shutil.move(to_bytes(tmpfile_obj.name, errors='surrogate_or_strict'), b_dest_filepath)
|
|
|
|
# Default to rw-r--r-- and only add execute if the tar file has execute.
|
|
tar_member = tar.getmember(to_native(filename, errors='surrogate_or_strict'))
|
|
new_mode = 0o644
|
|
if stat.S_IMODE(tar_member.mode) & stat.S_IXUSR:
|
|
new_mode |= 0o0111
|
|
|
|
os.chmod(b_dest_filepath, new_mode)
|
|
|
|
|
|
def _get_tar_file_member(tar, filename):
|
|
n_filename = to_native(filename, errors='surrogate_or_strict')
|
|
try:
|
|
member = tar.getmember(n_filename)
|
|
except KeyError:
|
|
raise AnsibleError("Collection tar at '%s' does not contain the expected file '%s'." % (
|
|
to_native(tar.name),
|
|
n_filename))
|
|
|
|
return _tarfile_extract(tar, member)
|
|
|
|
|
|
def _get_json_from_tar_file(b_path, filename):
|
|
file_contents = ''
|
|
|
|
with tarfile.open(b_path, mode='r') as collection_tar:
|
|
with _get_tar_file_member(collection_tar, filename) as (dummy, tar_obj):
|
|
bufsize = 65536
|
|
data = tar_obj.read(bufsize)
|
|
while data:
|
|
file_contents += to_text(data)
|
|
data = tar_obj.read(bufsize)
|
|
|
|
return json.loads(file_contents)
|
|
|
|
|
|
def _get_tar_file_hash(b_path, filename):
|
|
with tarfile.open(b_path, mode='r') as collection_tar:
|
|
with _get_tar_file_member(collection_tar, filename) as (dummy, tar_obj):
|
|
return _consume_file(tar_obj)
|
|
|
|
|
|
def _get_file_hash(b_path, filename): # type: (bytes, str) -> str
|
|
filepath = os.path.join(b_path, to_bytes(filename, errors='surrogate_or_strict'))
|
|
with open(filepath, 'rb') as fp:
|
|
return _consume_file(fp)
|
|
|
|
|
|
def _is_child_path(path, parent_path, link_name=None):
|
|
""" Checks that path is a path within the parent_path specified. """
|
|
b_path = to_bytes(path, errors='surrogate_or_strict')
|
|
|
|
if link_name and not os.path.isabs(b_path):
|
|
# If link_name is specified, path is the source of the link and we need to resolve the absolute path.
|
|
b_link_dir = os.path.dirname(to_bytes(link_name, errors='surrogate_or_strict'))
|
|
b_path = os.path.abspath(os.path.join(b_link_dir, b_path))
|
|
|
|
b_parent_path = to_bytes(parent_path, errors='surrogate_or_strict')
|
|
return b_path == b_parent_path or b_path.startswith(b_parent_path + to_bytes(os.path.sep))
|
|
|
|
|
|
def _resolve_depenency_map(
|
|
requested_requirements, # type: t.Iterable[Requirement]
|
|
galaxy_apis, # type: t.Iterable[GalaxyAPI]
|
|
concrete_artifacts_manager, # type: ConcreteArtifactsManager
|
|
preferred_candidates, # type: t.Iterable[Candidate] | None
|
|
no_deps, # type: bool
|
|
allow_pre_release, # type: bool
|
|
upgrade, # type: bool
|
|
include_signatures, # type: bool
|
|
offline, # type: bool
|
|
): # type: (...) -> dict[str, Candidate]
|
|
"""Return the resolved dependency map."""
|
|
if not HAS_RESOLVELIB:
|
|
raise AnsibleError("Failed to import resolvelib, check that a supported version is installed")
|
|
if not HAS_PACKAGING:
|
|
raise AnsibleError("Failed to import packaging, check that a supported version is installed")
|
|
|
|
req = None
|
|
|
|
try:
|
|
dist = distribution('ansible-core')
|
|
except Exception:
|
|
pass
|
|
else:
|
|
req = next((rr for r in (dist.requires or []) if (rr := PkgReq(r)).name == 'resolvelib'), None)
|
|
finally:
|
|
if req is None:
|
|
# TODO: replace the hardcoded versions with a warning if the dist info is missing
|
|
# display.warning("Unable to find 'ansible-core' distribution requirements to verify the resolvelib version is supported.")
|
|
if not RESOLVELIB_LOWERBOUND <= RESOLVELIB_VERSION < RESOLVELIB_UPPERBOUND:
|
|
raise AnsibleError(
|
|
f"ansible-galaxy requires resolvelib<{RESOLVELIB_UPPERBOUND.vstring},>={RESOLVELIB_LOWERBOUND.vstring}"
|
|
)
|
|
elif not req.specifier.contains(RESOLVELIB_VERSION.vstring):
|
|
raise AnsibleError(f"ansible-galaxy requires {req.name}{req.specifier}")
|
|
|
|
collection_dep_resolver = build_collection_dependency_resolver(
|
|
galaxy_apis=galaxy_apis,
|
|
concrete_artifacts_manager=concrete_artifacts_manager,
|
|
user_requirements=requested_requirements,
|
|
preferred_candidates=preferred_candidates,
|
|
with_deps=not no_deps,
|
|
with_pre_releases=allow_pre_release,
|
|
upgrade=upgrade,
|
|
include_signatures=include_signatures,
|
|
offline=offline,
|
|
)
|
|
try:
|
|
return collection_dep_resolver.resolve(
|
|
requested_requirements,
|
|
max_rounds=2000000, # NOTE: same constant pip uses
|
|
).mapping
|
|
except CollectionDependencyResolutionImpossible as dep_exc:
|
|
conflict_causes = (
|
|
'* {req.fqcn!s}:{req.ver!s} ({dep_origin!s})'.format(
|
|
req=req_inf.requirement,
|
|
dep_origin='direct request'
|
|
if req_inf.parent is None
|
|
else 'dependency of {parent!s}'.
|
|
format(parent=req_inf.parent),
|
|
)
|
|
for req_inf in dep_exc.causes
|
|
)
|
|
error_msg_lines = list(chain(
|
|
(
|
|
'Failed to resolve the requested '
|
|
'dependencies map. Could not satisfy the following '
|
|
'requirements:',
|
|
),
|
|
conflict_causes,
|
|
))
|
|
raise raise_from( # NOTE: Leading "raise" is a hack for mypy bug #9717
|
|
AnsibleError('\n'.join(error_msg_lines)),
|
|
dep_exc,
|
|
)
|
|
except CollectionDependencyInconsistentCandidate as dep_exc:
|
|
parents = [
|
|
"%s.%s:%s" % (p.namespace, p.name, p.ver)
|
|
for p in dep_exc.criterion.iter_parent()
|
|
if p is not None
|
|
]
|
|
|
|
error_msg_lines = [
|
|
(
|
|
'Failed to resolve the requested dependencies map. '
|
|
'Got the candidate {req.fqcn!s}:{req.ver!s} ({dep_origin!s}) '
|
|
'which didn\'t satisfy all of the following requirements:'.
|
|
format(
|
|
req=dep_exc.candidate,
|
|
dep_origin='direct request'
|
|
if not parents else 'dependency of {parent!s}'.
|
|
format(parent=', '.join(parents))
|
|
)
|
|
)
|
|
]
|
|
|
|
for req in dep_exc.criterion.iter_requirement():
|
|
error_msg_lines.append(
|
|
'* {req.fqcn!s}:{req.ver!s}'.format(req=req)
|
|
)
|
|
|
|
raise raise_from( # NOTE: Leading "raise" is a hack for mypy bug #9717
|
|
AnsibleError('\n'.join(error_msg_lines)),
|
|
dep_exc,
|
|
)
|
|
except ValueError as exc:
|
|
raise AnsibleError(to_native(exc)) from exc
|