Fix build_ignore when installing a collection from source (#75547)

* Make individual dirs that should exist instead of using shutil.copytree

* Test build_ignore when installing collections in git repos

* changelog

* Fix assertion
Fix git directory

* Fix nested content while building the collection dir

Test installing a collection from a git repo and artifact have the same result

Refactor to use variables
pull/75680/head
Sloane Hertel 3 years ago committed by GitHub
parent d6dededab8
commit f38a97cece
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -0,0 +1,2 @@
bugfixes:
- ansible-galaxy - Fix a bug with build_ignore when installing collections from source (https://github.com/ansible/ansible/issues/75528).

@ -994,23 +994,20 @@ def _build_collection_dir(b_collection_path, b_collection_output, collection_man
os.chmod(b_path, 0o0644)
base_directories = []
for file_info in file_manifest['files']:
for file_info in sorted(file_manifest['files'], key=lambda x: x['name']):
if file_info['name'] == '.':
continue
src_file = os.path.join(b_collection_path, to_bytes(file_info['name'], errors='surrogate_or_strict'))
dest_file = os.path.join(b_collection_output, to_bytes(file_info['name'], errors='surrogate_or_strict'))
if any(src_file.startswith(directory) for directory in base_directories):
continue
existing_is_exec = os.stat(src_file).st_mode & stat.S_IXUSR
mode = 0o0755 if existing_is_exec else 0o0644
if os.path.isdir(src_file):
mode = 0o0755
base_directories.append(src_file)
shutil.copytree(src_file, dest_file)
os.mkdir(dest_file, mode)
else:
shutil.copyfile(src_file, dest_file)

@ -1,7 +1,7 @@
- name: delete installed collections
file:
state: "{{ item }}"
path: "{{ galaxy_dir }}/ansible_collections"
state: absent
path: "{{ item }}"
loop:
- absent
- directory
- "{{ install_path }}"
- "{{ alt_install_path }}"

@ -1,12 +1,10 @@
- name: Clone a git repository
git:
repo: https://github.com/ansible-collections/amazon.aws.git
dest: '{{ galaxy_dir }}/development/amazon.aws/'
dest: '{{ scm_path }}/amazon.aws/'
- name: install
command: 'ansible-galaxy collection install git+file://{{galaxy_dir }}/development/amazon.aws/.git'
args:
chdir: '{{ galaxy_dir }}/development'
command: 'ansible-galaxy collection install git+file://{{ scm_path }}/amazon.aws/.git'
- name: list installed collections
command: 'ansible-galaxy collection list'

@ -5,7 +5,7 @@
- name: Test installing collections from git repositories
environment:
ANSIBLE_COLLECTIONS_PATHS: '{{ galaxy_dir }}'
ANSIBLE_COLLECTIONS_PATHS: "{{ galaxy_dir }}/collections"
vars:
cleanup: True
galaxy_dir: "{{ galaxy_dir }}"
@ -32,8 +32,9 @@
path: '{{ item }}'
state: absent
loop:
- '{{ galaxy_dir }}/ansible_collections'
- '{{ galaxy_dir }}/development'
- "{{ install_path }}"
- "{{ alt_install_path }}"
- "{{ scm_path }}"
- name: remove git
package:

@ -1,5 +1,5 @@
- name: Install all collections by default
command: 'ansible-galaxy collection install git+file://{{ galaxy_dir }}/development/ansible_test/.git'
command: 'ansible-galaxy collection install git+file://{{ test_repo_path }}/.git'
- name: list installed collections
command: 'ansible-galaxy collection list'
@ -10,5 +10,36 @@
- "'ansible_test.collection_1' in installed_collections.stdout"
- "'ansible_test.collection_2' in installed_collections.stdout"
- name: install from artifact to another path to compare contents
command: 'ansible-galaxy collection install {{ artifact_path }} -p {{ alt_install_path }} --no-deps'
vars:
artifact_path: "{{ galaxy_dir }}/ansible_test-collection_1-1.0.0.tar.gz"
- name: check if the files and folders in build_ignore were respected
stat:
path: "{{ install_path }}/ansible_test/collection_1/{{ item }}"
register: result
loop:
- foo.txt
- foobar/baz.txt
- foobar/qux
- assert:
that: result.results | map(attribute='stat') | map(attribute='exists') is not any
- name: check that directory with ignored files exists and is empty
stat:
path: "{{ install_path }}/ansible_test/collection_1/foobar"
register: result
- assert:
that: result.stat.exists
- name: test that there are no diff installing from a repo vs artifact
command: "diff -ur {{ collection_from_artifact }} {{ collection_from_repo }} -x *.json"
vars:
collection_from_repo: "{{ install_path }}/ansible_test/collection_1"
collection_from_artifact: "{{ alt_install_path }}/ansible_test/collection_1"
- include_tasks: ./empty_installed_collections.yml
when: cleanup

@ -1,5 +1,5 @@
- name: test installing one collection
command: 'ansible-galaxy collection install git+file://{{ galaxy_dir }}/development/ansible_test/.git#collection_2'
command: 'ansible-galaxy collection install git+file://{{ test_repo_path }}/.git#collection_2'
- name: list installed collections
command: 'ansible-galaxy collection list'

@ -1,5 +1,5 @@
- name: Rerun installing a collection with a dep
command: 'ansible-galaxy collection install git+file://{{ galaxy_dir }}/development/ansible_test/.git#/collection_1/'
command: 'ansible-galaxy collection install git+file://{{ test_repo_path }}/.git#/collection_1/'
register: installed
- name: SCM collections don't have a concrete artifact version so the collection should always be reinstalled
@ -9,7 +9,7 @@
- "'Created collection for ansible_test.collection_2' in installed.stdout"
- name: The collection should also be reinstalled when --force flag is used
command: 'ansible-galaxy collection install git+file://{{ galaxy_dir }}/development/ansible_test/.git#/collection_1/ --force'
command: 'ansible-galaxy collection install git+file://{{ test_repo_path }}/.git#/collection_1/ --force'
register: installed
- assert:
@ -19,7 +19,7 @@
- "'Created collection for ansible_test.collection_2' in installed.stdout"
- name: The collection should also be reinstalled when --force-with-deps is used
command: 'ansible-galaxy collection install git+file://{{ galaxy_dir }}/development/ansible_test/.git#/collection_1/ --force-with-deps'
command: 'ansible-galaxy collection install git+file://{{ test_repo_path }}/.git#/collection_1/ --force-with-deps'
register: installed
- assert:

@ -80,7 +80,7 @@
- name: Clone a git repository
git:
repo: https://github.com/ansible-collections/amazon.aws.git
dest: '{{ galaxy_dir }}/development/amazon.aws/'
dest: '{{ scm_path }}/amazon.aws/'
- name: test using name as a git repo
command: 'ansible-galaxy collection install -r git_prefix_name.yml'
@ -99,5 +99,5 @@
path: '{{ item }}'
state: absent
loop:
- '{{ galaxy_dir }}/development/amazon.aws/'
- '{{ scm_path }}/amazon.aws/'
- '{{ galaxy_dir }}/requirements'

@ -1,5 +1,5 @@
- name: test installing one collection that has a SCM dep with --no-deps
command: 'ansible-galaxy collection install git+file://{{ galaxy_dir }}/development/ansible_test/.git#/collection_1/ --no-deps'
command: 'ansible-galaxy collection install git+file://{{ test_repo_path }}/.git#/collection_1/ --no-deps'
- name: list installed collections
command: 'ansible-galaxy collection list'
@ -14,7 +14,7 @@
include_tasks: ./empty_installed_collections.yml
- name: test installing one collection that has a SCM dep
command: 'ansible-galaxy collection install git+file://{{ galaxy_dir }}/development/ansible_test/.git#/collection_1/'
command: 'ansible-galaxy collection install git+file://{{ test_repo_path }}/.git#/collection_1/'
- name: list installed collections
command: 'ansible-galaxy collection list'

@ -1,5 +1,5 @@
- name: Install all collections in a repo, one of which has a recursive dependency
command: 'ansible-galaxy collection install git+file://{{ galaxy_dir }}/development/namespace_1/.git'
command: 'ansible-galaxy collection install git+file://{{ scm_path }}/namespace_1/.git'
register: command
- assert:
@ -13,22 +13,22 @@
in command.stdout_lines
- >-
"Installing 'namespace_1.collection_1:1.0.0' to
'{{ galaxy_dir }}/ansible_collections/namespace_1/collection_1'"
'{{ install_path }}/namespace_1/collection_1'"
in command.stdout_lines
- >-
'Created collection for namespace_1.collection_1:1.0.0 at
{{ galaxy_dir }}/ansible_collections/namespace_1/collection_1'
{{ install_path }}/namespace_1/collection_1'
in command.stdout_lines
- >-
'namespace_1.collection_1:1.0.0 was installed successfully'
in command.stdout_lines
- >-
"Installing 'namespace_2.collection_2:1.0.0' to
'{{ galaxy_dir }}/ansible_collections/namespace_2/collection_2'"
'{{ install_path }}/namespace_2/collection_2'"
in command.stdout_lines
- >-
'Created collection for namespace_2.collection_2:1.0.0 at
{{ galaxy_dir }}/ansible_collections/namespace_2/collection_2'
{{ install_path }}/namespace_2/collection_2'
in command.stdout_lines
- >-
'namespace_2.collection_2:1.0.0 was installed successfully'
@ -44,7 +44,7 @@
- "'namespace_2.collection_2' in installed_collections.stdout"
- name: Install a specific collection in a repo with a recursive dependency
command: 'ansible-galaxy collection install git+file://{{ galaxy_dir }}/development/namespace_1/.git#/collection_1/ --force-with-deps'
command: 'ansible-galaxy collection install git+file://{{ scm_path }}/namespace_1/.git#/collection_1/ --force-with-deps'
register: command
- assert:
@ -58,22 +58,22 @@
in command.stdout_lines
- >-
"Installing 'namespace_1.collection_1:1.0.0' to
'{{ galaxy_dir }}/ansible_collections/namespace_1/collection_1'"
'{{ install_path }}/namespace_1/collection_1'"
in command.stdout_lines
- >-
'Created collection for namespace_1.collection_1:1.0.0 at
{{ galaxy_dir }}/ansible_collections/namespace_1/collection_1'
{{ install_path }}/namespace_1/collection_1'
in command.stdout_lines
- >-
'namespace_1.collection_1:1.0.0 was installed successfully'
in command.stdout_lines
- >-
"Installing 'namespace_2.collection_2:1.0.0' to
'{{ galaxy_dir }}/ansible_collections/namespace_2/collection_2'"
'{{ install_path }}/namespace_2/collection_2'"
in command.stdout_lines
- >-
'Created collection for namespace_2.collection_2:1.0.0 at
{{ galaxy_dir }}/ansible_collections/namespace_2/collection_2'
{{ install_path }}/namespace_2/collection_2'
in command.stdout_lines
- >-
'namespace_2.collection_2:1.0.0 was installed successfully'

@ -15,5 +15,5 @@
path: '{{ item }}'
state: directory
loop:
- '{{ galaxy_dir }}/ansible_collections'
- '{{ galaxy_dir }}/development/ansible_test'
- '{{ install_path }}'
- '{{ test_repo_path }}'

@ -1,27 +1,70 @@
- name: Initialize a git repo
command: 'git init {{ galaxy_dir }}/development/ansible_test'
command: 'git init {{ test_repo_path }}'
- stat:
path: "{{ galaxy_dir }}/development/ansible_test"
path: "{{ test_repo_path }}"
- name: Add a couple collections to the repository
command: 'ansible-galaxy collection init {{ item }}'
args:
chdir: '{{ galaxy_dir }}/development'
chdir: '{{ scm_path }}'
loop:
- 'ansible_test.collection_1'
- 'ansible_test.collection_2'
- name: Preserve the (empty) docs directory for the SCM collection
file:
path: '{{ test_repo_path }}/{{ item }}/docs/README.md'
state: touch
loop:
- collection_1
- collection_2
- name: Preserve the (empty) roles directory for the SCM collection
file:
path: '{{ test_repo_path }}/{{ item }}/roles/README.md'
state: touch
loop:
- collection_1
- collection_2
- name: create extra files and folders to test build_ignore
file:
path: '{{ test_repo_path }}/collection_1/{{ item.name }}'
state: '{{ item.state }}'
loop:
- name: foo.txt
state: touch
- name: foobar
state: directory
- name: foobar/qux
state: directory
- name: foobar/qux/bar
state: touch
- name: foobar/baz.txt
state: touch
- name: Add collection_2 as a dependency of collection_1
lineinfile:
path: '{{ galaxy_dir }}/development/ansible_test/collection_1/galaxy.yml'
path: '{{ test_repo_path }}/collection_1/galaxy.yml'
regexp: '^dependencies'
line: "dependencies: {'git+file://{{ galaxy_dir }}/development/ansible_test/.git#collection_2/': '*'}"
line: "dependencies: {'git+file://{{ test_repo_path }}/.git#collection_2/': '*'}"
- name: Ignore a directory and files
lineinfile:
path: '{{ test_repo_path }}/collection_1/galaxy.yml'
regexp: '^build_ignore'
line: "build_ignore: ['foo.txt', 'foobar/*']"
- name: Commit the changes
command: '{{ item }}'
args:
chdir: '{{ galaxy_dir }}/development/ansible_test'
chdir: '{{ test_repo_path }}'
loop:
- git add ./
- git commit -m 'add collections'
- name: Build the actual artifact for ansible_test.collection_1 for comparison
command: "ansible-galaxy collection build ansible_test/collection_1 --output-path {{ galaxy_dir }}"
args:
chdir: "{{ scm_path }}"

@ -1,5 +1,5 @@
- name: Initialize git repositories
command: 'git init {{ galaxy_dir }}/development/{{ item }}'
command: 'git init {{ scm_path }}/{{ item }}'
loop:
- namespace_1
- namespace_2
@ -7,27 +7,27 @@
- name: Add a couple collections to the repository
command: 'ansible-galaxy collection init {{ item }}'
args:
chdir: '{{ galaxy_dir }}/development'
chdir: '{{ scm_path }}'
loop:
- 'namespace_1.collection_1'
- 'namespace_2.collection_2'
- name: Add collection_2 as a dependency of collection_1
lineinfile:
path: '{{ galaxy_dir }}/development/namespace_1/collection_1/galaxy.yml'
path: '{{ scm_path }}/namespace_1/collection_1/galaxy.yml'
regexp: '^dependencies'
line: "dependencies: {'git+file://{{ galaxy_dir }}/development/namespace_2/.git#collection_2/': '*'}"
line: "dependencies: {'git+file://{{ scm_path }}/namespace_2/.git#collection_2/': '*'}"
- name: Add collection_1 as a dependency on collection_2
lineinfile:
path: '{{ galaxy_dir }}/development/namespace_2/collection_2/galaxy.yml'
path: '{{ scm_path }}/namespace_2/collection_2/galaxy.yml'
regexp: '^dependencies'
line: "dependencies: {'git+file://{{ galaxy_dir }}/development/namespace_1/.git#collection_1/': 'master'}"
line: "dependencies: {'git+file://{{ scm_path }}/namespace_1/.git#collection_1/': 'master'}"
- name: Commit the changes
shell: git add ./; git commit -m 'add collection'
args:
chdir: '{{ galaxy_dir }}/development/{{ item }}'
chdir: '{{ scm_path }}/{{ item }}'
loop:
- namespace_1
- namespace_2

@ -1,2 +1,2 @@
collections:
- name: git+file://{{ galaxy_dir }}/development/amazon.aws/.git
- name: git+file://{{ scm_path }}/amazon.aws/.git

@ -1,3 +1,3 @@
collections:
- name: file://{{ galaxy_dir }}/development/amazon.aws/.git
- name: file://{{ scm_path }}/amazon.aws/.git
type: git

@ -1,3 +1,3 @@
collections:
# should not work: git prefix or type is required
- name: file://{{ galaxy_dir }}/development/amazon.aws/.git
- name: file://{{ scm_path }}/amazon.aws/.git

@ -1,4 +1,4 @@
collections:
# should not work: source is expected to be a galaxy server name or URL
- source: git+file://{{ galaxy_dir }}/development/amazon.aws/.git
- source: git+file://{{ scm_path }}/amazon.aws/.git
name: ansible.nope

@ -1,5 +1,5 @@
collections:
# should not work: source is expected to be a galaxy server name or URL
- source: git+file://{{ galaxy_dir }}/development/amazon.aws/.git
- source: git+file://{{ scm_path }}/amazon.aws/.git
name: ansible.nope
type: git

@ -1,3 +1,3 @@
collections:
# should not work: source is expected to be a galaxy server name or URL
- source: git+file://{{ galaxy_dir }}/development/amazon.aws/.git
- source: git+file://{{ scm_path }}/amazon.aws/.git

@ -0,0 +1,4 @@
install_path: "{{ galaxy_dir }}/collections/ansible_collections"
alt_install_path: "{{ galaxy_dir }}/other_collections/ansible_collections"
scm_path: "{{ galaxy_dir }}/development"
test_repo_path: "{{ galaxy_dir }}/development/ansible_test"
Loading…
Cancel
Save