Add more include and yaml parsing tests (#70506)

These additional tests should provide coverage for features currently tested by the postgres incidental tests.

lib/ansible/parsing/utils/yaml.py

@@ -13,7 +13,7 @@ from yaml import YAMLError
 
 from ansible.errors import AnsibleParserError
 from ansible.errors.yaml_strings import YAML_SYNTAX_ERROR
-from ansible.module_utils._text import to_native, to_text
+from ansible.module_utils._text import to_native
 from ansible.parsing.yaml.loader import AnsibleLoader
 from ansible.parsing.yaml.objects import AnsibleBaseYAMLObject
 from ansible.parsing.ajson import AnsibleJSONDecoder
@@ -36,10 +36,11 @@ def _handle_error(json_exc, yaml_exc, file_name, show_content):
         err_obj = AnsibleBaseYAMLObject()
         err_obj.ansible_pos = (file_name, yaml_exc.problem_mark.line + 1, yaml_exc.problem_mark.column + 1)
 
-    err_msg = 'We were unable to read either as JSON nor YAML, these are the errors we got from each:\n' \
+    n_yaml_syntax_error = YAML_SYNTAX_ERROR % to_native(getattr(yaml_exc, 'problem', u''))
-              'JSON: %s\n\n' % to_text(json_exc) + YAML_SYNTAX_ERROR % getattr(yaml_exc, 'problem', '')
+    n_err_msg = 'We were unable to read either as JSON nor YAML, these are the errors we got from each:\n' \
+                'JSON: %s\n\n%s' % (to_native(json_exc), n_yaml_syntax_error)
 
-    raise AnsibleParserError(to_native(err_msg), obj=err_obj, show_content=show_content, orig_exc=yaml_exc)
+    raise AnsibleParserError(n_err_msg, obj=err_obj, show_content=show_content, orig_exc=yaml_exc)
 
 
 def _safe_load(stream, file_name=None, vault_secrets=None):

test/integration/targets/include_import/runme.sh

@@ -114,3 +114,5 @@ test "$(grep -c 'ok=3' test_allow_single_role_dup.out)" = 1
 
 # https://github.com/ansible/ansible/issues/66764
 ANSIBLE_HOST_PATTERN_MISMATCH=error ansible-playbook empty_group_warning/playbook.yml
+
+ansible-playbook test_include_loop.yml "$@"

test/integration/targets/include_import/test_include_loop.yml

@@ -0,0 +1,17 @@
+- hosts: localhost
+  gather_facts: false
+  tasks:
+    - name: skipped include undefined loop
+      include_tasks: doesnt_matter.yml
+      loop: '{{ lkjsdflkjsdlfkjsdlfkjsdf }}'
+      when: false
+      register: skipped_include
+
+    - debug:
+        var: skipped_include
+
+    - assert:
+        that:
+          - skipped_include.results is undefined
+          - skipped_include.skip_reason is defined
+          - skipped_include is skipped

test/integration/targets/yaml_parsing/aliases

@@ -0,0 +1 @@
+shippable/posix/group3

test/integration/targets/yaml_parsing/playbook.yml

@@ -0,0 +1,5 @@
+- hosts: localhost
+  gather_facts: false
+  vars:
+    foo: bar
+    foo: baz  # yamllint disable rule:key-duplicates

test/integration/targets/yaml_parsing/tasks/main.yml

@@ -0,0 +1,33 @@
+- name: Test ANSIBLE_DUPLICATE_YAML_DICT_KEY=warn
+  command: ansible-playbook {{ verbosity }} {{ role_path }}/playbook.yml
+  environment:
+    ANSIBLE_DUPLICATE_YAML_DICT_KEY: warn
+  register: duplicate_warn
+
+- assert:
+    that:
+      - '"found a duplicate dict key (foo)" in duplicate_warn.stderr'
+      - duplicate_warn.rc == 0
+
+- name: Test ANSIBLE_DUPLICATE_YAML_DICT_KEY=error
+  command: ansible-playbook {{ verbosity }} {{ role_path }}/playbook.yml
+  failed_when: duplicate_error.rc != 4
+  environment:
+    ANSIBLE_DUPLICATE_YAML_DICT_KEY: error
+  register: duplicate_error
+
+- assert:
+    that:
+      - '"found a duplicate dict key (foo)" in duplicate_error.stderr'
+      - duplicate_error.rc == 4
+
+- name: Test ANSIBLE_DUPLICATE_YAML_DICT_KEY=ignore
+  command: ansible-playbook {{ verbosity }} {{ role_path }}/playbook.yml
+  environment:
+    ANSIBLE_DUPLICATE_YAML_DICT_KEY: ignore
+  register: duplicate_ignore
+
+- assert:
+    that:
+      - '"found a duplicate dict key (foo)" not in duplicate_ignore.stderr'
+      - duplicate_ignore.rc == 0

test/integration/targets/yaml_parsing/vars/main.yml

@@ -0,0 +1 @@
+verbosity: "{{ '' if not ansible_verbosity else '-' ~ ('v' * ansible_verbosity) }}"

test/units/parsing/yaml/test_constructor.py

@@ -0,0 +1,84 @@
+# -*- coding: utf-8 -*-
+# (c) 2020 Matt Martz <matt@sivel.net>
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+import pytest
+from yaml import MappingNode, Mark, ScalarNode
+from yaml.constructor import ConstructorError
+
+import ansible.constants as C
+from ansible.utils.display import Display
+from ansible.parsing.yaml.constructor import AnsibleConstructor
+
+
+@pytest.fixture
+def dupe_node():
+    tag = 'tag:yaml.org,2002:map'
+    scalar_tag = 'tag:yaml.org,2002:str'
+    mark = Mark(tag, 0, 0, 0, None, None)
+    node = MappingNode(
+        tag,
+        [
+            (
+                ScalarNode(tag=scalar_tag, value='bar', start_mark=mark),
+                ScalarNode(tag=scalar_tag, value='baz', start_mark=mark)
+            ),
+            (
+                ScalarNode(tag=scalar_tag, value='bar', start_mark=mark),
+                ScalarNode(tag=scalar_tag, value='qux', start_mark=mark)
+            ),
+        ],
+        start_mark=mark
+    )
+
+    return node
+
+
+class Capture:
+    def __init__(self):
+        self.called = False
+        self.calls = []
+
+    def __call__(self, *args, **kwargs):
+        self.called = True
+        self.calls.append((
+            args,
+            kwargs
+        ))
+
+
+def test_duplicate_yaml_dict_key_ignore(dupe_node, monkeypatch):
+    monkeypatch.setattr(C, 'DUPLICATE_YAML_DICT_KEY', 'ignore')
+    cap = Capture()
+    monkeypatch.setattr(Display(), 'warning', cap)
+    ac = AnsibleConstructor()
+    ac.construct_mapping(dupe_node)
+    assert not cap.called
+
+
+def test_duplicate_yaml_dict_key_warn(dupe_node, monkeypatch):
+    monkeypatch.setattr(C, 'DUPLICATE_YAML_DICT_KEY', 'warn')
+    cap = Capture()
+    monkeypatch.setattr(Display(), 'warning', cap)
+    ac = AnsibleConstructor()
+    ac.construct_mapping(dupe_node)
+    assert cap.called
+    expected = [
+        (
+            (
+                'While constructing a mapping from tag:yaml.org,2002:map, line 1, column 1, '
+                'found a duplicate dict key (bar). Using last defined value only.',
+            ),
+            {}
+        )
+    ]
+    assert cap.calls == expected
+
+
+def test_duplicate_yaml_dict_key_error(dupe_node, monkeypatch, mocker):
+    monkeypatch.setattr(C, 'DUPLICATE_YAML_DICT_KEY', 'error')
+    ac = AnsibleConstructor()
+    pytest.raises(ConstructorError, ac.construct_mapping, dupe_node)