diff --git a/lib/ansible/plugins/action/include_vars.py b/lib/ansible/plugins/action/include_vars.py index e3b843ad973..b1867d48f54 100644 --- a/lib/ansible/plugins/action/include_vars.py +++ b/lib/ansible/plugins/action/include_vars.py @@ -1,4 +1,4 @@ -# (c) 2013-2014, Benno Joy +# (c) 2016, Allen Sanabria # # This file is part of Ansible # @@ -14,53 +14,281 @@ # # You should have received a copy of the GNU General Public License # along with Ansible. If not, see . + from __future__ import (absolute_import, division, print_function) __metaclass__ = type +from os import path, walk +import re + from ansible.errors import AnsibleError from ansible.plugins.action import ActionBase from ansible.utils.unicode import to_str + class ActionModule(ActionBase): TRANSFERS_FILES = False - def run(self, tmp=None, task_vars=None): + def _mutually_exclusive(self): + dir_arguments = [ + self.source_dir, self.files_matching, self.ignore_files, + self.depth + ] + if self.source_file and None not in dir_arguments: + err_msg = ( + "Can not include {0} with file argument" + .format(", ".join(self.VALID_DIR_ARGUMENTS)) + ) + raise AnsibleError(err_msg) + + elif self.source_dir and self.source_file: + err_msg = ( + "Need to pass either file or dir" + ) + raise AnsibleError(err_msg) + + def _set_dir_defaults(self): + if not self.depth: + self.depth = 0 + + if self.files_matching: + self.matcher = re.compile(r'{0}'.format(self.files_matching)) + else: + self.matcher = None + + if not self.ignore_files: + self.ignore_files = list() + + if isinstance(self.ignore_files, str): + self.ignore_files = self.ignore_files.split() - varname = self._task.args.get('name') - source = self._task.args.get('file') - if not source: - source = self._task.args.get('_raw_params') - if source is None: - raise AnsibleError("No filename was found for the included vars. " + \ - "Use `- include_vars: ` or the `file:` option " + \ - "to specify the vars filename.", self._task._ds) + elif isinstance(self.ignore_files, dict): + return { + 'failed': True, + 'message': '{0} must be a list'.format(self.ignore_files) + } - if task_vars is None: + def _set_args(self): + """ Set instance variables based on the arguments that were passed + """ + self.VALID_DIR_ARGUMENTS = [ + 'dir', 'depth', 'files_matching', 'ignore_files' + ] + self.VALID_FILE_ARGUMENTS = ['file', '_raw_params'] + self.GLOBAL_FILE_ARGUMENTS = ['name'] + + self.VALID_ARGUMENTS = ( + self.VALID_DIR_ARGUMENTS + self.VALID_FILE_ARGUMENTS + + self.GLOBAL_FILE_ARGUMENTS + ) + for arg in self._task.args: + if arg not in self.VALID_ARGUMENTS: + err_msg = '{0} is not a valid option in debug'.format(arg) + raise AnsibleError(err_msg) + + self.return_results_as_name = self._task.args.get('name', None) + self.source_dir = self._task.args.get('dir', None) + self.source_file = self._task.args.get('file', None) + if not self.source_dir and not self.source_file: + self.source_file = self._task.args.get('_raw_params') + + self.depth = self._task.args.get('depth', None) + self.files_matching = self._task.args.get('files_matching', None) + self.ignore_files = self._task.args.get('ignore_files', None) + + self._mutually_exclusive() + + def run(self, tmp=None, task_vars=None): + """ Load yml files recursively from a directory. + """ + self.VALID_FILE_EXTENSIONS = ['yaml', 'yml', '.json'] + if not task_vars: task_vars = dict() + self.show_content = True + self._set_args() + + results = dict() + if self.source_dir: + self._set_dir_defaults() + self._set_root_dir() + if path.exists(self.source_dir): + for root_dir, filenames in self._traverse_dir_depth(): + failed, err_msg, updated_results = ( + self._load_files_in_dir(root_dir, filenames) + ) + if not failed: + results.update(updated_results) + else: + break + else: + failed = True + err_msg = ( + '{0} directory does not exist'.format(self.source_dir) + ) + else: + try: + self.source_file = self._find_needle('vars', self.source_file) + failed, err_msg, updated_results = ( + self._load_files(self.source_file) + ) + if not failed: + results.update(updated_results) + + except AnsibleError as e: + err_msg = to_str(e) + raise AnsibleError(err_msg) + + if self.return_results_as_name: + scope = dict() + scope[self.return_results_as_name] = results + results = scope + result = super(ActionModule, self).run(tmp, task_vars) - try: - source = self._find_needle('vars', source) - except AnsibleError as e: - result['failed'] = True - result['message'] = to_str(e) - return result + if failed: + result['failed'] = failed + result['message'] = err_msg + + result['ansible_facts'] = results + result['_ansible_no_log'] = not self.show_content + + return result + + def _set_root_dir(self): + if self._task._role: + if self.source_dir.split('/')[0] == 'vars': + path_to_use = ( + path.join(self._task._role._role_path, self.source_dir) + ) + if path.exists(path_to_use): + self.source_dir = path_to_use + else: + path_to_use = ( + path.join( + self._task._role._role_path, 'vars', self.source_dir + ) + ) + self.source_dir = path_to_use + else: + current_dir = ( + "/".join(self._task._ds._data_source.split('/')[:-1]) + ) + self.source_dir = path.join(current_dir, self.source_dir) + + def _traverse_dir_depth(self): + """ Recursively iterate over a directory and sort the files in + alphabetical order. Do not iterate pass the set depth. + The default depth is unlimited. + """ + current_depth = 0 + sorted_walk = list(walk(self.source_dir)) + sorted_walk.sort(key=lambda x: x[0]) + for current_root, current_dir, current_files in sorted_walk: + current_depth += 1 + if current_depth <= self.depth or self.depth == 0: + current_files.sort() + yield (current_root, current_files) + else: + break + + def _ignore_file(self, filename): + """ Return True if a file matches the list of ignore_files. + Args: + filename (str): The filename that is being matched against. + + Returns: + Boolean + """ + for file_type in self.ignore_files: + try: + if re.search(r'{0}$'.format(file_type), filename): + return True + except Exception: + err_msg = 'Invalid regular expression: {0}'.format(file_type) + raise AnsibleError(err_msg) + return False - (data, show_content) = self._loader._get_file_contents(source) + def _is_valid_file_ext(self, source_file): + """ Verify if source file has a valid extension + Args: + source_file (str): The full path of source file or source file. + + Returns: + Bool + """ + success = False + file_ext = source_file.split('.') + if len(file_ext) >= 1: + if file_ext[-1] in self.VALID_FILE_EXTENSIONS: + success = True + return success + return success + + def _load_files(self, filename): + """ Loads a file and converts the output into a valid Python dict. + Args: + filename (str): The source file. + + Returns: + Tuple (bool, str, dict) + """ + results = dict() + failed = False + err_msg = '' + if not self._is_valid_file_ext(filename): + failed = True + err_msg = ( + '{0} does not have a valid extension: {1}' + .format(filename, ', '.join(self.VALID_FILE_EXTENSIONS)) + ) + return failed, err_msg, results + + data, show_content = self._loader._get_file_contents(filename) + self.show_content = show_content data = self._loader.load(data, show_content) - if data is None: - data = {} + if not data: + data = dict() if not isinstance(data, dict): - result['failed'] = True - result['message'] = "%s must be stored as a dictionary/hash" % source + failed = True + err_msg = ( + '{0} must be stored as a dictionary/hash' + .format(filename) + ) else: - if varname: - scope = {} - scope[varname] = data - data = scope - result['ansible_facts'] = data - result['_ansible_no_log'] = not show_content + results.update(data) + return failed, err_msg, results - return result + def _load_files_in_dir(self, root_dir, var_files): + """ Load the found yml files and update/overwrite the dictionary. + Args: + root_dir (str): The base directory of the list of files that is being passed. + var_files: (list): List of files to iterate over and load into a dictionary. + + Returns: + Tuple (bool, str, dict) + """ + results = dict() + failed = False + err_msg = '' + for filename in var_files: + stop_iter = False + # Never include main.yml from a role, as that is the default included by the role + if self._task._role: + if filename == 'main.yml': + stop_iter = True + continue + + filepath = path.join(root_dir, filename) + if self.files_matching: + if not self.matcher.search(filename): + stop_iter = True + + if not stop_iter and not failed: + if path.exists(filepath) and not self._ignore_file(filename): + failed, err_msg, loaded_data = self._load_files(filepath) + if not failed: + results.update(loaded_data) + + return failed, err_msg, results diff --git a/test/integration/roles/test_include_vars/defaults/main.yml b/test/integration/roles/test_include_vars/defaults/main.yml new file mode 100644 index 00000000000..901fb220d22 --- /dev/null +++ b/test/integration/roles/test_include_vars/defaults/main.yml @@ -0,0 +1,3 @@ +--- +testing: 1 +base_dir: defaults diff --git a/test/integration/roles/test_include_vars/tasks/main.yml b/test/integration/roles/test_include_vars/tasks/main.yml new file mode 100644 index 00000000000..5ef6b3ef05c --- /dev/null +++ b/test/integration/roles/test_include_vars/tasks/main.yml @@ -0,0 +1,86 @@ +--- +- name: verify that the default value is indeed 1 + assert: + that: + - "testing == 1" + - "base_dir == 'defaults'" + +- name: include the vars/environments/development/all.yml + include_vars: + file: environments/development/all.yml + +- name: verify that the default value is indeed 789 + assert: + that: + - "testing == 789" + - "base_dir == 'environments/development'" + +- name: include the vars/environments/development/all.yml and save results in all + include_vars: + file: environments/development/all.yml + name: all + +- name: verify that the values are stored in the all variable + assert: + that: + - "all['testing'] == 789" + - "all['base_dir'] == 'environments/development'" + +- name: include the all directory in vars + include_vars: + dir: all + depth: 1 + +- name: verify that the default value is indeed 123 + assert: + that: + - "testing == 123" + - "base_dir == 'all'" + +- name: include every directory in vars + include_vars: + dir: vars + +- name: verify that the variable overwrite based on alphabetical order + assert: + that: + - "testing == 456" + - "base_dir == 'services'" + - "webapp_containers == 10" + +- name: include every directory in vars except files matching webapp.yml + include_vars: + dir: vars + ignore_files: + - webapp.yml + +- name: verify that the webapp.yml file was not included + assert: + that: + - "testing == 789" + - "base_dir == 'environments/development'" + +- name: include only files matching webapp.yml + include_vars: + dir: environments + files_matching: webapp.yml + +- name: verify that only files matching webapp.yml and in the environments directory get loaded. + assert: + that: + - "testing == 101112" + - "base_dir == 'development/services'" + - "webapp_containers == 20" + +- name: include only files matching webapp.yml and store results in webapp + include_vars: + dir: environments + files_matching: webapp.yml + name: webapp + +- name: verify that only files matching webapp.yml and in the environments directory get loaded into stored variable webapp. + assert: + that: + - "webapp['testing'] == 101112" + - "webapp['base_dir'] == 'development/services'" + - "webapp['webapp_containers'] == 20" diff --git a/test/integration/roles/test_include_vars/vars/all/all.yml b/test/integration/roles/test_include_vars/vars/all/all.yml new file mode 100644 index 00000000000..14c3e92b8ec --- /dev/null +++ b/test/integration/roles/test_include_vars/vars/all/all.yml @@ -0,0 +1,3 @@ +--- +testing: 123 +base_dir: all diff --git a/test/integration/roles/test_include_vars/vars/environments/development/all.yml b/test/integration/roles/test_include_vars/vars/environments/development/all.yml new file mode 100644 index 00000000000..9f370de5499 --- /dev/null +++ b/test/integration/roles/test_include_vars/vars/environments/development/all.yml @@ -0,0 +1,3 @@ +--- +testing: 789 +base_dir: 'environments/development' diff --git a/test/integration/roles/test_include_vars/vars/environments/development/services/webapp.yml b/test/integration/roles/test_include_vars/vars/environments/development/services/webapp.yml new file mode 100644 index 00000000000..a0a809c9e54 --- /dev/null +++ b/test/integration/roles/test_include_vars/vars/environments/development/services/webapp.yml @@ -0,0 +1,4 @@ +--- +testing: 101112 +base_dir: 'development/services' +webapp_containers: 20 diff --git a/test/integration/roles/test_include_vars/vars/services/webapp.yml b/test/integration/roles/test_include_vars/vars/services/webapp.yml new file mode 100644 index 00000000000..f0dcc8b517b --- /dev/null +++ b/test/integration/roles/test_include_vars/vars/services/webapp.yml @@ -0,0 +1,4 @@ +--- +testing: 456 +base_dir: services +webapp_containers: 10 diff --git a/test/integration/test_include_vars.yml b/test/integration/test_include_vars.yml new file mode 100644 index 00000000000..cb6aa7ec8dd --- /dev/null +++ b/test/integration/test_include_vars.yml @@ -0,0 +1,5 @@ +--- +- hosts: 127.0.0.1 + gather_facts: False + roles: + - { role: test_include_vars }