From 1268f4778dd8cd75112f80c2307ce80c51851db5 Mon Sep 17 00:00:00 2001 From: Dag Wieers Date: Fri, 12 Aug 2016 05:33:54 +0200 Subject: [PATCH] Introduce new 'filetree' lookup plugin (#14332) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Introduce new 'filetree' lookup plugin The new "filetree" lookup plugin makes it possible to recurse over a tree of files within the task loop. This makes it possible to e.g. template a complete tree of files to a target system with little effort while retaining permissions and ownership. The module supports directories, files and symlinks. The item dictionary consists of: - src - root - path - mode - state - owner - group - seuser - serole - setype - selevel - uid - gid - size - mtime - ctime EXAMPLES: Here is an example of how we use with_filetree within a role: ```yaml - name: Create directories file: path: /web/{{ item.path }} state: directory mode: '{{ item.mode }}' owner: '{{ item.owner }}' group: '{{ item.group }}' force: yes with_filetree: web/ when: item.state == 'directory' - name: Template complete tree file: src: '{{ item.src }}' dest: /web/{{ item.path }} state: 'link' mode: '{{ item.mode }}' owner: '{{ item.owner }}' group: '{{ item.group }}' with_filetree: web/ when: item.state == 'link' - name: Template complete tree template: src: '{{ item.src }}' dest: /web/{{ item.path }} mode: '{{ item.mode }}' owner: '{{ item.owner }}' group: '{{ item.group }}' force: yes with_filetree: web/ when: item.state == 'file' ``` SPECIAL USE: The following properties also have its special use: - root: Makes it possible to filter by original location - path: Is the relative path to root - uid, gid: Makes it possible to force-create by exact id, rather than by name - size, mtime, ctime: Makes it possible to filter out files by size, mtime or ctime TODO: - Add snippets to documentation * Small fixes for Python 3 * Return the portion of the file’s mode that can be set by os.chmod() And remove the exists=True, which is redundant. * Use lstat() instead of stat() since we support symlinks * Avoid a few possible stat() calls * Bring in line with v1.9 and hybrid plugin * Remove glob module since we no longer use it * Included suggestions from @RussellLuo - Two blank lines will be better. See PEP 8 - I think if props is not None is more conventional :smile: * Support failed pwd/grp lookups * Implement first-found functionality in the path-order --- lib/ansible/plugins/lookup/filetree.py | 132 +++++++++++++++++++++++++ 1 file changed, 132 insertions(+) create mode 100644 lib/ansible/plugins/lookup/filetree.py diff --git a/lib/ansible/plugins/lookup/filetree.py b/lib/ansible/plugins/lookup/filetree.py new file mode 100644 index 00000000000..d0cbe298fc0 --- /dev/null +++ b/lib/ansible/plugins/lookup/filetree.py @@ -0,0 +1,132 @@ +# (c) 2016 Dag Wieers +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see . +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import os +import pwd +import grp +import stat + +from ansible.plugins.lookup import LookupBase +from __main__ import display +warning = display.warning + +HAVE_SELINUX=False +try: + import selinux + HAVE_SELINUX=True +except ImportError: + pass + +def _to_filesystem_str(path): + '''Returns filesystem path as a str, if it wasn't already. + + Used in selinux interactions because it cannot accept unicode + instances, and specifying complex args in a playbook leaves + you with unicode instances. This method currently assumes + that your filesystem encoding is UTF-8. + + ''' + if isinstance(path, unicode): + path = path.encode("utf-8") + return path + +# If selinux fails to find a default, return an array of None +def selinux_context(path): + context = [None, None, None, None] + if HAVE_SELINUX and selinux.is_selinux_enabled(): + try: + ret = selinux.lgetfilecon_raw(_to_filesystem_str(path)) + except OSError: + return context + if ret[0] != -1: + # Limit split to 4 because the selevel, the last in the list, + # may contain ':' characters + context = ret[1].split(':', 3) + return context + +def file_props(root, path): + ''' Returns dictionary with file properties, or return None on failure ''' + abspath = os.path.join(root, path) + + try: + st = os.lstat(abspath) + except OSError as e: + warning('filetree: Error using stat() on path %s (%s)' % (abspath, e)) + return None + + ret = dict(root=root, path=path) + + if stat.S_ISLNK(st.st_mode): + ret['state'] = 'link' + ret['src'] = os.readlink(abspath) + elif stat.S_ISDIR(st.st_mode): + ret['state'] = 'directory' + elif stat.S_ISREG(st.st_mode): + ret['state'] = 'file' + ret['src'] = abspath + else: + warning('filetree: Error file type of %s is not supported' % abspath) + return None + + ret['uid'] = st.st_uid + ret['gid'] = st.st_gid + try: + ret['owner'] = pwd.getpwuid(st.st_uid).pw_name + except KeyError: + ret['owner'] = st.st_uid + try: + ret['group'] = grp.getgrgid(st.st_gid).gr_name + except KeyError: + ret['group'] = st.st_gid + ret['mode'] = str(oct(stat.S_IMODE(st.st_mode))) + ret['size'] = st.st_size + ret['mtime'] = st.st_mtime + ret['ctime'] = st.st_ctime + + if HAVE_SELINUX and selinux.is_selinux_enabled() == 1: + context = selinux_context(abspath) + ret['seuser'] = context[0] + ret['serole'] = context[1] + ret['setype'] = context[2] + ret['selevel'] = context[3] + + return ret + + +class LookupModule(LookupBase): + + def run(self, terms, variables=None, **kwargs): + basedir = self.get_basedir(variables) + + ret = [] + for term in terms: + term_file = os.path.basename(term) + dwimmed_path = self._loader.path_dwim_relative(basedir, 'files', os.path.dirname(term)) + path = os.path.join(dwimmed_path, term_file) + for root, dirs, files in os.walk(path, topdown=True): + for entry in dirs + files: + relpath = os.path.relpath(os.path.join(root, entry), path) + + # Skip if relpath was already processed (from another root) + if relpath not in [ entry['path'] for entry in ret ]: + props = file_props(path, relpath) + if props is not None: + ret.append(props) + + return ret \ No newline at end of file