Merge branch 'lineinfile' of git://github.com/Tinche/ansible into lif

library/lineinfile

@@ -25,7 +25,8 @@ DOCUMENTATION = """
 ---
 module: lineinfile
 author: Daniel Hokka Zakrisson
-short_description: Ensure a particular line is in a file
+short_description: Ensure a particular line is in a file, or replace an
+                   existing line using a back-referenced regular expression.
 description:
   - This module will search a file for a line, and ensure that it is present or absent.
   - This is primarily useful when you want to change a single line in a
@@ -36,12 +37,13 @@ options:
     required: true
     aliases: [ name, destfile ]
     description:
-      - The file to modify
+      - The file to modify.
   regexp:
     required: true
     description:
-      - The regular expression to look for in the file. For C(state=present),
-        the pattern to replace. For C(state=absent), the pattern of the line
+      - The regular expression to look for in every line of the file. For
+        C(state=present), the pattern to replace if found; only the last line
+        found will be replaced. For C(state=absent), the pattern of the line
         to remove.  Uses Python regular expressions; see
         U(http://docs.python.org/2/library/re.html).
   state:
@@ -55,7 +57,22 @@ options:
     required: false
     description:
       - Required for C(state=present). The line to insert/replace into the
-        file. May contain backreferences.
+        file. If backrefs is set, may contain backreferences that will get
+        expanded with the regexp capture groups if the regexp matches. The
+        backreferences should be double escaped (see examples).
+  backrefs:
+    required: false
+    default: "no"
+    choices: [ "yes", "no" ]
+    version_added: 1.1
+    description:
+      - Used with C(state=present). If set, line can contain backreferences
+        (both positional and named) that will get populated if the regexp
+        matches. This flag changes the operation of the module slightly;
+        insertbefore) and insertafter will be ignored, and if the regexp
+        doesn't match anywhere in the file, the file will be left unchanged.
+        If the regexp does match, the last matching line will be replaced by
+        the expanded line parameter.
   insertafter:
     required: false
     default: EOF
@@ -63,6 +80,7 @@ options:
       - Used with C(state=present). If specified, the line will be inserted
         after the specified regular expression. A special value is
         available; C(EOF) for inserting the line at the end of the file.
+        May not be used with backrefs.
     choices: [ 'EOF', '*regex*' ]
   insertbefore:
     required: false
@@ -70,8 +88,8 @@ options:
     description:
       - Used with C(state=present). If specified, the line will be inserted
         before the specified regular expression. A value is available;
-         C(BOF) for inserting the line at the beginning of the
-        file.
+        C(BOF) for inserting the line at the beginning of the file.
+        May not be used with backrefs.
     choices: [ 'BOF', '*regex*' ]
   create:
     required: false
@@ -94,7 +112,7 @@ options:
      required: false
 """
 
-EXAMPLES = """
+EXAMPLES = r"""
    lineinfile: dest=/etc/selinux/config regexp=^SELINUX= line=SELINUX=disabled
 
    lineinfile: dest=/etc/sudoers state=absent regexp="^%wheel"
@@ -105,9 +123,9 @@ EXAMPLES = """
 
    lineinfile: dest=/etc/services regexp="^# port for http" insertbefore="^www.*80/tcp" line="# port for http by default"
 
-   lineinfile: \\\"dest=/etc/sudoers state=present regexp='^%wheel' line ='%wheel ALL=(ALL) NOPASSWD: ALL'\\\"
+   lineinfile: dest=/etc/sudoers state=present regexp='^%wheel' line ='%wheel ALL=(ALL) NOPASSWD: ALL'
 
-   lineinfile dest=/tmp/grub.conf state=present regexp='^(splashimage=.*)$' line="#\\1"
+   lineinfile: dest=/opt/jboss-as/bin/standalone.conf state=present regexp='^(.*)Xms(\d+)m(.*)$' line='\\1Xms${xms}m\\3'
 """
 
 
@@ -121,9 +139,11 @@ def check_file_attrs(module, changed, message):
         changed = True
         message += "ownership, perms or SE linux context changed"
 
-    return [ message, changed ]
+    return message, changed
 
-def present(module, dest, regexp, line, insertafter, insertbefore, create, backup):
+
+def present(module, dest, regexp, line, insertafter, insertbefore, create,
+            backup, backrefs):
 
     if os.path.isdir(dest):
         module.fail_json(rc=256, msg='Destination %s is a directory !' % dest)
@@ -143,9 +163,9 @@ def present(module, dest, regexp, line, insertafter, insertbefore, create, backu
 
     mre = re.compile(regexp)
 
-    if insertafter is not None and insertafter not in ('BOF', 'EOF'):
+    if insertafter not in (None, 'BOF', 'EOF'):
         insre = re.compile(insertafter)
-    elif insertbefore is not None and insertbefore not in ('BOF'):
+    elif insertbefore not in (None, 'BOF'):
         insre = re.compile(insertbefore)
     else:
         insre = None
@@ -154,12 +174,12 @@ def present(module, dest, regexp, line, insertafter, insertbefore, create, backu
     # index[1] is the line num where insertafter/inserbefore has been found
     index = [-1, -1]
     m = None
-    for lineno in range(0, len(lines)):
-        match_found = mre.search(lines[lineno])
+    for lineno, cur_line in enumerate(lines):
+        match_found = mre.search(cur_line)
         if match_found:
             index[0] = lineno
             m = match_found
-        elif insre is not None and insre.search(lines[lineno]):
+        elif insre is not None and insre.search(cur_line):
             if insertafter:
                 # + 1 for the next line
                 index[1] = lineno + 1
@@ -167,15 +187,24 @@ def present(module, dest, regexp, line, insertafter, insertbefore, create, backu
                 # + 1 for the previous line
                 index[1] = lineno
 
-    # Regexp matched a line in the file
-    if index[0] != -1:
-        if lines[index[0]] == m.expand(line) + os.linesep:
     msg = ''
     changed = False
+    # Regexp matched a line in the file
+    if index[0] != -1:
+        if backrefs:
+            new_line = m.expand(line)
         else:
-            lines[index[0]] = m.expand(line) + os.linesep
+            # Don't do backref expansion if not asked.
+            new_line = line
+
+        if lines[index[0]] != new_line + os.linesep:
+            lines[index[0]] = new_line + os.linesep
             msg = 'line replaced'
             changed = True
+    elif backrefs:
+        # Do absolutely nothing, since it's not safe generating the line
+        # without the regexp matching to populate the backrefs.
+        pass
     # Add it to the beginning of the file
     elif insertbefore == 'BOF' or insertafter == 'BOF':
         lines.insert(0, line + os.linesep)
@@ -188,11 +217,10 @@ def present(module, dest, regexp, line, insertafter, insertbefore, create, backu
         lines.append(line + os.linesep)
         msg = 'line added'
         changed = True
-    # Do nothing if regexp didn't match
+    # Do nothing if insert* didn't match
     elif index[1] == -1:
-        msg = ''
-        changed = False 
-    # insertafter/insertbefore= matched
+        pass
+    # insert* matched, but not the regexp
     else:
         lines.insert(index[1], line + os.linesep)
         msg = 'line added'
@@ -205,9 +233,10 @@ def present(module, dest, regexp, line, insertafter, insertbefore, create, backu
         f.writelines(lines)
         f.close()
 
-    [ msg, changed ] = check_file_attrs(module, changed, msg)
+    msg, changed = check_file_attrs(module, changed, msg)
     module.exit_json(changed=changed, msg=msg)
 
+
 def absent(module, dest, regexp, backup):
 
     if os.path.isdir(dest):
@@ -242,9 +271,10 @@ def absent(module, dest, regexp, backup):
     if changed:
         msg = "%s line(s) removed" % len(found)
 
-    [ msg, changed ] = check_file_attrs(module, changed, msg)
+    msg, changed = check_file_attrs(module, changed, msg)
     module.exit_json(changed=changed, found=len(found), msg=msg)
 
+
 def main():
     module = AnsibleModule(
         argument_spec=dict(
@@ -254,6 +284,7 @@ def main():
             line=dict(aliases=['value']),
             insertafter=dict(default=None),
             insertbefore=dict(default=None),
+            backrefs=dict(default=False, type='bool'),
             create=dict(default=False, type='bool'),
             backup=dict(default=False, type='bool'),
         ),
@@ -265,13 +296,21 @@ def main():
     params = module.params
     create = module.params['create']
     backup = module.params['backup']
+    backrefs = module.params['backrefs']
     dest = os.path.expanduser(params['dest'])
 
     if params['state'] == 'present':
         if 'line' not in params:
             module.fail_json(msg='line= is required with state=present')
+
+        # Deal with the insertafter default value manually, to avoid errors
+        # because of the mutually_exclusive mechanism.
+        ins_bef, ins_aft = params['insertbefore'], params['insertafter']
+        if ins_bef is None and ins_aft is None:
+            ins_aft = 'EOF'
+
         present(module, dest, params['regexp'], params['line'],
-                params['insertafter'], params['insertbefore'], create, backup)
+                ins_aft, ins_bef, create, backup, backrefs)
     else:
         absent(module, dest, params['regexp'], backup)
 
@@ -279,4 +318,3 @@ def main():
 #<<INCLUDE_ANSIBLE_MODULE_COMMON>>
 
 main()
-

test/TestRunner.py

@@ -10,10 +10,10 @@ import os
 import shutil
 import time
 import tempfile
-import urllib2
 
 from nose.plugins.skip import SkipTest
 
+
 def get_binary(name):
     for directory in os.environ["PATH"].split(os.pathsep):
         path = os.path.join(directory, name)
@@ -21,6 +21,7 @@ def get_binary(name):
             return path
     return None
 
+
 class TestRunner(unittest.TestCase):
 
     def setUp(self):
@@ -110,13 +111,13 @@ class TestRunner(unittest.TestCase):
         data_out = file(output).read()
         assert data_in == data_out
         assert 'failed' not in result
-        assert result['changed'] == True
+        assert result['changed'] is True
         assert 'md5sum' in result
         result = self._run('copy', [
             "src=%s" % input_,
             "dest=%s" % output,
         ])
-        assert result['changed'] == False
+        assert result['changed'] is False
 
     def test_command(self):
         # test command module, change trigger, etc
@@ -191,7 +192,6 @@ class TestRunner(unittest.TestCase):
         assert not os.path.exists(filedemo)
         assert not self._run('file', ['dest=' + filedemo, 'state=absent'])['changed']
 
-
         filedemo = tempfile.mkdtemp()
         assert self._run('file', ['dest=' + filedemo, 'state=file'])['failed']
         assert os.path.isdir(filedemo)
@@ -209,7 +209,6 @@ class TestRunner(unittest.TestCase):
         assert not os.path.exists(filedemo)
         assert not self._run('file', ['dest=' + filedemo, 'state=absent'])['changed']
 
-
         tmp_dir = tempfile.mkdtemp()
         filedemo = os.path.join(tmp_dir, 'link')
         os.symlink('/dev/zero', filedemo)
@@ -263,7 +262,7 @@ class TestRunner(unittest.TestCase):
     def test_fetch(self):
         input_ = self._get_test_file('sample.j2')
         output = os.path.join(self.stage_dir, 'localhost', input_)
-        result = self._run('fetch', [ "src=%s" % input_, "dest=%s" % self.stage_dir ])
+        self._run('fetch', ["src=%s" % input_, "dest=%s" % self.stage_dir])
         assert os.path.exists(output)
         assert open(input_).read() == open(output).read()
 
@@ -280,7 +279,7 @@ class TestRunner(unittest.TestCase):
         assert out.find("first") != -1
         assert out.find("second") != -1
         assert out.find("third") != -1
-        assert result['changed'] == True
+        assert result['changed'] is True
         assert 'md5sum' in result
         assert 'failed' not in result
         result = self._run('assemble', [
@@ -288,9 +287,10 @@ class TestRunner(unittest.TestCase):
             "dest=%s" % output,
         ])
         print result
-        assert result['changed'] == False
+        assert result['changed'] is False
 
     def test_lineinfile(self):
+        """Unit tests for the lineinfile module, without backref features."""
         sampleroot = 'rocannon'
         sample_origin = self._get_test_file(sampleroot + '.txt')
         sample = self._get_stage_file(sampleroot + '.out' + '.txt')
@@ -305,17 +305,17 @@ class TestRunner(unittest.TestCase):
                     "line='%s'" % testline
                     ])
         result = self._run(*testcase)
-        assert result['changed'] == True
+        assert result['changed']
         assert result['msg'] == 'line added'
-        artifact = [ x.strip() for x in open(sample).readlines() ]
+        artifact = [x.strip() for x in open(sample)]
         assert artifact[-1] == testline
         assert artifact.count(testline) == 1
 
         # run a second time, verify only one line has been added
         result = self._run(*testcase)
-        assert result['changed'] == False
+        assert not result['changed']
         assert result['msg'] == ''
-        artifact = [ x.strip() for x in open(sample).readlines() ]
+        artifact = [x.strip() for x in open(sample)]
         assert artifact.count(testline) == 1
 
         # insertafter with EOF
@@ -327,13 +327,15 @@ class TestRunner(unittest.TestCase):
                     "line='%s'" % testline
                     ])
         result = self._run(*testcase)
-        assert result['changed'] == True
+        assert result['changed']
         assert result['msg'] == 'line added'
-        artifact = [ x.strip() for x in open(sample).readlines() ]
+        artifact = [x.strip() for x in open(sample)]
         assert artifact[-1] == testline
         assert artifact.count(testline) == 1
 
         # with invalid insertafter regex
+        # If the regexp doesn't match and the insertafter doesn't match,
+        # do nothing.
         testline = 'Third: Line added with an invalid insertafter regex'
         testcase = ('lineinfile', [
                     "dest=%s" % sample,
@@ -342,13 +344,11 @@ class TestRunner(unittest.TestCase):
                     "line='%s'" % testline
                     ])
         result = self._run(*testcase)
-        assert result['changed'] == True
-        assert result['msg'] == 'line added'
-        artifact = [ x.strip() for x in open(sample).readlines() ]
-        assert artifact[-1] == testline
-        assert artifact.count(testline) == 1
+        assert not result['changed']
 
         # with an insertafter regex
+        # The regexp doesn't match, but the insertafter is specified and does,
+        # so insert after insertafter.
         testline = 'Fourth: Line added with a valid insertafter regex'
         testcase = ('lineinfile', [
                     "dest=%s" % sample,
@@ -357,16 +357,16 @@ class TestRunner(unittest.TestCase):
                     "line='%s'" % testline
                     ])
         result = self._run(*testcase)
-        assert result['changed'] == True
+        assert result['changed']
         assert result['msg'] == 'line added'
-        artifact = [ x.strip() for x in open(sample).readlines() ]
+        artifact = [x.strip() for x in open(sample)]
         assert artifact.count(testline) == 1
         idx = artifact.index('receive messages to and from a corresponding device over any distance')
         assert artifact[idx + 1] == testline
 
         # replacement of a line from a regex
         # we replace the line, so we need to get its idx before the run
-        artifact = [ x.strip() for x in open(sample).readlines() ]
+        artifact = [x.strip() for x in open(sample)]
         target_line = 'combination of microphone, speaker, keyboard and display. It can send and'
         idx = artifact.index(target_line)
 
@@ -377,16 +377,16 @@ class TestRunner(unittest.TestCase):
                     "line='%s'" % testline
                     ])
         result = self._run(*testcase)
-        assert result['changed'] == True
+        assert result['changed']
         assert result['msg'] == 'line replaced'
-        artifact = [ x.strip() for x in open(sample).readlines() ]
+        artifact = [x.strip() for x in open(sample)]
         assert artifact.count(testline) == 1
         assert artifact.index(testline) == idx
         assert target_line not in artifact
 
         # removal of a line
         # we replace the line, so we need to get its idx before the run
-        artifact = [ x.strip() for x in open(sample).readlines() ]
+        artifact = [x.strip() for x in open(sample)]
         target_line = 'receive messages to and from a corresponding device over any distance'
         idx = artifact.index(target_line)
 
@@ -396,11 +396,10 @@ class TestRunner(unittest.TestCase):
                     "state=absent"
                     ])
         result = self._run(*testcase)
-        assert result['changed'] == True
-        artifact = [ x.strip() for x in open(sample).readlines() ]
+        assert result['changed']
+        artifact = [x.strip() for x in open(sample)]
         assert target_line not in artifact
 
-
         # with both insertafter and insertbefore (should fail)
         testline = 'Seventh: this line should not be there'
         testcase = ('lineinfile', [
@@ -411,7 +410,7 @@ class TestRunner(unittest.TestCase):
                     "line='%s'" % testline
                     ])
         result = self._run(*testcase)
-        assert result['failed'] == True
+        assert result['failed']
 
         # insertbefore with BOF
         testline = 'Eighth: insertbefore BOF'
@@ -422,9 +421,9 @@ class TestRunner(unittest.TestCase):
                     "line='%s'" % testline
                     ])
         result = self._run(*testcase)
-        assert result['changed'] == True
+        assert result['changed']
         assert result['msg'] == 'line added'
-        artifact = [ x.strip() for x in open(sample).readlines() ]
+        artifact = [x.strip() for x in open(sample)]
         assert artifact.count(testline) == 1
         assert artifact[0] == testline
 
@@ -437,9 +436,9 @@ class TestRunner(unittest.TestCase):
                     "line='%s'" % testline
                     ])
         result = self._run(*testcase)
-        assert result['changed'] == True
+        assert result['changed']
         assert result['msg'] == 'line added'
-        artifact = [ x.strip() for x in open(sample).readlines() ]
+        artifact = [x.strip() for x in open(sample)]
         assert artifact.count(testline) == 1
         idx = artifact.index('communication. Typically it is depicted as a lunch-box sized object with some')
         assert artifact[idx - 1] == testline
@@ -447,4 +446,139 @@ class TestRunner(unittest.TestCase):
         # cleanup
         os.unlink(sample)
 
+    def test_lineinfile_backrefs(self):
+        """Unit tests for the lineinfile module, with backref features."""
+        sampleroot = 'rocannon'
+        sample_origin = self._get_test_file(sampleroot + '.txt')
+        origin_lines = [line.strip() for line in open(sample_origin)]
+        sample = self._get_stage_file(sampleroot + '.out' + '.txt')
+        shutil.copy(sample_origin, sample)
+        # The order of the test cases is important
 
+        # The regexp doesn't match, so the line will not be added anywhere.
+        testline = r'\\1: Line added by default at the end of the file.'
+        testcase = ('lineinfile', [
+                    "dest=%s" % sample,
+                    "regexp='^(First): '",
+                    "line='%s'" % testline,
+                    "backrefs=yes",
+                    ])
+        result = self._run(*testcase)
+        assert not result['changed']
+        assert result['msg'] == ''
+        artifact = [x.strip() for x in open(sample)]
+        assert artifact == origin_lines
+
+        # insertafter with EOF
+        # The regexp doesn't match, so the line will not be added anywhere.
+        testline = r'\\1: Line added with insertafter=EOF'
+        testcase = ('lineinfile', [
+                    "dest=%s" % sample,
+                    "insertafter=EOF",
+                    "regexp='^(Second): '",
+                    "line='%s'" % testline,
+                    "backrefs=yes",
+                    ])
+        result = self._run(*testcase)
+        assert not result['changed']
+        assert result['msg'] == ''
+        artifact = [x.strip() for x in open(sample)]
+        assert artifact == origin_lines
+
+        # with invalid insertafter regex
+        # The regexp doesn't match, so do nothing.
+        testline = r'\\1: Line added with an invalid insertafter regex'
+        testcase = ('lineinfile', [
+                    "dest=%s" % sample,
+                    "insertafter='^abcdefgh'",
+                    "regexp='^(Third): '",
+                    "line='%s'" % testline,
+                    "backrefs=yes",
+                    ])
+        result = self._run(*testcase)
+        assert not result['changed']
+        assert artifact == origin_lines
+
+        # with an insertafter regex
+        # The regexp doesn't match, so do nothing.
+        testline = r'\\1: Line added with a valid insertafter regex'
+        testcase = ('lineinfile', [
+                    "dest=%s" % sample,
+                    "insertafter='^receive messages to '",
+                    "regexp='^(Fourth): '",
+                    "line='%s'" % testline,
+                    "backrefs=yes",
+                    ])
+        result = self._run(*testcase)
+        assert not result['changed']
+        assert result['msg'] == ''
+        assert artifact == origin_lines
+
+        # replacement of a line from a regex
+        # we replace the line, so we need to get its idx before the run
+        artifact = [x.strip() for x in open(sample)]
+        target_line = 'combination of microphone, speaker, keyboard and display. It can send and'
+        idx = artifact.index(target_line)
+
+        testline = r'\\1 of megaphone'
+        testline_after = 'combination of megaphone'
+        testcase = ('lineinfile', [
+                    "dest=%s" % sample,
+                    "regexp='(combination) of microphone'",
+                    "line='%s'" % testline,
+                    "backrefs=yes",
+                    ])
+        result = self._run(*testcase)
+        assert result['changed']
+        assert result['msg'] == 'line replaced'
+        artifact = [x.strip() for x in open(sample)]
+        assert artifact.count(testline_after) == 1
+        assert artifact.index(testline_after) == idx
+        assert target_line not in artifact
+
+        # Go again, should be unchanged now.
+        testline = r'\\1 of megaphone'
+        testline_after = 'combination of megaphone'
+        testcase = ('lineinfile', [
+                    "dest=%s" % sample,
+                    "regexp='(combination) of megaphone'",
+                    "line='%s'" % testline,
+                    "backrefs=yes",
+                    ])
+        result = self._run(*testcase)
+        assert not result['changed']
+        assert result['msg'] == ''
+
+        # Try a numeric, named capture group example.
+        f = open(sample, 'a+')
+        f.write("1 + 1 = 3" + os.linesep)
+        f.close()
+        testline = r"2 + \\g<num> = 3"
+        testline_after = "2 + 1 = 3"
+        testcase = ('lineinfile', [
+                    "dest=%s" % sample,
+                    r"regexp='1 \\+ (?P<num>\\d) = 3'",
+                    "line='%s'" % testline,
+                    "backrefs=yes",
+                    ])
+        result = self._run(*testcase)
+        artifact = [x.strip() for x in open(sample)]
+        assert result['changed']
+        assert result['msg'] == 'line replaced'
+        artifact = [x.strip() for x in open(sample)]
+        assert '1 + 1 = 3' not in artifact
+        assert testline_after == artifact[-1]
+
+        # with both insertafter and insertbefore (should fail)
+        testline = 'Seventh: this line should not be there'
+        testcase = ('lineinfile', [
+                    "dest=%s" % sample,
+                    "insertafter='BOF'",
+                    "insertbefore='BOF'",
+                    "regexp='^communication. '",
+                    "line='%s'" % testline
+                    ])
+        result = self._run(*testcase)
+        assert result['failed']
+
+        os.unlink(sample)