Fixes #38174: Add remove-brick operation for gluster_volume (#38269)

* Fixes #38174: Add remove-brick operation for gluster_volume

Existing gluster_volume module does not support removal of bricks
from a gluster volume. Also, operation remove-brick needs to be
followed by a commit operation.

Signed-off-by: Devyani Kota <dkota@redhat.com>

* Updates #38174: Add documentation for remove-brick operation

Signed-off-by: Devyani Kota <dkota@redhat.com>

* Updates #38174: Add support to reduce cluster configuration.

An exceptional scenario exists where the user might want to
reduce the gluster cluster configuration for replicated
volumes from replica 3 to replica 2 that need to be handled
in a different manner than the generic gluster commands,
where the user is expected to mention the replica count
of the new configuration in order to remove the bricks.

Signed-off-by: Devyani Kota <dkota@redhat.com>

* Updates #38174: Add checks for self-heal status prior remove operation.

While reducing the cluster configuration from replica 3
to 2, it needs to be checked for status of self-heal
prior remove-brick operation is executed.

Signed-off-by: Devyani Kota <dkota@redhat.com>

lib/ansible/modules/storage/glusterfs/gluster_volume.py

@@ -153,6 +153,27 @@ EXAMPLES = """
       - 192.0.2.10
       - 192.0.2.11
   run_once: true
+
+- name: Remove the bricks from gluster volume
+  gluster_volume:
+    state: present
+    name: testvol
+    bricks: /bricks/brick1/b1,/bricks/brick2/b2
+    cluster:
+      - 10.70.42.85
+    force: true
+  run_once: true
+
+- name: Reduce cluster configuration
+  gluster_volume:
+    state: present
+    name: testvol
+    bricks: /bricks/brick3/b1,/bricks/brick4/b2
+    replicas: 2
+    cluster:
+      - 10.70.42.85
+    force: true
+  run_once: true
 """
 
 import re
@@ -243,6 +264,8 @@ def get_volumes():
                     volume['arbiters'] = []
                 value = value[:-10]
                 volume['arbiters'].append(value)
+            elif key.lower() == 'number of bricks':
+                volume['replicas'] = value[-1:]
             if key.lower() != 'bricks' and key.lower()[:5] == 'brick':
                 if 'bricks' not in volume:
                     volume['bricks'] = []
@@ -354,6 +377,57 @@ def add_bricks(name, new_bricks, stripe, replica, force):
     run_gluster(args)
 
 
+def remove_bricks(name, removed_bricks, force):
+    # max-tries=12 with default_interval=10 secs
+    max_tries = 12
+    retries = 0
+    success = False
+    args = ['volume', 'remove-brick', name]
+    args.extend(removed_bricks)
+    # create a copy of args to use for commit operation
+    args_c = args[:]
+    args.append('start')
+    run_gluster(args)
+    # remove-brick operation needs to be followed by commit operation.
+    if not force:
+        module.fail_json(msg="Force option is mandatory.")
+    else:
+        while retries < max_tries:
+            last_brick = removed_bricks[-1]
+            out = run_gluster(['volume', 'remove-brick', name, last_brick, 'status'])
+            for row in out.split('\n')[1:]:
+                if 'completed' in row:
+                    # remove-brick successful, call commit operation.
+                    args_c.append('commit')
+                    out = run_gluster(args_c)
+                    success = True
+                    break
+                else:
+                    time.sleep(10)
+            if success:
+                break
+            retries += 1
+        if not success:
+            # remove-brick still in process, needs to be committed after completion.
+            module.fail_json(msg="Exceeded number of tries, check remove-brick status.\n"
+                                 "Commit operation needs to be followed.")
+
+
+def reduce_config(name, removed_bricks, replicas, force):
+    out = run_gluster(['volume', 'heal', name, 'info'])
+    summary = out.split("\n")
+    for line in summary:
+        if 'Number' in line and int(line.split(":")[1].strip()) != 0:
+            module.fail_json(msg="Operation aborted, self-heal in progress.")
+    args = ['volume', 'remove-brick', name, 'replica', replicas]
+    args.extend(removed_bricks)
+    if force:
+        args.append('force')
+    else:
+        module.fail_json(msg="Force option is mandatory")
+    run_gluster(args)
+
+
 def do_rebalance(name):
     run_gluster(['volume', 'rebalance', name, 'start'])
 
@@ -465,22 +539,31 @@ def main():
             new_bricks = []
             removed_bricks = []
             all_bricks = []
+            bricks_in_volume = volumes[volume_name]['bricks']
+
             for node in cluster:
                 for brick_path in brick_paths:
                     brick = '%s:%s' % (node, brick_path)
                     all_bricks.append(brick)
-                    if brick not in volumes[volume_name]['bricks']:
+                    if brick not in bricks_in_volume:
                         new_bricks.append(brick)
 
-            # this module does not yet remove bricks, but we check those anyways
-            for brick in volumes[volume_name]['bricks']:
-                if brick not in all_bricks:
-                    removed_bricks.append(brick)
+            if not new_bricks and len(all_bricks) < bricks_in_volume:
+                for brick in bricks_in_volume:
+                    if brick not in all_bricks:
+                        removed_bricks.append(brick)
 
             if new_bricks:
                 add_bricks(volume_name, new_bricks, stripes, replicas, force)
                 changed = True
 
+            if removed_bricks:
+                if replicas and int(replicas) < int(volumes[volume_name]['replicas']):
+                    reduce_config(volume_name, removed_bricks, str(replicas), force)
+                else:
+                    remove_bricks(volume_name, removed_bricks, force)
+                changed = True
+
             # handle quotas
             if quota:
                 if not volumes[volume_name]['quota']: