Use apt-get as fallback for apt upgrade

In answer to #2540, `aptitude` was introduced as tool of choice for running
upgrades in the apt module and installing new packages that arise as
dependencies during upgrades.

This recently lead to problems, as for example Ubuntu Xenial (16.04) ships
without aptitude (installed).

Studying the man pages of both apt-get and aptitude, it appears that we can
achieve the effects of `aptitude safe-upgrade` using

```
apt-get upgrade --with-new-pkgs --autoremove
```

while `aptitude full-upgrade` seems to be identical to `apt-get dist-upgrade`.

We use `apt-get` as described above as a fall-back in case that `aptitude`
cannot be found, issuing a warning when it does so.

Furthermore it introduces a flag `force_apt_get` which may be used to enforce
usage of apt-get (which does not issue a warning).

The integration tests are updated accordingly.

Cf. also the discussion in #27370.

Fixes #18987

lib/ansible/modules/packaging/os/apt.py

@@ -89,6 +89,7 @@ options:
       - 'If full, performs an aptitude full-upgrade.'
       - 'If dist, performs an apt-get dist-upgrade.'
       - 'Note: This does not upgrade a specific package, use state=latest for that.'
+      - 'Note: Since 2.4, apt-get is used as a fall-back if aptitude is not present.'
     version_added: "1.1"
     required: false
     default: "no"
@@ -126,14 +127,21 @@ options:
     required: false
     default: false
     version_added: "2.1"
+  force_apt_get:
+    description:
+      - Force usage of apt-get instead of aptitude
+    required: false
+    default: false
+    version_added: "2.4"
 requirements:
    - python-apt (python 2)
    - python3-apt (python 3)
-   - aptitude
+   - aptitude (before 2.4)
 author: "Matthew Williams (@mgwilliams)"
 notes:
-   - Three of the upgrade modes (C(full), C(safe) and its alias C(yes)) require C(aptitude), otherwise
-     C(apt-get) suffices.
+   - Three of the upgrade modes (C(full), C(safe) and its alias C(yes))
+     required C(aptitude) up to 2.3, since 2.4 C(apt-get) is used as a
+     fall-back.
 '''
 
 EXAMPLES = '''
@@ -711,6 +719,7 @@ def cleanup(m, purge=False, force=False, operation=None,
 
 
 def upgrade(m, mode="yes", force=False, default_release=None,
+            use_apt_get=False,
             dpkg_options=expand_dpkg_options(DPKG_OPTIONS)):
     if m.check_mode:
         check_arg = '--simulate'
@@ -719,19 +728,23 @@ def upgrade(m, mode="yes", force=False, default_release=None,
 
     apt_cmd = None
     prompt_regex = None
-    if mode == "dist":
+    if mode == "dist" or (mode == "full" and use_apt_get):
         # apt-get dist-upgrade
         apt_cmd = APT_GET_CMD
         upgrade_command = "dist-upgrade"
-    elif mode == "full":
+    elif mode == "full" and not use_apt_get:
         # aptitude full-upgrade
         apt_cmd = APTITUDE_CMD
         upgrade_command = "full-upgrade"
     else:
-        # aptitude safe-upgrade # mode=yes # default
-        apt_cmd = APTITUDE_CMD
-        upgrade_command = "safe-upgrade"
-        prompt_regex = r"(^Do you want to ignore this warning and proceed anyway\?|^\*\*\*.*\[default=.*\])"
+        if use_apt_get:
+            apt_cmd = APT_GET_CMD
+            upgrade_command = "upgrade --with-new-pkgs --autoremove"
+        else:
+            # aptitude safe-upgrade # mode=yes # default
+            apt_cmd = APTITUDE_CMD
+            upgrade_command = "safe-upgrade"
+            prompt_regex = r"(^Do you want to ignore this warning and proceed anyway\?|^\*\*\*.*\[default=.*\])"
 
     if force:
         if apt_cmd == APT_GET_CMD:
@@ -860,6 +873,7 @@ def main():
             autoremove=dict(type='bool', default='no'),
             autoclean=dict(type='bool', default='no'),
             only_upgrade=dict(type='bool', default=False),
+            force_apt_get=dict(type='bool', default=False),
             allow_unauthenticated=dict(default='no', aliases=['allow-unauthenticated'], type='bool'),
         ),
         mutually_exclusive=[['package', 'upgrade', 'deb']],
@@ -894,8 +908,11 @@ def main():
     if p['upgrade'] == 'no':
         p['upgrade'] = None
 
-    if not APTITUDE_CMD and p.get('upgrade', None) in ['full', 'safe', 'yes']:
-        module.fail_json(msg="Could not find aptitude. Please ensure it is installed.")
+    use_apt_get = p['force_apt_get']
+
+    if not use_apt_get and not APTITUDE_CMD and p.get('upgrade', None) in ['full', 'safe', 'yes']:
+        module.warn("Could not find aptitude. Using apt-get instead")
+        use_apt_get = True
 
     updated_cache = False
     updated_cache_time = 0
@@ -956,7 +973,7 @@ def main():
         force_yes = p['force']
 
         if p['upgrade']:
-            upgrade(module, p['upgrade'], force_yes, p['default_release'], dpkg_options)
+            upgrade(module, p['upgrade'], force_yes, p['default_release'], use_apt_get, dpkg_options)
 
         if p['deb']:
             if p['state'] != 'present':
@@ -976,7 +993,7 @@ def main():
         if latest and all_installed:
             if packages:
                 module.fail_json(msg='unable to install additional packages when ugrading all installed packages')
-            upgrade(module, 'yes', force_yes, p['default_release'], dpkg_options)
+            upgrade(module, 'yes', force_yes, p['default_release'], use_apt_get, dpkg_options)
 
         if packages:
             for package in packages:

test/integration/inventory

@@ -10,6 +10,9 @@ facthost[0:20] ansible_host=1270.0.0.1 ansible_connection=local
 [binary_modules]
 testhost_binary_modules ansible_host=127.0.0.1 ansible_connection=local
 
+[local_group]
+kube-pippin.knf.local
+
 # the following inline declarations are accompanied
 # by (preferred) group_vars/ and host_vars/ variables
 # and are used in testing of variable precedence

test/integration/targets/apt/tasks/main.yml

@@ -24,20 +24,49 @@
  - include: 'apt-builddep.yml'
    when: ansible_distribution in ('Ubuntu', 'Debian')
 
- - name: Check if aptitude is installed
-   command: dpkg-query -l aptitude
-   register: deb_check
+ - include: upgrade.yml upgrade_type=dist
    when: ansible_distribution in ('Ubuntu', 'Debian')
 
- - include: upgrade.yml upgrade_type=dist
+ - name: Check if aptitude is installed
+   command: dpkg-query --show --showformat='${db:Status-Abbrev}' aptitude
+   register: aptitude_status
    when: ansible_distribution in ('Ubuntu', 'Debian')
 
- - include: upgrade.yml upgrade_type=safe
+ - debug: var=aptitude_status.stdout
+
+ - name: Remove aptitude, if installed, to test fall-back to apt-get
+   apt:
+     pkg: aptitude
+     state: absent
+   when:
+     - aptitude_status.stdout.find('ii') != -1
+
+ - include: "upgrade.yml aptitude_present={{ False | bool }} upgrade_type={{ item.upgrade_type }} force_apt_get={{ item.force_apt_get }}"
    when:
      - ansible_distribution in ('Ubuntu', 'Debian')
-     - deb_check.stdout.find('no packages found') != -1
+   with_items:
+     - { upgrade_type: safe, force_apt_get: False }
+     - { upgrade_type: full, force_apt_get: False }
+     - { upgrade_type: safe, force_apt_get: True }
+     - { upgrade_type: full, force_apt_get: True }
+
+ - name: (Re-)Install aptitude, run same tests again
+   apt:
+     pkg: aptitude
+     state: present
 
- - include: upgrade.yml upgrade_type=full
+ - include: upgrade.yml upgrade_type={{ item.upgrade_type }} force_apt_get={{ item.force_apt_get }}
    when:
      - ansible_distribution in ('Ubuntu', 'Debian')
-     - deb_check.stdout.find('no packages found') != -1
+   with_items:
+     - { upgrade_type: safe, force_apt_get: False }
+     - { upgrade_type: full, force_apt_get: False }
+     - { upgrade_type: safe, force_apt_get: True }
+     - { upgrade_type: full, force_apt_get: True }
+
+ - name: Remove aptitude if not originally present
+   apt:
+     pkg: aptitude
+     state: absent
+   when:
+     - aptitude_status.stdout.find('ii') == -1

test/integration/targets/apt/tasks/upgrade.yml

@@ -14,14 +14,35 @@
     that:
       - "{{ hello_version.stdout }}=={{ hello_old_version }}"
 
-- name: "(upgrade type: {{upgrade_type}}) upgrade packages to latest version"
+- name: "(upgrade type: {{upgrade_type}}) upgrade packages to latest version, force_apt_get: {{force_apt_get|default(False)}}"
   apt:
     upgrade: "{{ upgrade_type }}"
+    force_apt_get: "{{ force_apt_get | default(False) }}"
+  register: upgrade_result
 
 - name: check hello version
   shell: dpkg -s hello | grep Version | awk '{print $2}'
   register: hello_version
 
+- debug: var=upgrade_result.warnings|default([])
+- debug: var=aptitude_present
+- debug: var=force_apt_get
+
+- name: check that warning is not given when force_apt_get set
+  assert:
+    that:
+      - "'Could not find aptitude. Using apt-get instead' not in upgrade_result.warnings | default([])"
+  when:
+    - force_apt_get | default(False)
+
+- name: check that warning is given when aptitude not found and force_apt_get not set
+  assert:
+    that:
+      - "'Could not find aptitude. Using apt-get instead' in upgrade_result.warnings"
+  when:
+    - not aptitude_present|default(True)
+    - not force_apt_get|default(False)
+
 - name: check that old version upgraded correctly
   assert:
     that:
@@ -31,12 +52,13 @@
 - name: "(upgrade type: {{upgrade_type}}) upgrade packages to latest version (Idempotant)"
   apt:
     upgrade: "{{ upgrade_type }}"
-  register: result
+    force_apt_get: "{{ force_apt_get | default(False) }}"
+  register: second_upgrade_result
 
 - name: check that nothing has changed (Idempotant)
   assert:
     that:
-      - "result.changed == false"
+      - "second_upgrade_result.changed == false"
 
 - name: remove hello
   apt: