Support non-KV and KV v1/v2 secret engines (#64288)

* Support generic and KV v1/v2 secret engines.

Fixes #41132
Fixes #50598
Fixes #54462
Fixes #64150

* Update KV v2 compatibility notes/usage docs.

* Add tests for KV v1/v2 and generic secrets engines.

* Add changelog fragment.

* Add KV v2 behavior changes to porting guide.

* Fix pylint blacklisted name warnings.

changelogs/fragments/64288-fix-hashi-vault-kv-v2.yaml

@@ -0,0 +1,2 @@
+bugfixes:
+- "hashi_vault - Fix KV v2 lookup to always return latest version"

docs/docsite/rst/porting_guides/porting_guide_2.10.rst

@@ -102,8 +102,11 @@ Noteworthy module changes
 Plugins
 =======
 
-No notable changes
 
+Noteworthy plugin changes
+-------------------------
+
+* The ``hashi_vault`` lookup plugin now returns the latest version when using the KV v2 secrets engine. Previously, it returned all versions of the secret which required additional steps to extract and filter the desired version.
 
 Porting custom scripts
 ======================

lib/ansible/plugins/lookup/hashi_vault.py

@@ -16,6 +16,7 @@ DOCUMENTATION = """
     - retrieve secrets from HashiCorp's vault
   notes:
     - Due to a current limitation in the HVAC library there won't necessarily be an error if a bad endpoint is specified.
+    - As of Ansible 2.10, only the latest secret is returned when specifying a KV v2 path.
   options:
     secret:
       description: query you are making.
@@ -98,8 +99,9 @@ EXAMPLES = """
   debug:
     msg: "{{ lookup('hashi_vault', 'secret=secret/hello token=c975b780-d1be-8016-866b-01d0f9b688a5 url=http://myvault:8200 namespace=teama/admins')}}"
 
-# to work with kv v2 (vault api - for kv v2 -  GET method requires that PATH should be "secret/data/:path")
-- name: Return all kv v2 secrets from a path
+# When using KV v2 the PATH should include "data" between the secret engine mount and path (e.g. "secret/data/:path")
+# see: https://www.vaultproject.io/api/secret/kv/kv-v2.html#read-secret-version
+- name: Return latest KV v2 secret from path
   debug:
     msg: "{{ lookup('hashi_vault', 'secret=secret/data/hello token=my_vault_token url=http://myvault_url:8200') }}"
 
@@ -197,6 +199,18 @@ class HashiVault:
     def get(self):
         data = self.client.read(self.secret)
 
+        # Check response for KV v2 fields and flatten nested secret data.
+        #
+        # https://vaultproject.io/api/secret/kv/kv-v2.html#sample-response-1
+        try:
+            # sentinel field checks
+            check_dd = data['data']['data']
+            check_md = data['data']['metadata']
+            # unwrap nested data
+            data = data['data']
+        except KeyError:
+            pass
+
         if data is None:
             raise AnsibleError("The secret %s doesn't seem to exist for hashi_vault lookup" % self.secret)
 

test/integration/targets/lookup_hashi_vault/lookup_hashi_vault/defaults/main.yml

@@ -1,3 +1,4 @@
 ---
-vault_base_path: 'secret/data/testproject'
-vault_base_path_kv: 'secret/testproject'  # required by KV 2 engine
+vault_gen_path: 'gen/testproject'
+vault_kv1_path: 'kv1/testproject'
+vault_kv2_path: 'kv2/data/testproject'

test/integration/targets/lookup_hashi_vault/lookup_hashi_vault/tasks/approle_test.yml

@@ -4,17 +4,17 @@
   block:
     - name: 'Fetch secrets using "hashi_vault" lookup'
       set_fact:
-        secret1: "{{ lookup('hashi_vault', conn_params ~ 'secret=' ~ vault_base_path ~ '/secret1 auth_method=approle secret_id=' ~ secret_id ~ ' role_id=' ~ role_id) }}"
-        secret2: "{{ lookup('hashi_vault', conn_params ~ 'secret=' ~ vault_base_path ~ '/secret2 auth_method=approle secret_id=' ~ secret_id ~ ' role_id=' ~ role_id) }}"
+        secret1: "{{ lookup('hashi_vault', conn_params ~ 'secret=' ~ vault_kv2_path ~ '/secret1 auth_method=approle secret_id=' ~ secret_id ~ ' role_id=' ~ role_id) }}"
+        secret2: "{{ lookup('hashi_vault', conn_params ~ 'secret=' ~ vault_kv2_path ~ '/secret2 auth_method=approle secret_id=' ~ secret_id ~ ' role_id=' ~ role_id) }}"
 
     - name: 'Check secret values'
       fail:
         msg: 'unexpected secret values'
-      when: secret1['data']['value'] != 'foo1' or secret2['data']['value'] != 'foo2'
+      when: secret1['value'] != 'foo1' or secret2['value'] != 'foo2'
 
     - name: 'Failure expected when erroneous credentials are used'
       vars:
-        secret_wrong_cred: "{{ lookup('hashi_vault', conn_params ~ 'secret=' ~ vault_base_path ~ '/secret2 auth_method=approle secret_id=toto role_id=' ~ role_id) }}"
+        secret_wrong_cred: "{{ lookup('hashi_vault', conn_params ~ 'secret=' ~ vault_kv2_path ~ '/secret2 auth_method=approle secret_id=toto role_id=' ~ role_id) }}"
       debug:
         msg: 'Failure is expected ({{ secret_wrong_cred }})'
       register: test_wrong_cred
@@ -22,7 +22,7 @@
 
     - name: 'Failure expected when unauthorized secret is read'
       vars:
-        secret_unauthorized: "{{ lookup('hashi_vault', conn_params ~ 'secret=' ~ vault_base_path ~ '/secret3 auth_method=approle secret_id=' ~ secret_id ~ ' role_id=' ~ role_id) }}"
+        secret_unauthorized: "{{ lookup('hashi_vault', conn_params ~ 'secret=' ~ vault_kv2_path ~ '/secret3 auth_method=approle secret_id=' ~ secret_id ~ ' role_id=' ~ role_id) }}"
       debug:
         msg: 'Failure is expected ({{ secret_unauthorized }})'
       register: test_unauthorized
@@ -30,7 +30,7 @@
 
     - name: 'Failure expected when inexistent secret is read'
       vars:
-        secret_inexistent: "{{ lookup('hashi_vault', conn_params ~ 'secret=' ~ vault_base_path ~ '/secret4 auth_method=approle secret_id=' ~ secret_id ~ ' role_id=' ~ role_id) }}"
+        secret_inexistent: "{{ lookup('hashi_vault', conn_params ~ 'secret=' ~ vault_kv2_path ~ '/secret4 auth_method=approle secret_id=' ~ secret_id ~ ' role_id=' ~ role_id) }}"
       debug:
         msg: 'Failure is expected ({{ secret_inexistent }})'
       register: test_inexistent

test/integration/targets/lookup_hashi_vault/lookup_hashi_vault/tasks/main.yml

@@ -74,22 +74,57 @@
             - name: 'Start vault server (dev mode enabled)'
               shell: 'nohup {{ vault_cmd }} server -dev -config {{ local_temp_dir }}/vault_config.hcl </dev/null >/dev/null 2>&1 &'
 
+            - name: 'Create generic secrets engine'
+              command: '{{ vault_cmd }} secrets enable -path=gen generic'
+
+            - name: 'Create KV v1 secrets engine'
+              command: '{{ vault_cmd }} secrets enable -path=kv1 -version=1 kv'
+
+            - name: 'Create KV v2 secrets engine'
+              command: '{{ vault_cmd }} secrets enable -path=kv2 -version=2 kv'
+
             - name: 'Create a test policy'
               shell: "echo '{{ policy }}' | {{ vault_cmd }} policy write test-policy -"
               vars:
                 policy: |
-                  path "{{ vault_base_path }}/secret1" {
+                  path "{{ vault_gen_path }}/secret1" {
+                    capabilities = ["read"]
+                  }
+                  path "{{ vault_gen_path }}/secret2" {
+                    capabilities = ["read", "update"]
+                  }
+                  path "{{ vault_gen_path }}/secret3" {
+                    capabilities = ["deny"]
+                  }
+                  path "{{ vault_kv1_path }}/secret1" {
                     capabilities = ["read"]
                   }
-                  path "{{ vault_base_path }}/secret2" {
+                  path "{{ vault_kv1_path }}/secret2" {
                     capabilities = ["read", "update"]
                   }
-                  path "{{ vault_base_path }}/secret3" {
+                  path "{{ vault_kv1_path }}/secret3" {
                     capabilities = ["deny"]
                   }
+                  path "{{ vault_kv2_path }}/secret1" {
+                    capabilities = ["read"]
+                  }
+                  path "{{ vault_kv2_path }}/secret2" {
+                    capabilities = ["read", "update"]
+                  }
+                  path "{{ vault_kv2_path }}/secret3" {
+                    capabilities = ["deny"]
+                  }
+
+            - name: 'Create generic secrets'
+              command: '{{ vault_cmd }} write {{ vault_gen_path }}/secret{{ item }} value=foo{{ item }}'
+              loop: [1, 2, 3]
+
+            - name: 'Create KV v1 secrets'
+              command: '{{ vault_cmd }} kv put {{ vault_kv1_path }}/secret{{ item }} value=foo{{ item }}'
+              loop: [1, 2, 3]
 
-            - name: 'Create secrets'
-              command: '{{ vault_cmd }} kv put {{ vault_base_path_kv }}/secret{{ item }} value=foo{{ item }}'
+            - name: 'Create KV v2 secrets'
+              command: '{{ vault_cmd }} kv put {{ vault_kv2_path | regex_replace("/data") }}/secret{{ item }} value=foo{{ item }}'
               loop: [1, 2, 3]
 
             - name: setup approle auth

test/integration/targets/lookup_hashi_vault/lookup_hashi_vault/tasks/token_test.yml

@@ -3,18 +3,31 @@
   block:
     - name: 'Fetch secrets using "hashi_vault" lookup'
       set_fact:
-        secret1: "{{ lookup('hashi_vault', conn_params ~ 'secret=' ~ vault_base_path ~ '/secret1 auth_method=token token=' ~ user_token) }}"
-        secret2: "{{ lookup('hashi_vault', conn_params ~ 'secret=' ~ vault_base_path ~ '/secret2 token=' ~ user_token) }}"
-        secret3: "{{ lookup('hashi_vault', conn_params ~ ' secret=' ~ vault_base_path ~ '/secret2  token=' ~ user_token) }}"
+        gen_secret1: "{{ lookup('hashi_vault', conn_params ~ 'secret=' ~ vault_gen_path ~ '/secret1 auth_method=token token=' ~ user_token) }}"
+        gen_secret2: "{{ lookup('hashi_vault', conn_params ~ 'secret=' ~ vault_gen_path ~ '/secret2 token=' ~ user_token) }}"
+        kv1_secret1: "{{ lookup('hashi_vault', conn_params ~ 'secret=' ~ vault_kv1_path ~ '/secret1 auth_method=token token=' ~ user_token) }}"
+        kv1_secret2: "{{ lookup('hashi_vault', conn_params ~ 'secret=' ~ vault_kv1_path ~ '/secret2 token=' ~ user_token) }}"
+        kv2_secret1: "{{ lookup('hashi_vault', conn_params ~ 'secret=' ~ vault_kv2_path ~ '/secret1 auth_method=token token=' ~ user_token) }}"
+        kv2_secret2: "{{ lookup('hashi_vault', conn_params ~ 'secret=' ~ vault_kv2_path ~ '/secret2 token=' ~ user_token) }}"
 
-    - name: 'Check secret values'
+    - name: 'Check secret generic values'
       fail:
         msg: 'unexpected secret values'
-      when: secret1['data']['value'] != 'foo1' or secret2['data']['value'] != 'foo2' or secret3['data']['value'] != 'foo2'
+      when: gen_secret1['value'] != 'foo1' or gen_secret2['value'] != 'foo2'
+
+    - name: 'Check secret kv1 values'
+      fail:
+        msg: 'unexpected secret values'
+      when: kv1_secret1['value'] != 'foo1' or kv1_secret2['value'] != 'foo2'
+
+    - name: 'Check secret kv2 values'
+      fail:
+        msg: 'unexpected secret values'
+      when: kv2_secret1['value'] != 'foo1' or kv2_secret2['value'] != 'foo2'
 
     - name: 'Failure expected when erroneous credentials are used'
       vars:
-        secret_wrong_cred: "{{ lookup('hashi_vault', conn_params ~ 'secret=' ~ vault_base_path ~ '/secret2 auth_method=token token=wrong_token') }}"
+        secret_wrong_cred: "{{ lookup('hashi_vault', conn_params ~ 'secret=' ~ vault_kv2_path ~ '/secret2 auth_method=token token=wrong_token') }}"
       debug:
         msg: 'Failure is expected ({{ secret_wrong_cred }})'
       register: test_wrong_cred
@@ -22,7 +35,7 @@
 
     - name: 'Failure expected when unauthorized secret is read'
       vars:
-        secret_unauthorized: "{{ lookup('hashi_vault', conn_params ~ 'secret=' ~ vault_base_path ~ '/secret3 token=' ~ user_token) }}"
+        secret_unauthorized: "{{ lookup('hashi_vault', conn_params ~ 'secret=' ~ vault_kv2_path ~ '/secret3 token=' ~ user_token) }}"
       debug:
         msg: 'Failure is expected ({{ secret_unauthorized }})'
       register: test_unauthorized
@@ -30,7 +43,7 @@
 
     - name: 'Failure expected when inexistent secret is read'
       vars:
-        secret_inexistent: "{{ lookup('hashi_vault', conn_params ~ 'secret=' ~ vault_base_path ~ '/secret4 token=' ~ user_token) }}"
+        secret_inexistent: "{{ lookup('hashi_vault', conn_params ~ 'secret=' ~ vault_kv2_path ~ '/secret4 token=' ~ user_token) }}"
       debug:
         msg: 'Failure is expected ({{ secret_inexistent }})'
       register: test_inexistent