Conjur Lookup Plugin (#34280)

* Imported lookup plugin from Role

* Plugin cleanup, including:
* Use existing Python YAML parsing
* Remove environment variables as connection options
* Added initial debugging information

* Reworked the lookup plugin using the Python Request library.  As it's available through Ansible, it makes communication with Conjur much more straight forward.

* Removed un-used libraries

* Fixed linting issues

* Standardized output on `format` and insure it works for 2.6, 2.7, and 3.x.

* Use quote_plus from the six library for improved python 2/3 behavior.

* Refactored identity & configuration to prefer user's file. This also includes a refactor to remove an un-needed dictionary merge method.

* Removed `requests` in favor of `ansible.module_utils.urls`.

* Refactored netrc loading to warn if host is not present.

* Tests and a refactor to support easier testing.

* Added reference to website

* Fixed two linting errors

* Fixed an extra line found by linting

* Updated file write to use binary to insure config files are written correctly

* Resolved linting issues

* Refactored config & identity loading to take advantage of plugin options

* Cleanup a bunch of small items caught by linting

* Removed extra line caught by linting

* Swapped in pytest and added some tests with mocked network responses

* Pushing to see if this approach works better...

* Refactored be open_url mocking based on feedback

* Fixed a couple linting issues & refactored mocking into each method to attempt to resolve a failing test

* Use a generic MagicMock for python 2.6

* Fixes doc typo

require -> required

* Use `type: path` in identity_file and config_file

Also removes `expanduser` calls below (which will now be called automatically on
paths.)

* Defines maintainers for conjur_variable plugin

* BOTMETA.yml:
** defines $team_cyberark_conjur as maintainers of Conjur Variable plugin
** adds myself and @jvanderhoof to that team

* Adds URLs to relevant documentation for Conjur Variable lookup plugin

* Clarifies "the server," "the machine" -> "controlling host"

The machine identity used is that of the Ansible controlling host, not any
server being provisioned or instructed. This documentation change aims to make
that relationship clear.

* Adds response code to exception message on authentication failure

* Enhances exception messages to specify the controlling host

These error messages are less likely to confuse a user as to which machine is
associated with the files, identities, and configurations being described.

* Adds ANSIBLE_METADATA for Conjur variable lookup plugin

.github/BOTMETA.yml

@@ -1147,6 +1147,8 @@ files:
   lib/ansible/plugins/lookup/dig:
     maintainers: jpmens
     labels: community
+  lib/ansible/plugins/lookup/conjur_variable.py:
+    maintainers: $team_cyberark_conjur
   lib/ansible/plugins/netconf/:
     maintainers: $team_networking
     labels: networking
@@ -1233,6 +1235,7 @@ macros:
   team_avi: ericsysmin grastogi23 khaltore
   team_azure: haroldwongms nitzmahone trstringer yuwzho xscript zikalino
   team_cumulus: isharacomix jrrivers privateip
+  team_cyberark_conjur: jvanderhoof ryanprior
   team_manageiq: gtanzillo abellotti zgalor yaacov cben
   team_netapp: hulquest lmprice broncofan gouthampacha
   team_netscaler: chiradeep giorgos-nikolopoulos

lib/ansible/plugins/lookup/conjur_variable.py

@@ -0,0 +1,167 @@
+# (c) 2018, Jason Vanderhoof <jason.vanderhoof@cyberark.com>, Oren Ben Meir <oren.benmeir@cyberark.com>
+# (c) 2018 Ansible Project
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+ANSIBLE_METADATA = {'metadata_version': '1.1',
+                    'status': ['preview'],
+                    'supported_by': 'community'}
+
+DOCUMENTATION = """
+    lookup: conjur_variable
+    version_added: "2.5"
+    short_description: Fetch credentials from CyberArk Conjur.
+    description:
+      - Retrieves credentials from Conjur using the controlling host's Conjur identity. Conjur info: U(https://www.conjur.org/).
+    requirements:
+      - The controlling host running Ansible has a Conjur identity. (More: U(https://developer.conjur.net/key_concepts/machine_identity.html))
+    options:
+      _term:
+        description: Variable path
+        required: True
+      identity_file:
+        description: Path to the Conjur identity file. The identity file follows the netrc file format convention.
+        type: path
+        default: /etc/conjur.identity
+        required: False
+        ini:
+          - section: conjur,
+            key: identity_file_path
+        env:
+          - name: CONJUR_IDENTITY_FILE
+      config_file:
+        description: Path to the Conjur configuration file. The configuration file is a YAML file.
+        type: path
+        default: /etc/conjur.conf
+        required: False
+        ini:
+          - section: conjur,
+            key: config_file_path
+        env:
+          - name: CONJUR_CONFIG_FILE
+"""
+
+EXAMPLES = """
+  - debug
+    msg: {{ lookup('conjur_variable', '/path/to/secret') }}
+"""
+
+RETURN = """
+  _raw:
+    description:
+      - Value stored in Conjur.
+"""
+
+import os.path
+from ansible.errors import AnsibleError
+from ansible.plugins.lookup import LookupBase
+from base64 import b64encode
+from netrc import netrc
+from os import environ
+from time import time
+from ansible.module_utils.six.moves.urllib.parse import quote_plus
+import yaml
+
+from ansible.module_utils.urls import open_url
+
+
+try:
+    from __main__ import display
+except ImportError:
+    from ansible.utils.display import Display
+    display = Display()
+
+
+# Load configuration and return as dictionary if file is present on file system
+def _load_conf_from_file(conf_path):
+    display.vvv('conf file: {0}'.format(conf_path))
+
+    if not os.path.exists(conf_path):
+        raise AnsibleError('Conjur configuration file `{0}` was not found on the controlling host'
+                           .format(conf_path))
+
+    display.vvvv('Loading configuration from: {0}'.format(conf_path))
+    with open(conf_path) as f:
+        config = yaml.safe_load(f.read())
+        if 'account' not in config or 'appliance_url' not in config:
+            raise AnsibleError('{0} on the controlling host must contain an `account` and `appliance_url` entry'
+                               .format(conf_path))
+        return config
+
+
+# Load identity and return as dictionary if file is present on file system
+def _load_identity_from_file(identity_path, appliance_url):
+    display.vvvv('identity file: {0}'.format(identity_path))
+
+    if not os.path.exists(identity_path):
+        raise AnsibleError('Conjur identity file `{0}` was not found on the controlling host'
+                           .format(identity_path))
+
+    display.vvvv('Loading identity from: {0} for {1}'.format(identity_path, appliance_url))
+
+    conjur_authn_url = '{0}/authn'.format(appliance_url)
+    identity = netrc(identity_path)
+
+    if identity.authenticators(conjur_authn_url) is None:
+        raise AnsibleError('The netrc file on the controlling host does not contain an entry for: {0}'
+                           .format(conjur_authn_url))
+
+    id, account, api_key = identity.authenticators(conjur_authn_url)
+    if not id or not api_key:
+        raise AnsibleError('{0} on the controlling host must contain a `login` and `password` entry for {1}'
+                           .format(identity_path, appliance_url))
+
+    return {'id': id, 'api_key': api_key}
+
+
+# Use credentials to retrieve temporary authorization token
+def _fetch_conjur_token(conjur_url, account, username, api_key):
+    conjur_url = '{0}/authn/{1}/{2}/authenticate'.format(conjur_url, account, username)
+    display.vvvv('Authentication request to Conjur at: {0}, with user: {1}'.format(conjur_url, username))
+
+    response = open_url(conjur_url, data=api_key, method='POST')
+    code = response.getcode()
+    if code != 200:
+        raise AnsibleError('Failed to authenticate as \'{0}\' (got {1} response)'
+                           .format(username, code))
+
+    return response.read()
+
+
+# Retrieve Conjur variable using the temporary token
+def _fetch_conjur_variable(conjur_variable, token, conjur_url, account):
+    token = b64encode(token)
+    headers = {'Authorization': 'Token token="{0}"'.format(token)}
+    display.vvvv('Header: {0}'.format(headers))
+
+    url = '{0}/secrets/{1}/variable/{2}'.format(conjur_url, account, quote_plus(conjur_variable))
+    display.vvvv('Conjur Variable URL: {0}'.format(url))
+
+    response = open_url(url, headers=headers, method='GET')
+
+    if response.getcode() == 200:
+        display.vvvv('Conjur variable {0} was successfully retrieved'.format(conjur_variable))
+        return [response.read()]
+    if response.getcode() == 401:
+        raise AnsibleError('Conjur request has invalid authorization credentials')
+    if response.getcode() == 403:
+        raise AnsibleError('The controlling host\'s Conjur identity does not have authorization to retrieve {0}'
+                           .format(conjur_variable))
+    if response.getcode() == 404:
+        raise AnsibleError('The variable {0} does not exist'.format(conjur_variable))
+
+    return {}
+
+
+class LookupModule(LookupBase):
+
+    def run(self, terms, variables=None, **kwargs):
+        conf_file = self.get_option('config_file')
+        conf = _load_conf_from_file(conf_file)
+
+        identity_file = self.get_option('identity_file')
+        identity = _load_identity_from_file(identity_file, conf['appliance_url'])
+
+        token = _fetch_conjur_token(conf['appliance_url'], conf['account'], identity['id'], identity['api_key'])
+        return _fetch_conjur_variable(terms[0], token, conf['appliance_url'], conf['account'])

test/units/plugins/lookup/test_conjur_variable.py

@@ -0,0 +1,110 @@
+# -*- coding: utf-8 -*-
+# (c) 2018, Jason Vanderhoof <jason.vanderhoof@cyberark.com>
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+# Make coding more python3-ish
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+
+import pytest
+from ansible.compat.tests.mock import MagicMock
+from ansible.errors import AnsibleError
+from ansible.module_utils.six.moves import http_client
+from ansible.plugins.lookup import conjur_variable
+import tempfile
+
+
+class TestLookupModule:
+    def test_valid_netrc_file(self):
+        with tempfile.NamedTemporaryFile() as temp_netrc:
+            temp_netrc.write(b"machine http://localhost/authn\n")
+            temp_netrc.write(b"  login admin\n")
+            temp_netrc.write(b"  password my-pass\n")
+            temp_netrc.seek(0)
+
+            results = conjur_variable._load_identity_from_file(temp_netrc.name, 'http://localhost')
+
+            assert results['id'] == 'admin'
+            assert results['api_key'] == 'my-pass'
+
+    def test_netrc_without_host_file(self):
+        with tempfile.NamedTemporaryFile() as temp_netrc:
+            temp_netrc.write(b"machine http://localhost/authn\n")
+            temp_netrc.write(b"  login admin\n")
+            temp_netrc.write(b"  password my-pass\n")
+            temp_netrc.seek(0)
+
+            with pytest.raises(AnsibleError):
+                conjur_variable._load_identity_from_file(temp_netrc.name, 'http://foo')
+
+    def test_valid_configuration(self):
+        with tempfile.NamedTemporaryFile() as configuration_file:
+            configuration_file.write(b"---\n")
+            configuration_file.write(b"account: demo-policy\n")
+            configuration_file.write(b"plugins: []\n")
+            configuration_file.write(b"appliance_url: http://localhost:8080\n")
+            configuration_file.seek(0)
+
+            results = conjur_variable._load_conf_from_file(configuration_file.name)
+            assert results['account'] == 'demo-policy'
+            assert results['appliance_url'] == 'http://localhost:8080'
+
+    def test_valid_token_retrieval(self, mocker):
+        mock_response = MagicMock(spec_set=http_client.HTTPResponse)
+        try:
+            mock_response.getcode.return_value = 200
+        except:
+            # HTTPResponse is a Python 3 only feature. This uses a generic mock for python 2.6
+            mock_response = MagicMock()
+            mock_response.getcode.return_value = 200
+
+        mock_response.read.return_value = 'foo-bar-token'
+        mocker.patch.object(conjur_variable, 'open_url', return_value=mock_response)
+
+        response = conjur_variable._fetch_conjur_token('http://conjur', 'account', 'username', 'api_key')
+        assert response == 'foo-bar-token'
+
+    def test_valid_fetch_conjur_variable(self, mocker):
+        mock_response = MagicMock(spec_set=http_client.HTTPResponse)
+        try:
+            mock_response.getcode.return_value = 200
+        except:
+            # HTTPResponse is a Python 3 only feature. This uses a generic mock for python 2.6
+            mock_response = MagicMock()
+            mock_response.getcode.return_value = 200
+
+        mock_response.read.return_value = 'foo-bar'
+        mocker.patch.object(conjur_variable, 'open_url', return_value=mock_response)
+
+        response = conjur_variable._fetch_conjur_token('super-secret', 'token', 'http://conjur', 'account')
+        assert response == 'foo-bar'
+
+    def test_invalid_fetch_conjur_variable(self, mocker):
+        for code in [401, 403, 404]:
+            mock_response = MagicMock(spec_set=http_client.HTTPResponse)
+            try:
+                mock_response.getcode.return_value = code
+            except:
+                # HTTPResponse is a Python 3 only feature. This uses a generic mock for python 2.6
+                mock_response = MagicMock()
+                mock_response.getcode.return_value = code
+
+            mocker.patch.object(conjur_variable, 'open_url', return_value=mock_response)
+
+            with pytest.raises(AnsibleError):
+                response = conjur_variable._fetch_conjur_token('super-secret', 'token', 'http://conjur', 'account')