Merge pull request #13771 from sivel/binary-modules

First pass at allowing binary modules
pull/15843/head
Matt Martz 9 years ago
commit 196453b9b2

@ -204,6 +204,25 @@ This should return something like::
{"changed": true, "time": "2012-03-14 12:23:00.000307"} {"changed": true, "time": "2012-03-14 12:23:00.000307"}
.. _binary_module_reading_input:
Binary Modules Input
~~~~~~~~~~~~~~~~~~~~
Support for binary modules was added in Ansible 2.2. When Ansible detects a binary module, it will proceed to
supply the argument input as a file on ``argv[1]`` that is formatted as JSON. The JSON contents of that file
would resemble something similar to the following payload for a module accepting the same arguments as the
``ping`` module::
{
"data": "pong",
"_ansible_verbosity": 4,
"_ansible_diff": false,
"_ansible_debug": false,
"_ansible_check_mode": false,
"_ansible_no_log": false
}
.. _module_provided_facts: .. _module_provided_facts:
Module Provided 'Facts' Module Provided 'Facts'

@ -490,6 +490,11 @@ def recursive_finder(name, data, py_module_names, py_module_cache, zf):
# Save memory; the file won't have to be read again for this ansible module. # Save memory; the file won't have to be read again for this ansible module.
del py_module_cache[py_module_file] del py_module_cache[py_module_file]
def _is_binary(module_data):
textchars = bytearray(set([7, 8, 9, 10, 12, 13, 27]) | set(range(0x20, 0x100)) - set([0x7f]))
start = module_data[:1024]
return bool(start.translate(None, textchars))
def _find_snippet_imports(module_name, module_data, module_path, module_args, task_vars, module_compression): def _find_snippet_imports(module_name, module_data, module_path, module_args, task_vars, module_compression):
""" """
Given the source of the module, convert it to a Jinja2 template to insert Given the source of the module, convert it to a Jinja2 template to insert
@ -504,7 +509,9 @@ def _find_snippet_imports(module_name, module_data, module_path, module_args, ta
# module_substyle is extra information that's useful internally. It tells # module_substyle is extra information that's useful internally. It tells
# us what we have to look to substitute in the module files and whether # us what we have to look to substitute in the module files and whether
# we're using module replacer or ziploader to format the module itself. # we're using module replacer or ziploader to format the module itself.
if REPLACER in module_data: if _is_binary(module_data):
module_substyle = module_style = 'binary'
elif REPLACER in module_data:
# Do REPLACER before from ansible.module_utils because we need make sure # Do REPLACER before from ansible.module_utils because we need make sure
# we substitute "from ansible.module_utils basic" for REPLACER # we substitute "from ansible.module_utils basic" for REPLACER
module_style = 'new' module_style = 'new'
@ -523,9 +530,9 @@ def _find_snippet_imports(module_name, module_data, module_path, module_args, ta
module_substyle = module_style = 'non_native_want_json' module_substyle = module_style = 'non_native_want_json'
shebang = None shebang = None
# Neither old-style nor non_native_want_json modules should be modified # Neither old-style, non_native_want_json nor binary modules should be modified
# except for the shebang line (Done by modify_module) # except for the shebang line (Done by modify_module)
if module_style in ('old', 'non_native_want_json'): if module_style in ('old', 'non_native_want_json', 'binary'):
return module_data, module_style, shebang return module_data, module_style, shebang
output = BytesIO() output = BytesIO()
@ -731,7 +738,9 @@ def modify_module(module_name, module_path, module_args, task_vars=dict(), modul
(module_data, module_style, shebang) = _find_snippet_imports(module_name, module_data, module_path, module_args, task_vars, module_compression) (module_data, module_style, shebang) = _find_snippet_imports(module_name, module_data, module_path, module_args, task_vars, module_compression)
if shebang is None: if module_style == 'binary':
return (module_data, module_style, shebang)
elif shebang is None:
lines = module_data.split(b"\n", 1) lines = module_data.split(b"\n", 1)
if lines[0].startswith(b"#!"): if lines[0].startswith(b"#!"):
shebang = lines[0].strip() shebang = lines[0].strip()

@ -147,7 +147,7 @@ class ActionBase(with_metaclass(ABCMeta, object)):
# insert shared code and arguments into the module # insert shared code and arguments into the module
(module_data, module_style, module_shebang) = modify_module(module_name, module_path, module_args, task_vars=task_vars, module_compression=self._play_context.module_compression) (module_data, module_style, module_shebang) = modify_module(module_name, module_path, module_args, task_vars=task_vars, module_compression=self._play_context.module_compression)
return (module_style, module_shebang, module_data) return (module_style, module_shebang, module_data, module_path)
def _compute_environment_string(self): def _compute_environment_string(self):
''' '''
@ -292,7 +292,7 @@ class ActionBase(with_metaclass(ABCMeta, object)):
return remote_path return remote_path
def _fixup_perms(self, remote_path, remote_user, execute=False, recursive=True): def _fixup_perms(self, remote_path, remote_user, execute=True, recursive=True):
""" """
We need the files we upload to be readable (and sometimes executable) We need the files we upload to be readable (and sometimes executable)
by the user being sudo'd to but we want to limit other people's access by the user being sudo'd to but we want to limit other people's access
@ -570,8 +570,8 @@ class ActionBase(with_metaclass(ABCMeta, object)):
# let module know our verbosity # let module know our verbosity
module_args['_ansible_verbosity'] = display.verbosity module_args['_ansible_verbosity'] = display.verbosity
(module_style, shebang, module_data) = self._configure_module(module_name=module_name, module_args=module_args, task_vars=task_vars) (module_style, shebang, module_data, module_path) = self._configure_module(module_name=module_name, module_args=module_args, task_vars=task_vars)
if not shebang: if not shebang and module_style != 'binary':
raise AnsibleError("module (%s) is missing interpreter line" % module_name) raise AnsibleError("module (%s) is missing interpreter line" % module_name)
# a remote tmp path may be necessary and not already created # a remote tmp path may be necessary and not already created
@ -581,14 +581,17 @@ class ActionBase(with_metaclass(ABCMeta, object)):
tmp = self._make_tmp_path(remote_user) tmp = self._make_tmp_path(remote_user)
if tmp: if tmp:
remote_module_filename = self._connection._shell.get_remote_filename(module_name) remote_module_filename = self._connection._shell.get_remote_filename(module_path)
remote_module_path = self._connection._shell.join_path(tmp, remote_module_filename) remote_module_path = self._connection._shell.join_path(tmp, remote_module_filename)
if module_style in ['old', 'non_native_want_json']: if module_style in ('old', 'non_native_want_json', 'binary'):
# we'll also need a temp file to hold our module arguments # we'll also need a temp file to hold our module arguments
args_file_path = self._connection._shell.join_path(tmp, 'args') args_file_path = self._connection._shell.join_path(tmp, 'args')
if remote_module_path or module_style != 'new': if remote_module_path or module_style != 'new':
display.debug("transferring module to remote") display.debug("transferring module to remote")
if module_style == 'binary':
self._transfer_file(module_path, remote_module_path)
else:
self._transfer_data(remote_module_path, module_data) self._transfer_data(remote_module_path, module_data)
if module_style == 'old': if module_style == 'old':
# we need to dump the module args to a k=v string in a file on # we need to dump the module args to a k=v string in a file on
@ -597,7 +600,7 @@ class ActionBase(with_metaclass(ABCMeta, object)):
for k,v in iteritems(module_args): for k,v in iteritems(module_args):
args_data += '%s="%s" ' % (k, pipes.quote(text_type(v))) args_data += '%s="%s" ' % (k, pipes.quote(text_type(v)))
self._transfer_data(args_file_path, args_data) self._transfer_data(args_file_path, args_data)
elif module_style == 'non_native_want_json': elif module_style in ('non_native_want_json', 'binary'):
self._transfer_data(args_file_path, json.dumps(module_args)) self._transfer_data(args_file_path, json.dumps(module_args))
display.debug("done transferring module to remote") display.debug("done transferring module to remote")

@ -54,15 +54,18 @@ class ActionModule(ActionBase):
module_args['_ansible_no_log'] = True module_args['_ansible_no_log'] = True
# configure, upload, and chmod the target module # configure, upload, and chmod the target module
(module_style, shebang, module_data) = self._configure_module(module_name=module_name, module_args=module_args, task_vars=task_vars) (module_style, shebang, module_data, module_path) = self._configure_module(module_name=module_name, module_args=module_args, task_vars=task_vars)
if module_style == 'binary':
self._transfer_file(module_path, remote_module_path)
else:
self._transfer_data(remote_module_path, module_data) self._transfer_data(remote_module_path, module_data)
# configure, upload, and chmod the async_wrapper module # configure, upload, and chmod the async_wrapper module
(async_module_style, shebang, async_module_data) = self._configure_module(module_name='async_wrapper', module_args=dict(), task_vars=task_vars) (async_module_style, shebang, async_module_data, _) = self._configure_module(module_name='async_wrapper', module_args=dict(), task_vars=task_vars)
self._transfer_data(async_module_path, async_module_data) self._transfer_data(async_module_path, async_module_data)
argsfile = None argsfile = None
if module_style == 'non_native_want_json': if module_style in ('non_native_want_json', 'binary'):
argsfile = self._transfer_data(self._connection._shell.join_path(tmp, 'arguments'), json.dumps(module_args)) argsfile = self._transfer_data(self._connection._shell.join_path(tmp, 'arguments'), json.dumps(module_args))
elif module_style == 'old': elif module_style == 'old':
args_data = "" args_data = ""

@ -62,7 +62,7 @@ class Connection(ConnectionBase):
'''WinRM connections over HTTP/HTTPS.''' '''WinRM connections over HTTP/HTTPS.'''
transport = 'winrm' transport = 'winrm'
module_implementation_preferences = ('.ps1', '') module_implementation_preferences = ('.ps1', '.exe', '')
become_methods = [] become_methods = []
allow_executable = False allow_executable = False

@ -50,7 +50,8 @@ class ShellBase(object):
return os.path.join(*args) return os.path.join(*args)
# some shells (eg, powershell) are snooty about filenames/extensions, this lets the shell plugin have a say # some shells (eg, powershell) are snooty about filenames/extensions, this lets the shell plugin have a say
def get_remote_filename(self, base_name): def get_remote_filename(self, pathname):
base_name = os.path.basename(pathname.strip())
return base_name.strip() return base_name.strip()
def path_has_trailing_slash(self, path): def path_has_trailing_slash(self, path):
@ -164,7 +165,13 @@ class ShellBase(object):
# don't quote the cmd if it's an empty string, because this will break pipelining mode # don't quote the cmd if it's an empty string, because this will break pipelining mode
if cmd.strip() != '': if cmd.strip() != '':
cmd = pipes.quote(cmd) cmd = pipes.quote(cmd)
cmd_parts = [env_string.strip(), shebang.replace("#!", "").strip(), cmd]
cmd_parts = []
if shebang:
shebang = shebang.replace("#!", "").strip()
else:
shebang = ""
cmd_parts.extend([env_string.strip(), shebang, cmd])
if arg_path is not None: if arg_path is not None:
cmd_parts.append(arg_path) cmd_parts.append(arg_path)
new_cmd = " ".join(cmd_parts) new_cmd = " ".join(cmd_parts)

@ -54,10 +54,12 @@ class ShellModule(object):
return path return path
return '\'%s\'' % path return '\'%s\'' % path
def get_remote_filename(self, pathname):
# powershell requires that script files end with .ps1 # powershell requires that script files end with .ps1
def get_remote_filename(self, base_name): base_name = os.path.basename(pathname.strip())
if not base_name.strip().lower().endswith('.ps1'): name, ext = os.path.splitext(base_name.strip())
return base_name.strip() + '.ps1' if ext.lower() not in ['.ps1', '.exe']:
return name + '.ps1'
return base_name.strip() return base_name.strip()
@ -146,6 +148,10 @@ class ShellModule(object):
cmd_parts.insert(0, '&') cmd_parts.insert(0, '&')
elif shebang and shebang.startswith('#!'): elif shebang and shebang.startswith('#!'):
cmd_parts.insert(0, shebang[2:]) cmd_parts.insert(0, shebang[2:])
elif not shebang:
# The module is assumed to be a binary
cmd_parts[0] = self._unquote(cmd_parts[0])
cmd_parts.append(arg_path)
script = ''' script = '''
Try Try
{ {

@ -23,7 +23,9 @@ VAULT_PASSWORD_FILE = vault-password
CONSUL_RUNNING := $(shell python consul_running.py) CONSUL_RUNNING := $(shell python consul_running.py)
EUID := $(shell id -u -r) EUID := $(shell id -u -r)
all: setup test_test_infra parsing test_var_precedence unicode test_templating_settings environment test_connection non_destructive destructive includes blocks pull check_mode test_hash test_handlers test_group_by test_vault test_tags test_lookup_paths no_log test_gathering_facts UNAME := $(shell uname | tr '[:upper:]' '[:lower:]')
all: setup test_test_infra parsing test_var_precedence unicode test_templating_settings environment test_connection non_destructive destructive includes blocks pull check_mode test_hash test_handlers test_group_by test_vault test_tags test_lookup_paths no_log test_gathering_facts test_binary_modules
test_test_infra: test_test_infra:
# ensure fail/assert work locally and can stop execution with non-zero exit code # ensure fail/assert work locally and can stop execution with non-zero exit code
@ -284,3 +286,17 @@ test_lookup_paths: setup
no_log: setup no_log: setup
# This test expects 7 loggable vars and 0 non loggable ones, if either mismatches it fails, run the ansible-playbook command to debug # This test expects 7 loggable vars and 0 non loggable ones, if either mismatches it fails, run the ansible-playbook command to debug
[ "$$(ansible-playbook no_log_local.yml -i $(INVENTORY) -e outputdir=$(TEST_DIR) -vvvvv | awk --source 'BEGIN { logme = 0; nolog = 0; } /LOG_ME/ { logme += 1;} /DO_NOT_LOG/ { nolog += 1;} END { printf "%d/%d", logme, nolog; }')" = "6/0" ] [ "$$(ansible-playbook no_log_local.yml -i $(INVENTORY) -e outputdir=$(TEST_DIR) -vvvvv | awk --source 'BEGIN { logme = 0; nolog = 0; } /LOG_ME/ { logme += 1;} /DO_NOT_LOG/ { nolog += 1;} END { printf "%d/%d", logme, nolog; }')" = "6/0" ]
test_binary_modules:
mytmpdir=$(MYTMPDIR); \
ls -al $$mytmpdir; \
curl https://storage.googleapis.com/golang/go1.6.2.$(UNAME)-amd64.tar.gz | tar -xz -C $$mytmpdir; \
[ $$? != 0 ] && wget -qO- https://storage.googleapis.com/golang/go1.6.2.$(UNAME)-amd64.tar.gz | tar -xz -C $$mytmpdir; \
ls -al $$mytmpdir; \
cd library; \
GOROOT=$$mytmpdir/go GOOS=linux GOARCH=amd64 $$mytmpdir/go/bin/go build -o helloworld_linux helloworld.go; \
GOROOT=$$mytmpdir/go GOOS=windows GOARCH=amd64 $$mytmpdir/go/bin/go build -o helloworld_win32nt.exe helloworld.go; \
GOROOT=$$mytmpdir/go GOOS=darwin GOARCH=amd64 $$mytmpdir/go/bin/go build -o helloworld_darwin helloworld.go; \
cd ..; \
rm -rf $$mytmpdir; \
ANSIBLE_HOST_KEY_CHECKING=false ansible-playbook test_binary_modules.yml -i $(INVENTORY) -v $(TEST_FLAGS)

@ -0,0 +1 @@
helloworld_*

@ -0,0 +1,89 @@
// This file is part of Ansible
//
// Ansible is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// Ansible is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with Ansible. If not, see <http://www.gnu.org/licenses/>.
package main
import (
"encoding/json"
"fmt"
"io/ioutil"
"os"
)
type ModuleArgs struct {
Name string
}
type Response struct {
Msg string `json:"msg"`
Changed bool `json:"changed"`
Failed bool `json:"failed"`
}
func ExitJson(responseBody Response) {
returnResponse(responseBody)
}
func FailJson(responseBody Response) {
responseBody.Failed = true
returnResponse(responseBody)
}
func returnResponse(responseBody Response) {
var response []byte
var err error
response, err = json.Marshal(responseBody)
if err != nil {
response, _ = json.Marshal(Response{Msg: "Invalid response object"})
}
fmt.Println(string(response))
if responseBody.Failed {
os.Exit(1)
} else {
os.Exit(0)
}
}
func main() {
var response Response
if len(os.Args) != 2 {
response.Msg = "No argument file provided"
FailJson(response)
}
argsFile := os.Args[1]
text, err := ioutil.ReadFile(argsFile)
if err != nil {
response.Msg = "Could not read configuration file: " + argsFile
FailJson(response)
}
var moduleArgs ModuleArgs
err = json.Unmarshal(text, &moduleArgs)
if err != nil {
response.Msg = "Configuration file not valid JSON: " + argsFile
FailJson(response)
}
var name string = "World"
if moduleArgs.Name != "" {
name = moduleArgs.Name
}
response.Msg = "Hello, " + name + "!"
ExitJson(response)
}

@ -0,0 +1,54 @@
- debug: var=ansible_system
- name: ping
ping:
when: ansible_system != 'Win32NT'
- name: win_ping
win_ping:
when: ansible_system == 'Win32NT'
- name: Hello, World!
action: "helloworld_{{ ansible_system|lower }}"
register: hello_world
- assert:
that:
- 'hello_world.msg == "Hello, World!"'
- name: Hello, Ansible!
action: "helloworld_{{ ansible_system|lower }}"
args:
name: Ansible
register: hello_ansible
- assert:
that:
- 'hello_ansible.msg == "Hello, Ansible!"'
- name: Async Hello, World!
action: "helloworld_{{ ansible_system|lower }}"
async: 1
poll: 1
when: ansible_system != 'Win32NT'
register: async_hello_world
- assert:
that:
- 'async_hello_world.msg == "Hello, World!"'
when: not async_hello_world|skipped
- name: Async Hello, Ansible!
action: "helloworld_{{ ansible_system|lower }}"
args:
name: Ansible
async: 1
poll: 1
when: ansible_system != 'Win32NT'
register: async_hello_ansible
- assert:
that:
- 'async_hello_ansible.msg == "Hello, Ansible!"'
when: not async_hello_ansible|skipped

@ -0,0 +1,6 @@
- hosts: all
roles:
- role: test_binary_modules
tags:
- test_binary_modules

@ -217,7 +217,7 @@ class TestActionBase(unittest.TestCase):
with patch.object(os, 'rename') as m: with patch.object(os, 'rename') as m:
mock_task.args = dict(a=1, foo='fö〩') mock_task.args = dict(a=1, foo='fö〩')
mock_connection.module_implementation_preferences = ('',) mock_connection.module_implementation_preferences = ('',)
(style, shebang, data) = action_base._configure_module(mock_task.action, mock_task.args) (style, shebang, data, path) = action_base._configure_module(mock_task.action, mock_task.args)
self.assertEqual(style, "new") self.assertEqual(style, "new")
self.assertEqual(shebang, b"#!/usr/bin/python") self.assertEqual(shebang, b"#!/usr/bin/python")
@ -229,7 +229,7 @@ class TestActionBase(unittest.TestCase):
mock_task.action = 'win_copy' mock_task.action = 'win_copy'
mock_task.args = dict(b=2) mock_task.args = dict(b=2)
mock_connection.module_implementation_preferences = ('.ps1',) mock_connection.module_implementation_preferences = ('.ps1',)
(style, shebang, data) = action_base._configure_module('stat', mock_task.args) (style, shebang, data, path) = action_base._configure_module('stat', mock_task.args)
self.assertEqual(style, "new") self.assertEqual(style, "new")
self.assertEqual(shebang, None) self.assertEqual(shebang, None)
@ -572,7 +572,7 @@ class TestActionBase(unittest.TestCase):
action_base._low_level_execute_command = MagicMock() action_base._low_level_execute_command = MagicMock()
action_base._fixup_perms = MagicMock() action_base._fixup_perms = MagicMock()
action_base._configure_module.return_value = ('new', '#!/usr/bin/python', 'this is the module data') action_base._configure_module.return_value = ('new', '#!/usr/bin/python', 'this is the module data', 'path')
action_base._late_needs_tmp_path.return_value = False action_base._late_needs_tmp_path.return_value = False
action_base._compute_environment_string.return_value = '' action_base._compute_environment_string.return_value = ''
action_base._connection.has_pipelining = True action_base._connection.has_pipelining = True
@ -581,12 +581,12 @@ class TestActionBase(unittest.TestCase):
self.assertEqual(action_base._execute_module(module_name='foo', module_args=dict(z=9, y=8, x=7), task_vars=dict(a=1)), dict(rc=0, stdout="ok", stdout_lines=['ok'])) self.assertEqual(action_base._execute_module(module_name='foo', module_args=dict(z=9, y=8, x=7), task_vars=dict(a=1)), dict(rc=0, stdout="ok", stdout_lines=['ok']))
# test with needing/removing a remote tmp path # test with needing/removing a remote tmp path
action_base._configure_module.return_value = ('old', '#!/usr/bin/python', 'this is the module data') action_base._configure_module.return_value = ('old', '#!/usr/bin/python', 'this is the module data', 'path')
action_base._late_needs_tmp_path.return_value = True action_base._late_needs_tmp_path.return_value = True
action_base._make_tmp_path.return_value = '/the/tmp/path' action_base._make_tmp_path.return_value = '/the/tmp/path'
self.assertEqual(action_base._execute_module(), dict(rc=0, stdout="ok", stdout_lines=['ok'])) self.assertEqual(action_base._execute_module(), dict(rc=0, stdout="ok", stdout_lines=['ok']))
action_base._configure_module.return_value = ('non_native_want_json', '#!/usr/bin/python', 'this is the module data') action_base._configure_module.return_value = ('non_native_want_json', '#!/usr/bin/python', 'this is the module data', 'path')
self.assertEqual(action_base._execute_module(), dict(rc=0, stdout="ok", stdout_lines=['ok'])) self.assertEqual(action_base._execute_module(), dict(rc=0, stdout="ok", stdout_lines=['ok']))
play_context.become = True play_context.become = True
@ -594,14 +594,14 @@ class TestActionBase(unittest.TestCase):
self.assertEqual(action_base._execute_module(), dict(rc=0, stdout="ok", stdout_lines=['ok'])) self.assertEqual(action_base._execute_module(), dict(rc=0, stdout="ok", stdout_lines=['ok']))
# test an invalid shebang return # test an invalid shebang return
action_base._configure_module.return_value = ('new', '', 'this is the module data') action_base._configure_module.return_value = ('new', '', 'this is the module data', 'path')
action_base._late_needs_tmp_path.return_value = False action_base._late_needs_tmp_path.return_value = False
self.assertRaises(AnsibleError, action_base._execute_module) self.assertRaises(AnsibleError, action_base._execute_module)
# test with check mode enabled, once with support for check # test with check mode enabled, once with support for check
# mode and once with support disabled to raise an error # mode and once with support disabled to raise an error
play_context.check_mode = True play_context.check_mode = True
action_base._configure_module.return_value = ('new', '#!/usr/bin/python', 'this is the module data') action_base._configure_module.return_value = ('new', '#!/usr/bin/python', 'this is the module data', 'path')
self.assertEqual(action_base._execute_module(), dict(rc=0, stdout="ok", stdout_lines=['ok'])) self.assertEqual(action_base._execute_module(), dict(rc=0, stdout="ok", stdout_lines=['ok']))
action_base._supports_check_mode = False action_base._supports_check_mode = False
self.assertRaises(AnsibleError, action_base._execute_module) self.assertRaises(AnsibleError, action_base._execute_module)

Loading…
Cancel
Save