Move uses of to_bytes, to_text, to_native to use the module_utils version (#17423)

We couldn't copy to_unicode, to_bytes, to_str into module_utils because
of licensing.  So once created it we had two sets of functions that did
the same things but had different implementations.  To remedy that, this
change removes the ansible.utils.unicode versions of those functions.
pull/17433/head
Toshio Kuratomi 8 years ago committed by GitHub
parent 7a9395b5e0
commit 4ed88512e4

@ -43,7 +43,7 @@ from multiprocessing import Lock
import ansible.constants as C import ansible.constants as C
from ansible.errors import AnsibleError, AnsibleOptionsError, AnsibleParserError from ansible.errors import AnsibleError, AnsibleOptionsError, AnsibleParserError
from ansible.utils.display import Display from ansible.utils.display import Display
from ansible.utils.unicode import to_unicode from ansible.module_utils._text import to_text
######################################## ########################################
@ -97,10 +97,10 @@ if __name__ == '__main__':
except AnsibleOptionsError as e: except AnsibleOptionsError as e:
cli.parser.print_help() cli.parser.print_help()
display.error(to_unicode(e), wrap_text=False) display.error(to_text(e), wrap_text=False)
exit_code = 5 exit_code = 5
except AnsibleParserError as e: except AnsibleParserError as e:
display.error(to_unicode(e), wrap_text=False) display.error(to_text(e), wrap_text=False)
exit_code = 4 exit_code = 4
# TQM takes care of these, but leaving comment to reserve the exit codes # TQM takes care of these, but leaving comment to reserve the exit codes
# except AnsibleHostUnreachable as e: # except AnsibleHostUnreachable as e:
@ -110,16 +110,16 @@ if __name__ == '__main__':
# display.error(str(e)) # display.error(str(e))
# exit_code = 2 # exit_code = 2
except AnsibleError as e: except AnsibleError as e:
display.error(to_unicode(e), wrap_text=False) display.error(to_text(e), wrap_text=False)
exit_code = 1 exit_code = 1
except KeyboardInterrupt: except KeyboardInterrupt:
display.error("User interrupted execution") display.error("User interrupted execution")
exit_code = 99 exit_code = 99
except Exception as e: except Exception as e:
have_cli_options = cli is not None and cli.options is not None have_cli_options = cli is not None and cli.options is not None
display.error("Unexpected Exception: %s" % to_unicode(e), wrap_text=False) display.error("Unexpected Exception: %s" % to_text(e), wrap_text=False)
if not have_cli_options or have_cli_options and cli.options.verbosity > 2: if not have_cli_options or have_cli_options and cli.options.verbosity > 2:
display.display(u"the full traceback was:\n\n%s" % to_unicode(traceback.format_exc())) display.display(u"the full traceback was:\n\n%s" % to_text(traceback.format_exc()))
else: else:
display.display("to see the full traceback, use -vvv") display.display("to see the full traceback, use -vvv")
exit_code = 250 exit_code = 250

@ -215,6 +215,28 @@ Python3. We'll need to gather experience to see if this is going to work out
well for modules as well or if we should give the module_utils API explicit well for modules as well or if we should give the module_utils API explicit
switches so that modules can choose to operate with text type all of the time. switches so that modules can choose to operate with text type all of the time.
Helpers
~~~~~~~
For converting between bytes, text, and native strings we have three helper
functions. These are :func:`ansible.module_utils._text.to_bytes`,
:func:`ansible.module_utils._text.to_native`, and
:func:`ansible.module_utils._text.to_text`. These are similar to using
``bytes.decode()`` and ``unicode.encode()`` with a few differences.
* By default they try very hard not to traceback.
* The default encoding is "utf-8"
* There are two error strategies that don't correspond one-to-one with
a python codec error handler. These are ``surrogate_or_strict`` and
``surrogate_or_replace``. ``surrogate_or_strict`` will use the ``surrogateescape``
error handler if available (mostly on python3) or strict if not. It is most
appropriate to use when dealing with something that needs to round trip its
value like file paths database keys, etc. Without ``surrogateescape`` the best
thing these values can do is generate a traceback that our code can catch
and decide how to show an error message. ``surrogate_or_replace`` is for
when a value is going to be displayed to the user. If the
``surrogateescape`` error handler is not present, it will replace
undecodable byte sequences with a replacement character.
================================ ================================
Porting Core Ansible to Python 3 Porting Core Ansible to Python 3

@ -19,6 +19,7 @@
# #
from __future__ import print_function from __future__ import print_function
__metaclass__ = type
import os import os
import glob import glob
@ -34,10 +35,10 @@ from collections import defaultdict
from jinja2 import Environment, FileSystemLoader from jinja2 import Environment, FileSystemLoader
from six import iteritems from six import iteritems
from ansible.errors import AnsibleError
from ansible.module_utils._text import to_bytes
from ansible.utils import module_docs from ansible.utils import module_docs
from ansible.utils.vars import merge_hash from ansible.utils.vars import merge_hash
from ansible.utils.unicode import to_bytes
from ansible.errors import AnsibleError
##################################################################################### #####################################################################################
# constants and paths # constants and paths

@ -33,7 +33,7 @@ import subprocess
from ansible.release import __version__ from ansible.release import __version__
from ansible import constants as C from ansible import constants as C
from ansible.errors import AnsibleError, AnsibleOptionsError from ansible.errors import AnsibleError, AnsibleOptionsError
from ansible.utils.unicode import to_bytes, to_unicode from ansible.module_utils._text import to_bytes, to_text
try: try:
from __main__ import display from __main__ import display
@ -109,7 +109,7 @@ class CLI(object):
if self.options.verbosity > 0: if self.options.verbosity > 0:
if C.CONFIG_FILE: if C.CONFIG_FILE:
display.display(u"Using %s as config file" % to_unicode(C.CONFIG_FILE)) display.display(u"Using %s as config file" % to_text(C.CONFIG_FILE))
else: else:
display.display(u"No config file found; using defaults") display.display(u"No config file found; using defaults")

@ -27,13 +27,13 @@ from ansible.cli import CLI
from ansible.errors import AnsibleError, AnsibleOptionsError from ansible.errors import AnsibleError, AnsibleOptionsError
from ansible.executor.task_queue_manager import TaskQueueManager from ansible.executor.task_queue_manager import TaskQueueManager
from ansible.inventory import Inventory from ansible.inventory import Inventory
from ansible.module_utils._text import to_text
from ansible.parsing.dataloader import DataLoader from ansible.parsing.dataloader import DataLoader
from ansible.parsing.splitter import parse_kv from ansible.parsing.splitter import parse_kv
from ansible.playbook.play import Play from ansible.playbook.play import Play
from ansible.plugins import get_all_plugin_loaders from ansible.plugins import get_all_plugin_loaders
from ansible.utils.vars import load_extra_vars from ansible.utils.vars import load_extra_vars
from ansible.utils.vars import load_options_vars from ansible.utils.vars import load_options_vars
from ansible.utils.unicode import to_unicode
from ansible.vars import VariableManager from ansible.vars import VariableManager
try: try:
@ -99,7 +99,7 @@ class AdHocCLI(CLI):
super(AdHocCLI, self).run() super(AdHocCLI, self).run()
# only thing left should be host pattern # only thing left should be host pattern
pattern = to_unicode(self.args[0], errors='strict') pattern = to_text(self.args[0], errors='surrogate_or_strict')
# ignore connection password cause we are local # ignore connection password cause we are local
if self.options.connection == "local": if self.options.connection == "local":

@ -39,18 +39,16 @@ import sys
from ansible import constants as C from ansible import constants as C
from ansible.cli import CLI from ansible.cli import CLI
from ansible.errors import AnsibleError from ansible.errors import AnsibleError
from ansible.executor.task_queue_manager import TaskQueueManager from ansible.executor.task_queue_manager import TaskQueueManager
from ansible.inventory import Inventory from ansible.inventory import Inventory
from ansible.module_utils._text import to_native, to_text
from ansible.parsing.dataloader import DataLoader from ansible.parsing.dataloader import DataLoader
from ansible.parsing.splitter import parse_kv from ansible.parsing.splitter import parse_kv
from ansible.playbook.play import Play from ansible.playbook.play import Play
from ansible.vars import VariableManager from ansible.plugins import module_loader
from ansible.utils import module_docs from ansible.utils import module_docs
from ansible.utils.color import stringc from ansible.utils.color import stringc
from ansible.utils.unicode import to_unicode, to_str from ansible.vars import VariableManager
from ansible.plugins import module_loader
try: try:
from __main__ import display from __main__ import display
@ -192,7 +190,7 @@ class ConsoleCLI(CLI, cmd.Cmd):
) )
play = Play().load(play_ds, variable_manager=self.variable_manager, loader=self.loader) play = Play().load(play_ds, variable_manager=self.variable_manager, loader=self.loader)
except Exception as e: except Exception as e:
display.error(u"Unable to build command: %s" % to_unicode(e)) display.error(u"Unable to build command: %s" % to_text(e))
return False return False
try: try:
@ -225,7 +223,7 @@ class ConsoleCLI(CLI, cmd.Cmd):
display.error('User interrupted execution') display.error('User interrupted execution')
return False return False
except Exception as e: except Exception as e:
display.error(to_unicode(e)) display.error(to_text(e))
# FIXME: add traceback in very very verbose mode # FIXME: add traceback in very very verbose mode
return False return False
@ -379,7 +377,7 @@ class ConsoleCLI(CLI, cmd.Cmd):
else: else:
completions = [x.name for x in self.inventory.list_hosts(self.options.cwd)] completions = [x.name for x in self.inventory.list_hosts(self.options.cwd)]
return [to_str(s)[offs:] for s in completions if to_str(s).startswith(to_str(mline))] return [to_native(s)[offs:] for s in completions if to_native(s).startswith(to_native(mline))]
def completedefault(self, text, line, begidx, endidx): def completedefault(self, text, line, begidx, endidx):
if line.split()[0] in self.modules: if line.split()[0] in self.modules:
@ -394,7 +392,6 @@ class ConsoleCLI(CLI, cmd.Cmd):
oc, a, _ = module_docs.get_docstring(in_path) oc, a, _ = module_docs.get_docstring(in_path)
return oc['options'].keys() return oc['options'].keys()
def run(self): def run(self):
super(ConsoleCLI, self).run() super(ConsoleCLI, self).run()
@ -410,7 +407,6 @@ class ConsoleCLI(CLI, cmd.Cmd):
self.pattern = self.args[0] self.pattern = self.args[0]
self.options.cwd = self.pattern self.options.cwd = self.pattern
# dynamically add modules as commands # dynamically add modules as commands
self.modules = self.list_modules() self.modules = self.list_modules()
for module in self.modules: for module in self.modules:
@ -465,4 +461,3 @@ class ConsoleCLI(CLI, cmd.Cmd):
atexit.register(readline.write_history_file, histfile) atexit.register(readline.write_history_file, histfile)
self.set_prompt() self.set_prompt()
self.cmdloop() self.cmdloop()

@ -39,7 +39,7 @@ from ansible.galaxy.role import GalaxyRole
from ansible.galaxy.login import GalaxyLogin from ansible.galaxy.login import GalaxyLogin
from ansible.galaxy.token import GalaxyToken from ansible.galaxy.token import GalaxyToken
from ansible.playbook.role.requirement import RoleRequirement from ansible.playbook.role.requirement import RoleRequirement
from ansible.utils.unicode import to_bytes, to_unicode from ansible.module_utils._text import to_bytes, to_text
try: try:
from __main__ import display from __main__ import display
@ -47,6 +47,7 @@ except ImportError:
from ansible.utils.display import Display from ansible.utils.display import Display
display = Display() display = Display()
class GalaxyCLI(CLI): class GalaxyCLI(CLI):
SKIP_INFO_KEYS = ("name", "description", "readme_html", "related", "summary_fields", "average_aw_composite", "average_aw_score", "url" ) SKIP_INFO_KEYS = ("name", "description", "readme_html", "related", "summary_fields", "average_aw_composite", "average_aw_score", "url" )
@ -65,7 +66,6 @@ class GalaxyCLI(CLI):
epilog = "\nSee '%s <command> --help' for more information on a specific command.\n\n" % os.path.basename(sys.argv[0]) epilog = "\nSee '%s <command> --help' for more information on a specific command.\n\n" % os.path.basename(sys.argv[0])
) )
self.set_action() self.set_action()
# common # common
@ -111,7 +111,7 @@ class GalaxyCLI(CLI):
if self.action in ['init', 'info']: if self.action in ['init', 'info']:
self.parser.add_option( '--offline', dest='offline', default=False, action='store_true', help="Don't query the galaxy API when creating roles") self.parser.add_option( '--offline', dest='offline', default=False, action='store_true', help="Don't query the galaxy API when creating roles")
if not self.action in ("delete","import","init","login","setup"): if self.action not in ("delete","import","init","login","setup"):
# NOTE: while the option type=str, the default is a list, and the # NOTE: while the option type=str, the default is a list, and the
# callback will set the value to a list. # callback will set the value to a list.
self.parser.add_option('-p', '--roles-path', dest='roles_path', action="callback", callback=CLI.expand_paths, type=str, default=C.DEFAULT_ROLES_PATH, self.parser.add_option('-p', '--roles-path', dest='roles_path', action="callback", callback=CLI.expand_paths, type=str, default=C.DEFAULT_ROLES_PATH,
@ -142,7 +142,7 @@ class GalaxyCLI(CLI):
def _display_role_info(self, role_info): def _display_role_info(self, role_info):
text = [u"", u"Role: %s" % to_unicode(role_info['name'])] text = [u"", u"Role: %s" % to_text(role_info['name'])]
text.append(u"\tdescription: %s" % role_info.get('description', '')) text.append(u"\tdescription: %s" % role_info.get('description', ''))
for k in sorted(role_info.keys()): for k in sorted(role_info.keys()):
@ -578,8 +578,8 @@ class GalaxyCLI(CLI):
if len(self.args) < 2: if len(self.args) < 2:
raise AnsibleError("Expected a github_username and github_repository. Use --help.") raise AnsibleError("Expected a github_username and github_repository. Use --help.")
github_repo = self.args.pop() github_repo = to_text(self.args.pop(), errors='surrogate_or_strict')
github_user = self.args.pop() github_user = to_text(self.args.pop(), errors='surrogate_or_strict')
if self.options.check_status: if self.options.check_status:
task = self.api.get_import_task(github_user=github_user, github_repo=github_repo) task = self.api.get_import_task(github_user=github_user, github_repo=github_repo)
@ -594,7 +594,8 @@ class GalaxyCLI(CLI):
display.display("The following Galaxy roles are being updated:" + u'\n', color=C.COLOR_CHANGED) display.display("The following Galaxy roles are being updated:" + u'\n', color=C.COLOR_CHANGED)
for t in task: for t in task:
display.display('%s.%s' % (t['summary_fields']['role']['namespace'],t['summary_fields']['role']['name']), color=C.COLOR_CHANGED) display.display('%s.%s' % (t['summary_fields']['role']['namespace'],t['summary_fields']['role']['name']), color=C.COLOR_CHANGED)
display.display(u'\n' + "To properly namespace this role, remove each of the above and re-import %s/%s from scratch" % (github_user,github_repo), color=C.COLOR_CHANGED) display.display(u'\nTo properly namespace this role, remove each of the above and re-import %s/%s from scratch' % (github_user, github_repo),
color=C.COLOR_CHANGED)
return 0 return 0
# found a single role as expected # found a single role as expected
display.display("Successfully submitted import request %d" % task[0]['id']) display.display("Successfully submitted import request %d" % task[0]['id'])

@ -26,7 +26,7 @@ from ansible.errors import AnsibleError, AnsibleOptionsError
from ansible.parsing.dataloader import DataLoader from ansible.parsing.dataloader import DataLoader
from ansible.parsing.vault import VaultEditor from ansible.parsing.vault import VaultEditor
from ansible.cli import CLI from ansible.cli import CLI
from ansible.utils.unicode import to_unicode from ansible.module_utils._text import to_text
try: try:
from __main__ import display from __main__ import display
@ -163,7 +163,7 @@ class VaultCLI(CLI):
# unicode here because we are displaying it and therefore can make # unicode here because we are displaying it and therefore can make
# the decision that the display doesn't have to be precisely what # the decision that the display doesn't have to be precisely what
# the input was (leave that to decrypt instead) # the input was (leave that to decrypt instead)
self.pager(to_unicode(self.editor.plaintext(f))) self.pager(ansible.module_utils._text.to_text(self.editor.plaintext(f)))
def execute_rekey(self): def execute_rekey(self):
for f in self.args: for f in self.args:

@ -25,8 +25,7 @@ from ansible.errors.yaml_strings import ( YAML_POSITION_DETAILS,
YAML_COMMON_UNQUOTED_COLON_ERROR, YAML_COMMON_UNQUOTED_COLON_ERROR,
YAML_COMMON_PARTIALLY_QUOTED_LINE_ERROR, YAML_COMMON_PARTIALLY_QUOTED_LINE_ERROR,
YAML_COMMON_UNBALANCED_QUOTES_ERROR ) YAML_COMMON_UNBALANCED_QUOTES_ERROR )
from ansible.module_utils._text import to_native, to_text
from ansible.utils.unicode import to_unicode, to_str
class AnsibleError(Exception): class AnsibleError(Exception):
@ -54,11 +53,11 @@ class AnsibleError(Exception):
if obj and isinstance(obj, AnsibleBaseYAMLObject): if obj and isinstance(obj, AnsibleBaseYAMLObject):
extended_error = self._get_extended_error() extended_error = self._get_extended_error()
if extended_error and not suppress_extended_error: if extended_error and not suppress_extended_error:
self.message = '%s\n\n%s' % (to_str(message), to_str(extended_error)) self.message = '%s\n\n%s' % (to_native(message), to_native(extended_error))
else: else:
self.message = '%s' % to_str(message) self.message = '%s' % to_native(message)
else: else:
self.message = '%s' % to_str(message) self.message = '%s' % to_native(message)
def __str__(self): def __str__(self):
return self.message return self.message
@ -104,8 +103,8 @@ class AnsibleError(Exception):
error_message += YAML_POSITION_DETAILS % (src_file, line_number, col_number) error_message += YAML_POSITION_DETAILS % (src_file, line_number, col_number)
if src_file not in ('<string>', '<unicode>') and self._show_content: if src_file not in ('<string>', '<unicode>') and self._show_content:
(target_line, prev_line) = self._get_error_lines_from_file(src_file, line_number - 1) (target_line, prev_line) = self._get_error_lines_from_file(src_file, line_number - 1)
target_line = to_unicode(target_line) target_line = to_text(target_line)
prev_line = to_unicode(prev_line) prev_line = to_text(prev_line)
if target_line: if target_line:
stripped_line = target_line.replace(" ","") stripped_line = target_line.replace(" ","")
arrow_line = (" " * (col_number-1)) + "^ here" arrow_line = (" " * (col_number-1)) + "^ here"

@ -29,11 +29,10 @@ import shlex
import zipfile import zipfile
from io import BytesIO from io import BytesIO
# from Ansible
from ansible.release import __version__, __author__ from ansible.release import __version__, __author__
from ansible import constants as C from ansible import constants as C
from ansible.errors import AnsibleError from ansible.errors import AnsibleError
from ansible.utils.unicode import to_bytes, to_unicode from ansible.module_utils._text import to_bytes, to_text
# Must import strategy and use write_locks from there # Must import strategy and use write_locks from there
# If we import write_locks directly then we end up binding a # If we import write_locks directly then we end up binding a
# variable to the object and then it never gets updated. # variable to the object and then it never gets updated.
@ -45,6 +44,7 @@ except ImportError:
from ansible.utils.display import Display from ansible.utils.display import Display
display = Display() display = Display()
REPLACER = b"#<<INCLUDE_ANSIBLE_MODULE_COMMON>>" REPLACER = b"#<<INCLUDE_ANSIBLE_MODULE_COMMON>>"
REPLACER_VERSION = b"\"<<ANSIBLE_VERSION>>\"" REPLACER_VERSION = b"\"<<ANSIBLE_VERSION>>\""
REPLACER_COMPLEX = b"\"<<INCLUDE_ANSIBLE_MODULE_COMPLEX_ARGS>>\"" REPLACER_COMPLEX = b"\"<<INCLUDE_ANSIBLE_MODULE_COMPLEX_ARGS>>\""
@ -239,7 +239,9 @@ def debug(command, zipped_mod, json_params):
else: else:
os.environ['PYTHONPATH'] = basedir os.environ['PYTHONPATH'] = basedir
p = subprocess.Popen([%(interpreter)s, script_path, args_path], env=os.environ, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE) p = subprocess.Popen([%(interpreter)s, script_path, args_path],
env=os.environ, shell=False, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, stdin=subprocess.PIPE)
(stdout, stderr) = p.communicate() (stdout, stderr) = p.communicate()
if not isinstance(stderr, (bytes, unicode)): if not isinstance(stderr, (bytes, unicode)):
@ -328,6 +330,7 @@ if __name__ == '__main__':
sys.exit(exitcode) sys.exit(exitcode)
''' '''
def _strip_comments(source): def _strip_comments(source):
# Strip comments and blank lines from the wrapper # Strip comments and blank lines from the wrapper
buf = [] buf = []
@ -338,6 +341,7 @@ def _strip_comments(source):
buf.append(line) buf.append(line)
return u'\n'.join(buf) return u'\n'.join(buf)
if C.DEFAULT_KEEP_REMOTE_FILES: if C.DEFAULT_KEEP_REMOTE_FILES:
# Keep comments when KEEP_REMOTE_FILES is set. That way users will see # Keep comments when KEEP_REMOTE_FILES is set. That way users will see
# the comments with some nice usage instructions # the comments with some nice usage instructions
@ -346,6 +350,7 @@ else:
# ANSIBALLZ_TEMPLATE stripped of comments for smaller over the wire size # ANSIBALLZ_TEMPLATE stripped of comments for smaller over the wire size
ACTIVE_ANSIBALLZ_TEMPLATE = _strip_comments(ANSIBALLZ_TEMPLATE) ACTIVE_ANSIBALLZ_TEMPLATE = _strip_comments(ANSIBALLZ_TEMPLATE)
class ModuleDepFinder(ast.NodeVisitor): class ModuleDepFinder(ast.NodeVisitor):
# Caveats: # Caveats:
# This code currently does not handle: # This code currently does not handle:
@ -404,6 +409,7 @@ def _slurp(path):
fd.close() fd.close()
return data return data
def _get_shebang(interpreter, task_vars, args=tuple()): def _get_shebang(interpreter, task_vars, args=tuple()):
""" """
Note not stellar API: Note not stellar API:
@ -425,6 +431,7 @@ def _get_shebang(interpreter, task_vars, args=tuple()):
return (shebang, interpreter) return (shebang, interpreter)
def recursive_finder(name, data, py_module_names, py_module_cache, zf): def recursive_finder(name, data, py_module_names, py_module_cache, zf):
""" """
Using ModuleDepFinder, make sure we have all of the module_utils files that Using ModuleDepFinder, make sure we have all of the module_utils files that
@ -529,11 +536,13 @@ def recursive_finder(name, data, py_module_names, py_module_cache, zf):
# Save memory; the file won't have to be read again for this ansible module. # Save memory; the file won't have to be read again for this ansible module.
del py_module_cache[py_module_file] del py_module_cache[py_module_file]
def _is_binary(module_data): def _is_binary(module_data):
textchars = bytearray(set([7, 8, 9, 10, 12, 13, 27]) | set(range(0x20, 0x100)) - set([0x7f])) textchars = bytearray(set([7, 8, 9, 10, 12, 13, 27]) | set(range(0x20, 0x100)) - set([0x7f]))
start = module_data[:1024] start = module_data[:1024]
return bool(start.translate(None, textchars)) return bool(start.translate(None, textchars))
def _find_snippet_imports(module_name, module_data, module_path, module_args, task_vars, module_compression): def _find_snippet_imports(module_name, module_data, module_path, module_args, task_vars, module_compression):
""" """
Given the source of the module, convert it to a Jinja2 template to insert Given the source of the module, convert it to a Jinja2 template to insert
@ -617,9 +626,12 @@ def _find_snippet_imports(module_name, module_data, module_path, module_args, ta
# Create the module zip data # Create the module zip data
zipoutput = BytesIO() zipoutput = BytesIO()
zf = zipfile.ZipFile(zipoutput, mode='w', compression=compression_method) zf = zipfile.ZipFile(zipoutput, mode='w', compression=compression_method)
### Note: If we need to import from release.py first, # Note: If we need to import from release.py first,
### remember to catch all exceptions: https://github.com/ansible/ansible/issues/16523 # remember to catch all exceptions: https://github.com/ansible/ansible/issues/16523
zf.writestr('ansible/__init__.py', b'from pkgutil import extend_path\n__path__=extend_path(__path__,__name__)\n__version__="' + to_bytes(__version__) + b'"\n__author__="' + to_bytes(__author__) + b'"\n') zf.writestr('ansible/__init__.py',
b'from pkgutil import extend_path\n__path__=extend_path(__path__,__name__)\n__version__="' +
to_bytes(__version__) + b'"\n__author__="' +
to_bytes(__author__) + b'"\n')
zf.writestr('ansible/module_utils/__init__.py', b'from pkgutil import extend_path\n__path__=extend_path(__path__,__name__)\n') zf.writestr('ansible/module_utils/__init__.py', b'from pkgutil import extend_path\n__path__=extend_path(__path__,__name__)\n')
zf.writestr('ansible_module_%s.py' % module_name, module_data) zf.writestr('ansible_module_%s.py' % module_name, module_data)
@ -655,8 +667,9 @@ def _find_snippet_imports(module_name, module_data, module_path, module_args, ta
try: try:
zipdata = open(cached_module_filename, 'rb').read() zipdata = open(cached_module_filename, 'rb').read()
except IOError: except IOError:
raise AnsibleError('A different worker process failed to create module file. Look at traceback for that process for debugging information.') raise AnsibleError('A different worker process failed to create module file.'
zipdata = to_unicode(zipdata, errors='strict') ' Look at traceback for that process for debugging information.')
zipdata = to_text(zipdata, errors='surrogate_or_strict')
shebang, interpreter = _get_shebang(u'/usr/bin/python', task_vars) shebang, interpreter = _get_shebang(u'/usr/bin/python', task_vars)
if shebang is None: if shebang is None:
@ -721,12 +734,11 @@ def _find_snippet_imports(module_name, module_data, module_path, module_args, ta
# The main event -- substitute the JSON args string into the module # The main event -- substitute the JSON args string into the module
module_data = module_data.replace(REPLACER_JSONARGS, module_args_json) module_data = module_data.replace(REPLACER_JSONARGS, module_args_json)
facility = b'syslog.' + to_bytes(task_vars.get('ansible_syslog_facility', C.DEFAULT_SYSLOG_FACILITY), errors='strict') facility = b'syslog.' + to_bytes(task_vars.get('ansible_syslog_facility', C.DEFAULT_SYSLOG_FACILITY), errors='surrogate_or_strict')
module_data = module_data.replace(b'syslog.LOG_USER', facility) module_data = module_data.replace(b'syslog.LOG_USER', facility)
return (module_data, module_style, shebang) return (module_data, module_style, shebang)
# ******************************************************************************
def modify_module(module_name, module_path, module_args, task_vars=dict(), module_compression='ZIP_STORED'): def modify_module(module_name, module_path, module_args, task_vars=dict(), module_compression='ZIP_STORED'):
""" """
@ -760,7 +772,7 @@ def modify_module(module_name, module_path, module_args, task_vars=dict(), modul
(module_data, module_style, shebang) = _find_snippet_imports(module_name, module_data, module_path, module_args, task_vars, module_compression) (module_data, module_style, shebang) = _find_snippet_imports(module_name, module_data, module_path, module_args, task_vars, module_compression)
if module_style == 'binary': if module_style == 'binary':
return (module_data, module_style, to_unicode(shebang, nonstring='passthru')) return (module_data, module_style, to_text(shebang, nonstring='passthru'))
elif shebang is None: elif shebang is None:
lines = module_data.split(b"\n", 1) lines = module_data.split(b"\n", 1)
if lines[0].startswith(b"#!"): if lines[0].startswith(b"#!"):
@ -769,7 +781,7 @@ def modify_module(module_name, module_path, module_args, task_vars=dict(), modul
interpreter = args[0] interpreter = args[0]
interpreter = to_bytes(interpreter) interpreter = to_bytes(interpreter)
new_shebang = to_bytes(_get_shebang(interpreter, task_vars, args[1:])[0], errors='strict', nonstring='passthru') new_shebang = to_bytes(_get_shebang(interpreter, task_vars, args[1:])[0], errors='surrogate_or_strict', nonstring='passthru')
if new_shebang: if new_shebang:
lines[0] = shebang = new_shebang lines[0] = shebang = new_shebang
@ -781,6 +793,6 @@ def modify_module(module_name, module_path, module_args, task_vars=dict(), modul
module_data = b"\n".join(lines) module_data = b"\n".join(lines)
else: else:
shebang = to_bytes(shebang, errors='strict') shebang = to_bytes(shebang, errors='surrogate_or_strict')
return (module_data, module_style, to_unicode(shebang, nonstring='passthru')) return (module_data, module_style, to_text(shebang, nonstring='passthru'))

@ -21,15 +21,13 @@ __metaclass__ = type
import os import os
from ansible.compat.six import string_types
from ansible import constants as C from ansible import constants as C
from ansible.executor.task_queue_manager import TaskQueueManager from ansible.executor.task_queue_manager import TaskQueueManager
from ansible.module_utils._text import to_native, to_text
from ansible.playbook import Playbook from ansible.playbook import Playbook
from ansible.template import Templar from ansible.template import Templar
from ansible.utils.helpers import pct_to_int from ansible.utils.helpers import pct_to_int
from ansible.utils.path import makedirs_safe from ansible.utils.path import makedirs_safe
from ansible.utils.unicode import to_unicode, to_str
try: try:
from __main__ import display from __main__ import display
@ -84,7 +82,7 @@ class PlaybookExecutor:
i = 1 i = 1
plays = pb.get_plays() plays = pb.get_plays()
display.vv(u'%d plays in %s' % (len(plays), to_unicode(playbook_path))) display.vv(u'%d plays in %s' % (len(plays), to_text(playbook_path)))
for play in plays: for play in plays:
if play._included_path is not None: if play._included_path is not None:
@ -276,7 +274,7 @@ class PlaybookExecutor:
for x in replay_hosts: for x in replay_hosts:
fd.write("%s\n" % x) fd.write("%s\n" % x)
except Exception as e: except Exception as e:
display.warning("Could not create retry file '%s'.\n\t%s" % (retry_path, to_str(e))) display.warning("Could not create retry file '%s'.\n\t%s" % (retry_path, to_native(e)))
return False return False
return True return True

@ -19,16 +19,10 @@
from __future__ import (absolute_import, division, print_function) from __future__ import (absolute_import, division, print_function)
__metaclass__ = type __metaclass__ = type
from ansible.compat.six.moves import queue
import json
import multiprocessing import multiprocessing
import os import os
import signal
import sys import sys
import time
import traceback import traceback
import zlib
from jinja2.exceptions import TemplateNotFound from jinja2.exceptions import TemplateNotFound
@ -40,13 +34,10 @@ try:
except ImportError: except ImportError:
HAS_ATFORK=False HAS_ATFORK=False
from ansible.errors import AnsibleError, AnsibleConnectionFailure from ansible.errors import AnsibleConnectionFailure
from ansible.executor.task_executor import TaskExecutor from ansible.executor.task_executor import TaskExecutor
from ansible.executor.task_result import TaskResult from ansible.executor.task_result import TaskResult
from ansible.playbook.handler import Handler from ansible.module_utils._text import to_text
from ansible.playbook.task import Task
from ansible.vars.unsafe_proxy import AnsibleJSONUnsafeDecoder
from ansible.utils.unicode import to_unicode
try: try:
from __main__ import display from __main__ import display
@ -144,11 +135,11 @@ class WorkerProcess(multiprocessing.Process):
try: try:
self._host.vars = dict() self._host.vars = dict()
self._host.groups = [] self._host.groups = []
task_result = TaskResult(self._host.name, self._task._uuid, dict(failed=True, exception=to_unicode(traceback.format_exc()), stdout='')) task_result = TaskResult(self._host.name, self._task._uuid, dict(failed=True, exception=to_text(traceback.format_exc()), stdout=''))
self._rslt_q.put(task_result, block=False) self._rslt_q.put(task_result, block=False)
except: except:
display.debug(u"WORKER EXCEPTION: %s" % to_unicode(e)) display.debug(u"WORKER EXCEPTION: %s" % to_text(e))
display.debug(u"WORKER TRACEBACK: %s" % to_unicode(traceback.format_exc())) display.debug(u"WORKER TRACEBACK: %s" % to_text(traceback.format_exc()))
display.debug("WORKER PROCESS EXITING") display.debug("WORKER PROCESS EXITING")

@ -20,7 +20,6 @@ from __future__ import (absolute_import, division, print_function)
__metaclass__ = type __metaclass__ = type
import base64 import base64
import json
import subprocess import subprocess
import sys import sys
import time import time
@ -31,12 +30,12 @@ from ansible.compat.six import iteritems, string_types, binary_type
from ansible import constants as C from ansible import constants as C
from ansible.errors import AnsibleError, AnsibleParserError, AnsibleUndefinedVariable, AnsibleConnectionFailure from ansible.errors import AnsibleError, AnsibleParserError, AnsibleUndefinedVariable, AnsibleConnectionFailure
from ansible.executor.task_result import TaskResult from ansible.executor.task_result import TaskResult
from ansible.module_utils._text import to_bytes, to_text
from ansible.playbook.conditional import Conditional from ansible.playbook.conditional import Conditional
from ansible.playbook.task import Task from ansible.playbook.task import Task
from ansible.template import Templar from ansible.template import Templar
from ansible.utils.encrypt import key_for_hostname from ansible.utils.encrypt import key_for_hostname
from ansible.utils.listify import listify_lookup_plugin_terms from ansible.utils.listify import listify_lookup_plugin_terms
from ansible.utils.unicode import to_unicode, to_bytes
from ansible.vars.unsafe_proxy import UnsafeProxy, wrap_var from ansible.vars.unsafe_proxy import UnsafeProxy, wrap_var
try: try:
@ -130,7 +129,7 @@ class TaskExecutor:
if isinstance(res, UnsafeProxy): if isinstance(res, UnsafeProxy):
return res._obj return res._obj
elif isinstance(res, binary_type): elif isinstance(res, binary_type):
return to_unicode(res, errors='strict') return to_text(res, errors='surrogate_or_strict')
elif isinstance(res, dict): elif isinstance(res, dict):
for k in res: for k in res:
res[k] = _clean_res(res[k]) res[k] = _clean_res(res[k])
@ -144,16 +143,16 @@ class TaskExecutor:
display.debug("done dumping result, returning") display.debug("done dumping result, returning")
return res return res
except AnsibleError as e: except AnsibleError as e:
return dict(failed=True, msg=to_unicode(e, nonstring='simplerepr')) return dict(failed=True, msg=to_text(e, nonstring='simplerepr'))
except Exception as e: except Exception as e:
return dict(failed=True, msg='Unexpected failure during module execution.', exception=to_unicode(traceback.format_exc()), stdout='') return dict(failed=True, msg='Unexpected failure during module execution.', exception=to_text(traceback.format_exc()), stdout='')
finally: finally:
try: try:
self._connection.close() self._connection.close()
except AttributeError: except AttributeError:
pass pass
except Exception as e: except Exception as e:
display.debug(u"error closing connection: %s" % to_unicode(e)) display.debug(u"error closing connection: %s" % to_text(e))
def _get_loop_items(self): def _get_loop_items(self):
''' '''
@ -182,11 +181,13 @@ class TaskExecutor:
# first_found loops are special. If the item is undefined # first_found loops are special. If the item is undefined
# then we want to fall through to the next value rather # then we want to fall through to the next value rather
# than failing. # than failing.
loop_terms = listify_lookup_plugin_terms(terms=self._task.loop_args, templar=templar, loader=self._loader, fail_on_undefined=False, convert_bare=True) loop_terms = listify_lookup_plugin_terms(terms=self._task.loop_args, templar=templar,
loader=self._loader, fail_on_undefined=False, convert_bare=True)
loop_terms = [t for t in loop_terms if not templar._contains_vars(t)] loop_terms = [t for t in loop_terms if not templar._contains_vars(t)]
else: else:
try: try:
loop_terms = listify_lookup_plugin_terms(terms=self._task.loop_args, templar=templar, loader=self._loader, fail_on_undefined=True, convert_bare=True) loop_terms = listify_lookup_plugin_terms(terms=self._task.loop_args, templar=templar,
loader=self._loader, fail_on_undefined=True, convert_bare=True)
except AnsibleUndefinedVariable as e: except AnsibleUndefinedVariable as e:
display.deprecated("Skipping task due to undefined Error, in the future this will be a fatal error.: %s" % to_bytes(e)) display.deprecated("Skipping task due to undefined Error, in the future this will be a fatal error.: %s" % to_bytes(e))
return None return None
@ -245,7 +246,9 @@ class TaskExecutor:
loop_pause = self._task.loop_control.pause or 0 loop_pause = self._task.loop_control.pause or 0
if loop_var in task_vars: if loop_var in task_vars:
display.warning("The loop variable '%s' is already in use. You should set the `loop_var` value in the `loop_control` option for the task to something else to avoid variable collisions and unexpected behavior." % loop_var) display.warning(u"The loop variable '%s' is already in use."
u"You should set the `loop_var` value in the `loop_control` option for the task"
u" to something else to avoid variable collisions and unexpected behavior." % loop_var)
ran_once = False ran_once = False
items = self._squash_items(items, loop_var, task_vars) items = self._squash_items(items, loop_var, task_vars)
@ -263,7 +266,7 @@ class TaskExecutor:
tmp_task._parent = self._task._parent tmp_task._parent = self._task._parent
tmp_play_context = self._play_context.copy() tmp_play_context = self._play_context.copy()
except AnsibleParserError as e: except AnsibleParserError as e:
results.append(dict(failed=True, msg=to_unicode(e))) results.append(dict(failed=True, msg=to_text(e)))
continue continue
# now we swap the internal task and play context with their copies, # now we swap the internal task and play context with their copies,
@ -279,7 +282,7 @@ class TaskExecutor:
res[loop_var] = item res[loop_var] = item
res['_ansible_item_result'] = True res['_ansible_item_result'] = True
if not label is None: if label is not None:
templar = Templar(loader=self._loader, shared_loader_obj=self._shared_loader_obj, variables=self._job_vars) templar = Templar(loader=self._loader, shared_loader_obj=self._shared_loader_obj, variables=self._job_vars)
res['_ansible_item_label'] = templar.template(label, fail_on_undefined=False) res['_ansible_item_label'] = templar.template(label, fail_on_undefined=False)
@ -482,7 +485,7 @@ class TaskExecutor:
try: try:
result = self._handler.run(task_vars=variables) result = self._handler.run(task_vars=variables)
except AnsibleConnectionFailure as e: except AnsibleConnectionFailure as e:
return dict(unreachable=True, msg=to_unicode(e)) return dict(unreachable=True, msg=to_text(e))
display.debug("handler run complete") display.debug("handler run complete")
# preserve no log # preserve no log
@ -666,7 +669,7 @@ class TaskExecutor:
try: try:
cmd = subprocess.Popen(['ssh','-o','ControlPersist'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) cmd = subprocess.Popen(['ssh','-o','ControlPersist'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
(out, err) = cmd.communicate() (out, err) = cmd.communicate()
err = to_unicode(err) err = to_text(err)
if u"Bad configuration option" in err or u"Usage:" in err: if u"Bad configuration option" in err or u"Usage:" in err:
conn_type = "paramiko" conn_type = "paramiko"
except OSError: except OSError:

@ -28,21 +28,20 @@ import time
from collections import deque from collections import deque
from ansible import constants as C from ansible import constants as C
from ansible.compat.six import string_types
from ansible.errors import AnsibleError from ansible.errors import AnsibleError
from ansible.executor import action_write_locks from ansible.executor import action_write_locks
from ansible.executor.play_iterator import PlayIterator from ansible.executor.play_iterator import PlayIterator
from ansible.executor.process.worker import WorkerProcess from ansible.executor.process.worker import WorkerProcess
from ansible.executor.stats import AggregateStats from ansible.executor.stats import AggregateStats
from ansible.module_utils.facts import Facts from ansible.module_utils._text import to_text
from ansible.playbook.block import Block from ansible.playbook.block import Block
from ansible.playbook.play_context import PlayContext from ansible.playbook.play_context import PlayContext
from ansible.plugins import action_loader, callback_loader, connection_loader, filter_loader, lookup_loader, module_loader, strategy_loader, test_loader from ansible.plugins import action_loader, callback_loader, connection_loader, filter_loader, lookup_loader, module_loader, strategy_loader, test_loader
from ansible.template import Templar
from ansible.vars.hostvars import HostVars
from ansible.plugins.callback import CallbackBase from ansible.plugins.callback import CallbackBase
from ansible.template import Templar
from ansible.utils.helpers import pct_to_int from ansible.utils.helpers import pct_to_int
from ansible.utils.unicode import to_unicode from ansible.vars.hostvars import HostVars
from ansible.compat.six import string_types
try: try:
from __main__ import display from __main__ import display
@ -288,7 +287,8 @@ class TaskQueueManager:
stdout_callback_loaded = True stdout_callback_loaded = True
elif callback_name == 'tree' and self._run_tree: elif callback_name == 'tree' and self._run_tree:
pass pass
elif not self._run_additional_callbacks or (callback_needs_whitelist and (C.DEFAULT_CALLBACK_WHITELIST is None or callback_name not in C.DEFAULT_CALLBACK_WHITELIST)): elif not self._run_additional_callbacks or (callback_needs_whitelist and (
C.DEFAULT_CALLBACK_WHITELIST is None or callback_name not in C.DEFAULT_CALLBACK_WHITELIST)):
continue continue
self._callback_plugins.append(callback_plugin()) self._callback_plugins.append(callback_plugin())
@ -469,7 +469,7 @@ class TaskQueueManager:
method(*args, **kwargs) method(*args, **kwargs)
except Exception as e: except Exception as e:
# TODO: add config toggle to make this fatal or not? # TODO: add config toggle to make this fatal or not?
display.warning(u"Failure using method (%s) in callback plugin (%s): %s" % (to_unicode(method_name), to_unicode(callback_plugin), to_unicode(e))) display.warning(u"Failure using method (%s) in callback plugin (%s): %s" % (to_text(method_name), to_text(callback_plugin), to_text(e)))
from traceback import format_tb from traceback import format_tb
from sys import exc_info from sys import exc_info
display.debug('Callback Exception: \n' + ' '.join(format_tb(exc_info()[2]))) display.debug('Callback Exception: \n' + ' '.join(format_tb(exc_info()[2])))

@ -28,12 +28,12 @@ import json
import ansible.constants as C import ansible.constants as C
from ansible.compat.six import string_types from ansible.compat.six import string_types
from ansible.compat.six.moves.urllib.parse import quote as urlquote, urlencode
from ansible.compat.six.moves.urllib.error import HTTPError from ansible.compat.six.moves.urllib.error import HTTPError
from ansible.compat.six.moves.urllib.parse import quote as urlquote, urlencode
from ansible.errors import AnsibleError from ansible.errors import AnsibleError
from ansible.module_utils.urls import open_url
from ansible.galaxy.token import GalaxyToken from ansible.galaxy.token import GalaxyToken
from ansible.utils.unicode import to_str from ansible.module_utils._text import to_native
from ansible.module_utils.urls import open_url
try: try:
from __main__ import display from __main__ import display
@ -115,12 +115,12 @@ class GalaxyAPI(object):
try: try:
return_data = open_url(url, validate_certs=self._validate_certs) return_data = open_url(url, validate_certs=self._validate_certs)
except Exception as e: except Exception as e:
raise AnsibleError("Failed to get data from the API server (%s): %s " % (url, to_str(e))) raise AnsibleError("Failed to get data from the API server (%s): %s " % (url, to_native(e)))
try: try:
data = json.load(return_data) data = json.load(return_data)
except Exception as e: except Exception as e:
raise AnsibleError("Could not process data from the API server (%s): %s " % (url, to_str(e))) raise AnsibleError("Could not process data from the API server (%s): %s " % (url, to_native(e)))
if 'current_version' not in data: if 'current_version' not in data:
raise AnsibleError("missing required 'current_version' from server response (%s)" % url) raise AnsibleError("missing required 'current_version' from server response (%s)" % url)

@ -33,12 +33,10 @@ from ansible.errors import AnsibleError
from ansible.inventory.dir import InventoryDirectory, get_file_parser from ansible.inventory.dir import InventoryDirectory, get_file_parser
from ansible.inventory.group import Group from ansible.inventory.group import Group
from ansible.inventory.host import Host from ansible.inventory.host import Host
from ansible.module_utils._text import to_bytes, to_text
from ansible.parsing.utils.addresses import parse_address
from ansible.plugins import vars_loader from ansible.plugins import vars_loader
from ansible.utils.unicode import to_unicode, to_bytes
from ansible.utils.vars import combine_vars from ansible.utils.vars import combine_vars
from ansible.parsing.utils.addresses import parse_address
HOSTS_PATTERNS_CACHE = {}
try: try:
from __main__ import display from __main__ import display
@ -46,6 +44,10 @@ except ImportError:
from ansible.utils.display import Display from ansible.utils.display import Display
display = Display() display = Display()
HOSTS_PATTERNS_CACHE = {}
class Inventory(object): class Inventory(object):
""" """
Host inventory for ansible. Host inventory for ansible.
@ -125,7 +127,7 @@ class Inventory(object):
try: try:
(host, port) = parse_address(h, allow_ranges=False) (host, port) = parse_address(h, allow_ranges=False)
except AnsibleError as e: except AnsibleError as e:
display.vvv("Unable to parse address from hostname, leaving unchanged: %s" % to_unicode(e)) display.vvv("Unable to parse address from hostname, leaving unchanged: %s" % to_text(e))
host = h host = h
port = None port = None
@ -151,7 +153,7 @@ class Inventory(object):
# should never happen, but JIC # should never happen, but JIC
raise AnsibleError("Unable to parse %s as an inventory source" % host_list) raise AnsibleError("Unable to parse %s as an inventory source" % host_list)
else: else:
display.warning("Host file not found: %s" % to_unicode(host_list)) display.warning("Host file not found: %s" % to_text(host_list))
self._vars_plugins = [ x for x in vars_loader.all(self) ] self._vars_plugins = [ x for x in vars_loader.all(self) ]
@ -205,9 +207,9 @@ class Inventory(object):
if not ignore_limits_and_restrictions: if not ignore_limits_and_restrictions:
if self._subset: if self._subset:
pattern_hash += u":%s" % to_unicode(self._subset) pattern_hash += u":%s" % to_text(self._subset)
if self._restriction: if self._restriction:
pattern_hash += u":%s" % to_unicode(self._restriction) pattern_hash += u":%s" % to_text(self._restriction)
if pattern_hash not in HOSTS_PATTERNS_CACHE: if pattern_hash not in HOSTS_PATTERNS_CACHE:
@ -491,7 +493,8 @@ class Inventory(object):
py_interp = sys.executable py_interp = sys.executable
if not py_interp: if not py_interp:
# sys.executable is not set in some cornercases. #13585 # sys.executable is not set in some cornercases. #13585
display.warning('Unable to determine python interpreter from sys.executable. Using /usr/bin/python default. You can correct this by setting ansible_python_interpreter for localhost') display.warning('Unable to determine python interpreter from sys.executable. Using /usr/bin/python default.'
' You can correct this by setting ansible_python_interpreter for localhost')
py_interp = '/usr/bin/python' py_interp = '/usr/bin/python'
new_host.set_variable("ansible_python_interpreter", py_interp) new_host.set_variable("ansible_python_interpreter", py_interp)
self.get_group("ungrouped").add_host(new_host) self.get_group("ungrouped").add_host(new_host)
@ -781,7 +784,7 @@ class Inventory(object):
path = os.path.realpath(os.path.join(basedir, 'group_vars')) path = os.path.realpath(os.path.join(basedir, 'group_vars'))
found_vars = set() found_vars = set()
if os.path.exists(path): if os.path.exists(path):
found_vars = set(os.listdir(to_unicode(path))) found_vars = set(os.listdir(to_text(path)))
return found_vars return found_vars
def _find_host_vars_files(self, basedir): def _find_host_vars_files(self, basedir):
@ -791,7 +794,7 @@ class Inventory(object):
path = os.path.realpath(os.path.join(basedir, 'host_vars')) path = os.path.realpath(os.path.join(basedir, 'host_vars'))
found_vars = set() found_vars = set()
if os.path.exists(path): if os.path.exists(path):
found_vars = set(os.listdir(to_unicode(path))) found_vars = set(os.listdir(to_text(path)))
return found_vars return found_vars
def _get_hostgroup_vars(self, host=None, group=None, new_pb_basedir=False, return_results=False): def _get_hostgroup_vars(self, host=None, group=None, new_pb_basedir=False, return_results=False):
@ -832,13 +835,13 @@ class Inventory(object):
# Before trying to load vars from file, check that the directory contains relvant file names # Before trying to load vars from file, check that the directory contains relvant file names
if host is None and any(map(lambda ext: group.name + ext in self._group_vars_files, C.YAML_FILENAME_EXTENSIONS)): if host is None and any(map(lambda ext: group.name + ext in self._group_vars_files, C.YAML_FILENAME_EXTENSIONS)):
# load vars in dir/group_vars/name_of_group # load vars in dir/group_vars/name_of_group
base_path = to_unicode(os.path.abspath(os.path.join(to_bytes(basedir), b"group_vars/" + to_bytes(group.name))), errors='strict') base_path = to_text(os.path.abspath(os.path.join(to_bytes(basedir), b"group_vars/" + to_bytes(group.name))), errors='surrogate_or_strict')
host_results = self._variable_manager.add_group_vars_file(base_path, self._loader) host_results = self._variable_manager.add_group_vars_file(base_path, self._loader)
if return_results: if return_results:
results = combine_vars(results, host_results) results = combine_vars(results, host_results)
elif group is None and any(map(lambda ext: host.name + ext in self._host_vars_files, C.YAML_FILENAME_EXTENSIONS)): elif group is None and any(map(lambda ext: host.name + ext in self._host_vars_files, C.YAML_FILENAME_EXTENSIONS)):
# same for hostvars in dir/host_vars/name_of_host # same for hostvars in dir/host_vars/name_of_host
base_path = to_unicode(os.path.abspath(os.path.join(to_bytes(basedir), b"host_vars/" + to_bytes(host.name))), errors='strict') base_path = to_text(os.path.abspath(os.path.join(to_bytes(basedir), b"host_vars/" + to_bytes(host.name))), errors='surrogate_or_strict')
group_results = self._variable_manager.add_host_vars_file(base_path, self._loader) group_results = self._variable_manager.add_host_vars_file(base_path, self._loader)
if return_results: if return_results:
results = combine_vars(results, group_results) results = combine_vars(results, group_results)

@ -28,9 +28,10 @@ from ansible.inventory.host import Host
from ansible.inventory.group import Group from ansible.inventory.group import Group
from ansible.inventory.expand_hosts import detect_range from ansible.inventory.expand_hosts import detect_range
from ansible.inventory.expand_hosts import expand_hostname_range from ansible.inventory.expand_hosts import expand_hostname_range
from ansible.module_utils._text import to_text
from ansible.parsing.utils.addresses import parse_address from ansible.parsing.utils.addresses import parse_address
from ansible.utils.shlex import shlex_split from ansible.utils.shlex import shlex_split
from ansible.utils.unicode import to_unicode
class InventoryParser(object): class InventoryParser(object):
""" """
@ -56,7 +57,7 @@ class InventoryParser(object):
(data, private) = loader._get_file_contents(filename) (data, private) = loader._get_file_contents(filename)
else: else:
with open(filename) as fh: with open(filename) as fh:
data = to_unicode(fh.read()) data = to_text(fh.read())
data = data.split('\n') data = data.split('\n')
self._parse(data) self._parse(data)
@ -125,7 +126,7 @@ class InventoryParser(object):
continue continue
elif line.startswith('[') and line.endswith(']'): elif line.startswith('[') and line.endswith(']'):
self._raise_error("Invalid section entry: '%s'. Please make sure that there are no spaces" % line + \ self._raise_error("Invalid section entry: '%s'. Please make sure that there are no spaces" % line +
"in the section entry, and that there are no other invalid characters") "in the section entry, and that there are no other invalid characters")
# It's not a section, so the current state tells us what kind of # It's not a section, so the current state tells us what kind of
@ -188,7 +189,6 @@ class InventoryParser(object):
if group.depth == 0 and group.name not in ('all', 'ungrouped'): if group.depth == 0 and group.name not in ('all', 'ungrouped'):
self.groups['all'].add_child_group(group) self.groups['all'].add_child_group(group)
def _parse_group_name(self, line): def _parse_group_name(self, line):
''' '''
Takes a single line and tries to parse it as a group name. Returns the Takes a single line and tries to parse it as a group name. Returns the
@ -323,7 +323,7 @@ class InventoryParser(object):
except SyntaxError: except SyntaxError:
# Is this a hash with an equals at the end? # Is this a hash with an equals at the end?
pass pass
return to_unicode(v, nonstring='passthru', errors='strict') return to_text(v, nonstring='passthru', errors='surrogate_or_strict')
def get_host_variables(self, host): def get_host_variables(self, host):
return {} return {}

@ -31,7 +31,7 @@ from ansible.errors import AnsibleError
from ansible.inventory.host import Host from ansible.inventory.host import Host
from ansible.inventory.group import Group from ansible.inventory.group import Group
from ansible.module_utils.basic import json_dict_bytes_to_unicode from ansible.module_utils.basic import json_dict_bytes_to_unicode
from ansible.utils.unicode import to_str, to_unicode from ansible.module_utils._text import to_native, to_text
class InventoryScript: class InventoryScript:
@ -61,9 +61,9 @@ class InventoryScript:
# make sure script output is unicode so that json loader will output # make sure script output is unicode so that json loader will output
# unicode strings itself # unicode strings itself
try: try:
self.data = to_unicode(stdout, errors="strict") self.data = to_text(stdout, errors="strict")
except Exception as e: except Exception as e:
raise AnsibleError("inventory data from {0} contained characters that cannot be interpreted as UTF-8: {1}".format(to_str(self.filename), to_str(e))) raise AnsibleError("inventory data from {0} contained characters that cannot be interpreted as UTF-8: {1}".format(to_native(self.filename), to_native(e)))
# see comment about _meta below # see comment about _meta below
self.host_vars_from_top = None self.host_vars_from_top = None
@ -78,11 +78,11 @@ class InventoryScript:
self.raw = self._loader.load(self.data) self.raw = self._loader.load(self.data)
except Exception as e: except Exception as e:
sys.stderr.write(err + "\n") sys.stderr.write(err + "\n")
raise AnsibleError("failed to parse executable inventory script results from {0}: {1}".format(to_str(self.filename), to_str(e))) raise AnsibleError("failed to parse executable inventory script results from {0}: {1}".format(to_native(self.filename), to_native(e)))
if not isinstance(self.raw, Mapping): if not isinstance(self.raw, Mapping):
sys.stderr.write(err + "\n") sys.stderr.write(err + "\n")
raise AnsibleError("failed to parse executable inventory script results from {0}: data needs to be formatted as a json dict".format(to_str(self.filename))) raise AnsibleError("failed to parse executable inventory script results from {0}: data needs to be formatted as a json dict".format(to_native(self.filename)))
group = None group = None
for (group_name, data) in self.raw.items(): for (group_name, data) in self.raw.items():
@ -152,7 +152,7 @@ class InventoryScript:
try: try:
got = self.host_vars_from_top.get(host.name, {}) got = self.host_vars_from_top.get(host.name, {})
except AttributeError as e: except AttributeError as e:
raise AnsibleError("Improperly formated host information for %s: %s" % (host.name,to_str(e))) raise AnsibleError("Improperly formated host information for %s: %s" % (host.name,to_native(e)))
return got return got
cmd = [self.filename, "--host", host.name] cmd = [self.filename, "--host", host.name]

@ -32,16 +32,18 @@
making backwards compatibility guarantees. The API may change between making backwards compatibility guarantees. The API may change between
releases. Do not use this unless you are willing to port your module code. releases. Do not use this unless you are willing to port your module code.
""" """
import codecs
from ansible.module_utils.six import PY3, text_type, binary_type from ansible.module_utils.six import PY3, text_type, binary_type
import codecs
try: try:
codecs.lookup_error('surrogateescape') codecs.lookup_error('surrogateescape')
HAS_SURROGATEESCAPE = True HAS_SURROGATEESCAPE = True
except LookupError: except LookupError:
HAS_SURROGATEESCAPE = False HAS_SURROGATEESCAPE = False
def to_bytes(obj, encoding='utf-8', errors=None, nonstring='simplerepr'): def to_bytes(obj, encoding='utf-8', errors=None, nonstring='simplerepr'):
"""Make sure that a string is a byte string """Make sure that a string is a byte string
@ -109,7 +111,14 @@ def to_bytes(obj, encoding='utf-8', errors=None, nonstring='simplerepr'):
# Note: We do these last even though we have to call to_bytes again on the # Note: We do these last even though we have to call to_bytes again on the
# value because we're optimizing the common case # value because we're optimizing the common case
if nonstring == 'simplerepr': if nonstring == 'simplerepr':
try:
value = str(obj) value = str(obj)
except UnicodeError:
try:
value = repr(obj)
except UnicodeError:
# Giving up
return to_bytes('')
elif nonstring == 'passthru': elif nonstring == 'passthru':
return obj return obj
elif nonstring == 'empty': elif nonstring == 'empty':
@ -122,6 +131,7 @@ def to_bytes(obj, encoding='utf-8', errors=None, nonstring='simplerepr'):
return to_bytes(value, encoding, errors) return to_bytes(value, encoding, errors)
def to_text(obj, encoding='utf-8', errors=None, nonstring='simplerepr'): def to_text(obj, encoding='utf-8', errors=None, nonstring='simplerepr'):
"""Make sure that a string is a text string """Make sure that a string is a text string
@ -175,7 +185,14 @@ def to_text(obj, encoding='utf-8', errors=None, nonstring='simplerepr'):
# Note: We do these last even though we have to call to_text again on the # Note: We do these last even though we have to call to_text again on the
# value because we're optimizing the common case # value because we're optimizing the common case
if nonstring == 'simplerepr': if nonstring == 'simplerepr':
try:
value = str(obj) value = str(obj)
except UnicodeError:
try:
value = repr(obj)
except UnicodeError:
# Giving up
return u''
elif nonstring == 'passthru': elif nonstring == 'passthru':
return obj return obj
elif nonstring == 'empty': elif nonstring == 'empty':
@ -187,6 +204,7 @@ def to_text(obj, encoding='utf-8', errors=None, nonstring='simplerepr'):
return to_text(value, encoding, errors) return to_text(value, encoding, errors)
#: :py:func:`to_native` #: :py:func:`to_native`
#: Transform a variable into the native str type for the python version #: Transform a variable into the native str type for the python version
#: #:

@ -805,7 +805,7 @@ class AnsibleModule(object):
if not HAVE_SELINUX or not self.selinux_enabled(): if not HAVE_SELINUX or not self.selinux_enabled():
return context return context
try: try:
ret = selinux.matchpathcon(to_native(path, errors='strict'), mode) ret = selinux.matchpathcon(to_native(path, errors='surrogate_or_strict'), mode)
except OSError: except OSError:
return context return context
if ret[0] == -1: if ret[0] == -1:
@ -820,7 +820,7 @@ class AnsibleModule(object):
if not HAVE_SELINUX or not self.selinux_enabled(): if not HAVE_SELINUX or not self.selinux_enabled():
return context return context
try: try:
ret = selinux.lgetfilecon_raw(to_native(path, errors='strict')) ret = selinux.lgetfilecon_raw(to_native(path, errors='surrogate_or_strict'))
except OSError: except OSError:
e = get_exception() e = get_exception()
if e.errno == errno.ENOENT: if e.errno == errno.ENOENT:
@ -2121,10 +2121,10 @@ class AnsibleModule(object):
to_clean_args = args to_clean_args = args
if PY2: if PY2:
if isinstance(args, text_type): if isinstance(args, text_type):
to_clean_args = args.encode('utf-8') to_clean_args = to_bytes(args)
else: else:
if isinstance(args, binary_type): if isinstance(args, binary_type):
to_clean_args = args.decode('utf-8', errors='replace') to_clean_args = to_text(args)
if isinstance(args, (text_type, binary_type)): if isinstance(args, (text_type, binary_type)):
to_clean_args = shlex.split(to_clean_args) to_clean_args = shlex.split(to_clean_args)
@ -2193,11 +2193,7 @@ class AnsibleModule(object):
if not binary_data: if not binary_data:
data += '\n' data += '\n'
if isinstance(data, text_type): if isinstance(data, text_type):
if PY3: data = to_bytes(data)
errors = 'surrogateescape'
else:
errors = 'strict'
data = data.encode('utf-8', errors=errors)
cmd.stdin.write(data) cmd.stdin.write(data)
cmd.stdin.close() cmd.stdin.close()

@ -27,16 +27,15 @@ import tempfile
from yaml import YAMLError from yaml import YAMLError
from ansible.compat.six import text_type, string_types from ansible.compat.six import text_type, string_types
from ansible.errors import AnsibleFileNotFound, AnsibleParserError, AnsibleError from ansible.errors import AnsibleFileNotFound, AnsibleParserError, AnsibleError
from ansible.errors.yaml_strings import YAML_SYNTAX_ERROR from ansible.errors.yaml_strings import YAML_SYNTAX_ERROR
from ansible.module_utils.basic import is_executable
from ansible.module_utils._text import to_bytes, to_native, to_text
from ansible.parsing.vault import VaultLib from ansible.parsing.vault import VaultLib
from ansible.parsing.quoting import unquote from ansible.parsing.quoting import unquote
from ansible.parsing.yaml.loader import AnsibleLoader from ansible.parsing.yaml.loader import AnsibleLoader
from ansible.parsing.yaml.objects import AnsibleBaseYAMLObject, AnsibleUnicode from ansible.parsing.yaml.objects import AnsibleBaseYAMLObject, AnsibleUnicode
from ansible.module_utils.basic import is_executable
from ansible.utils.path import unfrackpath from ansible.utils.path import unfrackpath
from ansible.utils.unicode import to_unicode, to_bytes, to_str
try: try:
from __main__ import display from __main__ import display
@ -44,6 +43,7 @@ except ImportError:
from ansible.utils.display import Display from ansible.utils.display import Display
display = Display() display = Display()
class DataLoader(): class DataLoader():
''' '''
@ -127,15 +127,15 @@ class DataLoader():
def path_exists(self, path): def path_exists(self, path):
path = self.path_dwim(path) path = self.path_dwim(path)
return os.path.exists(to_bytes(path, errors='strict')) return os.path.exists(to_bytes(path, errors='surrogate_or_strict'))
def is_file(self, path): def is_file(self, path):
path = self.path_dwim(path) path = self.path_dwim(path)
return os.path.isfile(to_bytes(path, errors='strict')) or path == os.devnull return os.path.isfile(to_bytes(path, errors='surrogate_or_strict')) or path == os.devnull
def is_directory(self, path): def is_directory(self, path):
path = self.path_dwim(path) path = self.path_dwim(path)
return os.path.isdir(to_bytes(path, errors='strict')) return os.path.isdir(to_bytes(path, errors='surrogate_or_strict'))
def list_directory(self, path): def list_directory(self, path):
path = self.path_dwim(path) path = self.path_dwim(path)
@ -178,7 +178,7 @@ class DataLoader():
data = self._vault.decrypt(data, filename=b_file_name) data = self._vault.decrypt(data, filename=b_file_name)
show_content = False show_content = False
data = to_unicode(data, errors='strict') data = to_text(data, errors='surrogate_or_strict')
return (data, show_content) return (data, show_content)
except (IOError, OSError) as e: except (IOError, OSError) as e:
@ -208,7 +208,7 @@ class DataLoader():
''' sets the base directory, used to find files when a relative path is given ''' ''' sets the base directory, used to find files when a relative path is given '''
if basedir is not None: if basedir is not None:
self._basedir = to_unicode(basedir) self._basedir = to_text(basedir)
def path_dwim(self, given): def path_dwim(self, given):
''' '''
@ -216,14 +216,14 @@ class DataLoader():
''' '''
given = unquote(given) given = unquote(given)
given = to_unicode(given, errors='strict') given = to_text(given, errors='surrogate_or_strict')
if given.startswith(u"/"): if given.startswith(u"/"):
return os.path.abspath(given) return os.path.abspath(given)
elif given.startswith(u"~"): elif given.startswith(u"~"):
return os.path.abspath(os.path.expanduser(given)) return os.path.abspath(os.path.expanduser(given))
else: else:
basedir = to_unicode(self._basedir, errors='strict') basedir = to_text(self._basedir, errors='surrogate_or_strict')
return os.path.abspath(os.path.join(basedir, given)) return os.path.abspath(os.path.join(basedir, given))
def path_dwim_relative(self, path, dirname, source): def path_dwim_relative(self, path, dirname, source):
@ -247,8 +247,8 @@ class DataLoader():
basedir = unfrackpath(path) basedir = unfrackpath(path)
# is it a role and if so make sure you get correct base path # is it a role and if so make sure you get correct base path
if path.endswith('tasks') and os.path.exists(to_bytes(os.path.join(path,'main.yml'), errors='strict')) \ if path.endswith('tasks') and os.path.exists(to_bytes(os.path.join(path,'main.yml'), errors='surrogate_or_strict')) \
or os.path.exists(to_bytes(os.path.join(path,'tasks/main.yml'), errors='strict')): or os.path.exists(to_bytes(os.path.join(path,'tasks/main.yml'), errors='surrogate_or_strict')):
isrole = True isrole = True
if path.endswith('tasks'): if path.endswith('tasks'):
basedir = unfrackpath(os.path.dirname(path)) basedir = unfrackpath(os.path.dirname(path))
@ -271,7 +271,7 @@ class DataLoader():
search.append(self.path_dwim(source)) search.append(self.path_dwim(source))
for candidate in search: for candidate in search:
if os.path.exists(to_bytes(candidate, errors='strict')): if os.path.exists(to_bytes(candidate, errors='surrogate_or_strict')):
break break
return candidate return candidate
@ -296,13 +296,13 @@ class DataLoader():
elif source.startswith('~') or source.startswith(os.path.sep): elif source.startswith('~') or source.startswith(os.path.sep):
# path is absolute, no relative needed, check existence and return source # path is absolute, no relative needed, check existence and return source
test_path = unfrackpath(b_source) test_path = unfrackpath(b_source)
if os.path.exists(to_bytes(test_path, errors='strict')): if os.path.exists(to_bytes(test_path, errors='surrogate_or_strict')):
result = test_path result = test_path
else: else:
search = [] search = []
for path in paths: for path in paths:
upath = unfrackpath(path) upath = unfrackpath(path)
b_upath = to_bytes(upath, errors='strict') b_upath = to_bytes(upath, errors='surrogate_or_strict')
b_mydir = os.path.dirname(b_upath) b_mydir = os.path.dirname(b_upath)
# if path is in role and 'tasks' not there already, add it into the search # if path is in role and 'tasks' not there already, add it into the search
@ -324,11 +324,11 @@ class DataLoader():
search.append(os.path.join(to_bytes(self.get_basedir()), b_dirname, b_source)) search.append(os.path.join(to_bytes(self.get_basedir()), b_dirname, b_source))
search.append(os.path.join(to_bytes(self.get_basedir()), b_source)) search.append(os.path.join(to_bytes(self.get_basedir()), b_source))
display.debug(u'search_path:\n\t%s' % to_unicode(b'\n\t'.join(search), errors='replace')) display.debug(u'search_path:\n\t%s' % to_text(b'\n\t'.join(search)))
for b_candidate in search: for b_candidate in search:
display.vvvvv(u'looking for "%s" at "%s"' % (source, to_unicode(b_candidate))) display.vvvvv(u'looking for "%s" at "%s"' % (source, to_text(b_candidate)))
if os.path.exists(b_candidate): if os.path.exists(b_candidate):
result = to_unicode(b_candidate) result = to_text(b_candidate)
break break
return result return result
@ -339,8 +339,8 @@ class DataLoader():
retrieve password from STDOUT retrieve password from STDOUT
""" """
this_path = os.path.realpath(to_bytes(os.path.expanduser(vault_password_file), errors='strict')) this_path = os.path.realpath(to_bytes(os.path.expanduser(vault_password_file), errors='surrogate_or_strict'))
if not os.path.exists(to_bytes(this_path, errors='strict')): if not os.path.exists(to_bytes(this_path, errors='surrogate_or_strict')):
raise AnsibleFileNotFound("The vault password file %s was not found" % this_path) raise AnsibleFileNotFound("The vault password file %s was not found" % this_path)
if self.is_executable(this_path): if self.is_executable(this_path):
@ -348,7 +348,8 @@ class DataLoader():
# STDERR not captured to make it easier for users to prompt for input in their scripts # STDERR not captured to make it easier for users to prompt for input in their scripts
p = subprocess.Popen(this_path, stdout=subprocess.PIPE) p = subprocess.Popen(this_path, stdout=subprocess.PIPE)
except OSError as e: except OSError as e:
raise AnsibleError("Problem running vault password script %s (%s). If this is not a script, remove the executable bit from the file." % (' '.join(this_path), e)) raise AnsibleError("Problem running vault password script %s (%s)."
" If this is not a script, remove the executable bit from the file." % (' '.join(this_path), to_native(e)))
stdout, stderr = p.communicate() stdout, stderr = p.communicate()
self.set_vault_password(stdout.strip('\r\n')) self.set_vault_password(stdout.strip('\r\n'))
else: else:
@ -381,11 +382,11 @@ class DataLoader():
""" """
if not file_path or not isinstance(file_path, string_types): if not file_path or not isinstance(file_path, string_types):
raise AnsibleParserError("Invalid filename: '%s'" % to_str(file_path)) raise AnsibleParserError("Invalid filename: '%s'" % to_native(file_path))
b_file_path = to_bytes(file_path, errors='strict') b_file_path = to_bytes(file_path, errors='surrogate_or_strict')
if not self.path_exists(b_file_path) or not self.is_file(b_file_path): if not self.path_exists(b_file_path) or not self.is_file(b_file_path):
raise AnsibleFileNotFound("the file_name '%s' does not exist, or is not readable" % to_str(file_path)) raise AnsibleFileNotFound("the file_name '%s' does not exist, or is not readable" % to_native(file_path))
if not self._vault: if not self._vault:
self._vault = VaultLib(password="") self._vault = VaultLib(password="")
@ -410,7 +411,7 @@ class DataLoader():
return real_path return real_path
except (IOError, OSError) as e: except (IOError, OSError) as e:
raise AnsibleParserError("an error occurred while trying to read the file '%s': %s" % (to_str(real_path), to_str(e))) raise AnsibleParserError("an error occurred while trying to read the file '%s': %s" % (to_native(real_path), to_native(e)))
def cleanup_tmp_file(self, file_path): def cleanup_tmp_file(self, file_path):
""" """
@ -420,7 +421,7 @@ class DataLoader():
""" """
if file_path in self._tempfiles: if file_path in self._tempfiles:
os.unlink(file_path) os.unlink(file_path)
self._tempfiles.remove(file_path); self._tempfiles.remove(file_path)
def cleanup_all_tmp_files(self): def cleanup_all_tmp_files(self):
for f in self._tempfiles: for f in self._tempfiles:

@ -23,8 +23,10 @@ import re
import codecs import codecs
from ansible.errors import AnsibleParserError from ansible.errors import AnsibleParserError
from ansible.module_utils._text import to_text
from ansible.parsing.quoting import unquote from ansible.parsing.quoting import unquote
# Decode escapes adapted from rspeer's answer here: # Decode escapes adapted from rspeer's answer here:
# http://stackoverflow.com/questions/4020539/process-escape-sequences-in-a-string-in-python # http://stackoverflow.com/questions/4020539/process-escape-sequences-in-a-string-in-python
_HEXCHAR = '[a-fA-F0-9]' _HEXCHAR = '[a-fA-F0-9]'
@ -36,12 +38,14 @@ _ESCAPE_SEQUENCE_RE = re.compile(r'''
| \\[\\'"abfnrtv] # Single-character escapes | \\[\\'"abfnrtv] # Single-character escapes
)'''.format(_HEXCHAR*8, _HEXCHAR*4, _HEXCHAR*2), re.UNICODE | re.VERBOSE) )'''.format(_HEXCHAR*8, _HEXCHAR*4, _HEXCHAR*2), re.UNICODE | re.VERBOSE)
def _decode_escapes(s): def _decode_escapes(s):
def decode_match(match): def decode_match(match):
return codecs.decode(match.group(0), 'unicode-escape') return codecs.decode(match.group(0), 'unicode-escape')
return _ESCAPE_SEQUENCE_RE.sub(decode_match, s) return _ESCAPE_SEQUENCE_RE.sub(decode_match, s)
def parse_kv(args, check_raw=False): def parse_kv(args, check_raw=False):
''' '''
Convert a string of key/value items to a dict. If any free-form params Convert a string of key/value items to a dict. If any free-form params
@ -50,9 +54,7 @@ def parse_kv(args, check_raw=False):
they will simply be ignored. they will simply be ignored.
''' '''
### FIXME: args should already be a unicode string args = to_text(args, nonstring='passthru')
from ansible.utils.unicode import to_unicode
args = to_unicode(args, nonstring='passthru')
options = {} options = {}
if args is not None: if args is not None:
@ -60,7 +62,7 @@ def parse_kv(args, check_raw=False):
vargs = split_args(args) vargs = split_args(args)
except ValueError as ve: except ValueError as ve:
if 'no closing quotation' in str(ve).lower(): if 'no closing quotation' in str(ve).lower():
raise AnsibleParsingError("error parsing argument string, try quoting the entire line.") raise AnsibleParserError("error parsing argument string, try quoting the entire line.")
else: else:
raise raise
@ -99,6 +101,7 @@ def parse_kv(args, check_raw=False):
return options return options
def _get_quote_state(token, quote_char): def _get_quote_state(token, quote_char):
''' '''
the goal of this block is to determine if the quoted string the goal of this block is to determine if the quoted string
@ -118,6 +121,7 @@ def _get_quote_state(token, quote_char):
quote_char = cur_char quote_char = cur_char
return quote_char return quote_char
def _count_jinja2_blocks(token, cur_depth, open_token, close_token): def _count_jinja2_blocks(token, cur_depth, open_token, close_token):
''' '''
this function counts the number of opening/closing blocks for a this function counts the number of opening/closing blocks for a
@ -132,6 +136,7 @@ def _count_jinja2_blocks(token, cur_depth, open_token, close_token):
cur_depth = 0 cur_depth = 0
return cur_depth return cur_depth
def split_args(args): def split_args(args):
''' '''
Splits args on whitespace, but intelligently reassembles Splits args on whitespace, but intelligently reassembles

@ -29,16 +29,10 @@ from ansible.errors import AnsibleError
from hashlib import sha256 from hashlib import sha256
from binascii import hexlify from binascii import hexlify
from binascii import unhexlify from binascii import unhexlify
from hashlib import md5
try:
from __main__ import display
except ImportError:
from ansible.utils.display import Display
display = Display()
# Note: Only used for loading obsolete VaultAES files. All files are written # Note: Only used for loading obsolete VaultAES files. All files are written
# using the newer VaultAES256 which does not require md5 # using the newer VaultAES256 which does not require md5
from hashlib import md5
try: try:
from Crypto.Hash import SHA256, HMAC from Crypto.Hash import SHA256, HMAC
@ -67,6 +61,15 @@ try:
except ImportError: except ImportError:
HAS_AES = False HAS_AES = False
from ansible.compat.six import PY3
from ansible.module_utils._text import to_bytes, to_text
try:
from __main__ import display
except ImportError:
from ansible.utils.display import Display
display = Display()
# OpenSSL pbkdf2_hmac # OpenSSL pbkdf2_hmac
HAS_PBKDF2HMAC = False HAS_PBKDF2HMAC = False
try: try:
@ -81,12 +84,11 @@ except Exception as e:
import traceback import traceback
display.debug("Traceback from import of cryptography was {0}".format(traceback.format_exc())) display.debug("Traceback from import of cryptography was {0}".format(traceback.format_exc()))
from ansible.compat.six import PY3
from ansible.utils.unicode import to_unicode, to_bytes
HAS_ANY_PBKDF2HMAC = HAS_PBKDF2 or HAS_PBKDF2HMAC HAS_ANY_PBKDF2HMAC = HAS_PBKDF2 or HAS_PBKDF2HMAC
CRYPTO_UPGRADE = "ansible-vault requires a newer version of pycrypto than the one installed on your platform. You may fix this with OS-specific commands such as: yum install python-devel; rpm -e --nodeps python-crypto; pip install pycrypto" CRYPTO_UPGRADE = "ansible-vault requires a newer version of pycrypto than the one installed on your platform." \
" You may fix this with OS-specific commands such as: yum install python-devel; rpm -e --nodeps python-crypto; pip install pycrypto"
b_HEADER = b'$ANSIBLE_VAULT' b_HEADER = b'$ANSIBLE_VAULT'
HEADER = '$ANSIBLE_VAULT' HEADER = '$ANSIBLE_VAULT'
@ -105,6 +107,7 @@ def check_prereqs():
class AnsibleVaultError(AnsibleError): class AnsibleVaultError(AnsibleError):
pass pass
def is_encrypted(b_data): def is_encrypted(b_data):
""" Test if this is vault encrypted data blob """ Test if this is vault encrypted data blob
@ -116,6 +119,7 @@ def is_encrypted(b_data):
return True return True
return False return False
def is_encrypted_file(file_obj): def is_encrypted_file(file_obj):
"""Test if the contents of a file obj are a vault encrypted data blob. """Test if the contents of a file obj are a vault encrypted data blob.
@ -278,7 +282,7 @@ class VaultLib:
tmpheader = tmpdata[0].strip().split(b';') tmpheader = tmpdata[0].strip().split(b';')
self.b_version = tmpheader[1].strip() self.b_version = tmpheader[1].strip()
self.cipher_name = to_unicode(tmpheader[2].strip()) self.cipher_name = to_text(tmpheader[2].strip())
clean_data = b''.join(tmpdata[1:]) clean_data = b''.join(tmpdata[1:])
return clean_data return clean_data
@ -528,6 +532,7 @@ class VaultEditor:
return editor return editor
class VaultFile(object): class VaultFile(object):
def __init__(self, password, filename): def __init__(self, password, filename):
@ -568,6 +573,7 @@ class VaultFile(object):
else: else:
return self.filename return self.filename
######################################## ########################################
# CIPHERS # # CIPHERS #
######################################## ########################################

@ -22,12 +22,11 @@ __metaclass__ = type
from yaml.constructor import Constructor, ConstructorError from yaml.constructor import Constructor, ConstructorError
from yaml.nodes import MappingNode from yaml.nodes import MappingNode
from ansible.module_utils._text import to_bytes
from ansible.parsing.vault import VaultLib
from ansible.parsing.yaml.objects import AnsibleMapping, AnsibleSequence, AnsibleUnicode from ansible.parsing.yaml.objects import AnsibleMapping, AnsibleSequence, AnsibleUnicode
from ansible.parsing.yaml.objects import AnsibleVaultEncryptedUnicode from ansible.parsing.yaml.objects import AnsibleVaultEncryptedUnicode
from ansible.vars.unsafe_proxy import wrap_var from ansible.vars.unsafe_proxy import wrap_var
from ansible.parsing.vault import VaultLib
from ansible.utils.unicode import to_bytes
try: try:
from __main__ import display from __main__ import display
@ -74,7 +73,8 @@ class AnsibleConstructor(Constructor):
"found unacceptable key (%s)" % exc, key_node.start_mark) "found unacceptable key (%s)" % exc, key_node.start_mark)
if key in mapping: if key in mapping:
display.warning(u'While constructing a mapping from {1}, line {2}, column {3}, found a duplicate dict key ({0}). Using last defined value only.'.format(key, *mapping.ansible_pos)) display.warning(u'While constructing a mapping from {1}, line {2}, column {3}, found a duplicate dict key ({0}).'
u' Using last defined value only.'.format(key, *mapping.ansible_pos))
value = self.construct_object(value_node, deep=deep) value = self.construct_object(value_node, deep=deep)
mapping[key] = value mapping[key] = value

@ -22,8 +22,7 @@ __metaclass__ = type
import yaml import yaml
from ansible.compat.six import text_type from ansible.compat.six import text_type
from ansible.errors import AnsibleError from ansible.module_utils._text import to_bytes
from ansible.utils.unicode import to_bytes
class AnsibleBaseYAMLObject(object): class AnsibleBaseYAMLObject(object):

@ -19,25 +19,22 @@
from __future__ import (absolute_import, division, print_function) from __future__ import (absolute_import, division, print_function)
__metaclass__ = type __metaclass__ = type
import collections
import itertools import itertools
import operator import operator
import uuid import uuid
from copy import copy as shallowcopy, deepcopy from copy import copy as shallowcopy
from functools import partial from functools import partial
from inspect import getmembers
from ansible.compat.six import iteritems, string_types, with_metaclass
from jinja2.exceptions import UndefinedError from jinja2.exceptions import UndefinedError
from ansible.compat.six import iteritems, string_types, with_metaclass
from ansible.errors import AnsibleParserError, AnsibleUndefinedVariable from ansible.errors import AnsibleParserError, AnsibleUndefinedVariable
from ansible.parsing.dataloader import DataLoader from ansible.module_utils._text import to_text
from ansible.playbook.attribute import Attribute, FieldAttribute from ansible.playbook.attribute import Attribute, FieldAttribute
from ansible.parsing.dataloader import DataLoader
from ansible.utils.boolean import boolean from ansible.utils.boolean import boolean
from ansible.utils.vars import combine_vars, isidentifier from ansible.utils.vars import combine_vars, isidentifier
from ansible.utils.unicode import to_unicode
try: try:
from __main__ import display from __main__ import display
@ -52,6 +49,7 @@ def _generic_g(prop_name, self):
except KeyError: except KeyError:
raise AttributeError("'%s' object has no attribute '%s'" % (self.__class__.__name__, prop_name)) raise AttributeError("'%s' object has no attribute '%s'" % (self.__class__.__name__, prop_name))
def _generic_g_method(prop_name, self): def _generic_g_method(prop_name, self):
try: try:
if self._squashed: if self._squashed:
@ -61,6 +59,7 @@ def _generic_g_method(prop_name, self):
except KeyError: except KeyError:
raise AttributeError("'%s' object has no attribute '%s'" % (self.__class__.__name__, prop_name)) raise AttributeError("'%s' object has no attribute '%s'" % (self.__class__.__name__, prop_name))
def _generic_g_parent(prop_name, self): def _generic_g_parent(prop_name, self):
try: try:
value = self._attributes[prop_name] value = self._attributes[prop_name]
@ -74,12 +73,15 @@ def _generic_g_parent(prop_name, self):
return value return value
def _generic_s(prop_name, self, value): def _generic_s(prop_name, self, value):
self._attributes[prop_name] = value self._attributes[prop_name] = value
def _generic_d(prop_name, self): def _generic_d(prop_name, self):
del self._attributes[prop_name] del self._attributes[prop_name]
class BaseMeta(type): class BaseMeta(type):
""" """
@ -142,6 +144,7 @@ class BaseMeta(type):
return super(BaseMeta, cls).__new__(cls, name, parents, dct) return super(BaseMeta, cls).__new__(cls, name, parents, dct)
class Base(with_metaclass(BaseMeta, object)): class Base(with_metaclass(BaseMeta, object)):
# connection/transport # connection/transport
@ -376,7 +379,7 @@ class Base(with_metaclass(BaseMeta, object)):
# and make sure the attribute is of the type it should be # and make sure the attribute is of the type it should be
if value is not None: if value is not None:
if attribute.isa == 'string': if attribute.isa == 'string':
value = to_unicode(value) value = to_text(value)
elif attribute.isa == 'int': elif attribute.isa == 'int':
value = int(value) value = int(value)
elif attribute.isa == 'float': elif attribute.isa == 'float':
@ -395,8 +398,8 @@ class Base(with_metaclass(BaseMeta, object)):
elif not isinstance(value, list): elif not isinstance(value, list):
if isinstance(value, string_types) and attribute.isa == 'barelist': if isinstance(value, string_types) and attribute.isa == 'barelist':
display.deprecated( display.deprecated(
"Using comma separated values for a list has been deprecated. " \ "Using comma separated values for a list has been deprecated. "
"You should instead use the correct YAML syntax for lists. " \ "You should instead use the correct YAML syntax for lists. "
) )
value = value.split(',') value = value.split(',')
else: else:
@ -532,4 +535,3 @@ class Base(with_metaclass(BaseMeta, object)):
setattr(self, '_uuid', data.get('uuid')) setattr(self, '_uuid', data.get('uuid'))
self._finalized = data.get('finalized', False) self._finalized = data.get('finalized', False)
self._squashed = data.get('squashed', False) self._squashed = data.get('squashed', False)

@ -22,12 +22,10 @@ __metaclass__ = type
import os import os
from ansible.compat.six import iteritems, string_types from ansible.compat.six import iteritems, string_types
from ansible.errors import AnsibleError, AnsibleParserError from ansible.errors import AnsibleError, AnsibleParserError
from ansible.module_utils._text import to_native
from ansible.parsing.mod_args import ModuleArgsParser from ansible.parsing.mod_args import ModuleArgsParser
from ansible.parsing.yaml.objects import AnsibleBaseYAMLObject, AnsibleMapping, AnsibleUnicode from ansible.parsing.yaml.objects import AnsibleBaseYAMLObject, AnsibleMapping, AnsibleUnicode
from ansible.plugins import lookup_loader from ansible.plugins import lookup_loader
from ansible.playbook.attribute import FieldAttribute from ansible.playbook.attribute import FieldAttribute
from ansible.playbook.base import Base from ansible.playbook.base import Base
@ -38,7 +36,6 @@ from ansible.playbook.loop_control import LoopControl
from ansible.playbook.role import Role from ansible.playbook.role import Role
from ansible.playbook.taggable import Taggable from ansible.playbook.taggable import Taggable
from ansible.utils.unicode import to_str
try: try:
from __main__ import display from __main__ import display
@ -182,7 +179,7 @@ class Task(Base, Conditional, Taggable, Become):
try: try:
(action, args, delegate_to) = args_parser.parse() (action, args, delegate_to) = args_parser.parse()
except AnsibleParserError as e: except AnsibleParserError as e:
raise AnsibleParserError(to_str(e), obj=ds) raise AnsibleParserError(to_native(e), obj=ds)
# the command/shell/script modules used to support the `cmd` arg, # the command/shell/script modules used to support the `cmd` arg,
# which corresponds to what we now call _raw_params, so move that # which corresponds to what we now call _raw_params, so move that
@ -233,7 +230,7 @@ class Task(Base, Conditional, Taggable, Become):
def _load_loop_control(self, attr, ds): def _load_loop_control(self, attr, ds):
if not isinstance(ds, dict): if not isinstance(ds, dict):
raise AnsibleParserError( raise AnsibleParserError(
"the `loop_control` value must be specified as a dictionary and cannot " \ "the `loop_control` value must be specified as a dictionary and cannot "
"be a variable itself (though it can contain variables)", "be a variable itself (though it can contain variables)",
obj=ds, obj=ds,
) )
@ -452,4 +449,3 @@ class Task(Base, Conditional, Taggable, Become):
if self._parent: if self._parent:
return self._parent.all_parents_static() return self._parent.all_parents_static()
return True return True

@ -31,7 +31,8 @@ import warnings
from collections import defaultdict from collections import defaultdict
from ansible import constants as C from ansible import constants as C
from ansible.utils.unicode import to_unicode from ansible.module_utils._text import to_text
try: try:
from __main__ import display from __main__ import display
@ -44,9 +45,11 @@ MODULE_CACHE = {}
PATH_CACHE = {} PATH_CACHE = {}
PLUGIN_PATH_CACHE = {} PLUGIN_PATH_CACHE = {}
def get_all_plugin_loaders(): def get_all_plugin_loaders():
return [(name, obj) for (name, obj) in inspect.getmembers(sys.modules[__name__]) if isinstance(obj, PluginLoader)] return [(name, obj) for (name, obj) in inspect.getmembers(sys.modules[__name__]) if isinstance(obj, PluginLoader)]
class PluginLoader: class PluginLoader:
''' '''
@ -72,11 +75,11 @@ class PluginLoader:
self.config = config self.config = config
if not class_name in MODULE_CACHE: if class_name not in MODULE_CACHE:
MODULE_CACHE[class_name] = {} MODULE_CACHE[class_name] = {}
if not class_name in PATH_CACHE: if class_name not in PATH_CACHE:
PATH_CACHE[class_name] = None PATH_CACHE[class_name] = None
if not class_name in PLUGIN_PATH_CACHE: if class_name not in PLUGIN_PATH_CACHE:
PLUGIN_PATH_CACHE[class_name] = defaultdict(dict) PLUGIN_PATH_CACHE[class_name] = defaultdict(dict)
self._module_cache = MODULE_CACHE[class_name] self._module_cache = MODULE_CACHE[class_name]
@ -250,7 +253,7 @@ class PluginLoader:
try: try:
full_paths = (os.path.join(path, f) for f in os.listdir(path)) full_paths = (os.path.join(path, f) for f in os.listdir(path))
except OSError as e: except OSError as e:
display.warning("Error accessing plugin paths: %s" % to_unicode(e)) display.warning("Error accessing plugin paths: %s" % to_text(e))
for full_path in (f for f in full_paths if os.path.isfile(f) and not f.endswith('__init__.py')): for full_path in (f for f in full_paths if os.path.isfile(f) and not f.endswith('__init__.py')):
full_name = os.path.basename(full_path) full_name = os.path.basename(full_path)
@ -389,7 +392,7 @@ class PluginLoader:
try: try:
obj = getattr(self._module_cache[path], self.class_name) obj = getattr(self._module_cache[path], self.class_name)
except AttributeError as e: except AttributeError as e:
display.warning("Skipping plugin (%s) as it seems to be invalid: %s" % (path, to_unicode(e))) display.warning("Skipping plugin (%s) as it seems to be invalid: %s" % (path, to_text(e)))
continue continue
if self.base_class: if self.base_class:

@ -35,9 +35,10 @@ from ansible.compat.six import binary_type, text_type, iteritems, with_metaclass
from ansible import constants as C from ansible import constants as C
from ansible.errors import AnsibleError, AnsibleConnectionFailure from ansible.errors import AnsibleError, AnsibleConnectionFailure
from ansible.executor.module_common import modify_module from ansible.executor.module_common import modify_module
from ansible.release import __version__ from ansible.module_utils._text import to_bytes, to_native, to_text
from ansible.parsing.utils.jsonify import jsonify from ansible.parsing.utils.jsonify import jsonify
from ansible.utils.unicode import to_bytes, to_str, to_unicode from ansible.release import __version__
try: try:
from __main__ import display from __main__ import display
@ -146,7 +147,8 @@ class ActionBase(with_metaclass(ABCMeta, object)):
"run 'git submodule update --init --recursive' to correct this problem." % (module_name)) "run 'git submodule update --init --recursive' to correct this problem." % (module_name))
# insert shared code and arguments into the module # insert shared code and arguments into the module
(module_data, module_style, module_shebang) = modify_module(module_name, module_path, module_args, task_vars=task_vars, module_compression=self._play_context.module_compression) (module_data, module_style, module_shebang) = modify_module(module_name, module_path, module_args,
task_vars=task_vars, module_compression=self._play_context.module_compression)
return (module_style, module_shebang, module_data, module_path) return (module_style, module_shebang, module_data, module_path)
@ -283,10 +285,10 @@ class ActionBase(with_metaclass(ABCMeta, object)):
afd, afile = tempfile.mkstemp() afd, afile = tempfile.mkstemp()
afo = os.fdopen(afd, 'wb') afo = os.fdopen(afd, 'wb')
try: try:
data = to_bytes(data, errors='strict') data = to_bytes(data, errors='surrogate_or_strict')
afo.write(data) afo.write(data)
except Exception as e: except Exception as e:
raise AnsibleError("failure writing module data to temporary file for transfer: %s" % str(e)) raise AnsibleError("failure writing module data to temporary file for transfer: %s" % to_native(e))
afo.flush() afo.flush()
afo.close() afo.close()
@ -372,17 +374,22 @@ class ActionBase(with_metaclass(ABCMeta, object)):
res = self._remote_chown(remote_paths, self._play_context.become_user) res = self._remote_chown(remote_paths, self._play_context.become_user)
if res['rc'] != 0 and remote_user == 'root': if res['rc'] != 0 and remote_user == 'root':
# chown failed even if remove_user is root # chown failed even if remove_user is root
raise AnsibleError('Failed to change ownership of the temporary files Ansible needs to create despite connecting as root. Unprivileged become user would be unable to read the file.') raise AnsibleError('Failed to change ownership of the temporary files Ansible needs to create despite connecting as root.'
' Unprivileged become user would be unable to read the file.')
elif res['rc'] != 0: elif res['rc'] != 0:
if C.ALLOW_WORLD_READABLE_TMPFILES: if C.ALLOW_WORLD_READABLE_TMPFILES:
# chown and fs acls failed -- do things this insecure # chown and fs acls failed -- do things this insecure
# way only if the user opted in in the config file # way only if the user opted in in the config file
display.warning('Using world-readable permissions for temporary files Ansible needs to create when becoming an unprivileged user which may be insecure. For information on securing this, see https://docs.ansible.com/ansible/become.html#becoming-an-unprivileged-user') display.warning('Using world-readable permissions for temporary files Ansible needs to create when becoming an unprivileged user.'
' This may be insecure. For information on securing this, see'
' https://docs.ansible.com/ansible/become.html#becoming-an-unprivileged-user')
res = self._remote_chmod(remote_paths, 'a+%s' % mode) res = self._remote_chmod(remote_paths, 'a+%s' % mode)
if res['rc'] != 0: if res['rc'] != 0:
raise AnsibleError('Failed to set file mode on remote files (rc: {0}, err: {1})'.format(res['rc'], res['stderr'])) raise AnsibleError('Failed to set file mode on remote files (rc: {0}, err: {1})'.format(res['rc'], res['stderr']))
else: else:
raise AnsibleError('Failed to set permissions on the temporary files Ansible needs to create when becoming an unprivileged user (rc: {0}, err: {1}). For information on working around this, see https://docs.ansible.com/ansible/become.html#becoming-an-unprivileged-user'.format(res['rc'], res['stderr'])) raise AnsibleError('Failed to set permissions on the temporary files Ansible needs to create when becoming an unprivileged user'
' (rc: {0}, err: {1}). For information on working around this,'
' see https://docs.ansible.com/ansible/become.html#becoming-an-unprivileged-user'.format(res['rc'], res['stderr']))
elif execute: elif execute:
# Can't depend on the file being transferred with execute # Can't depend on the file being transferred with execute
# permissions. Only need user perms because no become was # permissions. Only need user perms because no become was
@ -438,7 +445,7 @@ class ActionBase(with_metaclass(ABCMeta, object)):
mystat['stat']['checksum'] = '1' mystat['stat']['checksum'] = '1'
# happens sometimes when it is a dir and not on bsd # happens sometimes when it is a dir and not on bsd
if not 'checksum' in mystat['stat']: if 'checksum' not in mystat['stat']:
mystat['stat']['checksum'] = '' mystat['stat']['checksum'] = ''
return mystat['stat'] return mystat['stat']
@ -461,15 +468,14 @@ class ActionBase(with_metaclass(ABCMeta, object)):
else: else:
x = remote_stat['checksum'] # if 1, file is missing x = remote_stat['checksum'] # if 1, file is missing
except AnsibleError as e: except AnsibleError as e:
errormsg = to_unicode(e) errormsg = to_text(e)
if errormsg.endswith('Permission denied'): if errormsg.endswith(u'Permission denied'):
x = "2" # cannot read file x = "2" # cannot read file
elif errormsg.endswith('MODULE FAILURE'): elif errormsg.endswith(u'MODULE FAILURE'):
x = "4" # python not found or module uncaught exception x = "4" # python not found or module uncaught exception
finally: finally:
return x return x
def _remote_expand_user(self, path): def _remote_expand_user(self, path):
''' takes a remote path and performs tilde expansion on the remote host ''' ''' takes a remote path and performs tilde expansion on the remote host '''
if not path.startswith('~'): # FIXME: Windows paths may start with "~ instead of just ~ if not path.startswith('~'): # FIXME: Windows paths may start with "~ instead of just ~
@ -681,7 +687,8 @@ class ActionBase(with_metaclass(ABCMeta, object)):
tmp_rm_res = self._low_level_execute_command(tmp_rm_cmd, sudoable=False) tmp_rm_res = self._low_level_execute_command(tmp_rm_cmd, sudoable=False)
tmp_rm_data = self._parse_returned_data(tmp_rm_res) tmp_rm_data = self._parse_returned_data(tmp_rm_res)
if tmp_rm_data.get('rc', 0) != 0: if tmp_rm_data.get('rc', 0) != 0:
display.warning('Error deleting remote temporary files (rc: {0}, stderr: {1})'.format(tmp_rm_res.get('rc'), tmp_rm_res.get('stderr', 'No error string available.'))) display.warning('Error deleting remote temporary files (rc: {0}, stderr: {1})'.format(tmp_rm_res.get('rc'),
tmp_rm_res.get('stderr', 'No error string available.')))
# parse the main result # parse the main result
data = self._parse_returned_data(res) data = self._parse_returned_data(res)
@ -709,7 +716,7 @@ class ActionBase(with_metaclass(ABCMeta, object)):
data['exception'] = res['stderr'] data['exception'] = res['stderr']
return data return data
def _low_level_execute_command(self, cmd, sudoable=True, in_data=None, executable=None, encoding_errors='replace'): def _low_level_execute_command(self, cmd, sudoable=True, in_data=None, executable=None, encoding_errors='surrogate_or_replace'):
''' '''
This is the function which executes the low level shell command, which This is the function which executes the low level shell command, which
may be commands to create/remove directories for temporary files, or to may be commands to create/remove directories for temporary files, or to
@ -758,16 +765,16 @@ class ActionBase(with_metaclass(ABCMeta, object)):
# stdout and stderr may be either a file-like or a bytes object. # stdout and stderr may be either a file-like or a bytes object.
# Convert either one to a text type # Convert either one to a text type
if isinstance(stdout, binary_type): if isinstance(stdout, binary_type):
out = to_unicode(stdout, errors=encoding_errors) out = to_text(stdout, errors=encoding_errors)
elif not isinstance(stdout, text_type): elif not isinstance(stdout, text_type):
out = to_unicode(b''.join(stdout.readlines()), errors=encoding_errors) out = to_text(b''.join(stdout.readlines()), errors=encoding_errors)
else: else:
out = stdout out = stdout
if isinstance(stderr, binary_type): if isinstance(stderr, binary_type):
err = to_unicode(stderr, errors=encoding_errors) err = to_text(stderr, errors=encoding_errors)
elif not isinstance(stderr, text_type): elif not isinstance(stderr, text_type):
err = to_unicode(b''.join(stderr.readlines()), errors=encoding_errors) err = to_text(b''.join(stderr.readlines()), errors=encoding_errors)
else: else:
err = stderr err = stderr
@ -871,7 +878,6 @@ class ActionBase(with_metaclass(ABCMeta, object)):
result = self._loader.path_dwim_relative_stack(path_stack, dirname, needle) result = self._loader.path_dwim_relative_stack(path_stack, dirname, needle)
if result is None: if result is None:
raise AnsibleError("Unable to find '%s' in expected paths." % to_str(needle)) raise AnsibleError("Unable to find '%s' in expected paths." % to_native(needle))
return result return result

@ -24,10 +24,10 @@ import tempfile
import re import re
from ansible.errors import AnsibleError from ansible.errors import AnsibleError
from ansible.module_utils._text import to_native, to_text
from ansible.plugins.action import ActionBase from ansible.plugins.action import ActionBase
from ansible.utils.boolean import boolean from ansible.utils.boolean import boolean
from ansible.utils.hashing import checksum_s from ansible.utils.hashing import checksum_s
from ansible.utils.unicode import to_str, to_unicode
class ActionModule(ActionBase): class ActionModule(ActionBase):
@ -42,7 +42,7 @@ class ActionModule(ActionBase):
delimit_me = False delimit_me = False
add_newline = False add_newline = False
for f in (to_unicode(p, errors='strict') for p in sorted(os.listdir(src_path))): for f in (to_text(p, errors='surrogate_or_strict') for p in sorted(os.listdir(src_path))):
if compiled_regexp and not compiled_regexp.search(f): if compiled_regexp and not compiled_regexp.search(f):
continue continue
fragment = u"%s/%s" % (src_path, f) fragment = u"%s/%s" % (src_path, f)
@ -114,7 +114,7 @@ class ActionModule(ActionBase):
src = self._find_needle('files', src) src = self._find_needle('files', src)
except AnsibleError as e: except AnsibleError as e:
result['failed'] = True result['failed'] = True
result['msg'] = to_str(e) result['msg'] = to_native(e)
return result return result
if not os.path.isdir(src): if not os.path.isdir(src):

@ -22,9 +22,10 @@ import pipes
import random import random
from ansible import constants as C from ansible import constants as C
from ansible.plugins.action import ActionBase
from ansible.compat.six import iteritems from ansible.compat.six import iteritems
from ansible.utils.unicode import to_unicode from ansible.module_utils._text import to_text
from ansible.plugins.action import ActionBase
class ActionModule(ActionBase): class ActionModule(ActionBase):
@ -76,7 +77,7 @@ class ActionModule(ActionBase):
elif module_style == 'old': elif module_style == 'old':
args_data = "" args_data = ""
for k, v in iteritems(module_args): for k, v in iteritems(module_args):
args_data += '%s="%s" ' % (k, pipes.quote(to_unicode(v))) args_data += '%s="%s" ' % (k, pipes.quote(to_text(v)))
argsfile = self._transfer_data(self._connection._shell.join_path(tmp, 'arguments'), args_data) argsfile = self._transfer_data(self._connection._shell.join_path(tmp, 'arguments'), args_data)
remote_paths = tmp, remote_module_path, remote_async_module_path remote_paths = tmp, remote_module_path, remote_async_module_path
@ -100,7 +101,7 @@ class ActionModule(ActionBase):
if not self._should_remove_tmp_path(tmp): if not self._should_remove_tmp_path(tmp):
async_cmd.append("-preserve_tmp") async_cmd.append("-preserve_tmp")
async_cmd = " ".join([to_unicode(x) for x in async_cmd]) async_cmd = " ".join(to_text(x) for x in async_cmd)
result.update(self._low_level_execute_command(cmd=async_cmd)) result.update(self._low_level_execute_command(cmd=async_cmd))
result['changed'] = True result['changed'] = True

@ -24,10 +24,10 @@ import os
import tempfile import tempfile
from ansible.errors import AnsibleError from ansible.errors import AnsibleError
from ansible.module_utils._text import to_bytes, to_native, to_text
from ansible.plugins.action import ActionBase from ansible.plugins.action import ActionBase
from ansible.utils.boolean import boolean from ansible.utils.boolean import boolean
from ansible.utils.hashing import checksum from ansible.utils.hashing import checksum
from ansible.utils.unicode import to_bytes, to_str, to_unicode
class ActionModule(ActionBase): class ActionModule(ActionBase):
@ -81,7 +81,7 @@ class ActionModule(ActionBase):
source = content_tempfile source = content_tempfile
except Exception as err: except Exception as err:
result['failed'] = True result['failed'] = True
result['msg'] = "could not write content temp file: %s" % to_str(err) result['msg'] = "could not write content temp file: %s" % to_native(err)
return result return result
# if we have first_available_file in our vars # if we have first_available_file in our vars
@ -96,14 +96,14 @@ class ActionModule(ActionBase):
source = self._find_needle('files', source) source = self._find_needle('files', source)
except AnsibleError as e: except AnsibleError as e:
result['failed'] = True result['failed'] = True
result['msg'] = to_unicode(e) result['msg'] = to_text(e)
return result return result
# A list of source file tuples (full_path, relative_path) which will try to copy to the destination # A list of source file tuples (full_path, relative_path) which will try to copy to the destination
source_files = [] source_files = []
# If source is a directory populate our list else source is a file and translate it to a tuple. # If source is a directory populate our list else source is a file and translate it to a tuple.
if os.path.isdir(to_bytes(source, errors='strict')): if os.path.isdir(to_bytes(source, errors='surrogate_or_strict')):
# Get the amount of spaces to remove to get the relative path. # Get the amount of spaces to remove to get the relative path.
if source_trailing_slash: if source_trailing_slash:
sz = len(source) sz = len(source)
@ -113,7 +113,7 @@ class ActionModule(ActionBase):
# Walk the directory and append the file tuples to source_files. # Walk the directory and append the file tuples to source_files.
for base_path, sub_folders, files in os.walk(to_bytes(source)): for base_path, sub_folders, files in os.walk(to_bytes(source)):
for file in files: for file in files:
full_path = to_unicode(os.path.join(base_path, file), errors='strict') full_path = to_text(os.path.join(base_path, file), errors='surrogate_or_strict')
rel_path = full_path[sz:] rel_path = full_path[sz:]
if rel_path.startswith('/'): if rel_path.startswith('/'):
rel_path = rel_path[1:] rel_path = rel_path[1:]
@ -247,7 +247,9 @@ class ActionModule(ActionBase):
if 'content' in new_module_args: if 'content' in new_module_args:
del new_module_args['content'] del new_module_args['content']
module_return = self._execute_module(module_name='copy', module_args=new_module_args, task_vars=task_vars, tmp=tmp, delete_remote_tmp=delete_remote_tmp) module_return = self._execute_module(module_name='copy',
module_args=new_module_args, task_vars=task_vars,
tmp=tmp, delete_remote_tmp=delete_remote_tmp)
module_executed = True module_executed = True
else: else:
@ -272,7 +274,9 @@ class ActionModule(ActionBase):
) )
# Execute the file module. # Execute the file module.
module_return = self._execute_module(module_name='file', module_args=new_module_args, task_vars=task_vars, tmp=tmp, delete_remote_tmp=delete_remote_tmp) module_return = self._execute_module(module_name='file',
module_args=new_module_args, task_vars=task_vars,
tmp=tmp, delete_remote_tmp=delete_remote_tmp)
module_executed = True module_executed = True
if not module_return.get('checksum'): if not module_return.get('checksum'):

@ -20,8 +20,8 @@ __metaclass__ = type
from ansible.compat.six import string_types from ansible.compat.six import string_types
from ansible.errors import AnsibleUndefinedVariable from ansible.errors import AnsibleUndefinedVariable
from ansible.module_utils._text import to_text
from ansible.plugins.action import ActionBase from ansible.plugins.action import ActionBase
from ansible.utils.unicode import to_unicode
class ActionModule(ActionBase): class ActionModule(ActionBase):
@ -66,7 +66,7 @@ class ActionModule(ActionBase):
if isinstance(self._task.args['var'], (list, dict)): if isinstance(self._task.args['var'], (list, dict)):
# If var is a list or dict, use the type as key to display # If var is a list or dict, use the type as key to display
result[to_unicode(type(self._task.args['var']))] = results result[to_text(type(self._task.args['var']))] = results
else: else:
result[self._task.args['var']] = results result[self._task.args['var']] = results
else: else:

@ -21,11 +21,11 @@ import os
import base64 import base64
from ansible.errors import AnsibleError from ansible.errors import AnsibleError
from ansible.module_utils._text import to_bytes
from ansible.plugins.action import ActionBase from ansible.plugins.action import ActionBase
from ansible.utils.boolean import boolean from ansible.utils.boolean import boolean
from ansible.utils.hashing import checksum, checksum_s, md5, secure_hash from ansible.utils.hashing import checksum, checksum_s, md5, secure_hash
from ansible.utils.path import makedirs_safe from ansible.utils.path import makedirs_safe
from ansible.utils.unicode import to_bytes
class ActionModule(ActionBase): class ActionModule(ActionBase):
@ -160,7 +160,7 @@ class ActionModule(ActionBase):
self._connection.fetch_file(source, dest) self._connection.fetch_file(source, dest)
else: else:
try: try:
f = open(to_bytes(dest, errors='strict'), 'w') f = open(to_bytes(dest, errors='surrogate_or_strict'), 'wb')
f.write(remote_data) f.write(remote_data)
f.close() f.close()
except (IOError, OSError) as e: except (IOError, OSError) as e:

@ -22,8 +22,8 @@ from os import path, walk
import re import re
from ansible.errors import AnsibleError from ansible.errors import AnsibleError
from ansible.module_utils._text import to_native
from ansible.plugins.action import ActionBase from ansible.plugins.action import ActionBase
from ansible.utils.unicode import to_str
class ActionModule(ActionBase): class ActionModule(ActionBase):
@ -137,7 +137,7 @@ class ActionModule(ActionBase):
results.update(updated_results) results.update(updated_results)
except AnsibleError as e: except AnsibleError as e:
err_msg = to_str(e) err_msg = to_native(e)
raise AnsibleError(err_msg) raise AnsibleError(err_msg)
if self.return_results_as_name: if self.return_results_as_name:

@ -26,10 +26,12 @@ import glob
import urlparse import urlparse
from ansible.plugins.action import ActionBase from ansible.plugins.action import ActionBase
from ansible.utils.unicode import to_unicode from ansible.module_utils._text import to_text
PRIVATE_KEYS_RE = re.compile('__.+__') PRIVATE_KEYS_RE = re.compile('__.+__')
class ActionModule(ActionBase): class ActionModule(ActionBase):
TRANSFERS_FILES = False TRANSFERS_FILES = False
@ -56,7 +58,6 @@ class ActionModule(ActionBase):
result['__backup__']) result['__backup__'])
result['backup_path'] = filepath result['backup_path'] = filepath
# strip out any keys that have two leading and two trailing # strip out any keys that have two leading and two trailing
# underscore characters # underscore characters
for key in result.keys(): for key in result.keys():
@ -98,7 +99,7 @@ class ActionModule(ActionBase):
try: try:
with open(source, 'r') as f: with open(source, 'r') as f:
template_data = to_unicode(f.read()) template_data = to_text(f.read())
except IOError: except IOError:
return dict(failed=True, msg='unable to load src file') return dict(failed=True, msg='unable to load src file')
@ -114,5 +115,3 @@ class ActionModule(ActionBase):
searchpath.append(os.path.dirname(source)) searchpath.append(os.path.dirname(source))
self._templar.environment.loader.searchpath = searchpath self._templar.environment.loader.searchpath = searchpath
self._task.args['src'] = self._templar.template(template_data) self._task.args['src'] = self._templar.template(template_data)

@ -19,18 +19,18 @@
from __future__ import (absolute_import, division, print_function) from __future__ import (absolute_import, division, print_function)
__metaclass__ = type __metaclass__ = type
import sys
import os import os
import time import time
import glob import glob
import urlparse import urlparse
from ansible.module_utils._text import to_text
from ansible.plugins.action import ActionBase from ansible.plugins.action import ActionBase
from ansible.utils.boolean import boolean
from ansible.utils.unicode import to_unicode
BOOLEANS = ('true', 'false', 'yes', 'no') BOOLEANS = ('true', 'false', 'yes', 'no')
class ActionModule(ActionBase): class ActionModule(ActionBase):
TRANSFERS_FILES = False TRANSFERS_FILES = False
@ -92,7 +92,7 @@ class ActionModule(ActionBase):
try: try:
with open(source, 'r') as f: with open(source, 'r') as f:
template_data = to_unicode(f.read()) template_data = to_text(f.read())
except IOError: except IOError:
return dict(failed=True, msg='unable to load src file') return dict(failed=True, msg='unable to load src file')
@ -108,5 +108,3 @@ class ActionModule(ActionBase):
searchpath.append(os.path.dirname(source)) searchpath.append(os.path.dirname(source))
self._templar.environment.loader.searchpath = searchpath self._templar.environment.loader.searchpath = searchpath
self._task.args['src'] = self._templar.template(template_data) self._task.args['src'] = self._templar.template(template_data)

@ -20,10 +20,10 @@ __metaclass__ = type
import os import os
from ansible.errors import AnsibleError
from ansible.module_utils._text import to_native
from ansible.plugins.action import ActionBase from ansible.plugins.action import ActionBase
from ansible.utils.boolean import boolean from ansible.utils.boolean import boolean
from ansible.errors import AnsibleError
from ansible.utils.unicode import to_str
class ActionModule(ActionBase): class ActionModule(ActionBase):
@ -52,7 +52,7 @@ class ActionModule(ActionBase):
src = self._find_needle('files', src) src = self._find_needle('files', src)
except AnsibleError as e: except AnsibleError as e:
result['failed'] = True result['failed'] = True
result['msg'] = to_str(e) result['msg'] = to_native(e)
return result return result
# create the remote tmp dir if needed, and put the source file there # create the remote tmp dir if needed, and put the source file there

@ -19,15 +19,14 @@ __metaclass__ = type
import os import os
from ansible.plugins.action import ActionBase
from ansible.errors import AnsibleError from ansible.errors import AnsibleError
from ansible.utils.unicode import to_str from ansible.module_utils._text import to_native
from ansible.plugins.action import ActionBase
class ActionModule(ActionBase): class ActionModule(ActionBase):
TRANSFERS_FILES = True TRANSFERS_FILES = True
def run(self, tmp=None, task_vars=None): def run(self, tmp=None, task_vars=None):
''' handler for file transfer operations ''' ''' handler for file transfer operations '''
if task_vars is None: if task_vars is None:
@ -74,7 +73,7 @@ class ActionModule(ActionBase):
try: try:
source = self._loader.get_real_file(self._find_needle('files', source)) source = self._loader.get_real_file(self._find_needle('files', source))
except AnsibleError as e: except AnsibleError as e:
return dict(failed=True, msg=to_str(e)) return dict(failed=True, msg=to_native(e))
# transfer the file to a remote tmp location # transfer the file to a remote tmp location
tmp_src = self._connection._shell.join_path(tmp, os.path.basename(source)) tmp_src = self._connection._shell.join_path(tmp, os.path.basename(source))

@ -23,12 +23,11 @@ import pwd
import time import time
from ansible import constants as C from ansible import constants as C
from ansible.errors import AnsibleError
from ansible.module_utils._text import to_bytes, to_native, to_text
from ansible.plugins.action import ActionBase from ansible.plugins.action import ActionBase
from ansible.utils.hashing import checksum_s from ansible.utils.hashing import checksum_s
from ansible.utils.boolean import boolean from ansible.utils.boolean import boolean
from ansible.utils.unicode import to_bytes, to_unicode, to_str
from ansible.errors import AnsibleError
class ActionModule(ActionBase): class ActionModule(ActionBase):
@ -78,7 +77,7 @@ class ActionModule(ActionBase):
source = self._find_needle('templates', source) source = self._find_needle('templates', source)
except AnsibleError as e: except AnsibleError as e:
result['failed'] = True result['failed'] = True
result['msg'] = to_str(e) result['msg'] = to_native(e)
if 'failed' in result: if 'failed' in result:
return result return result
@ -96,7 +95,7 @@ class ActionModule(ActionBase):
b_source = to_bytes(source) b_source = to_bytes(source)
try: try:
with open(b_source, 'r') as f: with open(b_source, 'r') as f:
template_data = to_unicode(f.read()) template_data = to_text(f.read())
try: try:
template_uid = pwd.getpwuid(os.stat(b_source).st_uid).pw_name template_uid = pwd.getpwuid(os.stat(b_source).st_uid).pw_name
@ -163,7 +162,7 @@ class ActionModule(ActionBase):
if self._play_context.diff: if self._play_context.diff:
diff = self._get_diff_data(dest, resultant, task_vars, source_file=False) diff = self._get_diff_data(dest, resultant, task_vars, source_file=False)
if not self._play_context.check_mode: # do actual work thorugh copy if not self._play_context.check_mode: # do actual work through copy
xfered = self._transfer_data(self._connection._shell.join_path(tmp, 'source'), resultant) xfered = self._transfer_data(self._connection._shell.join_path(tmp, 'source'), resultant)
# fix file permissions when the copy is done as a different user # fix file permissions when the copy is done as a different user

@ -20,10 +20,11 @@ __metaclass__ = type
import os import os
from ansible.errors import AnsibleError
from ansible.module_utils._text import to_native
from ansible.plugins.action import ActionBase from ansible.plugins.action import ActionBase
from ansible.utils.boolean import boolean from ansible.utils.boolean import boolean
from ansible.errors import AnsibleError
from ansible.utils.unicode import to_str
class ActionModule(ActionBase): class ActionModule(ActionBase):
@ -82,7 +83,7 @@ class ActionModule(ActionBase):
source = self._loader.get_real_file(self._find_needle('files', source)) source = self._loader.get_real_file(self._find_needle('files', source))
except AnsibleError as e: except AnsibleError as e:
result['failed'] = True result['failed'] = True
result['msg'] = to_str(e) result['msg'] = to_native(e)
self._remove_tmp_path(tmp) self._remove_tmp_path(tmp)
return result return result

@ -19,26 +19,24 @@
from __future__ import (absolute_import, division, print_function) from __future__ import (absolute_import, division, print_function)
__metaclass__ = type __metaclass__ = type
from ansible.plugins.action import ActionBase
from ansible.utils.boolean import boolean
from ansible.utils.unicode import to_unicode
from ansible.errors import AnsibleUndefinedVariable
import socket import socket
import time import time
import traceback
from datetime import datetime, timedelta from datetime import datetime, timedelta
from ansible.plugins.action import ActionBase
try: try:
from __main__ import display from __main__ import display
except ImportError: except ImportError:
from ansible.utils.display import Display from ansible.utils.display import Display
display = Display() display = Display()
class TimedOutException(Exception): class TimedOutException(Exception):
pass pass
class ActionModule(ActionBase): class ActionModule(ActionBase):
TRANSFERS_FILES = False TRANSFERS_FILES = False
@ -137,4 +135,3 @@ class ActionModule(ActionBase):
result['msg'] = toex.message result['msg'] = toex.message
return result return result

@ -31,9 +31,9 @@ except ImportError:
from ansible import constants as C from ansible import constants as C
from ansible.errors import AnsibleError from ansible.errors import AnsibleError
from ansible.module_utils._text import to_bytes
from ansible.parsing.utils.jsonify import jsonify from ansible.parsing.utils.jsonify import jsonify
from ansible.plugins.cache.base import BaseCacheModule from ansible.plugins.cache.base import BaseCacheModule
from ansible.utils.unicode import to_bytes
try: try:
from __main__ import display from __main__ import display

@ -24,12 +24,10 @@ import difflib
import warnings import warnings
from copy import deepcopy from copy import deepcopy
from ansible.compat.six import string_types
from ansible import constants as C from ansible import constants as C
from ansible.vars import strip_internal_keys from ansible.module_utils._text import to_text
from ansible.utils.color import stringc from ansible.utils.color import stringc
from ansible.utils.unicode import to_unicode from ansible.vars import strip_internal_keys
try: try:
from __main__ import display as global_display from __main__ import display as global_display
@ -37,14 +35,15 @@ except ImportError:
from ansible.utils.display import Display from ansible.utils.display import Display
global_display = Display() global_display = Display()
__all__ = ["CallbackBase"]
try: try:
from __main__ import cli from __main__ import cli
except ImportError: except ImportError:
# using API w/o cli # using API w/o cli
cli = False cli = False
__all__ = ["CallbackBase"]
class CallbackBase: class CallbackBase:
''' '''
@ -146,8 +145,8 @@ class CallbackBase:
after_header = "after: %s" % diff['after_header'] after_header = "after: %s" % diff['after_header']
else: else:
after_header = 'after' after_header = 'after'
differ = difflib.unified_diff(to_unicode(diff['before']).splitlines(True), differ = difflib.unified_diff(to_text(diff['before']).splitlines(True),
to_unicode(diff['after']).splitlines(True), to_text(diff['after']).splitlines(True),
fromfile=before_header, fromfile=before_header,
tofile=after_header, tofile=after_header,
fromfiledate='', fromfiledate='',
@ -166,7 +165,7 @@ class CallbackBase:
if has_diff: if has_diff:
ret.append('\n') ret.append('\n')
if 'prepared' in diff: if 'prepared' in diff:
ret.append(to_unicode(diff['prepared'])) ret.append(to_text(diff['prepared']))
except UnicodeDecodeError: except UnicodeDecodeError:
ret.append(">> the files are different, but the diff library cannot compare unicode strings\n\n") ret.append(">> the files are different, but the diff library cannot compare unicode strings\n\n")
return u''.join(ret) return u''.join(ret)
@ -362,4 +361,3 @@ class CallbackBase:
def v2_runner_retry(self, result): def v2_runner_retry(self, result):
pass pass

@ -21,8 +21,8 @@ __metaclass__ = type
import os import os
import time import time
from ansible.module_utils._text import to_bytes
from ansible.plugins.callback import CallbackBase from ansible.plugins.callback import CallbackBase
from ansible.utils.unicode import to_bytes
try: try:
from junit_xml import TestSuite, TestCase from junit_xml import TestSuite, TestCase
@ -40,6 +40,7 @@ except ImportError:
except ImportError: except ImportError:
HAS_ORDERED_DICT = False HAS_ORDERED_DICT = False
class CallbackModule(CallbackBase): class CallbackModule(CallbackBase):
""" """
This callback writes playbook output to a JUnit formatted XML file. This callback writes playbook output to a JUnit formatted XML file.
@ -181,7 +182,7 @@ class CallbackModule(CallbackBase):
output_file = os.path.join(self._output_dir, '%s-%s.xml' % (self._playbook_name, time.time())) output_file = os.path.join(self._output_dir, '%s-%s.xml' % (self._playbook_name, time.time()))
with open(output_file, 'wb') as xml: with open(output_file, 'wb') as xml:
xml.write(to_bytes(report, errors='strict')) xml.write(to_bytes(report, errors='surrogate_or_strict'))
def v2_playbook_on_start(self, playbook): def v2_playbook_on_start(self, playbook):
self._playbook_path = playbook._file_name self._playbook_path = playbook._file_name

@ -23,9 +23,10 @@ import os
import time import time
import json import json
from ansible.utils.unicode import to_bytes from ansible.module_utils._text import to_bytes
from ansible.plugins.callback import CallbackBase from ansible.plugins.callback import CallbackBase
# NOTE: in Ansible 1.2 or later general logging is available without # NOTE: in Ansible 1.2 or later general logging is available without
# this plugin, just set ANSIBLE_LOG_PATH as an environment variable # this plugin, just set ANSIBLE_LOG_PATH as an environment variable
# or log_path in the DEFAULTS section of your ansible configuration # or log_path in the DEFAULTS section of your ansible configuration

@ -25,9 +25,10 @@ import smtplib
import json import json
from ansible.compat.six import string_types from ansible.compat.six import string_types
from ansible.utils.unicode import to_bytes from ansible.module_utils._text import to_bytes
from ansible.plugins.callback import CallbackBase from ansible.plugins.callback import CallbackBase
def mail(subject='Ansible error mail', sender=None, to=None, cc=None, bcc=None, body=None, smtphost=None): def mail(subject='Ansible error mail', sender=None, to=None, cc=None, bcc=None, body=None, smtphost=None):
if sender is None: if sender is None:

@ -20,10 +20,10 @@ __metaclass__ = type
import os import os
from ansible.constants import TREE_DIR
from ansible.module_utils._text import to_bytes
from ansible.plugins.callback import CallbackBase from ansible.plugins.callback import CallbackBase
from ansible.utils.path import makedirs_safe from ansible.utils.path import makedirs_safe
from ansible.utils.unicode import to_bytes
from ansible.constants import TREE_DIR
class CallbackModule(CallbackBase): class CallbackModule(CallbackBase):
@ -68,4 +68,3 @@ class CallbackModule(CallbackBase):
def v2_runner_on_unreachable(self, result): def v2_runner_on_unreachable(self, result):
self.result_to_tree(result) self.result_to_tree(result)

@ -1,4 +1,3 @@
# (c) 2015 Toshio Kuratomi <tkuratomi@ansible.com> # (c) 2015 Toshio Kuratomi <tkuratomi@ansible.com>
# #
# This file is part of Ansible # This file is part of Ansible
@ -32,8 +31,9 @@ from ansible.compat.six import with_metaclass
from ansible import constants as C from ansible import constants as C
from ansible.compat.six import string_types from ansible.compat.six import string_types
from ansible.errors import AnsibleError from ansible.errors import AnsibleError
from ansible.module_utils._text import to_bytes, to_text
from ansible.plugins import shell_loader from ansible.plugins import shell_loader
from ansible.utils.unicode import to_bytes, to_unicode
try: try:
from __main__ import display from __main__ import display
@ -138,9 +138,9 @@ class ConnectionBase(with_metaclass(ABCMeta, object)):
# exception, it merely mangles the output: # exception, it merely mangles the output:
# >>> shlex.split(u't e') # >>> shlex.split(u't e')
# ['t\x00\x00\x00', '\x00\x00\x00e\x00\x00\x00'] # ['t\x00\x00\x00', '\x00\x00\x00e\x00\x00\x00']
return [to_unicode(x.strip()) for x in shlex.split(to_bytes(argstring)) if x.strip()] return [to_text(x.strip()) for x in shlex.split(to_bytes(argstring)) if x.strip()]
except AttributeError: except AttributeError:
return [to_unicode(x.strip()) for x in shlex.split(argstring) if x.strip()] return [to_text(x.strip()) for x in shlex.split(argstring) if x.strip()]
@abstractproperty @abstractproperty
def transport(self): def transport(self):

@ -27,10 +27,11 @@ import time
from ansible import constants as C from ansible import constants as C
from ansible.errors import AnsibleError, AnsibleFileNotFound, AnsibleConnectionFailure from ansible.errors import AnsibleError, AnsibleFileNotFound, AnsibleConnectionFailure
from ansible.module_utils._text import to_bytes
from ansible.parsing.utils.jsonify import jsonify from ansible.parsing.utils.jsonify import jsonify
from ansible.plugins.connection import ConnectionBase from ansible.plugins.connection import ConnectionBase
from ansible.utils.encrypt import key_for_hostname, keyczar_encrypt, keyczar_decrypt from ansible.utils.encrypt import key_for_hostname, keyczar_encrypt, keyczar_decrypt
from ansible.utils.unicode import to_bytes
try: try:
from __main__ import display from __main__ import display
@ -211,7 +212,7 @@ class Connection(ConnectionBase):
''' transfer a file from local to remote ''' ''' transfer a file from local to remote '''
display.vvv("PUT %s TO %s" % (in_path, out_path), host=self._play_context.remote_addr) display.vvv("PUT %s TO %s" % (in_path, out_path), host=self._play_context.remote_addr)
in_path = to_bytes(in_path, errors='strict') in_path = to_bytes(in_path, errors='surrogate_or_strict')
if not os.path.exists(in_path): if not os.path.exists(in_path):
raise AnsibleFileNotFound("file or module does not exist: %s" % in_path) raise AnsibleFileNotFound("file or module does not exist: %s" % in_path)
@ -265,7 +266,7 @@ class Connection(ConnectionBase):
if self.send_data(data): if self.send_data(data):
raise AnsibleError("failed to initiate the file fetch with %s" % self._play_context.remote_addr) raise AnsibleError("failed to initiate the file fetch with %s" % self._play_context.remote_addr)
fh = open(to_bytes(out_path, errors='strict'), "w") fh = open(to_bytes(out_path, errors='surrogate_or_strict'), "w")
try: try:
bytes = 0 bytes = 0
while True: while True:

@ -28,9 +28,10 @@ import traceback
from ansible import constants as C from ansible import constants as C
from ansible.errors import AnsibleError from ansible.errors import AnsibleError
from ansible.plugins.connection import ConnectionBase, BUFSIZE
from ansible.module_utils.basic import is_executable from ansible.module_utils.basic import is_executable
from ansible.utils.unicode import to_bytes from ansible.module_utils._text import to_bytes
from ansible.plugins.connection import ConnectionBase, BUFSIZE
try: try:
from __main__ import display from __main__ import display
@ -93,7 +94,7 @@ class Connection(ConnectionBase):
local_cmd = [self.chroot_cmd, self.chroot, executable, '-c', cmd] local_cmd = [self.chroot_cmd, self.chroot, executable, '-c', cmd]
display.vvv("EXEC %s" % (local_cmd), host=self.chroot) display.vvv("EXEC %s" % (local_cmd), host=self.chroot)
local_cmd = [to_bytes(i, errors='strict') for i in local_cmd] local_cmd = [to_bytes(i, errors='surrogate_or_strict') for i in local_cmd]
p = subprocess.Popen(local_cmd, shell=False, stdin=stdin, p = subprocess.Popen(local_cmd, shell=False, stdin=stdin,
stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout=subprocess.PIPE, stderr=subprocess.PIPE)
@ -129,7 +130,7 @@ class Connection(ConnectionBase):
out_path = pipes.quote(self._prefix_login_path(out_path)) out_path = pipes.quote(self._prefix_login_path(out_path))
try: try:
with open(to_bytes(in_path, errors='strict'), 'rb') as in_file: with open(to_bytes(in_path, errors='surrogate_or_strict'), 'rb') as in_file:
try: try:
p = self._buffered_exec_command('dd of=%s bs=%s' % (out_path, BUFSIZE), stdin=in_file) p = self._buffered_exec_command('dd of=%s bs=%s' % (out_path, BUFSIZE), stdin=in_file)
except OSError: except OSError:
@ -155,7 +156,7 @@ class Connection(ConnectionBase):
except OSError: except OSError:
raise AnsibleError("chroot connection requires dd command in the chroot") raise AnsibleError("chroot connection requires dd command in the chroot")
with open(to_bytes(out_path, errors='strict'), 'wb+') as out_file: with open(to_bytes(out_path, errors='surrogate_or_strict'), 'wb+') as out_file:
try: try:
chunk = p.stdout.read(BUFSIZE) chunk = p.stdout.read(BUFSIZE)
while chunk: while chunk:

@ -35,8 +35,9 @@ from distutils.version import LooseVersion
import ansible.constants as C import ansible.constants as C
from ansible.errors import AnsibleError, AnsibleFileNotFound from ansible.errors import AnsibleError, AnsibleFileNotFound
from ansible.module_utils._text import to_bytes
from ansible.plugins.connection import ConnectionBase, BUFSIZE from ansible.plugins.connection import ConnectionBase, BUFSIZE
from ansible.utils.unicode import to_bytes
try: try:
from __main__ import display from __main__ import display
@ -196,7 +197,7 @@ class Connection(ConnectionBase):
local_cmd = self._build_exec_cmd([self._play_context.executable, '-c', cmd]) local_cmd = self._build_exec_cmd([self._play_context.executable, '-c', cmd])
display.vvv("EXEC %s" % (local_cmd,), host=self._play_context.remote_addr) display.vvv("EXEC %s" % (local_cmd,), host=self._play_context.remote_addr)
local_cmd = [to_bytes(i, errors='strict') for i in local_cmd] local_cmd = [to_bytes(i, errors='surrogate_or_strict') for i in local_cmd]
p = subprocess.Popen(local_cmd, shell=False, stdin=subprocess.PIPE, p = subprocess.Popen(local_cmd, shell=False, stdin=subprocess.PIPE,
stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout=subprocess.PIPE, stderr=subprocess.PIPE)
@ -223,7 +224,7 @@ class Connection(ConnectionBase):
display.vvv("PUT %s TO %s" % (in_path, out_path), host=self._play_context.remote_addr) display.vvv("PUT %s TO %s" % (in_path, out_path), host=self._play_context.remote_addr)
out_path = self._prefix_login_path(out_path) out_path = self._prefix_login_path(out_path)
if not os.path.exists(to_bytes(in_path, errors='strict')): if not os.path.exists(to_bytes(in_path, errors='surrogate_or_strict')):
raise AnsibleFileNotFound( raise AnsibleFileNotFound(
"file or module does not exist: %s" % in_path) "file or module does not exist: %s" % in_path)
@ -233,8 +234,8 @@ class Connection(ConnectionBase):
# Although docker version 1.8 and later provide support, the # Although docker version 1.8 and later provide support, the
# owner and group of the files are always set to root # owner and group of the files are always set to root
args = self._build_exec_cmd([self._play_context.executable, "-c", "dd of=%s bs=%s" % (out_path, BUFSIZE)]) args = self._build_exec_cmd([self._play_context.executable, "-c", "dd of=%s bs=%s" % (out_path, BUFSIZE)])
args = [to_bytes(i, errors='strict') for i in args] args = [to_bytes(i, errors='surrogate_or_strict') for i in args]
with open(to_bytes(in_path, errors='strict'), 'rb') as in_file: with open(to_bytes(in_path, errors='surrogate_or_strict'), 'rb') as in_file:
try: try:
p = subprocess.Popen(args, stdin=in_file, p = subprocess.Popen(args, stdin=in_file,
stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout=subprocess.PIPE, stderr=subprocess.PIPE)
@ -256,7 +257,7 @@ class Connection(ConnectionBase):
out_dir = os.path.dirname(out_path) out_dir = os.path.dirname(out_path)
args = [self.docker_cmd, "cp", "%s:%s" % (self._play_context.remote_addr, in_path), out_dir] args = [self.docker_cmd, "cp", "%s:%s" % (self._play_context.remote_addr, in_path), out_dir]
args = [to_bytes(i, errors='strict') for i in args] args = [to_bytes(i, errors='surrogate_or_strict') for i in args]
p = subprocess.Popen(args, stdin=subprocess.PIPE, p = subprocess.Popen(args, stdin=subprocess.PIPE,
stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout=subprocess.PIPE, stderr=subprocess.PIPE)

@ -27,10 +27,9 @@ import pipes
import subprocess import subprocess
import traceback import traceback
from ansible import constants as C
from ansible.errors import AnsibleError from ansible.errors import AnsibleError
from ansible.module_utils._text import to_bytes
from ansible.plugins.connection import ConnectionBase, BUFSIZE from ansible.plugins.connection import ConnectionBase, BUFSIZE
from ansible.utils.unicode import to_bytes
try: try:
from __main__ import display from __main__ import display
@ -117,7 +116,7 @@ class Connection(ConnectionBase):
local_cmd += [self.jail, self._play_context.executable, '-c', set_env + cmd] local_cmd += [self.jail, self._play_context.executable, '-c', set_env + cmd]
display.vvv("EXEC %s" % (local_cmd,), host=self.jail) display.vvv("EXEC %s" % (local_cmd,), host=self.jail)
local_cmd = [to_bytes(i, errors='strict') for i in local_cmd] local_cmd = [to_bytes(i, errors='surrogate_or_strict') for i in local_cmd]
p = subprocess.Popen(local_cmd, shell=False, stdin=stdin, p = subprocess.Popen(local_cmd, shell=False, stdin=stdin,
stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout=subprocess.PIPE, stderr=subprocess.PIPE)
@ -153,7 +152,7 @@ class Connection(ConnectionBase):
out_path = pipes.quote(self._prefix_login_path(out_path)) out_path = pipes.quote(self._prefix_login_path(out_path))
try: try:
with open(to_bytes(in_path, errors='strict'), 'rb') as in_file: with open(to_bytes(in_path, errors='surrogate_or_strict'), 'rb') as in_file:
try: try:
p = self._buffered_exec_command('dd of=%s bs=%s' % (out_path, BUFSIZE), stdin=in_file) p = self._buffered_exec_command('dd of=%s bs=%s' % (out_path, BUFSIZE), stdin=in_file)
except OSError: except OSError:
@ -179,7 +178,7 @@ class Connection(ConnectionBase):
except OSError: except OSError:
raise AnsibleError("jail connection requires dd command in the jail") raise AnsibleError("jail connection requires dd command in the jail")
with open(to_bytes(out_path, errors='strict'), 'wb+') as out_file: with open(to_bytes(out_path, errors='surrogate_or_strict'), 'wb+') as out_file:
try: try:
chunk = p.stdout.read(BUFSIZE) chunk = p.stdout.read(BUFSIZE)
while chunk: while chunk:

@ -29,8 +29,9 @@ import traceback
from ansible import constants as C from ansible import constants as C
from ansible.errors import AnsibleError from ansible.errors import AnsibleError
from ansible.module_utils._text import to_bytes
from ansible.plugins.connection import ConnectionBase, BUFSIZE from ansible.plugins.connection import ConnectionBase, BUFSIZE
from ansible.utils.unicode import to_bytes
try: try:
from __main__ import display from __main__ import display
@ -94,7 +95,7 @@ class Connection(ConnectionBase):
local_cmd += [self.lxc, '--', executable, '-c', cmd] local_cmd += [self.lxc, '--', executable, '-c', cmd]
display.vvv("EXEC %s" % (local_cmd,), host=self.lxc) display.vvv("EXEC %s" % (local_cmd,), host=self.lxc)
local_cmd = [to_bytes(i, errors='strict') for i in local_cmd] local_cmd = [to_bytes(i, errors='surrogate_or_strict') for i in local_cmd]
p = subprocess.Popen(local_cmd, shell=False, stdin=stdin, p = subprocess.Popen(local_cmd, shell=False, stdin=stdin,
stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout=subprocess.PIPE, stderr=subprocess.PIPE)
@ -130,7 +131,7 @@ class Connection(ConnectionBase):
out_path = pipes.quote(self._prefix_login_path(out_path)) out_path = pipes.quote(self._prefix_login_path(out_path))
try: try:
with open(to_bytes(in_path, errors='strict'), 'rb') as in_file: with open(to_bytes(in_path, errors='surrogate_or_strict'), 'rb') as in_file:
try: try:
p = self._buffered_exec_command('dd of=%s bs=%s' % (out_path, BUFSIZE), stdin=in_file) p = self._buffered_exec_command('dd of=%s bs=%s' % (out_path, BUFSIZE), stdin=in_file)
except OSError: except OSError:
@ -156,7 +157,7 @@ class Connection(ConnectionBase):
except OSError: except OSError:
raise AnsibleError("chroot connection requires dd command in the chroot") raise AnsibleError("chroot connection requires dd command in the chroot")
with open(to_bytes(out_path, errors='strict'), 'wb+') as out_file: with open(to_bytes(out_path, errors='surrogate_or_strict'), 'wb+') as out_file:
try: try:
chunk = p.stdout.read(BUFSIZE) chunk = p.stdout.read(BUFSIZE)
while chunk: while chunk:

@ -30,8 +30,9 @@ from ansible.compat.six import text_type, binary_type
import ansible.constants as C import ansible.constants as C
from ansible.errors import AnsibleError, AnsibleFileNotFound from ansible.errors import AnsibleError, AnsibleFileNotFound
from ansible.module_utils._text import to_bytes, to_native
from ansible.plugins.connection import ConnectionBase from ansible.plugins.connection import ConnectionBase
from ansible.utils.unicode import to_bytes, to_str
try: try:
from __main__ import display from __main__ import display
@ -122,14 +123,14 @@ class Connection(ConnectionBase):
super(Connection, self).put_file(in_path, out_path) super(Connection, self).put_file(in_path, out_path)
display.vvv(u"PUT {0} TO {1}".format(in_path, out_path), host=self._play_context.remote_addr) display.vvv(u"PUT {0} TO {1}".format(in_path, out_path), host=self._play_context.remote_addr)
if not os.path.exists(to_bytes(in_path, errors='strict')): if not os.path.exists(to_bytes(in_path, errors='surrogate_or_strict')):
raise AnsibleFileNotFound("file or module does not exist: {0}".format(to_str(in_path))) raise AnsibleFileNotFound("file or module does not exist: {0}".format(to_native(in_path)))
try: try:
shutil.copyfile(to_bytes(in_path, errors='strict'), to_bytes(out_path, errors='strict')) shutil.copyfile(to_bytes(in_path, errors='surrogate_or_strict'), to_bytes(out_path, errors='surrogate_or_strict'))
except shutil.Error: except shutil.Error:
raise AnsibleError("failed to copy: {0} and {1} are the same".format(to_str(in_path), to_str(out_path))) raise AnsibleError("failed to copy: {0} and {1} are the same".format(to_native(in_path), to_native(out_path)))
except IOError as e: except IOError as e:
raise AnsibleError("failed to transfer file to {0}: {1}".format(to_str(out_path), to_str(e))) raise AnsibleError("failed to transfer file to {0}: {1}".format(to_native(out_path), to_native(e)))
def fetch_file(self, in_path, out_path): def fetch_file(self, in_path, out_path):
''' fetch a file from local to local -- for copatibility ''' ''' fetch a file from local to local -- for copatibility '''

@ -24,10 +24,6 @@ import traceback
import select import select
import fcntl import fcntl
import errno import errno
from ansible import errors
from ansible import constants as C
from ansible.plugins.connection import ConnectionBase
from ansible.utils.unicode import to_bytes
HAS_LIBLXC = False HAS_LIBLXC = False
try: try:
@ -36,6 +32,12 @@ try:
except ImportError: except ImportError:
pass pass
from ansible import constants as C
from ansible import errors
from ansible.module_utils._text import to_bytes
from ansible.plugins.connection import ConnectionBase
class Connection(ConnectionBase): class Connection(ConnectionBase):
''' Local lxc based connections ''' ''' Local lxc based connections '''
@ -102,8 +104,8 @@ class Connection(ConnectionBase):
''' run a command on the chroot ''' ''' run a command on the chroot '''
super(Connection, self).exec_command(cmd, in_data=in_data, sudoable=sudoable) super(Connection, self).exec_command(cmd, in_data=in_data, sudoable=sudoable)
executable = to_bytes(self._play_context.executable, errors='strict') executable = to_bytes(self._play_context.executable, errors='surrogate_or_strict')
local_cmd = [executable, '-c', to_bytes(cmd, errors='strict')] local_cmd = [executable, '-c', to_bytes(cmd, errors='surrogate_or_strict')]
read_stdout, write_stdout = None, None read_stdout, write_stdout = None, None
read_stderr, write_stderr = None, None read_stderr, write_stderr = None, None
@ -154,8 +156,8 @@ class Connection(ConnectionBase):
''' transfer a file from local to lxc ''' ''' transfer a file from local to lxc '''
super(Connection, self).put_file(in_path, out_path) super(Connection, self).put_file(in_path, out_path)
self._display.vvv("PUT %s TO %s" % (in_path, out_path), host=self.container_name) self._display.vvv("PUT %s TO %s" % (in_path, out_path), host=self.container_name)
in_path = to_bytes(in_path, errors='strict') in_path = to_bytes(in_path, errors='surrogate_or_strict')
out_path = to_bytes(out_path, errors='strict') out_path = to_bytes(out_path, errors='surrogate_or_strict')
if not os.path.exists(in_path): if not os.path.exists(in_path):
msg = "file or module does not exist: %s" % in_path msg = "file or module does not exist: %s" % in_path
@ -182,8 +184,8 @@ class Connection(ConnectionBase):
''' fetch a file from lxc to local ''' ''' fetch a file from lxc to local '''
super(Connection, self).fetch_file(in_path, out_path) super(Connection, self).fetch_file(in_path, out_path)
self._display.vvv("FETCH %s TO %s" % (in_path, out_path), host=self.container_name) self._display.vvv("FETCH %s TO %s" % (in_path, out_path), host=self.container_name)
in_path = to_bytes(in_path, errors='strict') in_path = to_bytes(in_path, errors='surrogate_or_strict')
out_path = to_bytes(out_path, errors='strict') out_path = to_bytes(out_path, errors='surrogate_or_strict')
try: try:
dst_file = open(out_path, "wb") dst_file = open(out_path, "wb")

@ -23,8 +23,8 @@ from distutils.spawn import find_executable
from subprocess import call, Popen, PIPE from subprocess import call, Popen, PIPE
from ansible.errors import AnsibleError, AnsibleConnectionFailure, AnsibleFileNotFound from ansible.errors import AnsibleError, AnsibleConnectionFailure, AnsibleFileNotFound
from ansible.module_utils._text import to_bytes, to_text
from ansible.plugins.connection import ConnectionBase from ansible.plugins.connection import ConnectionBase
from ansible.utils.unicode import to_bytes, to_unicode
class Connection(ConnectionBase): class Connection(ConnectionBase):
@ -61,14 +61,14 @@ class Connection(ConnectionBase):
local_cmd = [self._lxc_cmd, "exec", self._host, "--", self._play_context.executable, "-c", cmd] local_cmd = [self._lxc_cmd, "exec", self._host, "--", self._play_context.executable, "-c", cmd]
local_cmd = [to_bytes(i, errors='strict') for i in local_cmd] local_cmd = [to_bytes(i, errors='surrogate_or_strict') for i in local_cmd]
in_data = to_bytes(in_data, errors='strict', nonstring='passthru') in_data = to_bytes(in_data, errors='surrogate_or_strict', nonstring='passthru')
process = Popen(local_cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE) process = Popen(local_cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE)
stdout, stderr = process.communicate(in_data) stdout, stderr = process.communicate(in_data)
stdout = to_unicode(stdout) stdout = to_text(stdout)
stderr = to_unicode(stderr) stderr = to_text(stderr)
if stderr == "error: Container is not running.\n": if stderr == "error: Container is not running.\n":
raise AnsibleConnectionFailure("container not running: %s" % self._host) raise AnsibleConnectionFailure("container not running: %s" % self._host)
@ -84,12 +84,12 @@ class Connection(ConnectionBase):
self._display.vvv(u"PUT {0} TO {1}".format(in_path, out_path), host=self._host) self._display.vvv(u"PUT {0} TO {1}".format(in_path, out_path), host=self._host)
if not os.path.isfile(to_bytes(in_path, errors='strict')): if not os.path.isfile(to_bytes(in_path, errors='surrogate_or_strict')):
raise AnsibleFileNotFound("input path is not a file: %s" % in_path) raise AnsibleFileNotFound("input path is not a file: %s" % in_path)
local_cmd = [self._lxc_cmd, "file", "push", in_path, self._host + "/" + out_path] local_cmd = [self._lxc_cmd, "file", "push", in_path, self._host + "/" + out_path]
local_cmd = [to_bytes(i, errors='strict') for i in local_cmd] local_cmd = [to_bytes(i, errors='surrogate_or_strict') for i in local_cmd]
call(local_cmd) call(local_cmd)
@ -101,7 +101,7 @@ class Connection(ConnectionBase):
local_cmd = [self._lxc_cmd, "file", "pull", self._host + "/" + in_path, out_path] local_cmd = [self._lxc_cmd, "file", "pull", self._host + "/" + in_path, out_path]
local_cmd = [to_bytes(i, errors='strict') for i in local_cmd] local_cmd = [to_bytes(i, errors='surrogate_or_strict') for i in local_cmd]
call(local_cmd) call(local_cmd)

@ -44,7 +44,7 @@ from ansible.compat.six.moves import input
from ansible.errors import AnsibleError, AnsibleConnectionFailure, AnsibleFileNotFound from ansible.errors import AnsibleError, AnsibleConnectionFailure, AnsibleFileNotFound
from ansible.plugins.connection import ConnectionBase from ansible.plugins.connection import ConnectionBase
from ansible.utils.path import makedirs_safe from ansible.utils.path import makedirs_safe
from ansible.utils.unicode import to_bytes from ansible.module_utils._text import to_bytes
try: try:
from __main__ import display from __main__ import display
@ -52,6 +52,7 @@ except ImportError:
from ansible.utils.display import Display from ansible.utils.display import Display
display = Display() display = Display()
AUTHENTICITY_MSG=""" AUTHENTICITY_MSG="""
paramiko: The authenticity of host '%s' can't be established. paramiko: The authenticity of host '%s' can't be established.
The %s key fingerprint is %s. The %s key fingerprint is %s.
@ -273,7 +274,7 @@ class Connection(ConnectionBase):
display.vvv("EXEC %s" % cmd, host=self._play_context.remote_addr) display.vvv("EXEC %s" % cmd, host=self._play_context.remote_addr)
cmd = to_bytes(cmd, errors='strict') cmd = to_bytes(cmd, errors='surrogate_or_strict')
no_prompt_out = b'' no_prompt_out = b''
no_prompt_err = b'' no_prompt_err = b''
@ -330,7 +331,7 @@ class Connection(ConnectionBase):
display.vvv("PUT %s TO %s" % (in_path, out_path), host=self._play_context.remote_addr) display.vvv("PUT %s TO %s" % (in_path, out_path), host=self._play_context.remote_addr)
if not os.path.exists(to_bytes(in_path, errors='strict')): if not os.path.exists(to_bytes(in_path, errors='surrogate_or_strict')):
raise AnsibleFileNotFound("file or module does not exist: %s" % in_path) raise AnsibleFileNotFound("file or module does not exist: %s" % in_path)
try: try:
@ -339,7 +340,7 @@ class Connection(ConnectionBase):
raise AnsibleError("failed to open a SFTP connection (%s)" % e) raise AnsibleError("failed to open a SFTP connection (%s)" % e)
try: try:
self.sftp.put(to_bytes(in_path, errors='strict'), to_bytes(out_path, errors='strict')) self.sftp.put(to_bytes(in_path, errors='surrogate_or_strict'), to_bytes(out_path, errors='surrogate_or_strict'))
except IOError: except IOError:
raise AnsibleError("failed to transfer file to %s" % out_path) raise AnsibleError("failed to transfer file to %s" % out_path)
@ -365,7 +366,7 @@ class Connection(ConnectionBase):
raise AnsibleError("failed to open a SFTP connection (%s)", e) raise AnsibleError("failed to open a SFTP connection (%s)", e)
try: try:
self.sftp.get(to_bytes(in_path, errors='strict'), to_bytes(out_path, errors='strict')) self.sftp.get(to_bytes(in_path, errors='surrogate_or_strict'), to_bytes(out_path, errors='surrogate_or_strict'))
except IOError: except IOError:
raise AnsibleError("failed to transfer file from %s" % in_path) raise AnsibleError("failed to transfer file from %s" % in_path)

@ -29,11 +29,11 @@ import subprocess
import time import time
from ansible import constants as C from ansible import constants as C
from ansible.compat.six import text_type, binary_type
from ansible.errors import AnsibleError, AnsibleConnectionFailure, AnsibleFileNotFound from ansible.errors import AnsibleError, AnsibleConnectionFailure, AnsibleFileNotFound
from ansible.module_utils._text import to_bytes, to_native, to_text
from ansible.plugins.connection import ConnectionBase from ansible.plugins.connection import ConnectionBase
from ansible.utils.path import unfrackpath, makedirs_safe from ansible.utils.path import unfrackpath, makedirs_safe
from ansible.utils.unicode import to_bytes, to_unicode, to_str
from ansible.compat.six import text_type, binary_type
try: try:
from __main__ import display from __main__ import display
@ -107,7 +107,7 @@ class Connection(ConnectionBase):
explanation of why they were added. explanation of why they were added.
""" """
self._command += args self._command += args
display.vvvvv('SSH: ' + explanation + ': (%s)' % ')('.join(map(to_unicode, args)), host=self._play_context.remote_addr) display.vvvvv('SSH: ' + explanation + ': (%s)' % ')('.join(map(to_text, args)), host=self._play_context.remote_addr)
def _build_command(self, binary, *other_args): def _build_command(self, binary, *other_args):
''' '''
@ -222,7 +222,7 @@ class Connection(ConnectionBase):
# The directory must exist and be writable. # The directory must exist and be writable.
makedirs_safe(b_cpdir, 0o700) makedirs_safe(b_cpdir, 0o700)
if not os.access(b_cpdir, os.W_OK): if not os.access(b_cpdir, os.W_OK):
raise AnsibleError("Cannot write to ControlPath %s" % to_str(cpdir)) raise AnsibleError("Cannot write to ControlPath %s" % to_native(cpdir))
args = ("-o", "ControlPath=" + C.ANSIBLE_SSH_CONTROL_PATH % dict(directory=cpdir)) args = ("-o", "ControlPath=" + C.ANSIBLE_SSH_CONTROL_PATH % dict(directory=cpdir))
self._add_args("found only ControlPersist; added ControlPath", args) self._add_args("found only ControlPersist; added ControlPath", args)
@ -275,7 +275,7 @@ class Connection(ConnectionBase):
output = [] output = []
for b_line in b_chunk.splitlines(True): for b_line in b_chunk.splitlines(True):
display_line = to_unicode(b_line, errors='replace').rstrip('\r\n') display_line = to_text(b_line).rstrip('\r\n')
suppress_output = False suppress_output = False
#display.debug("Examining line (source=%s, state=%s): '%s'" % (source, state, display_line)) #display.debug("Examining line (source=%s, state=%s): '%s'" % (source, state, display_line))
@ -314,7 +314,7 @@ class Connection(ConnectionBase):
Starts the command and communicates with it until it ends. Starts the command and communicates with it until it ends.
''' '''
display_cmd = list(map(pipes.quote, map(to_unicode, cmd))) display_cmd = list(map(pipes.quote, map(to_text, cmd)))
display.vvv(u'SSH: EXEC {0}'.format(u' '.join(display_cmd)), host=self.host) display.vvv(u'SSH: EXEC {0}'.format(u' '.join(display_cmd)), host=self.host)
# Start the given command. If we don't need to pipeline data, we can try # Start the given command. If we don't need to pipeline data, we can try
@ -424,7 +424,7 @@ class Connection(ConnectionBase):
if p.poll() is not None: if p.poll() is not None:
break break
self._terminate_process(p) self._terminate_process(p)
raise AnsibleError('Timeout (%ds) waiting for privilege escalation prompt: %s' % (timeout, to_str(b_stdout))) raise AnsibleError('Timeout (%ds) waiting for privilege escalation prompt: %s' % (timeout, to_native(b_stdout)))
# Read whatever output is available on stdout and stderr, and stop # Read whatever output is available on stdout and stderr, and stop
# listening to the pipe if it's been closed. # listening to the pipe if it's been closed.
@ -434,14 +434,14 @@ class Connection(ConnectionBase):
if b_chunk == b'': if b_chunk == b'':
rpipes.remove(p.stdout) rpipes.remove(p.stdout)
b_tmp_stdout += b_chunk b_tmp_stdout += b_chunk
display.debug("stdout chunk (state=%s):\n>>>%s<<<\n" % (state, to_unicode(b_chunk, errors='replace'))) display.debug("stdout chunk (state=%s):\n>>>%s<<<\n" % (state, to_text(b_chunk)))
if p.stderr in rfd: if p.stderr in rfd:
b_chunk = p.stderr.read() b_chunk = p.stderr.read()
if b_chunk == b'': if b_chunk == b'':
rpipes.remove(p.stderr) rpipes.remove(p.stderr)
b_tmp_stderr += b_chunk b_tmp_stderr += b_chunk
display.debug("stderr chunk (state=%s):\n>>>%s<<<\n" % (state, to_unicode(b_chunk, errors='replace'))) display.debug("stderr chunk (state=%s):\n>>>%s<<<\n" % (state, to_text(b_chunk)))
# We examine the output line-by-line until we have negotiated any # We examine the output line-by-line until we have negotiated any
# privilege escalation prompt and subsequent success/error message. # privilege escalation prompt and subsequent success/error message.
@ -631,8 +631,8 @@ class Connection(ConnectionBase):
super(Connection, self).put_file(in_path, out_path) super(Connection, self).put_file(in_path, out_path)
display.vvv(u"PUT {0} TO {1}".format(in_path, out_path), host=self.host) display.vvv(u"PUT {0} TO {1}".format(in_path, out_path), host=self.host)
if not os.path.exists(to_bytes(in_path, errors='strict')): if not os.path.exists(to_bytes(in_path, errors='surrogate_or_strict')):
raise AnsibleFileNotFound("file or module does not exist: {0}".format(to_str(in_path))) raise AnsibleFileNotFound("file or module does not exist: {0}".format(to_native(in_path)))
# scp and sftp require square brackets for IPv6 addresses, but # scp and sftp require square brackets for IPv6 addresses, but
# accept them for hostnames and IPv4 addresses too. # accept them for hostnames and IPv4 addresses too.
@ -649,7 +649,7 @@ class Connection(ConnectionBase):
(returncode, stdout, stderr) = self._run(cmd, in_data) (returncode, stdout, stderr) = self._run(cmd, in_data)
if returncode != 0: if returncode != 0:
raise AnsibleError("failed to transfer file to {0}:\n{1}\n{2}".format(to_str(out_path), to_str(stdout), to_str(stderr))) raise AnsibleError("failed to transfer file to {0}:\n{1}\n{2}".format(to_native(out_path), to_native(stdout), to_native(stderr)))
def fetch_file(self, in_path, out_path): def fetch_file(self, in_path, out_path):
''' fetch a file from remote to local ''' ''' fetch a file from remote to local '''

@ -26,9 +26,21 @@ import shlex
import traceback import traceback
import json import json
HAVE_KERBEROS = False
try:
import kerberos
HAVE_KERBEROS = True
except ImportError:
pass
from ansible.compat.six import string_types from ansible.compat.six import string_types
from ansible.compat.six.moves.urllib.parse import urlunsplit from ansible.compat.six.moves.urllib.parse import urlunsplit
from ansible.errors import AnsibleError, AnsibleConnectionFailure from ansible.errors import AnsibleError, AnsibleConnectionFailure
from ansible.errors import AnsibleFileNotFound
from ansible.module_utils._text import to_bytes, to_native, to_text
from ansible.plugins.connection import ConnectionBase
from ansible.utils.hashing import secure_hash
from ansible.utils.path import makedirs_safe
try: try:
import winrm import winrm
@ -42,25 +54,13 @@ try:
except ImportError: except ImportError:
raise AnsibleError("xmltodict is not installed") raise AnsibleError("xmltodict is not installed")
HAVE_KERBEROS = False
try:
import kerberos
HAVE_KERBEROS = True
except ImportError:
pass
from ansible.errors import AnsibleFileNotFound
from ansible.plugins.connection import ConnectionBase
from ansible.utils.hashing import secure_hash
from ansible.utils.path import makedirs_safe
from ansible.utils.unicode import to_bytes, to_unicode, to_str
try: try:
from __main__ import display from __main__ import display
except ImportError: except ImportError:
from ansible.utils.display import Display from ansible.utils.display import Display
display = Display() display = Display()
class Connection(ConnectionBase): class Connection(ConnectionBase):
'''WinRM connections over HTTP/HTTPS.''' '''WinRM connections over HTTP/HTTPS.'''
@ -156,10 +156,10 @@ class Connection(ConnectionBase):
return protocol return protocol
except Exception as e: except Exception as e:
err_msg = to_unicode(e).strip() err_msg = to_text(e).strip()
if re.search(to_unicode(r'Operation\s+?timed\s+?out'), err_msg, re.I): if re.search(to_text(r'Operation\s+?timed\s+?out'), err_msg, re.I):
raise AnsibleError('the connection attempt timed out') raise AnsibleError('the connection attempt timed out')
m = re.search(to_unicode(r'Code\s+?(\d{3})'), err_msg) m = re.search(to_text(r'Code\s+?(\d{3})'), err_msg)
if m: if m:
code = int(m.groups()[0]) code = int(m.groups()[0])
if code == 401: if code == 401:
@ -167,9 +167,9 @@ class Connection(ConnectionBase):
elif code == 411: elif code == 411:
return protocol return protocol
errors.append(u'%s: %s' % (transport, err_msg)) errors.append(u'%s: %s' % (transport, err_msg))
display.vvvvv(u'WINRM CONNECTION ERROR: %s\n%s' % (err_msg, to_unicode(traceback.format_exc())), host=self._winrm_host) display.vvvvv(u'WINRM CONNECTION ERROR: %s\n%s' % (err_msg, to_text(traceback.format_exc())), host=self._winrm_host)
if errors: if errors:
raise AnsibleConnectionFailure(', '.join(map(to_str, errors))) raise AnsibleConnectionFailure(', '.join(map(to_native, errors)))
else: else:
raise AnsibleError('No transport found for WinRM connection') raise AnsibleError('No transport found for WinRM connection')
@ -220,12 +220,12 @@ class Connection(ConnectionBase):
# TODO: check result from response and set stdin_push_failed if we have nonzero # TODO: check result from response and set stdin_push_failed if we have nonzero
if from_exec: if from_exec:
display.vvvvv('WINRM RESULT %r' % to_unicode(response), host=self._winrm_host) display.vvvvv('WINRM RESULT %r' % to_text(response), host=self._winrm_host)
else: else:
display.vvvvvv('WINRM RESULT %r' % to_unicode(response), host=self._winrm_host) display.vvvvvv('WINRM RESULT %r' % to_text(response), host=self._winrm_host)
display.vvvvvv('WINRM STDOUT %s' % to_unicode(response.std_out), host=self._winrm_host) display.vvvvvv('WINRM STDOUT %s' % to_text(response.std_out), host=self._winrm_host)
display.vvvvvv('WINRM STDERR %s' % to_unicode(response.std_err), host=self._winrm_host) display.vvvvvv('WINRM STDERR %s' % to_text(response.std_err), host=self._winrm_host)
if stdin_push_failed: if stdin_push_failed:
raise AnsibleError('winrm send_input failed; \nstdout: %s\nstderr %s' % (response.std_out, response.std_err)) raise AnsibleError('winrm send_input failed; \nstdout: %s\nstderr %s' % (response.std_out, response.std_err))
@ -250,7 +250,7 @@ class Connection(ConnectionBase):
def exec_command(self, cmd, in_data=None, sudoable=True): def exec_command(self, cmd, in_data=None, sudoable=True):
super(Connection, self).exec_command(cmd, in_data=in_data, sudoable=sudoable) super(Connection, self).exec_command(cmd, in_data=in_data, sudoable=sudoable)
cmd_parts = shlex.split(to_bytes(cmd), posix=False) cmd_parts = shlex.split(to_bytes(cmd), posix=False)
cmd_parts = map(to_unicode, cmd_parts) cmd_parts = map(to_text, cmd_parts)
script = None script = None
cmd_ext = cmd_parts and self._shell._unquote(cmd_parts[0]).lower()[-4:] or '' cmd_ext = cmd_parts and self._shell._unquote(cmd_parts[0]).lower()[-4:] or ''
# Support running .ps1 files (via script/raw). # Support running .ps1 files (via script/raw).
@ -266,7 +266,7 @@ class Connection(ConnectionBase):
cmd_parts = self._shell._encode_script(script, as_list=True, strict_mode=False) cmd_parts = self._shell._encode_script(script, as_list=True, strict_mode=False)
if '-EncodedCommand' in cmd_parts: if '-EncodedCommand' in cmd_parts:
encoded_cmd = cmd_parts[cmd_parts.index('-EncodedCommand') + 1] encoded_cmd = cmd_parts[cmd_parts.index('-EncodedCommand') + 1]
decoded_cmd = to_unicode(base64.b64decode(encoded_cmd).decode('utf-16-le')) decoded_cmd = to_text(base64.b64decode(encoded_cmd).decode('utf-16-le'))
display.vvv("EXEC %s" % decoded_cmd, host=self._winrm_host) display.vvv("EXEC %s" % decoded_cmd, host=self._winrm_host)
else: else:
display.vvv("EXEC %s" % cmd, host=self._winrm_host) display.vvv("EXEC %s" % cmd, host=self._winrm_host)
@ -300,9 +300,9 @@ class Connection(ConnectionBase):
# FUTURE: determine buffer size at runtime via remote winrm config? # FUTURE: determine buffer size at runtime via remote winrm config?
def _put_file_stdin_iterator(self, in_path, out_path, buffer_size=250000): def _put_file_stdin_iterator(self, in_path, out_path, buffer_size=250000):
in_size = os.path.getsize(to_bytes(in_path, errors='strict')) in_size = os.path.getsize(to_bytes(in_path, errors='surrogate_or_strict'))
offset = 0 offset = 0
with open(to_bytes(in_path, errors='strict'), 'rb') as in_file: with open(to_bytes(in_path, errors='surrogate_or_strict'), 'rb') as in_file:
for out_data in iter((lambda:in_file.read(buffer_size)), ''): for out_data in iter((lambda:in_file.read(buffer_size)), ''):
offset += len(out_data) offset += len(out_data)
self._display.vvvvv('WINRM PUT "%s" to "%s" (offset=%d size=%d)' % (in_path, out_path, offset, len(out_data)), host=self._winrm_host) self._display.vvvvv('WINRM PUT "%s" to "%s" (offset=%d size=%d)' % (in_path, out_path, offset, len(out_data)), host=self._winrm_host)
@ -318,7 +318,7 @@ class Connection(ConnectionBase):
super(Connection, self).put_file(in_path, out_path) super(Connection, self).put_file(in_path, out_path)
out_path = self._shell._unquote(out_path) out_path = self._shell._unquote(out_path)
display.vvv('PUT "%s" TO "%s"' % (in_path, out_path), host=self._winrm_host) display.vvv('PUT "%s" TO "%s"' % (in_path, out_path), host=self._winrm_host)
if not os.path.exists(to_bytes(in_path, errors='strict')): if not os.path.exists(to_bytes(in_path, errors='surrogate_or_strict')):
raise AnsibleFileNotFound('file or module does not exist: "%s"' % in_path) raise AnsibleFileNotFound('file or module does not exist: "%s"' % in_path)
script_template = u''' script_template = u'''
@ -357,7 +357,7 @@ class Connection(ConnectionBase):
result = self._winrm_exec(cmd_parts[0], cmd_parts[1:], stdin_iterator=self._put_file_stdin_iterator(in_path, out_path)) result = self._winrm_exec(cmd_parts[0], cmd_parts[1:], stdin_iterator=self._put_file_stdin_iterator(in_path, out_path))
# TODO: improve error handling # TODO: improve error handling
if result.status_code != 0: if result.status_code != 0:
raise AnsibleError(to_str(result.std_err)) raise AnsibleError(to_native(result.std_err))
put_output = json.loads(result.std_out) put_output = json.loads(result.std_out)
remote_sha1 = put_output.get("sha1") remote_sha1 = put_output.get("sha1")
@ -368,7 +368,7 @@ class Connection(ConnectionBase):
local_sha1 = secure_hash(in_path) local_sha1 = secure_hash(in_path)
if not remote_sha1 == local_sha1: if not remote_sha1 == local_sha1:
raise AnsibleError("Remote sha1 hash {0} does not match local hash {1}".format(to_str(remote_sha1), to_str(local_sha1))) raise AnsibleError("Remote sha1 hash {0} does not match local hash {1}".format(to_native(remote_sha1), to_native(local_sha1)))
def fetch_file(self, in_path, out_path): def fetch_file(self, in_path, out_path):
super(Connection, self).fetch_file(in_path, out_path) super(Connection, self).fetch_file(in_path, out_path)
@ -407,7 +407,7 @@ class Connection(ConnectionBase):
cmd_parts = self._shell._encode_script(script, as_list=True) cmd_parts = self._shell._encode_script(script, as_list=True)
result = self._winrm_exec(cmd_parts[0], cmd_parts[1:]) result = self._winrm_exec(cmd_parts[0], cmd_parts[1:])
if result.status_code != 0: if result.status_code != 0:
raise IOError(to_str(result.std_err)) raise IOError(to_native(result.std_err))
if result.std_out.strip() == '[DIR]': if result.std_out.strip() == '[DIR]':
data = None data = None
else: else:
@ -418,9 +418,9 @@ class Connection(ConnectionBase):
else: else:
if not out_file: if not out_file:
# If out_path is a directory and we're expecting a file, bail out now. # If out_path is a directory and we're expecting a file, bail out now.
if os.path.isdir(to_bytes(out_path, errors='strict')): if os.path.isdir(to_bytes(out_path, errors='surrogate_or_strict')):
break break
out_file = open(to_bytes(out_path, errors='strict'), 'wb') out_file = open(to_bytes(out_path, errors='surrogate_or_strict'), 'wb')
out_file.write(data) out_file.write(data)
if len(data) < buffer_size: if len(data) < buffer_size:
break break

@ -31,7 +31,8 @@ import traceback
from ansible import constants as C from ansible import constants as C
from ansible.errors import AnsibleError from ansible.errors import AnsibleError
from ansible.plugins.connection import ConnectionBase, BUFSIZE from ansible.plugins.connection import ConnectionBase, BUFSIZE
from ansible.utils.unicode import to_bytes from ansible.module_utils._text import to_bytes
try: try:
from __main__ import display from __main__ import display

@ -19,7 +19,6 @@
from __future__ import (absolute_import, division, print_function) from __future__ import (absolute_import, division, print_function)
__metaclass__ = type __metaclass__ = type
import sys import sys
import base64 import base64
import itertools import itertools
@ -39,15 +38,6 @@ import uuid
import yaml import yaml
from jinja2.filters import environmentfilter from jinja2.filters import environmentfilter
from ansible.compat.six import iteritems, string_types
from ansible import errors
from ansible.parsing.yaml.dumper import AnsibleDumper
from ansible.utils.hashing import md5s, checksum_s
from ansible.utils.unicode import unicode_wrap, to_unicode
from ansible.utils.vars import merge_hash
from ansible.vars.hostvars import HostVars
from ansible.compat.six.moves import reduce
try: try:
import passlib.hash import passlib.hash
@ -55,6 +45,16 @@ try:
except: except:
HAS_PASSLIB = False HAS_PASSLIB = False
from ansible import errors
from ansible.compat.six import iteritems, string_types
from ansible.compat.six.moves import reduce
from ansible.module_utils._text import to_text
from ansible.parsing.yaml.dumper import AnsibleDumper
from ansible.utils.hashing import md5s, checksum_s
from ansible.utils.unicode import unicode_wrap
from ansible.utils.vars import merge_hash
from ansible.vars.hostvars import HostVars
UUID_NAMESPACE_ANSIBLE = uuid.UUID('361E6D51-FAEC-444A-9079-341386DA8E2E') UUID_NAMESPACE_ANSIBLE = uuid.UUID('361E6D51-FAEC-444A-9079-341386DA8E2E')
@ -72,12 +72,12 @@ class AnsibleJSONEncoder(json.JSONEncoder):
def to_yaml(a, *args, **kw): def to_yaml(a, *args, **kw):
'''Make verbose, human readable yaml''' '''Make verbose, human readable yaml'''
transformed = yaml.dump(a, Dumper=AnsibleDumper, allow_unicode=True, **kw) transformed = yaml.dump(a, Dumper=AnsibleDumper, allow_unicode=True, **kw)
return to_unicode(transformed) return to_text(transformed)
def to_nice_yaml(a, indent=4, *args, **kw): def to_nice_yaml(a, indent=4, *args, **kw):
'''Make verbose, human readable yaml''' '''Make verbose, human readable yaml'''
transformed = yaml.dump(a, Dumper=AnsibleDumper, indent=indent, allow_unicode=True, default_flow_style=False, **kw) transformed = yaml.dump(a, Dumper=AnsibleDumper, indent=indent, allow_unicode=True, default_flow_style=False, **kw)
return to_unicode(transformed) return to_text(transformed)
def to_json(a, *args, **kw): def to_json(a, *args, **kw):
''' Convert the value to JSON ''' ''' Convert the value to JSON '''
@ -132,7 +132,7 @@ def fileglob(pathname):
def regex_replace(value='', pattern='', replacement='', ignorecase=False): def regex_replace(value='', pattern='', replacement='', ignorecase=False):
''' Perform a `re.sub` returning a string ''' ''' Perform a `re.sub` returning a string '''
value = to_unicode(value, errors='strict', nonstring='simplerepr') value = to_text(value, errors='surrogate_or_strict', nonstring='simplerepr')
if ignorecase: if ignorecase:
flags = re.I flags = re.I

@ -98,8 +98,8 @@ class LookupBase(with_metaclass(ABCMeta, object)):
must be converted into python's unicode type as the strings will be run must be converted into python's unicode type as the strings will be run
through jinja2 which has this requirement. You can use:: through jinja2 which has this requirement. You can use::
from ansible.utils.unicode import to_unicode from ansible.module_utils._text import to_text
result_string = to_unicode(result_string) result_string = to_text(result_string)
""" """
pass pass

@ -22,7 +22,8 @@ import csv
from ansible.errors import AnsibleError from ansible.errors import AnsibleError
from ansible.plugins.lookup import LookupBase from ansible.plugins.lookup import LookupBase
from ansible.utils.unicode import to_bytes, to_str, to_unicode from ansible.module_utils._text import to_bytes, to_native, to_text
class CSVRecoder: class CSVRecoder:
""" """
@ -49,7 +50,7 @@ class CSVReader:
def next(self): def next(self):
row = self.reader.next() row = self.reader.next()
return [to_unicode(s) for s in row] return [to_text(s) for s in row]
def __iter__(self): def __iter__(self):
return self return self
@ -66,7 +67,7 @@ class LookupModule(LookupBase):
if row[0] == key: if row[0] == key:
return row[int(col)] return row[int(col)]
except Exception as e: except Exception as e:
raise AnsibleError("csvfile: %s" % to_str(e)) raise AnsibleError("csvfile: %s" % to_native(e))
return dflt return dflt

@ -22,7 +22,8 @@ import glob
from ansible.plugins.lookup import LookupBase from ansible.plugins.lookup import LookupBase
from ansible.errors import AnsibleFileNotFound from ansible.errors import AnsibleFileNotFound
from ansible.utils.unicode import to_bytes, to_unicode from ansible.module_utils._text import to_bytes, to_text
class LookupModule(LookupBase): class LookupModule(LookupBase):
@ -36,6 +37,6 @@ class LookupModule(LookupBase):
except AnsibleFileNotFound: except AnsibleFileNotFound:
dwimmed_path = None dwimmed_path = None
if dwimmed_path: if dwimmed_path:
globbed = glob.glob(to_bytes(os.path.join(dwimmed_path, term_file), errors='strict')) globbed = glob.glob(to_bytes(os.path.join(dwimmed_path, term_file), errors='surrogate_or_strict'))
ret.extend(to_unicode(g, errors='strict') for g in globbed if os.path.isfile(g)) ret.extend(to_text(g, errors='surrogate_or_strict') for g in globbed if os.path.isfile(g))
return ret return ret

@ -22,12 +22,6 @@ import pwd
import grp import grp
import stat import stat
from ansible.plugins.lookup import LookupBase
from ansible.utils.unicode import to_str
from __main__ import display
warning = display.warning
HAVE_SELINUX=False HAVE_SELINUX=False
try: try:
import selinux import selinux
@ -35,6 +29,11 @@ try:
except ImportError: except ImportError:
pass pass
from ansible.plugins.lookup import LookupBase
from ansible.module_utils._text import to_native
from __main__ import display
# If selinux fails to find a default, return an array of None # If selinux fails to find a default, return an array of None
def selinux_context(path): def selinux_context(path):
@ -43,7 +42,7 @@ def selinux_context(path):
try: try:
# note: the selinux module uses byte strings on python2 and text # note: the selinux module uses byte strings on python2 and text
# strings on python3 # strings on python3
ret = selinux.lgetfilecon_raw(to_str(path)) ret = selinux.lgetfilecon_raw(to_native(path))
except OSError: except OSError:
return context return context
if ret[0] != -1: if ret[0] != -1:
@ -60,7 +59,7 @@ def file_props(root, path):
try: try:
st = os.lstat(abspath) st = os.lstat(abspath)
except OSError as e: except OSError as e:
warning('filetree: Error using stat() on path %s (%s)' % (abspath, e)) display.warning('filetree: Error using stat() on path %s (%s)' % (abspath, e))
return None return None
ret = dict(root=root, path=path) ret = dict(root=root, path=path)
@ -74,7 +73,7 @@ def file_props(root, path):
ret['state'] = 'file' ret['state'] = 'file'
ret['src'] = abspath ret['src'] = abspath
else: else:
warning('filetree: Error file type of %s is not supported' % abspath) display.warning('filetree: Error file type of %s is not supported' % abspath)
return None return None
ret['uid'] = st.st_uid ret['uid'] = st.st_uid

@ -30,7 +30,7 @@ except ImportError:
from ansible.errors import AnsibleError from ansible.errors import AnsibleError
from ansible.plugins.lookup import LookupBase from ansible.plugins.lookup import LookupBase
from ansible.utils.unicode import to_bytes from ansible.module_utils._text import to_bytes, to_text
def _parse_params(term): def _parse_params(term):
@ -59,13 +59,15 @@ class LookupModule(LookupBase):
def read_properties(self, filename, key, dflt, is_regexp): def read_properties(self, filename, key, dflt, is_regexp):
config = StringIO() config = StringIO()
config.write(u'[java_properties]\n' + open(to_bytes(filename, errors='strict')).read()) current_cfg_file = open(to_bytes(filename, errors='surrogate_or_strict'), 'rb')
config.write(u'[java_properties]\n' + to_text(current_cfg_file.read(), errors='surrogate_or_strict'))
config.seek(0, os.SEEK_SET) config.seek(0, os.SEEK_SET)
self.cp.readfp(config) self.cp.readfp(config)
return self.get_value(key, 'java_properties', dflt, is_regexp) return self.get_value(key, 'java_properties', dflt, is_regexp)
def read_ini(self, filename, key, section, dflt, is_regexp): def read_ini(self, filename, key, section, dflt, is_regexp):
self.cp.readfp(open(to_bytes(filename, errors='strict'))) self.cp.readfp(open(to_bytes(filename, errors='surrogate_or_strict')))
return self.get_value(key, section, dflt, is_regexp) return self.get_value(key, section, dflt, is_regexp)
def get_value(self, key, section, dflt, is_regexp): def get_value(self, key, section, dflt, is_regexp):

@ -21,10 +21,10 @@ import shelve
from ansible.errors import AnsibleError from ansible.errors import AnsibleError
from ansible.plugins.lookup import LookupBase from ansible.plugins.lookup import LookupBase
from ansible.utils.unicode import to_bytes, to_unicode from ansible.module_utils._text import to_bytes, to_text
class LookupModule(LookupBase):
class LookupModule(LookupBase):
def read_shelve(self, shelve_filename, key): def read_shelve(self, shelve_filename, key):
""" """
@ -66,7 +66,7 @@ class LookupModule(LookupBase):
if res is None: if res is None:
raise AnsibleError("Key %s not found in shelve file %s" % (key, file)) raise AnsibleError("Key %s not found in shelve file %s" % (key, file))
# Convert the value read to string # Convert the value read to string
ret.append(to_unicode(res)) ret.append(to_text(res))
break break
else: else:
raise AnsibleError("Could not locate shelve file in lookup: %s" % file) raise AnsibleError("Could not locate shelve file in lookup: %s" % file)

@ -21,7 +21,7 @@ import os
from ansible.errors import AnsibleError from ansible.errors import AnsibleError
from ansible.plugins.lookup import LookupBase from ansible.plugins.lookup import LookupBase
from ansible.utils.unicode import to_unicode, to_bytes from ansible.module_utils._text import to_bytes, to_text
try: try:
from __main__ import display from __main__ import display
@ -43,8 +43,8 @@ class LookupModule(LookupBase):
lookupfile = self.find_file_in_search_path(variables, 'templates', term) lookupfile = self.find_file_in_search_path(variables, 'templates', term)
display.vvvv("File lookup using %s as file" % lookupfile) display.vvvv("File lookup using %s as file" % lookupfile)
if lookupfile: if lookupfile:
with open(to_bytes(lookupfile, errors='strict'), 'r') as f: with open(to_bytes(lookupfile, errors='surrogate_or_strict'), 'rb') as f:
template_data = to_unicode(f.read()) template_data = to_text(f.read(), errors='surrogate_or_strict')
# set jinja2 internal search path for includes # set jinja2 internal search path for includes
if 'ansible_search_path' in variables: if 'ansible_search_path' in variables:

@ -18,11 +18,11 @@ from __future__ import (absolute_import, division, print_function)
__metaclass__ = type __metaclass__ = type
from ansible.compat.six.moves.urllib.error import HTTPError, URLError
from ansible.errors import AnsibleError from ansible.errors import AnsibleError
from ansible.plugins.lookup import LookupBase from ansible.module_utils._text import to_text
from ansible.module_utils.urls import open_url, ConnectionError, SSLValidationError from ansible.module_utils.urls import open_url, ConnectionError, SSLValidationError
from ansible.utils.unicode import to_unicode from ansible.plugins.lookup import LookupBase
from ansible.compat.six.moves.urllib.error import HTTPError, URLError
try: try:
from __main__ import display from __main__ import display
@ -52,5 +52,5 @@ class LookupModule(LookupBase):
raise AnsibleError("Error connecting to %s: %s" % (term, str(e))) raise AnsibleError("Error connecting to %s: %s" % (term, str(e)))
for line in response.read().splitlines(): for line in response.read().splitlines():
ret.append(to_unicode(line)) ret.append(to_text(line))
return ret return ret

@ -22,9 +22,9 @@ import os
import re import re
import shlex import shlex
from ansible.compat.six import text_type
from ansible.errors import AnsibleError from ansible.errors import AnsibleError
from ansible.utils.unicode import to_bytes, to_unicode from ansible.module_utils._text import to_bytes, to_text
_common_args = ['PowerShell', '-NoProfile', '-NonInteractive', '-ExecutionPolicy', 'Unrestricted'] _common_args = ['PowerShell', '-NoProfile', '-NonInteractive', '-ExecutionPolicy', 'Unrestricted']
@ -34,6 +34,7 @@ _powershell_version = os.environ.get('POWERSHELL_VERSION', None)
if _powershell_version: if _powershell_version:
_common_args = ['PowerShell', '-Version', _powershell_version] + _common_args[1:] _common_args = ['PowerShell', '-Version', _powershell_version] + _common_args[1:]
class ShellModule(object): class ShellModule(object):
# Common shell filenames that this plugin handles # Common shell filenames that this plugin handles
@ -60,7 +61,7 @@ class ShellModule(object):
raise AnsibleError("PowerShell environment value for key '%s' exceeds 32767 characters in length" % key) raise AnsibleError("PowerShell environment value for key '%s' exceeds 32767 characters in length" % key)
# powershell single quoted literals need single-quote doubling as their only escaping # powershell single quoted literals need single-quote doubling as their only escaping
value = value.replace("'", "''") value = value.replace("'", "''")
return text_type(value) return to_text(value, errors='surrogate_or_strict')
def env_prefix(self, **kwargs): def env_prefix(self, **kwargs):
env = self.env.copy() env = self.env.copy()
@ -164,7 +165,7 @@ class ShellModule(object):
def build_module_command(self, env_string, shebang, cmd, arg_path=None, rm_tmp=None): def build_module_command(self, env_string, shebang, cmd, arg_path=None, rm_tmp=None):
cmd_parts = shlex.split(to_bytes(cmd), posix=False) cmd_parts = shlex.split(to_bytes(cmd), posix=False)
cmd_parts = map(to_unicode, cmd_parts) cmd_parts = map(to_text, cmd_parts)
if shebang and shebang.lower() == '#!powershell': if shebang and shebang.lower() == '#!powershell':
if not self._unquote(cmd_parts[0]).lower().endswith('.ps1'): if not self._unquote(cmd_parts[0]).lower().endswith('.ps1'):
cmd_parts[0] = '"%s.ps1"' % self._unquote(cmd_parts[0]) cmd_parts[0] = '"%s.ps1"' % self._unquote(cmd_parts[0])
@ -219,7 +220,7 @@ class ShellModule(object):
def _unquote(self, value): def _unquote(self, value):
'''Remove any matching quotes that wrap the given value.''' '''Remove any matching quotes that wrap the given value.'''
value = to_unicode(value or '') value = to_text(value or '')
m = re.match(r'^\s*?\'(.*?)\'\s*?$', value) m = re.match(r'^\s*?\'(.*?)\'\s*?$', value)
if m: if m:
return m.group(1) return m.group(1)
@ -244,7 +245,7 @@ class ShellModule(object):
def _encode_script(self, script, as_list=False, strict_mode=True): def _encode_script(self, script, as_list=False, strict_mode=True):
'''Convert a PowerShell script to a single base64-encoded command.''' '''Convert a PowerShell script to a single base64-encoded command.'''
script = to_unicode(script) script = to_text(script)
if strict_mode: if strict_mode:
script = u'Set-StrictMode -Version Latest\r\n%s' % script script = u'Set-StrictMode -Version Latest\r\n%s' % script
script = '\n'.join([x.strip() for x in script.splitlines() if x.strip()]) script = '\n'.join([x.strip() for x in script.splitlines() if x.strip()])

@ -19,18 +19,11 @@
from __future__ import (absolute_import, division, print_function) from __future__ import (absolute_import, division, print_function)
__metaclass__ = type __metaclass__ = type
import json
import time
import zlib
from collections import defaultdict
from jinja2.exceptions import UndefinedError from jinja2.exceptions import UndefinedError
from ansible.compat.six.moves import queue as Queue from ansible.compat.six.moves import queue as Queue
from ansible.compat.six import iteritems, text_type, string_types from ansible.compat.six import iteritems, string_types
from ansible import constants as C
from ansible.errors import AnsibleError, AnsibleParserError, AnsibleUndefinedVariable from ansible.errors import AnsibleError, AnsibleParserError, AnsibleUndefinedVariable
from ansible.executor.play_iterator import PlayIterator
from ansible.executor.task_result import TaskResult from ansible.executor.task_result import TaskResult
from ansible.inventory.host import Host from ansible.inventory.host import Host
from ansible.inventory.group import Group from ansible.inventory.group import Group
@ -38,11 +31,10 @@ from ansible.playbook.helpers import load_list_of_blocks
from ansible.playbook.included_file import IncludedFile from ansible.playbook.included_file import IncludedFile
from ansible.playbook.task_include import TaskInclude from ansible.playbook.task_include import TaskInclude
from ansible.playbook.role_include import IncludeRole from ansible.playbook.role_include import IncludeRole
from ansible.plugins import action_loader, connection_loader, filter_loader, lookup_loader, module_loader, test_loader from ansible.plugins import action_loader
from ansible.template import Templar from ansible.template import Templar
from ansible.utils.unicode import to_unicode
from ansible.vars.unsafe_proxy import wrap_var
from ansible.vars import combine_vars, strip_internal_keys from ansible.vars import combine_vars, strip_internal_keys
from ansible.module_utils._text import to_text
try: try:
@ -138,7 +130,7 @@ class StrategyBase:
ret_results = [] ret_results = []
def get_original_host(host_name): def get_original_host(host_name):
host_name = to_unicode(host_name) host_name = to_text(host_name)
if host_name in self._inventory._hosts_cache: if host_name in self._inventory._hosts_cache:
return self._inventory._hosts_cache[host_name] return self._inventory._hosts_cache[host_name]
else: else:
@ -161,7 +153,7 @@ class StrategyBase:
target_handler_name = templar.template(handler_task.get_name()) target_handler_name = templar.template(handler_task.get_name())
if target_handler_name == handler_name: if target_handler_name == handler_name:
return handler_task return handler_task
except (UndefinedError, AnsibleUndefinedVariable) as e: except (UndefinedError, AnsibleUndefinedVariable):
# We skip this handler due to the fact that it may be using # We skip this handler due to the fact that it may be using
# a variable in the name that was conditionally included via # a variable in the name that was conditionally included via
# set_fact or some other method, and we don't want to error # set_fact or some other method, and we don't want to error
@ -182,7 +174,7 @@ class StrategyBase:
target_handler_name = templar.template(target_handler.get_name()) target_handler_name = templar.template(target_handler.get_name())
if target_handler_name == handler_name: if target_handler_name == handler_name:
return True return True
except (UndefinedError, AnsibleUndefinedVariable) as e: except (UndefinedError, AnsibleUndefinedVariable):
pass pass
return parent_handler_match(target_handler._parent, handler_name) return parent_handler_match(target_handler._parent, handler_name)
else: else:
@ -567,14 +559,13 @@ class StrategyBase:
# mark all of the hosts including this file as failed, send callbacks, # mark all of the hosts including this file as failed, send callbacks,
# and increment the stats for this host # and increment the stats for this host
for host in included_file._hosts: for host in included_file._hosts:
tr = TaskResult(host=host, task=included_file._task, return_data=dict(failed=True, reason=to_unicode(e))) tr = TaskResult(host=host, task=included_file._task, return_data=dict(failed=True, reason=to_text(e)))
iterator.mark_host_failed(host) iterator.mark_host_failed(host)
self._tqm._failed_hosts[host.name] = True self._tqm._failed_hosts[host.name] = True
self._tqm._stats.increment('failures', host.name) self._tqm._stats.increment('failures', host.name)
self._tqm.send_callback('v2_runner_on_failed', tr) self._tqm.send_callback('v2_runner_on_failed', tr)
return [] return []
# finally, send the callback and return the list of blocks loaded # finally, send the callback and return the list of blocks loaded
self._tqm.send_callback('v2_playbook_on_include', included_file) self._tqm.send_callback('v2_playbook_on_include', included_file)
display.debug("done processing included file") display.debug("done processing included file")

@ -26,7 +26,8 @@ from ansible.playbook.included_file import IncludedFile
from ansible.plugins import action_loader from ansible.plugins import action_loader
from ansible.plugins.strategy import StrategyBase from ansible.plugins.strategy import StrategyBase
from ansible.template import Templar from ansible.template import Templar
from ansible.utils.unicode import to_unicode from ansible.module_utils._text import to_text
try: try:
from __main__ import display from __main__ import display
@ -66,8 +67,7 @@ class StrategyModule(StrategyBase):
break break
work_to_do = False # assume we have no more work to do work_to_do = False # assume we have no more work to do
starting_host = last_host # save current position so we know when we've starting_host = last_host # save current position so we know when we've looped back around and need to break
# looped back around and need to break
# try and find an unblocked host with a task to run # try and find an unblocked host with a task to run
host_results = [] host_results = []
@ -109,7 +109,7 @@ class StrategyModule(StrategyBase):
display.debug("done getting variables") display.debug("done getting variables")
try: try:
task.name = to_unicode(templar.template(task.name, fail_on_undefined=False), nonstring='empty') task.name = to_text(templar.template(task.name, fail_on_undefined=False), nonstring='empty')
display.debug("done templating") display.debug("done templating")
except: except:
# just ignore any errors during task name templating, # just ignore any errors during task name templating,
@ -120,10 +120,10 @@ class StrategyModule(StrategyBase):
run_once = templar.template(task.run_once) or action and getattr(action, 'BYPASS_HOST_LOOP', False) run_once = templar.template(task.run_once) or action and getattr(action, 'BYPASS_HOST_LOOP', False)
if run_once: if run_once:
if action and getattr(action, 'BYPASS_HOST_LOOP', False): if action and getattr(action, 'BYPASS_HOST_LOOP', False):
raise AnsibleError("The '%s' module bypasses the host loop, which is currently not supported in the free strategy " \ raise AnsibleError("The '%s' module bypasses the host loop, which is currently not supported in the free strategy "
"and would instead execute for every host in the inventory list." % task.action, obj=task._ds) "and would instead execute for every host in the inventory list." % task.action, obj=task._ds)
else: else:
display.warning("Using run_once with the free strategy is not currently supported. This task will still be " \ display.warning("Using run_once with the free strategy is not currently supported. This task will still be "
"executed for every host in the inventory list.") "executed for every host in the inventory list.")
# check to see if this task should be skipped, due to it being a member of a # check to see if this task should be skipped, due to it being a member of a
@ -143,7 +143,8 @@ class StrategyModule(StrategyBase):
# handle step if needed, skip meta actions as they are used internally # handle step if needed, skip meta actions as they are used internally
if not self._step or self._take_step(task, host_name): if not self._step or self._take_step(task, host_name):
if task.any_errors_fatal: if task.any_errors_fatal:
display.warning("Using any_errors_fatal with the free strategy is not supported, as tasks are executed independently on each host") display.warning("Using any_errors_fatal with the free strategy is not supported,"
" as tasks are executed independently on each host")
self._tqm.send_callback('v2_playbook_on_task_start', task, is_conditional=False) self._tqm.send_callback('v2_playbook_on_task_start', task, is_conditional=False)
self._queue_task(host, task, task_vars, play_context) self._queue_task(host, task, task_vars, play_context)
del task_vars del task_vars

@ -29,7 +29,8 @@ from ansible.playbook.task import Task
from ansible.plugins import action_loader from ansible.plugins import action_loader
from ansible.plugins.strategy import StrategyBase from ansible.plugins.strategy import StrategyBase
from ansible.template import Templar from ansible.template import Templar
from ansible.utils.unicode import to_unicode from ansible.module_utils._text import to_text
try: try:
from __main__ import display from __main__ import display
@ -243,7 +244,7 @@ class StrategyModule(StrategyBase):
saved_name = task.name saved_name = task.name
display.debug("done copying, going to template now") display.debug("done copying, going to template now")
try: try:
task.name = to_unicode(templar.template(task.name, fail_on_undefined=False), nonstring='empty') task.name = to_text(templar.template(task.name, fail_on_undefined=False), nonstring='empty')
display.debug("done templating") display.debug("done templating")
except: except:
# just ignore any errors during task name templating, # just ignore any errors during task name templating,
@ -368,7 +369,7 @@ class StrategyModule(StrategyBase):
for host in included_file._hosts: for host in included_file._hosts:
self._tqm._failed_hosts[host.name] = True self._tqm._failed_hosts[host.name] = True
iterator.mark_host_failed(host) iterator.mark_host_failed(host)
display.error(to_unicode(e), wrap_text=False) display.error(to_text(e), wrap_text=False)
include_failure = True include_failure = True
continue continue

@ -25,8 +25,8 @@ import os
import re import re
from io import StringIO from io import StringIO
from numbers import Number
from ansible.compat.six import string_types, text_type, binary_type
from jinja2 import Environment from jinja2 import Environment
from jinja2.loaders import FileSystemLoader from jinja2.loaders import FileSystemLoader
from jinja2.exceptions import TemplateSyntaxError, UndefinedError from jinja2.exceptions import TemplateSyntaxError, UndefinedError
@ -34,19 +34,20 @@ from jinja2.utils import concat as j2_concat
from jinja2.runtime import StrictUndefined from jinja2.runtime import StrictUndefined
from ansible import constants as C from ansible import constants as C
from ansible.compat.six import string_types, text_type
from ansible.errors import AnsibleError, AnsibleFilterError, AnsibleUndefinedVariable from ansible.errors import AnsibleError, AnsibleFilterError, AnsibleUndefinedVariable
from ansible.plugins import filter_loader, lookup_loader, test_loader from ansible.plugins import filter_loader, lookup_loader, test_loader
from ansible.template.safe_eval import safe_eval from ansible.template.safe_eval import safe_eval
from ansible.template.template import AnsibleJ2Template from ansible.template.template import AnsibleJ2Template
from ansible.template.vars import AnsibleJ2Vars from ansible.template.vars import AnsibleJ2Vars
from ansible.utils.unicode import to_unicode, to_str from ansible.module_utils._text import to_native, to_text
try: try:
from hashlib import sha1 from hashlib import sha1
except ImportError: except ImportError:
from sha import sha as sha1 from sha import sha as sha1
from numbers import Number
try: try:
from __main__ import display from __main__ import display
@ -107,6 +108,7 @@ def _escape_backslashes(data, jinja_env):
return data return data
def _count_newlines_from_end(in_str): def _count_newlines_from_end(in_str):
''' '''
Counts the number of newlines at the end of a string. This is used during Counts the number of newlines at the end of a string. This is used during
@ -255,10 +257,10 @@ class Templar:
if prev_idx is not None: if prev_idx is not None:
# replace the opening # replace the opening
data.seek(prev_idx, os.SEEK_SET) data.seek(prev_idx, os.SEEK_SET)
data.write(to_unicode(self.environment.comment_start_string)) data.write(to_text(self.environment.comment_start_string))
# replace the closing # replace the closing
data.seek(token_start, os.SEEK_SET) data.seek(token_start, os.SEEK_SET)
data.write(to_unicode(self.environment.comment_end_string)) data.write(to_text(self.environment.comment_end_string))
else: else:
raise AnsibleError("Error while cleaning data for safety: unhandled regex match") raise AnsibleError("Error while cleaning data for safety: unhandled regex match")
@ -293,7 +295,7 @@ class Templar:
return self._clean_data(variable) return self._clean_data(variable)
else: else:
# Do we need to convert these into text_type as well? # Do we need to convert these into text_type as well?
# return self._clean_data(to_unicode(variable._obj, nonstring='passthru')) # return self._clean_data(to_text(variable._obj, nonstring='passthru'))
return self._clean_data(variable._obj) return self._clean_data(variable._obj)
try: try:
@ -383,7 +385,7 @@ class Templar:
returns True if the data contains a variable pattern returns True if the data contains a variable pattern
''' '''
if isinstance(data, string_types): if isinstance(data, string_types):
for marker in [self.environment.block_start_string, self.environment.variable_start_string, self.environment.comment_start_string]: for marker in (self.environment.block_start_string, self.environment.variable_start_string, self.environment.comment_start_string):
if marker in data: if marker in data:
return True return True
return False return False
@ -399,7 +401,8 @@ class Templar:
first_part = variable.split("|")[0].split(".")[0].split("[")[0] first_part = variable.split("|")[0].split(".")[0].split("[")[0]
if (contains_filters or first_part in self._available_variables) and self.environment.variable_start_string not in variable: if (contains_filters or first_part in self._available_variables) and self.environment.variable_start_string not in variable:
if bare_deprecated: if bare_deprecated:
display.deprecated("Using bare variables is deprecated. Update your playbooks so that the environment value uses the full variable syntax ('%s%s%s')" % display.deprecated("Using bare variables is deprecated."
" Update your playbooks so that the environment value uses the full variable syntax ('%s%s%s')" %
(self.environment.variable_start_string, variable, self.environment.variable_end_string)) (self.environment.variable_start_string, variable, self.environment.variable_end_string))
return "%s%s%s" % (self.environment.variable_start_string, variable, self.environment.variable_end_string) return "%s%s%s" % (self.environment.variable_start_string, variable, self.environment.variable_end_string)
@ -485,10 +488,10 @@ class Templar:
try: try:
t = myenv.from_string(data) t = myenv.from_string(data)
except TemplateSyntaxError as e: except TemplateSyntaxError as e:
raise AnsibleError("template error while templating string: %s. String: %s" % (to_str(e), to_str(data))) raise AnsibleError("template error while templating string: %s. String: %s" % (to_native(e), to_native(data)))
except Exception as e: except Exception as e:
if 'recursion' in to_str(e): if 'recursion' in to_native(e):
raise AnsibleError("recursive loop detected in template string: %s" % to_str(data)) raise AnsibleError("recursive loop detected in template string: %s" % to_native(data))
else: else:
return data return data
@ -503,13 +506,13 @@ class Templar:
try: try:
res = j2_concat(rf) res = j2_concat(rf)
except TypeError as te: except TypeError as te:
if 'StrictUndefined' in to_str(te): if 'StrictUndefined' in to_native(te):
errmsg = "Unable to look up a name or access an attribute in template string (%s).\n" % to_str(data) errmsg = "Unable to look up a name or access an attribute in template string (%s).\n" % to_native(data)
errmsg += "Make sure your variable name does not contain invalid characters like '-': %s" % to_str(te) errmsg += "Make sure your variable name does not contain invalid characters like '-': %s" % to_native(te)
raise AnsibleUndefinedVariable(errmsg) raise AnsibleUndefinedVariable(errmsg)
else: else:
display.debug("failing because of a type error, template data is: %s" % to_str(data)) display.debug("failing because of a type error, template data is: %s" % to_native(data))
raise AnsibleError("Unexpected templating type error occurred on (%s): %s" % (to_str(data),to_str(te))) raise AnsibleError("Unexpected templating type error occurred on (%s): %s" % (to_native(data),to_native(te)))
if preserve_trailing_newlines: if preserve_trailing_newlines:
# The low level calls above do not preserve the newline # The low level calls above do not preserve the newline
@ -534,4 +537,3 @@ class Templar:
else: else:
#TODO: return warning about undefined var #TODO: return warning about undefined var
return data return data

@ -21,7 +21,8 @@ __metaclass__ = type
from ansible.compat.six import iteritems from ansible.compat.six import iteritems
from jinja2.utils import missing from jinja2.utils import missing
from ansible.utils.unicode import to_unicode from ansible.module_utils._text import to_native
__all__ = ['AnsibleJ2Vars'] __all__ = ['AnsibleJ2Vars']
@ -88,7 +89,7 @@ class AnsibleJ2Vars:
try: try:
value = self._templar.template(variable) value = self._templar.template(variable)
except Exception as e: except Exception as e:
raise type(e)(to_unicode(variable) + ': ' + e.message) raise type(e)(to_native(variable) + ': ' + e.message)
return value return value
def add_locals(self, locals): def add_locals(self, locals):
@ -99,4 +100,3 @@ class AnsibleJ2Vars:
if locals is None: if locals is None:
return self return self
return AnsibleJ2Vars(self._templar, self._globals, locals=locals, *self._extras) return AnsibleJ2Vars(self._templar, self._globals, locals=locals, *self._extras)

@ -35,7 +35,8 @@ from termios import TIOCGWINSZ
from ansible import constants as C from ansible import constants as C
from ansible.errors import AnsibleError from ansible.errors import AnsibleError
from ansible.utils.color import stringc from ansible.utils.color import stringc
from ansible.utils.unicode import to_bytes, to_unicode from ansible.module_utils._text import to_bytes, to_text
try: try:
# Python 2 # Python 2
@ -45,7 +46,6 @@ except NameError:
pass pass
logger = None logger = None
#TODO: make this a logging callback instead #TODO: make this a logging callback instead
if C.DEFAULT_LOG_PATH: if C.DEFAULT_LOG_PATH:
@ -124,7 +124,7 @@ class Display:
# Convert back to text string on python3 # Convert back to text string on python3
# We first convert to a byte string so that we get rid of # We first convert to a byte string so that we get rid of
# characters that are invalid in the user's locale # characters that are invalid in the user's locale
msg2 = to_unicode(msg2, self._output_encoding(stderr=stderr)) msg2 = to_text(msg2, self._output_encoding(stderr=stderr))
if not stderr: if not stderr:
fileobj = sys.stdout fileobj = sys.stdout
@ -149,7 +149,7 @@ class Display:
# Convert back to text string on python3 # Convert back to text string on python3
# We first convert to a byte string so that we get rid of # We first convert to a byte string so that we get rid of
# characters that are invalid in the user's locale # characters that are invalid in the user's locale
msg2 = to_unicode(msg2, self._output_encoding(stderr=stderr)) msg2 = to_text(msg2, self._output_encoding(stderr=stderr))
if color == C.COLOR_ERROR: if color == C.COLOR_ERROR:
logger.error(msg2) logger.error(msg2)
@ -279,7 +279,7 @@ class Display:
if sys.version_info >= (3,): if sys.version_info >= (3,):
# Convert back into text on python3. We do this double conversion # Convert back into text on python3. We do this double conversion
# to get rid of characters that are illegal in the user's locale # to get rid of characters that are illegal in the user's locale
prompt_string = to_unicode(prompt_string) prompt_string = to_text(prompt_string)
if private: if private:
return getpass.getpass(msg) return getpass.getpass(msg)
@ -323,7 +323,7 @@ class Display:
result = do_encrypt(result, encrypt, salt_size, salt) result = do_encrypt(result, encrypt, salt_size, salt)
# handle utf-8 chars # handle utf-8 chars
result = to_unicode(result, errors='strict') result = to_text(result, errors='surrogate_or_strict')
return result return result
@staticmethod @staticmethod

@ -39,14 +39,14 @@ except ImportError:
_md5 = None _md5 = None
from ansible.errors import AnsibleError from ansible.errors import AnsibleError
from ansible.utils.unicode import to_bytes from ansible.module_utils._text import to_bytes
def secure_hash_s(data, hash_func=sha1): def secure_hash_s(data, hash_func=sha1):
''' Return a secure hash hex digest of data. ''' ''' Return a secure hash hex digest of data. '''
digest = hash_func() digest = hash_func()
data = to_bytes(data, errors='strict') data = to_bytes(data, errors='surrogate_or_strict')
digest.update(data) digest.update(data)
return digest.hexdigest() return digest.hexdigest()
@ -54,12 +54,12 @@ def secure_hash_s(data, hash_func=sha1):
def secure_hash(filename, hash_func=sha1): def secure_hash(filename, hash_func=sha1):
''' Return a secure hash hex digest of local file, None if file is not present or a directory. ''' ''' Return a secure hash hex digest of local file, None if file is not present or a directory. '''
if not os.path.exists(to_bytes(filename, errors='strict')) or os.path.isdir(to_bytes(filename, errors='strict')): if not os.path.exists(to_bytes(filename, errors='surrogate_or_strict')) or os.path.isdir(to_bytes(filename, errors='strict')):
return None return None
digest = hash_func() digest = hash_func()
blocksize = 64 * 1024 blocksize = 64 * 1024
try: try:
infile = open(to_bytes(filename, errors='strict'), 'rb') infile = open(to_bytes(filename, errors='surrogate_or_strict'), 'rb')
block = infile.read(blocksize) block = infile.read(blocksize)
while block: while block:
digest.update(block) digest.update(block)

@ -20,11 +20,12 @@ __metaclass__ = type
import os import os
from errno import EEXIST from errno import EEXIST
from ansible.errors import AnsibleError from ansible.errors import AnsibleError
from ansible.utils.unicode import to_bytes, to_str, to_unicode from ansible.module_utils._text import to_bytes, to_native, to_text
from ansible.compat.six import PY2
__all__ = ['unfrackpath', 'makedirs_safe'] __all__ = ['unfrackpath', 'makedirs_safe']
def unfrackpath(path): def unfrackpath(path):
''' '''
Returns a path that is free of symlinks, environment Returns a path that is free of symlinks, environment
@ -40,10 +41,9 @@ def unfrackpath(path):
example:: example::
'$HOME/../../var/mail' becomes '/var/spool/mail' '$HOME/../../var/mail' becomes '/var/spool/mail'
''' '''
canonical_path = os.path.normpath(os.path.realpath(os.path.expanduser(os.path.expandvars(to_bytes(path, errors='strict'))))) canonical_path = os.path.normpath(os.path.realpath(os.path.expanduser(os.path.expandvars(to_bytes(path, errors='surrogate_or_strict')))))
if PY2: return to_text(canonical_path, errors='surrogate_or_strict')
return to_unicode(canonical_path, errors='strict')
return to_unicode(canonical_path, errors='surrogateescape')
def makedirs_safe(path, mode=None): def makedirs_safe(path, mode=None):
'''Safe way to create dirs in muliprocess/thread environments. '''Safe way to create dirs in muliprocess/thread environments.
@ -64,4 +64,4 @@ def makedirs_safe(path, mode=None):
os.makedirs(b_rpath) os.makedirs(b_rpath)
except OSError as e: except OSError as e:
if e.errno != EEXIST: if e.errno != EEXIST:
raise AnsibleError("Unable to create local directories(%s): %s" % (to_str(rpath), to_str(e))) raise AnsibleError("Unable to create local directories(%s): %s" % (to_native(rpath), to_native(e)))

@ -21,8 +21,7 @@ __metaclass__ = type
import shlex import shlex
from ansible.compat.six import PY3 from ansible.compat.six import PY3
from ansible.module_utils._text import to_bytes, to_text
from ansible.utils.unicode import to_bytes, to_unicode
if PY3: if PY3:
@ -31,5 +30,5 @@ if PY3:
else: else:
# shlex.split() wants bytes (i.e. ``str``) input on Python 2 # shlex.split() wants bytes (i.e. ``str``) input on Python 2
def shlex_split(s, comments=False, posix=True): def shlex_split(s, comments=False, posix=True):
return map(to_unicode, shlex.split(to_bytes(s), comments, posix)) return map(to_text, shlex.split(to_bytes(s), comments, posix))
shlex_split.__doc__ = shlex.split.__doc__ shlex_split.__doc__ = shlex.split.__doc__

@ -19,264 +19,48 @@
from __future__ import (absolute_import, division, print_function) from __future__ import (absolute_import, division, print_function)
__metaclass__ = type __metaclass__ = type
from ansible.compat.six import string_types, text_type, binary_type, PY3 from ansible.module_utils._text import to_bytes as _to_bytes, to_text, to_native
# to_bytes and to_unicode were written by Toshio Kuratomi for the
# python-kitchen library https://pypi.python.org/pypi/kitchen
# They are licensed in kitchen under the terms of the GPLv2+
# They were copied and modified for use in ansible by Toshio in Jan 2015
# (simply removing the deprecated features)
#: Aliases for the utf-8 codec
_UTF8_ALIASES = frozenset(('utf-8', 'UTF-8', 'utf8', 'UTF8', 'utf_8', 'UTF_8',
'utf', 'UTF', 'u8', 'U8'))
#: Aliases for the latin-1 codec
_LATIN1_ALIASES = frozenset(('latin-1', 'LATIN-1', 'latin1', 'LATIN1',
'latin', 'LATIN', 'l1', 'L1', 'cp819', 'CP819', '8859', 'iso8859-1',
'ISO8859-1', 'iso-8859-1', 'ISO-8859-1'))
# EXCEPTION_CONVERTERS is defined below due to using to_unicode
def to_unicode(obj, encoding='utf-8', errors='replace', nonstring=None):
'''Convert an object into a :class:`unicode` string
:arg obj: Object to convert to a :class:`unicode` string. This should
normally be a byte :class:`str`
:kwarg encoding: What encoding to try converting the byte :class:`str` as.
Defaults to :term:`utf-8`
:kwarg errors: If errors are found while decoding, perform this action.
Defaults to ``replace`` which replaces the invalid bytes with
a character that means the bytes were unable to be decoded. Other
values are the same as the error handling schemes in the `codec base
classes
<http://docs.python.org/library/codecs.html#codec-base-classes>`_.
For instance ``strict`` which raises an exception and ``ignore`` which
simply omits the non-decodable characters.
:kwarg nonstring: How to treat nonstring values. Possible values are:
:simplerepr: Attempt to call the object's "simple representation"
method and return that value. Python-2.3+ has two methods that
try to return a simple representation: :meth:`object.__unicode__`
and :meth:`object.__str__`. We first try to get a usable value
from :meth:`object.__unicode__`. If that fails we try the same
with :meth:`object.__str__`.
:empty: Return an empty :class:`unicode` string
:strict: Raise a :exc:`TypeError`
:passthru: Return the object unchanged
:repr: Attempt to return a :class:`unicode` string of the repr of the
object
Default is ``simplerepr``
:raises TypeError: if :attr:`nonstring` is ``strict`` and
a non-:class:`basestring` object is passed in or if :attr:`nonstring`
is set to an unknown value
:raises UnicodeDecodeError: if :attr:`errors` is ``strict`` and
:attr:`obj` is not decodable using the given encoding
:returns: :class:`unicode` string or the original object depending on the
value of :attr:`nonstring`.
Usually this should be used on a byte :class:`str` but it can take both
byte :class:`str` and :class:`unicode` strings intelligently. Nonstring
objects are handled in different ways depending on the setting of the
:attr:`nonstring` parameter.
The default values of this function are set so as to always return
a :class:`unicode` string and never raise an error when converting from
a byte :class:`str` to a :class:`unicode` string. However, when you do
not pass validly encoded text (or a nonstring object), you may end up with
output that you don't expect. Be sure you understand the requirements of
your data, not just ignore errors by passing it through this function.
'''
# Could use isbasestring/isunicode here but we want this code to be as
# fast as possible
if isinstance(obj, text_type):
return obj
if isinstance(obj, binary_type):
if encoding in _UTF8_ALIASES:
return text_type(obj, 'utf-8', errors)
if encoding in _LATIN1_ALIASES:
return text_type(obj, 'latin-1', errors)
return obj.decode(encoding, errors)
if not nonstring:
nonstring = 'simplerepr'
if nonstring == 'empty':
return u''
elif nonstring == 'passthru':
return obj
elif nonstring == 'simplerepr':
try: try:
simple = obj.__unicode__() from __main__ import display
except (AttributeError, UnicodeError): except ImportError:
simple = None from ansible.utils.display import Display
if not simple: display = Display()
try:
simple = text_type(obj)
except UnicodeError:
try:
simple = obj.__str__()
except (UnicodeError, AttributeError):
simple = u''
if isinstance(simple, binary_type):
return text_type(simple, encoding, errors)
return simple
elif nonstring in ('repr', 'strict'):
obj_repr = repr(obj)
if isinstance(obj_repr, binary_type):
obj_repr = text_type(obj_repr, encoding, errors)
if nonstring == 'repr':
return obj_repr
raise TypeError('to_unicode was given "%(obj)s" which is neither'
' a byte string (str) or a unicode string' %
{'obj': obj_repr.encode(encoding, 'replace')})
raise TypeError('nonstring value, %(param)s, is not set to a valid'
' action' % {'param': nonstring})
def to_bytes(obj, encoding='utf-8', errors='replace', nonstring=None):
'''Convert an object into a byte :class:`str`
:arg obj: Object to convert to a byte :class:`str`. This should normally __all__ = ('to_bytes', 'to_unicode', 'to_str', 'unicode_wrap')
be a :class:`unicode` string.
:kwarg encoding: Encoding to use to convert the :class:`unicode` string
into a byte :class:`str`. Defaults to :term:`utf-8`.
:kwarg errors: If errors are found while encoding, perform this action.
Defaults to ``replace`` which replaces the invalid bytes with
a character that means the bytes were unable to be encoded. Other
values are the same as the error handling schemes in the `codec base
classes
<http://docs.python.org/library/codecs.html#codec-base-classes>`_.
For instance ``strict`` which raises an exception and ``ignore`` which
simply omits the non-encodable characters.
:kwarg nonstring: How to treat nonstring values. Possible values are:
:simplerepr: Attempt to call the object's "simple representation"
method and return that value. Python-2.3+ has two methods that
try to return a simple representation: :meth:`object.__unicode__`
and :meth:`object.__str__`. We first try to get a usable value
from :meth:`object.__str__`. If that fails we try the same
with :meth:`object.__unicode__`.
:empty: Return an empty byte :class:`str`
:strict: Raise a :exc:`TypeError`
:passthru: Return the object unchanged
:repr: Attempt to return a byte :class:`str` of the :func:`repr` of the
object
Default is ``simplerepr``. ###
### Backwards compat
###
:raises TypeError: if :attr:`nonstring` is ``strict`` and def to_bytes(*args, **kwargs):
a non-:class:`basestring` object is passed in or if :attr:`nonstring` display.deprecated(u'ansible.utils.unicode.to_bytes is deprecated. Use ansible.module_utils._text.to_bytes instead', version=u'2.4')
is set to an unknown value. if 'errors' not in kwargs:
:raises UnicodeEncodeError: if :attr:`errors` is ``strict`` and all of the kwargs['errors'] = 'replace'
bytes of :attr:`obj` are unable to be encoded using :attr:`encoding`. return _to_bytes(*args, **kwargs)
:returns: byte :class:`str` or the original object depending on the value
of :attr:`nonstring`.
.. warning::
If you pass a byte :class:`str` into this function the byte def to_unicode(*args, **kwargs):
:class:`str` is returned unmodified. It is **not** re-encoded with display.deprecated(u'ansible.utils.unicode.to_unicode is deprecated. Use ansible.module_utils._text.to_text instead', version=u'2.4')
the specified :attr:`encoding`. The easiest way to achieve that is:: if 'errors' not in kwargs:
kwargs['errors'] = 'replace'
return to_text(*args, **kwargs)
to_bytes(to_unicode(text), encoding='utf-8')
The initial :func:`to_unicode` call will ensure text is def to_str(*args, **kwargs):
a :class:`unicode` string. Then, :func:`to_bytes` will turn that into display.deprecated(u'ansible.utils.unicode.to_str is deprecated. Use ansible.module_utils._text.to_native instead', version=u'2.4')
a byte :class:`str` with the specified encoding. if 'errors' not in kwargs:
kwargs['errors'] = 'replace'
return to_native(*args, **kwargs)
Usually, this should be used on a :class:`unicode` string but it can take ### End Backwards compat
either a byte :class:`str` or a :class:`unicode` string intelligently.
Nonstring objects are handled in different ways depending on the setting
of the :attr:`nonstring` parameter.
The default values of this function are set so as to always return a byte
:class:`str` and never raise an error when converting from unicode to
bytes. However, when you do not pass an encoding that can validly encode
the object (or a non-string object), you may end up with output that you
don't expect. Be sure you understand the requirements of your data, not
just ignore errors by passing it through this function.
'''
# Could use isbasestring, isbytestring here but we want this to be as fast
# as possible
if isinstance(obj, binary_type):
return obj
if isinstance(obj, text_type):
return obj.encode(encoding, errors)
if not nonstring:
nonstring = 'simplerepr'
if nonstring == 'empty':
return b''
elif nonstring == 'passthru':
return obj
elif nonstring == 'simplerepr':
try:
simple = str(obj)
except UnicodeError:
try:
simple = obj.__str__()
except (AttributeError, UnicodeError):
simple = None
if not simple:
try:
simple = obj.__unicode__()
except (AttributeError, UnicodeError):
simple = b''
if isinstance(simple, text_type):
simple = simple.encode(encoding, 'replace')
return simple
elif nonstring in ('repr', 'strict'):
try:
obj_repr = obj.__repr__()
except (AttributeError, UnicodeError):
obj_repr = b''
if isinstance(obj_repr, text_type):
obj_repr = obj_repr.encode(encoding, errors)
else:
obj_repr = binary_type(obj_repr)
if nonstring == 'repr':
return obj_repr
raise TypeError('to_bytes was given "%(obj)s" which is neither'
' a unicode string or a byte string (str)' % {'obj': obj_repr})
raise TypeError('nonstring value, %(param)s, is not set to a valid'
' action' % {'param': nonstring})
# force the return value of a function to be unicode. Use with partial to
# ensure that a filter will return unicode values.
def unicode_wrap(func, *args, **kwargs): def unicode_wrap(func, *args, **kwargs):
return to_unicode(func(*args, **kwargs), nonstring='passthru') """If a function returns a string, force it to be a text string.
Use with partial to ensure that filter plugins will return text values.
# Alias for converting to native strings. """
# Native strings are the default string type for the particular version of return to_text(func(*args, **kwargs), nonstring='passthru')
# python. The objects are called "str" in both py2 and py3 but they mean
# different things. In py2, it's a byte string like in C. In py3 it's an
# abstract text type (like py2's unicode type).
#
# Use this when raising exceptions and wanting to get the string
# representation of an object for the exception message. For example:
#
# try:
# do_something()
# except Exception as e:
# raise AnsibleError(to_str(e))
#
# Note that this is because python's exception handling expects native strings
# and doe the wrong thing if given the other sort of string (in py2, if given
# unicode strings, it could traceback or omit the message. in py3, if given
# byte strings it prints their repr (so the message ends up as b'message').
#
# If you use ansible's API instead of re-raising an exception, use to_unicode
# instead:
#
# try:
# do_something()
# except Exception as e:
# display.warn(to_unicode(e))
if PY3:
to_str = to_unicode
else:
to_str = to_bytes

@ -28,7 +28,7 @@ from ansible.compat.six import iteritems, string_types
from ansible import constants as C from ansible import constants as C
from ansible.errors import AnsibleError from ansible.errors import AnsibleError
from ansible.parsing.splitter import parse_kv from ansible.parsing.splitter import parse_kv
from ansible.utils.unicode import to_unicode, to_str from ansible.module_utils._text import to_native, to_text
def _validate_mutable_mappings(a, b): def _validate_mutable_mappings(a, b):
@ -49,11 +49,12 @@ def _validate_mutable_mappings(a, b):
try: try:
myvars.append(dumps(x)) myvars.append(dumps(x))
except: except:
myvars.append(to_str(x)) myvars.append(to_native(x))
raise AnsibleError("failed to combine variables, expected dicts but got a '{0}' and a '{1}': \n{2}\n{3}".format( raise AnsibleError("failed to combine variables, expected dicts but got a '{0}' and a '{1}': \n{2}\n{3}".format(
a.__class__.__name__, b.__class__.__name__, myvars[0], myvars[1]) a.__class__.__name__, b.__class__.__name__, myvars[0], myvars[1])
) )
def combine_vars(a, b): def combine_vars(a, b):
""" """
Return a copy of dictionaries of variables based on configured hash behavior Return a copy of dictionaries of variables based on configured hash behavior
@ -68,6 +69,7 @@ def combine_vars(a, b):
result.update(b) result.update(b)
return result return result
def merge_hash(a, b): def merge_hash(a, b):
""" """
Recursively merges hash b into a so that keys from b take precedence over keys from a Recursively merges hash b into a so that keys from b take precedence over keys from a
@ -95,10 +97,11 @@ def merge_hash(a, b):
return result return result
def load_extra_vars(loader, options): def load_extra_vars(loader, options):
extra_vars = {} extra_vars = {}
for extra_vars_opt in options.extra_vars: for extra_vars_opt in options.extra_vars:
extra_vars_opt = to_unicode(extra_vars_opt, errors='strict') extra_vars_opt = to_text(extra_vars_opt, errors='surrogate_or_strict')
if extra_vars_opt.startswith(u"@"): if extra_vars_opt.startswith(u"@"):
# Argument is a YAML file (JSON is a subset of YAML) # Argument is a YAML file (JSON is a subset of YAML)
data = loader.load_from_file(extra_vars_opt[1:]) data = loader.load_from_file(extra_vars_opt[1:])
@ -111,6 +114,7 @@ def load_extra_vars(loader, options):
extra_vars = combine_vars(extra_vars, data) extra_vars = combine_vars(extra_vars, data)
return extra_vars return extra_vars
def load_options_vars(options): def load_options_vars(options):
options_vars = {} options_vars = {}
# For now only return check mode, but we can easily return more # For now only return check mode, but we can easily return more
@ -118,6 +122,7 @@ def load_options_vars(options):
options_vars['ansible_check_mode'] = options.check options_vars['ansible_check_mode'] = options.check
return options_vars return options_vars
def isidentifier(ident): def isidentifier(ident):
""" """
Determines, if string is valid Python identifier using the ast module. Determines, if string is valid Python identifier using the ast module.
@ -148,4 +153,3 @@ def isidentifier(ident):
return False return False
return True return True

@ -54,18 +54,21 @@ from __future__ import (absolute_import, division, print_function)
__metaclass__ = type __metaclass__ = type
import json import json
from ansible.utils.unicode import to_unicode
from ansible.compat.six import string_types, text_type from ansible.compat.six import string_types, text_type
from ansible.module_utils._text import to_text
__all__ = ['UnsafeProxy', 'AnsibleUnsafe', 'AnsibleJSONUnsafeEncoder', 'AnsibleJSONUnsafeDecoder', 'wrap_var'] __all__ = ['UnsafeProxy', 'AnsibleUnsafe', 'AnsibleJSONUnsafeEncoder', 'AnsibleJSONUnsafeDecoder', 'wrap_var']
class AnsibleUnsafe(object): class AnsibleUnsafe(object):
__UNSAFE__ = True __UNSAFE__ = True
class AnsibleUnsafeText(text_type, AnsibleUnsafe): class AnsibleUnsafeText(text_type, AnsibleUnsafe):
pass pass
class UnsafeProxy(object): class UnsafeProxy(object):
def __new__(cls, obj, *args, **kwargs): def __new__(cls, obj, *args, **kwargs):
# In our usage we should only receive unicode strings. # In our usage we should only receive unicode strings.
@ -73,10 +76,11 @@ class UnsafeProxy(object):
# we're given but we may want to take it out for testing and sanitize # we're given but we may want to take it out for testing and sanitize
# our input instead. # our input instead.
if isinstance(obj, string_types): if isinstance(obj, string_types):
obj = to_unicode(obj, errors='strict') obj = to_text(obj, errors='surrogate_or_strict')
return AnsibleUnsafeText(obj) return AnsibleUnsafeText(obj)
return obj return obj
class AnsibleJSONUnsafeEncoder(json.JSONEncoder): class AnsibleJSONUnsafeEncoder(json.JSONEncoder):
def encode(self, obj): def encode(self, obj):
if isinstance(obj, AnsibleUnsafe): if isinstance(obj, AnsibleUnsafe):
@ -84,6 +88,7 @@ class AnsibleJSONUnsafeEncoder(json.JSONEncoder):
else: else:
return super(AnsibleJSONUnsafeEncoder, self).encode(obj) return super(AnsibleJSONUnsafeEncoder, self).encode(obj)
class AnsibleJSONUnsafeDecoder(json.JSONDecoder): class AnsibleJSONUnsafeDecoder(json.JSONDecoder):
def decode(self, obj): def decode(self, obj):
value = super(AnsibleJSONUnsafeDecoder, self).decode(obj) value = super(AnsibleJSONUnsafeDecoder, self).decode(obj)
@ -92,6 +97,7 @@ class AnsibleJSONUnsafeDecoder(json.JSONDecoder):
else: else:
return value return value
def _wrap_dict(v): def _wrap_dict(v):
for k in v.keys(): for k in v.keys():
if v[k] is not None: if v[k] is not None:
@ -115,4 +121,3 @@ def wrap_var(v):
if v is not None and not isinstance(v, AnsibleUnsafe): if v is not None and not isinstance(v, AnsibleUnsafe):
v = UnsafeProxy(v) v = UnsafeProxy(v)
return v return v

@ -27,7 +27,8 @@ from contextlib import contextmanager
from io import BytesIO, StringIO from io import BytesIO, StringIO
from ansible.compat.six import PY3 from ansible.compat.six import PY3
from ansible.compat.tests import unittest from ansible.compat.tests import unittest
from ansible.utils.unicode import to_bytes from ansible.module_utils._text import to_bytes
@contextmanager @contextmanager
def swap_stdin_and_argv(stdin_data='', argv_data=tuple()): def swap_stdin_and_argv(stdin_data='', argv_data=tuple()):
@ -48,6 +49,7 @@ def swap_stdin_and_argv(stdin_data='', argv_data=tuple()):
sys.stdin = real_stdin sys.stdin = real_stdin
sys.argv = real_argv sys.argv = real_argv
@contextmanager @contextmanager
def swap_stdout(): def swap_stdout():
""" """
@ -62,6 +64,7 @@ def swap_stdout():
yield fake_stream yield fake_stream
sys.stdout = old_stdout sys.stdout = old_stdout
class ModuleTestCase(unittest.TestCase): class ModuleTestCase(unittest.TestCase):
def setUp(self, module_args=None): def setUp(self, module_args=None):
if module_args is None: if module_args is None:

@ -30,11 +30,12 @@ from binascii import hexlify
from nose.plugins.skip import SkipTest from nose.plugins.skip import SkipTest
from ansible.compat.tests import unittest from ansible.compat.tests import unittest
from ansible.utils.unicode import to_bytes, to_unicode
from ansible import errors from ansible import errors
from ansible.parsing.vault import VaultLib from ansible.parsing.vault import VaultLib
from ansible.parsing import vault from ansible.parsing import vault
from ansible.module_utils._text import to_bytes
# Counter import fails for 2.0.1, requires >= 2.6.1 from pip # Counter import fails for 2.0.1, requires >= 2.6.1 from pip
try: try:

@ -27,11 +27,12 @@ from nose.plugins.skip import SkipTest
from ansible.compat.tests import unittest from ansible.compat.tests import unittest
from ansible.compat.tests.mock import patch from ansible.compat.tests.mock import patch
from ansible.utils.unicode import to_bytes, to_unicode
from ansible import errors from ansible import errors
from ansible.parsing.vault import VaultLib from ansible.parsing.vault import VaultLib
from ansible.parsing.vault import VaultEditor from ansible.parsing.vault import VaultEditor
from ansible.module_utils._text import to_bytes, to_text
# Counter import fails for 2.0.1, requires >= 2.6.1 from pip # Counter import fails for 2.0.1, requires >= 2.6.1 from pip
try: try:
@ -66,6 +67,7 @@ v11_data = """$ANSIBLE_VAULT;1.1;AES256
3631633031323837340a396530313963373030343933616133393566366137363761373930663833 3631633031323837340a396530313963373030343933616133393566366137363761373930663833
3739""" 3739"""
class TestVaultEditor(unittest.TestCase): class TestVaultEditor(unittest.TestCase):
def setUp(self): def setUp(self):
@ -121,20 +123,19 @@ class TestVaultEditor(unittest.TestCase):
error_hit = False error_hit = False
try: try:
ve.decrypt_file(v10_file.name) ve.decrypt_file(v10_file.name)
except errors.AnsibleError as e: except errors.AnsibleError:
error_hit = True error_hit = True
# verify decrypted content # verify decrypted content
f = open(v10_file.name, "rb") f = open(v10_file.name, "rb")
fdata = to_unicode(f.read()) fdata = to_text(f.read())
f.close() f.close()
os.unlink(v10_file.name) os.unlink(v10_file.name)
assert error_hit == False, "error decrypting 1.0 file" assert error_hit is False, "error decrypting 1.0 file"
assert fdata.strip() == "foo", "incorrect decryption of 1.0 file: %s" % fdata.strip() assert fdata.strip() == "foo", "incorrect decryption of 1.0 file: %s" % fdata.strip()
def test_decrypt_1_1(self): def test_decrypt_1_1(self):
if not HAS_AES or not HAS_COUNTER or not HAS_PBKDF2: if not HAS_AES or not HAS_COUNTER or not HAS_PBKDF2:
raise SkipTest raise SkipTest
@ -149,20 +150,19 @@ class TestVaultEditor(unittest.TestCase):
error_hit = False error_hit = False
try: try:
ve.decrypt_file(v11_file.name) ve.decrypt_file(v11_file.name)
except errors.AnsibleError as e: except errors.AnsibleError:
error_hit = True error_hit = True
# verify decrypted content # verify decrypted content
f = open(v11_file.name, "rb") f = open(v11_file.name, "rb")
fdata = to_unicode(f.read()) fdata = to_text(f.read())
f.close() f.close()
os.unlink(v11_file.name) os.unlink(v11_file.name)
assert error_hit == False, "error decrypting 1.0 file" assert error_hit is False, "error decrypting 1.0 file"
assert fdata.strip() == "foo", "incorrect decryption of 1.0 file: %s" % fdata.strip() assert fdata.strip() == "foo", "incorrect decryption of 1.0 file: %s" % fdata.strip()
@unittest.skipIf(sys.version_info[0] >= 3, "VaultAES still needs to be ported to Python 3") @unittest.skipIf(sys.version_info[0] >= 3, "VaultAES still needs to be ported to Python 3")
def test_rekey_migration(self): def test_rekey_migration(self):
""" """
@ -182,7 +182,7 @@ class TestVaultEditor(unittest.TestCase):
error_hit = False error_hit = False
try: try:
ve.rekey_file(v10_file.name, 'ansible2') ve.rekey_file(v10_file.name, 'ansible2')
except errors.AnsibleError as e: except errors.AnsibleError:
error_hit = True error_hit = True
# verify decrypted content # verify decrypted content
@ -190,7 +190,7 @@ class TestVaultEditor(unittest.TestCase):
fdata = f.read() fdata = f.read()
f.close() f.close()
assert error_hit == False, "error rekeying 1.0 file to 1.1" assert error_hit is False, "error rekeying 1.0 file to 1.1"
# ensure filedata can be decrypted, is 1.1 and is AES256 # ensure filedata can be decrypted, is 1.1 and is AES256
vl = VaultLib("ansible2") vl = VaultLib("ansible2")
@ -198,13 +198,11 @@ class TestVaultEditor(unittest.TestCase):
error_hit = False error_hit = False
try: try:
dec_data = vl.decrypt(fdata) dec_data = vl.decrypt(fdata)
except errors.AnsibleError as e: except errors.AnsibleError:
error_hit = True error_hit = True
os.unlink(v10_file.name) os.unlink(v10_file.name)
assert vl.cipher_name == "AES256", "wrong cipher name set after rekey: %s" % vl.cipher_name assert vl.cipher_name == "AES256", "wrong cipher name set after rekey: %s" % vl.cipher_name
assert error_hit == False, "error decrypting migrated 1.0 file" assert error_hit is False, "error decrypting migrated 1.0 file"
assert dec_data.strip() == "foo", "incorrect decryption of rekeyed/migrated file: %s" % dec_data assert dec_data.strip() == "foo", "incorrect decryption of rekeyed/migrated file: %s" % dec_data

@ -32,7 +32,6 @@ from ansible.parsing.yaml.loader import AnsibleLoader
from ansible.parsing import vault from ansible.parsing import vault
from ansible.parsing.yaml.objects import AnsibleVaultEncryptedUnicode from ansible.parsing.yaml.objects import AnsibleVaultEncryptedUnicode
from ansible.parsing.yaml.dumper import AnsibleDumper from ansible.parsing.yaml.dumper import AnsibleDumper
from ansible.utils.unicode import to_bytes
from units.mock.yaml_helper import YamlTestUtils from units.mock.yaml_helper import YamlTestUtils
@ -41,6 +40,7 @@ try:
except ImportError: except ImportError:
from yaml.parser import ParserError from yaml.parser import ParserError
class NameStringIO(StringIO): class NameStringIO(StringIO):
"""In py2.6, StringIO doesn't let you set name because a baseclass has it """In py2.6, StringIO doesn't let you set name because a baseclass has it
as readonly property""" as readonly property"""
@ -49,6 +49,7 @@ class NameStringIO(StringIO):
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):
super(NameStringIO, self).__init__(*args, **kwargs) super(NameStringIO, self).__init__(*args, **kwargs)
class TestAnsibleLoaderBasic(unittest.TestCase): class TestAnsibleLoaderBasic(unittest.TestCase):
def setUp(self): def setUp(self):
@ -283,6 +284,7 @@ class TestAnsibleLoaderVault(unittest.TestCase, YamlTestUtils):
self.assertFalse(plaintext_var != vault_string) self.assertFalse(plaintext_var != vault_string)
self.assertFalse(vault_string != plaintext_var) self.assertFalse(vault_string != plaintext_var)
class TestAnsibleLoaderPlay(unittest.TestCase): class TestAnsibleLoaderPlay(unittest.TestCase):
def setUp(self): def setUp(self):

@ -21,8 +21,6 @@
from __future__ import (absolute_import, division, print_function) from __future__ import (absolute_import, division, print_function)
__metaclass__ = type __metaclass__ = type
import ast
import json
import pipes import pipes
import os import os
@ -33,7 +31,6 @@ except ImportError:
from nose.tools import eq_, raises from nose.tools import eq_, raises
from ansible.release import __version__ as ansible_version
from ansible import constants as C from ansible import constants as C
from ansible.compat.six import text_type from ansible.compat.six import text_type
from ansible.compat.tests import unittest from ansible.compat.tests import unittest
@ -41,12 +38,12 @@ from ansible.compat.tests.mock import patch, MagicMock, mock_open
from ansible.errors import AnsibleError from ansible.errors import AnsibleError
from ansible.playbook.play_context import PlayContext from ansible.playbook.play_context import PlayContext
from ansible.plugins import PluginLoader
from ansible.plugins.action import ActionBase from ansible.plugins.action import ActionBase
from ansible.template import Templar from ansible.template import Templar
from ansible.utils.unicode import to_bytes
from units.mock.loader import DictDataLoader from units.mock.loader import DictDataLoader
from ansible.module_utils._text import to_bytes
python_module_replacers = b""" python_module_replacers = b"""
#!/usr/bin/python #!/usr/bin/python
@ -67,11 +64,13 @@ WINDOWS_ARGS = "<<INCLUDE_ANSIBLE_MODULE_JSON_ARGS>>"
class DerivedActionBase(ActionBase): class DerivedActionBase(ActionBase):
TRANSFERS_FILES = False TRANSFERS_FILES = False
def run(self, tmp=None, task_vars=None): def run(self, tmp=None, task_vars=None):
# We're not testing the plugin run() method, just the helper # We're not testing the plugin run() method, just the helper
# methods ActionBase defines # methods ActionBase defines
return super(DerivedActionBase, self).run(tmp=tmp, task_vars=task_vars) return super(DerivedActionBase, self).run(tmp=tmp, task_vars=task_vars)
class TestActionBase(unittest.TestCase): class TestActionBase(unittest.TestCase):
def test_action_base_run(self): def test_action_base_run(self):
@ -144,7 +143,7 @@ class TestActionBase(unittest.TestCase):
self.assertRaises(AnsibleError, action_base._configure_module, 'badmodule', mock_task.args) self.assertRaises(AnsibleError, action_base._configure_module, 'badmodule', mock_task.args)
# test powershell module formatting # test powershell module formatting
with patch.object(builtins, 'open', mock_open(read_data=to_bytes(powershell_module_replacers.strip(), encoding='utf-8'))) as m: with patch.object(builtins, 'open', mock_open(read_data=to_bytes(powershell_module_replacers.strip(), encoding='utf-8'))):
mock_task.action = 'win_copy' mock_task.action = 'win_copy'
mock_task.args = dict(b=2) mock_task.args = dict(b=2)
mock_connection.module_implementation_preferences = ('.ps1',) mock_connection.module_implementation_preferences = ('.ps1',)
@ -497,7 +496,10 @@ class TestActionBase(unittest.TestCase):
action_base._connection.has_pipelining = True action_base._connection.has_pipelining = True
action_base._low_level_execute_command.return_value = dict(stdout='{"rc": 0, "stdout": "ok"}') action_base._low_level_execute_command.return_value = dict(stdout='{"rc": 0, "stdout": "ok"}')
self.assertEqual(action_base._execute_module(module_name=None, module_args=None), dict(_ansible_parsed=True, rc=0, stdout="ok", stdout_lines=['ok'])) self.assertEqual(action_base._execute_module(module_name=None, module_args=None), dict(_ansible_parsed=True, rc=0, stdout="ok", stdout_lines=['ok']))
self.assertEqual(action_base._execute_module(module_name='foo', module_args=dict(z=9, y=8, x=7), task_vars=dict(a=1)), dict(_ansible_parsed=True, rc=0, stdout="ok", stdout_lines=['ok'])) self.assertEqual(action_base._execute_module(module_name='foo',
module_args=dict(z=9, y=8, x=7), task_vars=dict(a=1)),
dict(_ansible_parsed=True, rc=0, stdout="ok",
stdout_lines=['ok']))
# test with needing/removing a remote tmp path # test with needing/removing a remote tmp path
action_base._configure_module.return_value = ('old', '#!/usr/bin/python', 'this is the module data', 'path') action_base._configure_module.return_value = ('old', '#!/usr/bin/python', 'this is the module data', 'path')
@ -555,6 +557,7 @@ class TestActionBase(unittest.TestCase):
finally: finally:
C.BECOME_ALLOW_SAME_USER = become_allow_same_user C.BECOME_ALLOW_SAME_USER = become_allow_same_user
# Note: Using nose's generator test cases here so we can't inherit from # Note: Using nose's generator test cases here so we can't inherit from
# unittest.TestCase # unittest.TestCase
class TestFilterNonJsonLines(object): class TestFilterNonJsonLines(object):
@ -592,4 +595,3 @@ class TestFilterNonJsonLines(object):
def test_unparsable_filter_non_json_lines(self): def test_unparsable_filter_non_json_lines(self):
for stdout_line in self.unparsable_cases: for stdout_line in self.unparsable_cases:
yield self.check_unparsable_filter_non_json_lines, stdout_line yield self.check_unparsable_filter_non_json_lines, stdout_line

@ -22,17 +22,17 @@ from __future__ import (absolute_import, division, print_function)
__metaclass__ = type __metaclass__ = type
import pipes import pipes
import sys
from io import StringIO from io import StringIO
from ansible.compat.tests import unittest from ansible.compat.tests import unittest
from ansible.compat.tests.mock import patch, MagicMock, mock_open from ansible.compat.tests.mock import patch, MagicMock
from ansible import constants as C from ansible import constants as C
from ansible.errors import AnsibleError, AnsibleConnectionFailure, AnsibleFileNotFound from ansible.errors import AnsibleError, AnsibleConnectionFailure, AnsibleFileNotFound
from ansible.playbook.play_context import PlayContext from ansible.playbook.play_context import PlayContext
from ansible.plugins.connection import ssh from ansible.plugins.connection import ssh
from ansible.utils.unicode import to_bytes, to_unicode from ansible.module_utils._text import to_bytes
class TestConnectionBaseClass(unittest.TestCase): class TestConnectionBaseClass(unittest.TestCase):
@ -277,16 +277,18 @@ class TestConnectionBaseClass(unittest.TestCase):
C.ANSIBLE_SSH_RETRIES = 9 C.ANSIBLE_SSH_RETRIES = 9
# test a regular, successful execution # test a regular, successful execution
conn._exec_command.return_value = (0, 'stdout', '') conn._exec_command.return_value = (0, b'stdout', b'')
res = conn.exec_command('ssh', 'some data') res = conn.exec_command('ssh', 'some data')
self.assertEquals(res, (0, b'stdout', b''), msg='exec_command did not return what the _exec_command helper returned')
# test a retry, followed by success # test a retry, followed by success
conn._exec_command.return_value = None conn._exec_command.return_value = None
conn._exec_command.side_effect = [(255, '', ''), (0, 'stdout', '')] conn._exec_command.side_effect = [(255, '', ''), (0, b'stdout', b'')]
res = conn.exec_command('ssh', 'some data') res = conn.exec_command('ssh', 'some data')
self.assertEquals(res, (0, b'stdout', b''), msg='exec_command did not return what the _exec_command helper returned')
# test multiple failures # test multiple failures
conn._exec_command.side_effect = [(255, '', '')]*10 conn._exec_command.side_effect = [(255, b'', b'')] * 10
self.assertRaises(AnsibleConnectionFailure, conn.exec_command, 'ssh', 'some data') self.assertRaises(AnsibleConnectionFailure, conn.exec_command, 'ssh', 'some data')
# test other failure from exec_command # test other failure from exec_command
@ -308,20 +310,22 @@ class TestConnectionBaseClass(unittest.TestCase):
# test with C.DEFAULT_SCP_IF_SSH enabled # test with C.DEFAULT_SCP_IF_SSH enabled
C.DEFAULT_SCP_IF_SSH = True C.DEFAULT_SCP_IF_SSH = True
res = conn.put_file('/path/to/in/file', '/path/to/dest/file') conn.put_file('/path/to/in/file', '/path/to/dest/file')
conn._run.assert_called_with('some command to run', None) conn._run.assert_called_with('some command to run', None)
res = conn.put_file(u'/path/to/in/file/with/unicode-fö〩', u'/path/to/dest/file/with/unicode-fö〩') conn.put_file(u'/path/to/in/file/with/unicode-fö〩', u'/path/to/dest/file/with/unicode-fö〩')
conn._run.assert_called_with('some command to run', None) conn._run.assert_called_with('some command to run', None)
# test with C.DEFAULT_SCP_IF_SSH disabled # test with C.DEFAULT_SCP_IF_SSH disabled
C.DEFAULT_SCP_IF_SSH = False C.DEFAULT_SCP_IF_SSH = False
expected_in_data = b' '.join((b'put', to_bytes(pipes.quote('/path/to/in/file')), to_bytes(pipes.quote('/path/to/dest/file')))) + b'\n' expected_in_data = b' '.join((b'put', to_bytes(pipes.quote('/path/to/in/file')), to_bytes(pipes.quote('/path/to/dest/file')))) + b'\n'
res = conn.put_file('/path/to/in/file', '/path/to/dest/file') conn.put_file('/path/to/in/file', '/path/to/dest/file')
conn._run.assert_called_with('some command to run', expected_in_data) conn._run.assert_called_with('some command to run', expected_in_data)
expected_in_data = b' '.join((b'put', to_bytes(pipes.quote('/path/to/in/file/with/unicode-fö〩')), to_bytes(pipes.quote('/path/to/dest/file/with/unicode-fö〩')))) + b'\n' expected_in_data = b' '.join((b'put',
res = conn.put_file(u'/path/to/in/file/with/unicode-fö〩', u'/path/to/dest/file/with/unicode-fö〩') to_bytes(pipes.quote('/path/to/in/file/with/unicode-fö〩')),
to_bytes(pipes.quote('/path/to/dest/file/with/unicode-fö〩')))) + b'\n'
conn.put_file(u'/path/to/in/file/with/unicode-fö〩', u'/path/to/dest/file/with/unicode-fö〩')
conn._run.assert_called_with('some command to run', expected_in_data) conn._run.assert_called_with('some command to run', expected_in_data)
# test that a non-zero rc raises an error # test that a non-zero rc raises an error
@ -346,23 +350,24 @@ class TestConnectionBaseClass(unittest.TestCase):
# test with C.DEFAULT_SCP_IF_SSH enabled # test with C.DEFAULT_SCP_IF_SSH enabled
C.DEFAULT_SCP_IF_SSH = True C.DEFAULT_SCP_IF_SSH = True
res = conn.fetch_file('/path/to/in/file', '/path/to/dest/file') conn.fetch_file('/path/to/in/file', '/path/to/dest/file')
conn._run.assert_called_with('some command to run', None) conn._run.assert_called_with('some command to run', None)
res = conn.fetch_file(u'/path/to/in/file/with/unicode-fö〩', u'/path/to/dest/file/with/unicode-fö〩') conn.fetch_file(u'/path/to/in/file/with/unicode-fö〩', u'/path/to/dest/file/with/unicode-fö〩')
conn._run.assert_called_with('some command to run', None) conn._run.assert_called_with('some command to run', None)
# test with C.DEFAULT_SCP_IF_SSH disabled # test with C.DEFAULT_SCP_IF_SSH disabled
C.DEFAULT_SCP_IF_SSH = False C.DEFAULT_SCP_IF_SSH = False
expected_in_data = b' '.join((b'get', to_bytes(pipes.quote('/path/to/in/file')), to_bytes(pipes.quote('/path/to/dest/file')))) + b'\n' expected_in_data = b' '.join((b'get', to_bytes(pipes.quote('/path/to/in/file')), to_bytes(pipes.quote('/path/to/dest/file')))) + b'\n'
res = conn.fetch_file('/path/to/in/file', '/path/to/dest/file') conn.fetch_file('/path/to/in/file', '/path/to/dest/file')
conn._run.assert_called_with('some command to run', expected_in_data) conn._run.assert_called_with('some command to run', expected_in_data)
expected_in_data = b' '.join((b'get', to_bytes(pipes.quote('/path/to/in/file/with/unicode-fö〩')), to_bytes(pipes.quote('/path/to/dest/file/with/unicode-fö〩')))) + b'\n' expected_in_data = b' '.join((b'get',
res = conn.fetch_file(u'/path/to/in/file/with/unicode-fö〩', u'/path/to/dest/file/with/unicode-fö〩') to_bytes(pipes.quote('/path/to/in/file/with/unicode-fö〩')),
to_bytes(pipes.quote('/path/to/dest/file/with/unicode-fö〩')))) + b'\n'
conn.fetch_file(u'/path/to/in/file/with/unicode-fö〩', u'/path/to/dest/file/with/unicode-fö〩')
conn._run.assert_called_with('some command to run', expected_in_data) conn._run.assert_called_with('some command to run', expected_in_data)
# test that a non-zero rc raises an error # test that a non-zero rc raises an error
conn._run.return_value = (1, 'stdout', 'some errors') conn._run.return_value = (1, 'stdout', 'some errors')
self.assertRaises(AnsibleError, conn.fetch_file, '/path/to/bad/file', '/remote/path/to/file') self.assertRaises(AnsibleError, conn.fetch_file, '/path/to/bad/file', '/remote/path/to/file')

Loading…
Cancel
Save