archive
/
ansible
mirror of https://github.com/ansible/ansible.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Move uses of to_bytes, to_text, to_native to use the module_utils version (#17423)

Browse Source 
We couldn't copy to_unicode, to_bytes, to_str into module_utils because
of licensing.  So once created it we had two sets of functions that did
the same things but had different implementations.  To remedy that, this
change removes the ansible.utils.unicode versions of those functions.
pull/17433/head
Toshio Kuratomi 8 years ago committed by GitHub
parent 7a9395b5e0
commit 4ed88512e4
89 changed files with 747 additions and 882 deletions
Show Stats Download Patch File Download Diff File

12
bin/ansible
Unescape Escape View File

@ -43,7 +43,7 @@ from multiprocessing import Lock
import ansible.constants as C
from ansible.errors import AnsibleError, AnsibleOptionsError, AnsibleParserError
from ansible.utils.display import Display
from ansible.utils.unicode import to_unicode
from ansible.module_utils._text import to_text
########################################
@ -97,10 +97,10 @@ if __name__ == '__main__':
except AnsibleOptionsError as e:
cli.parser.print_help()
display.error(to_unicode(e), wrap_text=False)
display.error(to_text(e), wrap_text=False)
exit_code = 5
except AnsibleParserError as e:
display.error(to_unicode(e), wrap_text=False)
display.error(to_text(e), wrap_text=False)
exit_code = 4
# TQM takes care of these, but leaving comment to reserve the exit codes
# except AnsibleHostUnreachable as e:
@ -110,16 +110,16 @@ if __name__ == '__main__':
# display.error(str(e))
# exit_code = 2
except AnsibleError as e:
display.error(to_unicode(e), wrap_text=False)
display.error(to_text(e), wrap_text=False)
exit_code = 1
except KeyboardInterrupt:
display.error("User interrupted execution")
exit_code = 99
except Exception as e:
have_cli_options = cli is not None and cli.options is not None
display.error("Unexpected Exception: %s" % to_unicode(e), wrap_text=False)
display.error("Unexpected Exception: %s" % to_text(e), wrap_text=False)
if not have_cli_options or have_cli_options and cli.options.verbosity > 2:
display.display(u"the full traceback was:\n\n%s" % to_unicode(traceback.format_exc()))
display.display(u"the full traceback was:\n\n%s" % to_text(traceback.format_exc()))
else:
display.display("to see the full traceback, use -vvv")
exit_code = 250

22
docsite/rst/developing_modules_python3.rst
Unescape Escape View File

@ -215,6 +215,28 @@ Python3. We'll need to gather experience to see if this is going to work out
well for modules as well or if we should give the module_utils API explicit
switches so that modules can choose to operate with text type all of the time.
Helpers
~~~~~~~
For converting between bytes, text, and native strings we have three helper
functions. These are :func:`ansible.module_utils._text.to_bytes`,
:func:`ansible.module_utils._text.to_native`, and
:func:`ansible.module_utils._text.to_text`. These are similar to using
``bytes.decode()`` and ``unicode.encode()`` with a few differences.
* By default they try very hard not to traceback.
* The default encoding is "utf-8"
* There are two error strategies that don't correspond one-to-one with
a python codec error handler. These are ``surrogate_or_strict`` and
``surrogate_or_replace``. ``surrogate_or_strict`` will use the ``surrogateescape``
error handler if available (mostly on python3) or strict if not. It is most
appropriate to use when dealing with something that needs to round trip its
value like file paths database keys, etc. Without ``surrogateescape`` the best
thing these values can do is generate a traceback that our code can catch
and decide how to show an error message. ``surrogate_or_replace`` is for
when a value is going to be displayed to the user. If the
``surrogateescape`` error handler is not present, it will replace
undecodable byte sequences with a replacement character.
================================
Porting Core Ansible to Python 3

5
hacking/module_formatter.py
Unescape Escape View File

@ -19,6 +19,7 @@
#
from __future__ import print_function
__metaclass__ = type
import os
import glob
@ -34,10 +35,10 @@ from collections import defaultdict
from jinja2 import Environment, FileSystemLoader
from six import iteritems
from ansible.errors import AnsibleError
from ansible.module_utils._text import to_bytes
from ansible.utils import module_docs
from ansible.utils.vars import merge_hash
from ansible.utils.unicode import to_bytes
from ansible.errors import AnsibleError
#####################################################################################
# constants and paths

4
lib/ansible/cli/__init__.py
Unescape Escape View File

@ -33,7 +33,7 @@ import subprocess
from ansible.release import __version__
from ansible import constants as C
from ansible.errors import AnsibleError, AnsibleOptionsError
from ansible.utils.unicode import to_bytes, to_unicode
from ansible.module_utils._text import to_bytes, to_text
try:
from __main__ import display
@ -109,7 +109,7 @@ class CLI(object):
if self.options.verbosity > 0:
if C.CONFIG_FILE:
display.display(u"Using %s as config file" % to_unicode(C.CONFIG_FILE))
display.display(u"Using %s as config file" % to_text(C.CONFIG_FILE))
else:
display.display(u"No config file found; using defaults")

4
lib/ansible/cli/adhoc.py
Unescape Escape View File

@ -27,13 +27,13 @@ from ansible.cli import CLI
from ansible.errors import AnsibleError, AnsibleOptionsError
from ansible.executor.task_queue_manager import TaskQueueManager
from ansible.inventory import Inventory
from ansible.module_utils._text import to_text
from ansible.parsing.dataloader import DataLoader
from ansible.parsing.splitter import parse_kv
from ansible.playbook.play import Play
from ansible.plugins import get_all_plugin_loaders
from ansible.utils.vars import load_extra_vars
from ansible.utils.vars import load_options_vars
from ansible.utils.unicode import to_unicode
from ansible.vars import VariableManager
try:
@ -99,7 +99,7 @@ class AdHocCLI(CLI):
super(AdHocCLI, self).run()
# only thing left should be host pattern
pattern = to_unicode(self.args[0], errors='strict')
pattern = to_text(self.args[0], errors='surrogate_or_strict')
# ignore connection password cause we are local
if self.options.connection == "local":

21
lib/ansible/cli/console.py
Unescape Escape View File

@ -39,18 +39,16 @@ import sys
from ansible import constants as C
from ansible.cli import CLI
from ansible.errors import AnsibleError
from ansible.executor.task_queue_manager import TaskQueueManager
from ansible.inventory import Inventory
from ansible.module_utils._text import to_native, to_text
from ansible.parsing.dataloader import DataLoader
from ansible.parsing.splitter import parse_kv
from ansible.playbook.play import Play
from ansible.vars import VariableManager
from ansible.plugins import module_loader
from ansible.utils import module_docs
from ansible.utils.color import stringc
from ansible.utils.unicode import to_unicode, to_str
from ansible.plugins import module_loader
from ansible.vars import VariableManager
try:
from __main__ import display
@ -192,11 +190,11 @@ class ConsoleCLI(CLI, cmd.Cmd):
)
play = Play().load(play_ds, variable_manager=self.variable_manager, loader=self.loader)
except Exception as e:
display.error(u"Unable to build command: %s" % to_unicode(e))
display.error(u"Unable to build command: %s" % to_text(e))
return False
try:
cb = 'minimal' #FIXME: make callbacks configurable
cb = 'minimal' # FIXME: make callbacks configurable
# now create a task queue manager to execute the play
self._tqm = None
try:
@ -225,8 +223,8 @@ class ConsoleCLI(CLI, cmd.Cmd):
display.error('User interrupted execution')
return False
except Exception as e:
display.error(to_unicode(e))
#FIXME: add traceback in very very verbose mode
display.error(to_text(e))
# FIXME: add traceback in very very verbose mode
return False
def emptyline(self):
@ -379,7 +377,7 @@ class ConsoleCLI(CLI, cmd.Cmd):
else:
completions = [x.name for x in self.inventory.list_hosts(self.options.cwd)]
return [to_str(s)[offs:] for s in completions if to_str(s).startswith(to_str(mline))]
return [to_native(s)[offs:] for s in completions if to_native(s).startswith(to_native(mline))]
def completedefault(self, text, line, begidx, endidx):
if line.split()[0] in self.modules:
@ -394,7 +392,6 @@ class ConsoleCLI(CLI, cmd.Cmd):
oc, a, _ = module_docs.get_docstring(in_path)
return oc['options'].keys()
def run(self):
super(ConsoleCLI, self).run()
@ -410,7 +407,6 @@ class ConsoleCLI(CLI, cmd.Cmd):
self.pattern = self.args[0]
self.options.cwd = self.pattern
# dynamically add modules as commands
self.modules = self.list_modules()
for module in self.modules:
@ -465,4 +461,3 @@ class ConsoleCLI(CLI, cmd.Cmd):
atexit.register(readline.write_history_file, histfile)
self.set_prompt()
self.cmdloop()

15
lib/ansible/cli/galaxy.py
Unescape Escape View File

@ -39,7 +39,7 @@ from ansible.galaxy.role import GalaxyRole
from ansible.galaxy.login import GalaxyLogin
from ansible.galaxy.token import GalaxyToken
from ansible.playbook.role.requirement import RoleRequirement
from ansible.utils.unicode import to_bytes, to_unicode
from ansible.module_utils._text import to_bytes, to_text
try:
from __main__ import display
@ -47,6 +47,7 @@ except ImportError:
from ansible.utils.display import Display
display = Display()
class GalaxyCLI(CLI):
SKIP_INFO_KEYS = ("name", "description", "readme_html", "related", "summary_fields", "average_aw_composite", "average_aw_score", "url" )
@ -65,7 +66,6 @@ class GalaxyCLI(CLI):
epilog = "\nSee '%s <command> --help' for more information on a specific command.\n\n" % os.path.basename(sys.argv[0])
)
self.set_action()
# common
@ -111,7 +111,7 @@ class GalaxyCLI(CLI):
if self.action in ['init', 'info']:
self.parser.add_option( '--offline', dest='offline', default=False, action='store_true', help="Don't query the galaxy API when creating roles")
if not self.action in ("delete","import","init","login","setup"):
if self.action not in ("delete","import","init","login","setup"):
# NOTE: while the option type=str, the default is a list, and the
# callback will set the value to a list.
self.parser.add_option('-p', '--roles-path', dest='roles_path', action="callback", callback=CLI.expand_paths, type=str, default=C.DEFAULT_ROLES_PATH,
@ -142,7 +142,7 @@ class GalaxyCLI(CLI):
def _display_role_info(self, role_info):
text = [u"", u"Role: %s" % to_unicode(role_info['name'])]
text = [u"", u"Role: %s" % to_text(role_info['name'])]
text.append(u"\tdescription: %s" % role_info.get('description', ''))
for k in sorted(role_info.keys()):
@ -578,8 +578,8 @@ class GalaxyCLI(CLI):
if len(self.args) < 2:
raise AnsibleError("Expected a github_username and github_repository. Use --help.")
github_repo = self.args.pop()
github_user = self.args.pop()
github_repo = to_text(self.args.pop(), errors='surrogate_or_strict')
github_user = to_text(self.args.pop(), errors='surrogate_or_strict')
if self.options.check_status:
task = self.api.get_import_task(github_user=github_user, github_repo=github_repo)
@ -594,7 +594,8 @@ class GalaxyCLI(CLI):
display.display("The following Galaxy roles are being updated:" + u'\n', color=C.COLOR_CHANGED)
for t in task:
display.display('%s.%s' % (t['summary_fields']['role']['namespace'],t['summary_fields']['role']['name']), color=C.COLOR_CHANGED)
display.display(u'\n' + "To properly namespace this role, remove each of the above and re-import %s/%s from scratch" % (github_user,github_repo), color=C.COLOR_CHANGED)
display.display(u'\nTo properly namespace this role, remove each of the above and re-import %s/%s from scratch' % (github_user, github_repo),
color=C.COLOR_CHANGED)
return 0
# found a single role as expected
display.display("Successfully submitted import request %d" % task[0]['id'])

4
lib/ansible/cli/vault.py
Unescape Escape View File

@ -26,7 +26,7 @@ from ansible.errors import AnsibleError, AnsibleOptionsError
from ansible.parsing.dataloader import DataLoader
from ansible.parsing.vault import VaultEditor
from ansible.cli import CLI
from ansible.utils.unicode import to_unicode
from ansible.module_utils._text import to_text
try:
from __main__ import display
@ -163,7 +163,7 @@ class VaultCLI(CLI):
# unicode here because we are displaying it and therefore can make
# the decision that the display doesn't have to be precisely what
# the input was (leave that to decrypt instead)
self.pager(to_unicode(self.editor.plaintext(f)))
self.pager(ansible.module_utils._text.to_text(self.editor.plaintext(f)))
def execute_rekey(self):
for f in self.args:

13
lib/ansible/errors/__init__.py
Unescape Escape View File

@ -25,8 +25,7 @@ from ansible.errors.yaml_strings import ( YAML_POSITION_DETAILS,
YAML_COMMON_UNQUOTED_COLON_ERROR,
YAML_COMMON_PARTIALLY_QUOTED_LINE_ERROR,
YAML_COMMON_UNBALANCED_QUOTES_ERROR )
from ansible.utils.unicode import to_unicode, to_str
from ansible.module_utils._text import to_native, to_text
class AnsibleError(Exception):
@ -54,11 +53,11 @@ class AnsibleError(Exception):
if obj and isinstance(obj, AnsibleBaseYAMLObject):
extended_error = self._get_extended_error()
if extended_error and not suppress_extended_error:
self.message = '%s\n\n%s' % (to_str(message), to_str(extended_error))
self.message = '%s\n\n%s' % (to_native(message), to_native(extended_error))
else:
self.message = '%s' % to_str(message)
self.message = '%s' % to_native(message)
else:
self.message = '%s' % to_str(message)
self.message = '%s' % to_native(message)
def __str__(self):
return self.message
@ -104,8 +103,8 @@ class AnsibleError(Exception):
error_message += YAML_POSITION_DETAILS % (src_file, line_number, col_number)
if src_file not in ('<string>', '<unicode>') and self._show_content:
(target_line, prev_line) = self._get_error_lines_from_file(src_file, line_number - 1)
target_line = to_unicode(target_line)
prev_line = to_unicode(prev_line)
target_line = to_text(target_line)
prev_line = to_text(prev_line)
if target_line:
stripped_line = target_line.replace(" ","")
arrow_line = (" " * (col_number-1)) + "^ here"

40
lib/ansible/executor/module_common.py
Unescape Escape View File

@ -29,11 +29,10 @@ import shlex
import zipfile
from io import BytesIO
# from Ansible
from ansible.release import __version__, __author__
from ansible import constants as C
from ansible.errors import AnsibleError
from ansible.utils.unicode import to_bytes, to_unicode
from ansible.module_utils._text import to_bytes, to_text
# Must import strategy and use write_locks from there
# If we import write_locks directly then we end up binding a
# variable to the object and then it never gets updated.
@ -45,6 +44,7 @@ except ImportError:
from ansible.utils.display import Display
display = Display()
REPLACER = b"#<<INCLUDE_ANSIBLE_MODULE_COMMON>>"
REPLACER_VERSION = b"\"<<ANSIBLE_VERSION>>\""
REPLACER_COMPLEX = b"\"<<INCLUDE_ANSIBLE_MODULE_COMPLEX_ARGS>>\""
@ -239,7 +239,9 @@ def debug(command, zipped_mod, json_params):
else:
os.environ['PYTHONPATH'] = basedir
p = subprocess.Popen([%(interpreter)s, script_path, args_path], env=os.environ, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE)
p = subprocess.Popen([%(interpreter)s, script_path, args_path],
env=os.environ, shell=False, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, stdin=subprocess.PIPE)
(stdout, stderr) = p.communicate()
if not isinstance(stderr, (bytes, unicode)):
@ -328,6 +330,7 @@ if __name__ == '__main__':
sys.exit(exitcode)
'''
def _strip_comments(source):
# Strip comments and blank lines from the wrapper
buf = []
@ -338,6 +341,7 @@ def _strip_comments(source):
buf.append(line)
return u'\n'.join(buf)
if C.DEFAULT_KEEP_REMOTE_FILES:
# Keep comments when KEEP_REMOTE_FILES is set. That way users will see
# the comments with some nice usage instructions
@ -346,6 +350,7 @@ else:
# ANSIBALLZ_TEMPLATE stripped of comments for smaller over the wire size
ACTIVE_ANSIBALLZ_TEMPLATE = _strip_comments(ANSIBALLZ_TEMPLATE)
class ModuleDepFinder(ast.NodeVisitor):
# Caveats:
# This code currently does not handle:
@ -404,6 +409,7 @@ def _slurp(path):
fd.close()
return data
def _get_shebang(interpreter, task_vars, args=tuple()):
"""
Note not stellar API:
@ -425,6 +431,7 @@ def _get_shebang(interpreter, task_vars, args=tuple()):
return (shebang, interpreter)
def recursive_finder(name, data, py_module_names, py_module_cache, zf):
"""
Using ModuleDepFinder, make sure we have all of the module_utils files that
@ -529,11 +536,13 @@ def recursive_finder(name, data, py_module_names, py_module_cache, zf):
# Save memory; the file won't have to be read again for this ansible module.
del py_module_cache[py_module_file]
def _is_binary(module_data):
textchars = bytearray(set([7, 8, 9, 10, 12, 13, 27]) | set(range(0x20, 0x100)) - set([0x7f]))
start = module_data[:1024]
return bool(start.translate(None, textchars))
def _find_snippet_imports(module_name, module_data, module_path, module_args, task_vars, module_compression):
"""
Given the source of the module, convert it to a Jinja2 template to insert
@ -617,9 +626,12 @@ def _find_snippet_imports(module_name, module_data, module_path, module_args, ta
# Create the module zip data
zipoutput = BytesIO()
zf = zipfile.ZipFile(zipoutput, mode='w', compression=compression_method)
### Note: If we need to import from release.py first,
### remember to catch all exceptions: https://github.com/ansible/ansible/issues/16523
zf.writestr('ansible/__init__.py', b'from pkgutil import extend_path\n__path__=extend_path(__path__,__name__)\n__version__="' + to_bytes(__version__) + b'"\n__author__="' + to_bytes(__author__) + b'"\n')
# Note: If we need to import from release.py first,
# remember to catch all exceptions: https://github.com/ansible/ansible/issues/16523
zf.writestr('ansible/__init__.py',
b'from pkgutil import extend_path\n__path__=extend_path(__path__,__name__)\n__version__="' +
to_bytes(__version__) + b'"\n__author__="' +
to_bytes(__author__) + b'"\n')
zf.writestr('ansible/module_utils/__init__.py', b'from pkgutil import extend_path\n__path__=extend_path(__path__,__name__)\n')
zf.writestr('ansible_module_%s.py' % module_name, module_data)
@ -655,8 +667,9 @@ def _find_snippet_imports(module_name, module_data, module_path, module_args, ta
try:
zipdata = open(cached_module_filename, 'rb').read()
except IOError:
raise AnsibleError('A different worker process failed to create module file. Look at traceback for that process for debugging information.')
zipdata = to_unicode(zipdata, errors='strict')
raise AnsibleError('A different worker process failed to create module file.'
' Look at traceback for that process for debugging information.')
zipdata = to_text(zipdata, errors='surrogate_or_strict')
shebang, interpreter = _get_shebang(u'/usr/bin/python', task_vars)
if shebang is None:
@ -721,12 +734,11 @@ def _find_snippet_imports(module_name, module_data, module_path, module_args, ta
# The main event -- substitute the JSON args string into the module
module_data = module_data.replace(REPLACER_JSONARGS, module_args_json)
facility = b'syslog.' + to_bytes(task_vars.get('ansible_syslog_facility', C.DEFAULT_SYSLOG_FACILITY), errors='strict')
facility = b'syslog.' + to_bytes(task_vars.get('ansible_syslog_facility', C.DEFAULT_SYSLOG_FACILITY), errors='surrogate_or_strict')
module_data = module_data.replace(b'syslog.LOG_USER', facility)
return (module_data, module_style, shebang)
# ******************************************************************************
def modify_module(module_name, module_path, module_args, task_vars=dict(), module_compression='ZIP_STORED'):
"""
@ -760,7 +772,7 @@ def modify_module(module_name, module_path, module_args, task_vars=dict(), modul
(module_data, module_style, shebang) = _find_snippet_imports(module_name, module_data, module_path, module_args, task_vars, module_compression)
if module_style == 'binary':
return (module_data, module_style, to_unicode(shebang, nonstring='passthru'))
return (module_data, module_style, to_text(shebang, nonstring='passthru'))
elif shebang is None:
lines = module_data.split(b"\n", 1)
if lines[0].startswith(b"#!"):
@ -769,7 +781,7 @@ def modify_module(module_name, module_path, module_args, task_vars=dict(), modul
interpreter = args[0]
interpreter = to_bytes(interpreter)
new_shebang = to_bytes(_get_shebang(interpreter, task_vars, args[1:])[0], errors='strict', nonstring='passthru')
new_shebang = to_bytes(_get_shebang(interpreter, task_vars, args[1:])[0], errors='surrogate_or_strict', nonstring='passthru')
if new_shebang:
lines[0] = shebang = new_shebang
@ -781,6 +793,6 @@ def modify_module(module_name, module_path, module_args, task_vars=dict(), modul
module_data = b"\n".join(lines)
else:
shebang = to_bytes(shebang, errors='strict')
shebang = to_bytes(shebang, errors='surrogate_or_strict')
return (module_data, module_style, to_unicode(shebang, nonstring='passthru'))
return (module_data, module_style, to_text(shebang, nonstring='passthru'))

8
lib/ansible/executor/playbook_executor.py
Unescape Escape View File

@ -21,15 +21,13 @@ __metaclass__ = type
import os
from ansible.compat.six import string_types
from ansible import constants as C
from ansible.executor.task_queue_manager import TaskQueueManager
from ansible.module_utils._text import to_native, to_text
from ansible.playbook import Playbook
from ansible.template import Templar
from ansible.utils.helpers import pct_to_int
from ansible.utils.path import makedirs_safe
from ansible.utils.unicode import to_unicode, to_str
try:
from __main__ import display
@ -84,7 +82,7 @@ class PlaybookExecutor:
i = 1
plays = pb.get_plays()
display.vv(u'%d plays in %s' % (len(plays), to_unicode(playbook_path)))
display.vv(u'%d plays in %s' % (len(plays), to_text(playbook_path)))
for play in plays:
if play._included_path is not None:
@ -276,7 +274,7 @@ class PlaybookExecutor:
for x in replay_hosts:
fd.write("%s\n" % x)
except Exception as e:
display.warning("Could not create retry file '%s'.\n\t%s" % (retry_path, to_str(e)))
display.warning("Could not create retry file '%s'.\n\t%s" % (retry_path, to_native(e)))
return False
return True

19
lib/ansible/executor/process/worker.py
Unescape Escape View File

@ -19,16 +19,10 @@
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type
from ansible.compat.six.moves import queue
import json
import multiprocessing
import os
import signal
import sys
import time
import traceback
import zlib
from jinja2.exceptions import TemplateNotFound
@ -40,13 +34,10 @@ try:
except ImportError:
HAS_ATFORK=False
from ansible.errors import AnsibleError, AnsibleConnectionFailure
from ansible.errors import AnsibleConnectionFailure
from ansible.executor.task_executor import TaskExecutor
from ansible.executor.task_result import TaskResult
from ansible.playbook.handler import Handler
from ansible.playbook.task import Task
from ansible.vars.unsafe_proxy import AnsibleJSONUnsafeDecoder
from ansible.utils.unicode import to_unicode
from ansible.module_utils._text import to_text
try:
from __main__ import display
@ -144,11 +135,11 @@ class WorkerProcess(multiprocessing.Process):
try:
self._host.vars = dict()
self._host.groups = []
task_result = TaskResult(self._host.name, self._task._uuid, dict(failed=True, exception=to_unicode(traceback.format_exc()), stdout=''))
task_result = TaskResult(self._host.name, self._task._uuid, dict(failed=True, exception=to_text(traceback.format_exc()), stdout=''))
self._rslt_q.put(task_result, block=False)
except:
display.debug(u"WORKER EXCEPTION: %s" % to_unicode(e))
display.debug(u"WORKER TRACEBACK: %s" % to_unicode(traceback.format_exc()))
display.debug(u"WORKER EXCEPTION: %s" % to_text(e))
display.debug(u"WORKER TRACEBACK: %s" % to_text(traceback.format_exc()))
display.debug("WORKER PROCESS EXITING")

35
lib/ansible/executor/task_executor.py
Unescape Escape View File

@ -20,7 +20,6 @@ from __future__ import (absolute_import, division, print_function)
__metaclass__ = type
import base64
import json
import subprocess
import sys
import time
@ -31,12 +30,12 @@ from ansible.compat.six import iteritems, string_types, binary_type
from ansible import constants as C
from ansible.errors import AnsibleError, AnsibleParserError, AnsibleUndefinedVariable, AnsibleConnectionFailure
from ansible.executor.task_result import TaskResult
from ansible.module_utils._text import to_bytes, to_text
from ansible.playbook.conditional import Conditional
from ansible.playbook.task import Task
from ansible.template import Templar
from ansible.utils.encrypt import key_for_hostname
from ansible.utils.listify import listify_lookup_plugin_terms
from ansible.utils.unicode import to_unicode, to_bytes
from ansible.vars.unsafe_proxy import UnsafeProxy, wrap_var
try:
@ -130,7 +129,7 @@ class TaskExecutor:
if isinstance(res, UnsafeProxy):
return res._obj
elif isinstance(res, binary_type):
return to_unicode(res, errors='strict')
return to_text(res, errors='surrogate_or_strict')
elif isinstance(res, dict):
for k in res:
res[k] = _clean_res(res[k])
@ -144,16 +143,16 @@ class TaskExecutor:
display.debug("done dumping result, returning")
return res
except AnsibleError as e:
return dict(failed=True, msg=to_unicode(e, nonstring='simplerepr'))
return dict(failed=True, msg=to_text(e, nonstring='simplerepr'))
except Exception as e:
return dict(failed=True, msg='Unexpected failure during module execution.', exception=to_unicode(traceback.format_exc()), stdout='')
return dict(failed=True, msg='Unexpected failure during module execution.', exception=to_text(traceback.format_exc()), stdout='')
finally:
try:
self._connection.close()
except AttributeError:
pass
except Exception as e:
display.debug(u"error closing connection: %s" % to_unicode(e))
display.debug(u"error closing connection: %s" % to_text(e))
def _get_loop_items(self):
'''
@ -177,16 +176,18 @@ class TaskExecutor:
items = None
if self._task.loop:
if self._task.loop in self._shared_loader_obj.lookup_loader:
#TODO: remove convert_bare true and deprecate this in with_
# TODO: remove convert_bare true and deprecate this in with_
if self._task.loop == 'first_found':
# first_found loops are special. If the item is undefined
# then we want to fall through to the next value rather
# than failing.
loop_terms = listify_lookup_plugin_terms(terms=self._task.loop_args, templar=templar, loader=self._loader, fail_on_undefined=False, convert_bare=True)
loop_terms = listify_lookup_plugin_terms(terms=self._task.loop_args, templar=templar,
loader=self._loader, fail_on_undefined=False, convert_bare=True)
loop_terms = [t for t in loop_terms if not templar._contains_vars(t)]
else:
try:
loop_terms = listify_lookup_plugin_terms(terms=self._task.loop_args, templar=templar, loader=self._loader, fail_on_undefined=True, convert_bare=True)
loop_terms = listify_lookup_plugin_terms(terms=self._task.loop_args, templar=templar,
loader=self._loader, fail_on_undefined=True, convert_bare=True)
except AnsibleUndefinedVariable as e:
display.deprecated("Skipping task due to undefined Error, in the future this will be a fatal error.: %s" % to_bytes(e))
return None
@ -195,7 +196,7 @@ class TaskExecutor:
mylookup = self._shared_loader_obj.lookup_loader.get(self._task.loop, loader=self._loader, templar=templar)
# give lookup task 'context' for subdir (mostly needed for first_found)
for subdir in ['template', 'var', 'file']: #TODO: move this to constants?
for subdir in ['template', 'var', 'file']: # TODO: move this to constants?
if subdir in self._task.action:
break
setattr(mylookup,'_subdir', subdir + 's')
@ -245,7 +246,9 @@ class TaskExecutor:
loop_pause = self._task.loop_control.pause or 0
if loop_var in task_vars:
display.warning("The loop variable '%s' is already in use. You should set the `loop_var` value in the `loop_control` option for the task to something else to avoid variable collisions and unexpected behavior." % loop_var)
display.warning(u"The loop variable '%s' is already in use."
u"You should set the `loop_var` value in the `loop_control` option for the task"
u" to something else to avoid variable collisions and unexpected behavior." % loop_var)
ran_once = False
items = self._squash_items(items, loop_var, task_vars)
@ -263,7 +266,7 @@ class TaskExecutor:
tmp_task._parent = self._task._parent
tmp_play_context = self._play_context.copy()
except AnsibleParserError as e:
results.append(dict(failed=True, msg=to_unicode(e)))
results.append(dict(failed=True, msg=to_text(e)))
continue
# now we swap the internal task and play context with their copies,
@ -279,7 +282,7 @@ class TaskExecutor:
res[loop_var] = item
res['_ansible_item_result'] = True
if not label is None:
if label is not None:
templar = Templar(loader=self._loader, shared_loader_obj=self._shared_loader_obj, variables=self._job_vars)
res['_ansible_item_label'] = templar.template(label, fail_on_undefined=False)
@ -421,7 +424,7 @@ class TaskExecutor:
include_file = templar.template(include_file)
return dict(include=include_file, include_variables=include_variables)
#TODO: not needed?
# TODO: not needed?
# if this task is a IncludeRole, we just return now with a success code so the main thread can expand the task list for the given host
elif self._task.action == 'include_role':
include_variables = self._task.args.copy()
@ -482,7 +485,7 @@ class TaskExecutor:
try:
result = self._handler.run(task_vars=variables)
except AnsibleConnectionFailure as e:
return dict(unreachable=True, msg=to_unicode(e))
return dict(unreachable=True, msg=to_text(e))
display.debug("handler run complete")
# preserve no log
@ -666,7 +669,7 @@ class TaskExecutor:
try:
cmd = subprocess.Popen(['ssh','-o','ControlPersist'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
(out, err) = cmd.communicate()
err = to_unicode(err)
err = to_text(err)
if u"Bad configuration option" in err or u"Usage:" in err:
conn_type = "paramiko"
except OSError:

16
lib/ansible/executor/task_queue_manager.py
Unescape Escape View File

@ -28,21 +28,20 @@ import time
from collections import deque
from ansible import constants as C
from ansible.compat.six import string_types
from ansible.errors import AnsibleError
from ansible.executor import action_write_locks
from ansible.executor.play_iterator import PlayIterator
from ansible.executor.process.worker import WorkerProcess
from ansible.executor.stats import AggregateStats
from ansible.module_utils.facts import Facts
from ansible.module_utils._text import to_text
from ansible.playbook.block import Block
from ansible.playbook.play_context import PlayContext
from ansible.plugins import action_loader, callback_loader, connection_loader, filter_loader, lookup_loader, module_loader, strategy_loader, test_loader
from ansible.template import Templar
from ansible.vars.hostvars import HostVars
from ansible.plugins.callback import CallbackBase
from ansible.template import Templar
from ansible.utils.helpers import pct_to_int
from ansible.utils.unicode import to_unicode
from ansible.compat.six import string_types
from ansible.vars.hostvars import HostVars
try:
from __main__ import display
@ -288,7 +287,8 @@ class TaskQueueManager:
stdout_callback_loaded = True
elif callback_name == 'tree' and self._run_tree:
pass
elif not self._run_additional_callbacks or (callback_needs_whitelist and (C.DEFAULT_CALLBACK_WHITELIST is None or callback_name not in C.DEFAULT_CALLBACK_WHITELIST)):
elif not self._run_additional_callbacks or (callback_needs_whitelist and (
C.DEFAULT_CALLBACK_WHITELIST is None or callback_name not in C.DEFAULT_CALLBACK_WHITELIST)):
continue
self._callback_plugins.append(callback_plugin())
@ -468,8 +468,8 @@ class TaskQueueManager:
else:
method(*args, **kwargs)
except Exception as e:
#TODO: add config toggle to make this fatal or not?
display.warning(u"Failure using method (%s) in callback plugin (%s): %s" % (to_unicode(method_name), to_unicode(callback_plugin), to_unicode(e)))
# TODO: add config toggle to make this fatal or not?
display.warning(u"Failure using method (%s) in callback plugin (%s): %s" % (to_text(method_name), to_text(callback_plugin), to_text(e)))
from traceback import format_tb
from sys import exc_info
display.debug('Callback Exception: \n' + ' '.join(format_tb(exc_info()[2])))

10
lib/ansible/galaxy/api.py
Unescape Escape View File

@ -28,12 +28,12 @@ import json
import ansible.constants as C
from ansible.compat.six import string_types
from ansible.compat.six.moves.urllib.parse import quote as urlquote, urlencode
from ansible.compat.six.moves.urllib.error import HTTPError
from ansible.compat.six.moves.urllib.parse import quote as urlquote, urlencode
from ansible.errors import AnsibleError
from ansible.module_utils.urls import open_url
from ansible.galaxy.token import GalaxyToken
from ansible.utils.unicode import to_str
from ansible.module_utils._text import to_native
from ansible.module_utils.urls import open_url
try:
from __main__ import display
@ -115,12 +115,12 @@ class GalaxyAPI(object):
try:
return_data = open_url(url, validate_certs=self._validate_certs)
except Exception as e:
raise AnsibleError("Failed to get data from the API server (%s): %s " % (url, to_str(e)))
raise AnsibleError("Failed to get data from the API server (%s): %s " % (url, to_native(e)))
try:
data = json.load(return_data)
except Exception as e:
raise AnsibleError("Could not process data from the API server (%s): %s " % (url, to_str(e)))
raise AnsibleError("Could not process data from the API server (%s): %s " % (url, to_native(e)))
if 'current_version' not in data:
raise AnsibleError("missing required 'current_version' from server response (%s)" % url)

31
lib/ansible/inventory/__init__.py
Unescape Escape View File

@ -33,12 +33,10 @@ from ansible.errors import AnsibleError
from ansible.inventory.dir import InventoryDirectory, get_file_parser
from ansible.inventory.group import Group
from ansible.inventory.host import Host
from ansible.module_utils._text import to_bytes, to_text
from ansible.parsing.utils.addresses import parse_address
from ansible.plugins import vars_loader
from ansible.utils.unicode import to_unicode, to_bytes
from ansible.utils.vars import combine_vars
from ansible.parsing.utils.addresses import parse_address
HOSTS_PATTERNS_CACHE = {}
try:
from __main__ import display
@ -46,6 +44,10 @@ except ImportError:
from ansible.utils.display import Display
display = Display()
HOSTS_PATTERNS_CACHE = {}
class Inventory(object):
"""
Host inventory for ansible.
@ -125,7 +127,7 @@ class Inventory(object):
try:
(host, port) = parse_address(h, allow_ranges=False)
except AnsibleError as e:
display.vvv("Unable to parse address from hostname, leaving unchanged: %s" % to_unicode(e))
display.vvv("Unable to parse address from hostname, leaving unchanged: %s" % to_text(e))
host = h
port = None
@ -138,7 +140,7 @@ class Inventory(object):
self.localhost = new_host
all.add_host(new_host)
elif self._loader.path_exists(host_list):
#TODO: switch this to a plugin loader and a 'condition' per plugin on which it should be tried, restoring 'inventory pllugins'
# TODO: switch this to a plugin loader and a 'condition' per plugin on which it should be tried, restoring 'inventory pllugins'
if self.is_directory(host_list):
# Ensure basedir is inside the directory
host_list = os.path.join(self.host_list, "")
@ -151,7 +153,7 @@ class Inventory(object):
# should never happen, but JIC
raise AnsibleError("Unable to parse %s as an inventory source" % host_list)
else:
display.warning("Host file not found: %s" % to_unicode(host_list))
display.warning("Host file not found: %s" % to_text(host_list))
self._vars_plugins = [ x for x in vars_loader.all(self) ]
@ -205,9 +207,9 @@ class Inventory(object):
if not ignore_limits_and_restrictions:
if self._subset:
pattern_hash += u":%s" % to_unicode(self._subset)
pattern_hash += u":%s" % to_text(self._subset)
if self._restriction:
pattern_hash += u":%s" % to_unicode(self._restriction)
pattern_hash += u":%s" % to_text(self._restriction)
if pattern_hash not in HOSTS_PATTERNS_CACHE:
@ -491,7 +493,8 @@ class Inventory(object):
py_interp = sys.executable
if not py_interp:
# sys.executable is not set in some cornercases. #13585
display.warning('Unable to determine python interpreter from sys.executable. Using /usr/bin/python default. You can correct this by setting ansible_python_interpreter for localhost')
display.warning('Unable to determine python interpreter from sys.executable. Using /usr/bin/python default.'
' You can correct this by setting ansible_python_interpreter for localhost')
py_interp = '/usr/bin/python'
new_host.set_variable("ansible_python_interpreter", py_interp)
self.get_group("ungrouped").add_host(new_host)
@ -781,7 +784,7 @@ class Inventory(object):
path = os.path.realpath(os.path.join(basedir, 'group_vars'))
found_vars = set()
if os.path.exists(path):
found_vars = set(os.listdir(to_unicode(path)))
found_vars = set(os.listdir(to_text(path)))
return found_vars
def _find_host_vars_files(self, basedir):
@ -791,7 +794,7 @@ class Inventory(object):
path = os.path.realpath(os.path.join(basedir, 'host_vars'))
found_vars = set()
if os.path.exists(path):
found_vars = set(os.listdir(to_unicode(path)))
found_vars = set(os.listdir(to_text(path)))
return found_vars
def _get_hostgroup_vars(self, host=None, group=None, new_pb_basedir=False, return_results=False):
@ -832,13 +835,13 @@ class Inventory(object):
# Before trying to load vars from file, check that the directory contains relvant file names
if host is None and any(map(lambda ext: group.name + ext in self._group_vars_files, C.YAML_FILENAME_EXTENSIONS)):
# load vars in dir/group_vars/name_of_group
base_path = to_unicode(os.path.abspath(os.path.join(to_bytes(basedir), b"group_vars/" + to_bytes(group.name))), errors='strict')
base_path = to_text(os.path.abspath(os.path.join(to_bytes(basedir), b"group_vars/" + to_bytes(group.name))), errors='surrogate_or_strict')
host_results = self._variable_manager.add_group_vars_file(base_path, self._loader)
if return_results:
results = combine_vars(results, host_results)
elif group is None and any(map(lambda ext: host.name + ext in self._host_vars_files, C.YAML_FILENAME_EXTENSIONS)):
# same for hostvars in dir/host_vars/name_of_host
base_path = to_unicode(os.path.abspath(os.path.join(to_bytes(basedir), b"host_vars/" + to_bytes(host.name))), errors='strict')
base_path = to_text(os.path.abspath(os.path.join(to_bytes(basedir), b"host_vars/" + to_bytes(host.name))), errors='surrogate_or_strict')
group_results = self._variable_manager.add_host_vars_file(base_path, self._loader)
if return_results:
results = combine_vars(results, group_results)

10
lib/ansible/inventory/ini.py
Unescape Escape View File

@ -28,9 +28,10 @@ from ansible.inventory.host import Host
from ansible.inventory.group import Group
from ansible.inventory.expand_hosts import detect_range
from ansible.inventory.expand_hosts import expand_hostname_range
from ansible.module_utils._text import to_text
from ansible.parsing.utils.addresses import parse_address
from ansible.utils.shlex import shlex_split
from ansible.utils.unicode import to_unicode
class InventoryParser(object):
"""
@ -56,7 +57,7 @@ class InventoryParser(object):
(data, private) = loader._get_file_contents(filename)
else:
with open(filename) as fh:
data = to_unicode(fh.read())
data = to_text(fh.read())
data = data.split('\n')
self._parse(data)
@ -125,7 +126,7 @@ class InventoryParser(object):
continue
elif line.startswith('[') and line.endswith(']'):
self._raise_error("Invalid section entry: '%s'. Please make sure that there are no spaces" % line + \
self._raise_error("Invalid section entry: '%s'. Please make sure that there are no spaces" % line +
"in the section entry, and that there are no other invalid characters")
# It's not a section, so the current state tells us what kind of
@ -188,7 +189,6 @@ class InventoryParser(object):
if group.depth == 0 and group.name not in ('all', 'ungrouped'):
self.groups['all'].add_child_group(group)
def _parse_group_name(self, line):
'''
Takes a single line and tries to parse it as a group name. Returns the
@ -323,7 +323,7 @@ class InventoryParser(object):
except SyntaxError:
# Is this a hash with an equals at the end?
pass
return to_unicode(v, nonstring='passthru', errors='strict')
return to_text(v, nonstring='passthru', errors='surrogate_or_strict')
def get_host_variables(self, host):
return {}

12
lib/ansible/inventory/script.py
Unescape Escape View File

@ -31,7 +31,7 @@ from ansible.errors import AnsibleError
from ansible.inventory.host import Host
from ansible.inventory.group import Group
from ansible.module_utils.basic import json_dict_bytes_to_unicode
from ansible.utils.unicode import to_str, to_unicode
from ansible.module_utils._text import to_native, to_text
class InventoryScript:
@ -61,9 +61,9 @@ class InventoryScript:
# make sure script output is unicode so that json loader will output
# unicode strings itself
try:
self.data = to_unicode(stdout, errors="strict")
self.data = to_text(stdout, errors="strict")
except Exception as e:
raise AnsibleError("inventory data from {0} contained characters that cannot be interpreted as UTF-8: {1}".format(to_str(self.filename), to_str(e)))
raise AnsibleError("inventory data from {0} contained characters that cannot be interpreted as UTF-8: {1}".format(to_native(self.filename), to_native(e)))
# see comment about _meta below
self.host_vars_from_top = None
@ -78,11 +78,11 @@ class InventoryScript:
self.raw = self._loader.load(self.data)
except Exception as e:
sys.stderr.write(err + "\n")
raise AnsibleError("failed to parse executable inventory script results from {0}: {1}".format(to_str(self.filename), to_str(e)))
raise AnsibleError("failed to parse executable inventory script results from {0}: {1}".format(to_native(self.filename), to_native(e)))
if not isinstance(self.raw, Mapping):
sys.stderr.write(err + "\n")
raise AnsibleError("failed to parse executable inventory script results from {0}: data needs to be formatted as a json dict".format(to_str(self.filename)))
raise AnsibleError("failed to parse executable inventory script results from {0}: data needs to be formatted as a json dict".format(to_native(self.filename)))
group = None
for (group_name, data) in self.raw.items():
@ -152,7 +152,7 @@ class InventoryScript:
try:
got = self.host_vars_from_top.get(host.name, {})
except AttributeError as e:
raise AnsibleError("Improperly formated host information for %s: %s" % (host.name,to_str(e)))
raise AnsibleError("Improperly formated host information for %s: %s" % (host.name,to_native(e)))
return got
cmd = [self.filename, "--host", host.name]

20
lib/ansible/module_utils/_text.py
Unescape Escape View File

@ -32,16 +32,18 @@
making backwards compatibility guarantees. The API may change between
releases. Do not use this unless you are willing to port your module code.
"""
import codecs
from ansible.module_utils.six import PY3, text_type, binary_type
import codecs
try:
codecs.lookup_error('surrogateescape')
HAS_SURROGATEESCAPE = True
except LookupError:
HAS_SURROGATEESCAPE = False
def to_bytes(obj, encoding='utf-8', errors=None, nonstring='simplerepr'):
"""Make sure that a string is a byte string
@ -109,7 +111,14 @@ def to_bytes(obj, encoding='utf-8', errors=None, nonstring='simplerepr'):
# Note: We do these last even though we have to call to_bytes again on the
# value because we're optimizing the common case
if nonstring == 'simplerepr':
try:
value = str(obj)
except UnicodeError:
try:
value = repr(obj)
except UnicodeError:
# Giving up
return to_bytes('')
elif nonstring == 'passthru':
return obj
elif nonstring == 'empty':
@ -122,6 +131,7 @@ def to_bytes(obj, encoding='utf-8', errors=None, nonstring='simplerepr'):
return to_bytes(value, encoding, errors)
def to_text(obj, encoding='utf-8', errors=None, nonstring='simplerepr'):
"""Make sure that a string is a text string
@ -175,7 +185,14 @@ def to_text(obj, encoding='utf-8', errors=None, nonstring='simplerepr'):
# Note: We do these last even though we have to call to_text again on the
# value because we're optimizing the common case
if nonstring == 'simplerepr':
try:
value = str(obj)
except UnicodeError:
try:
value = repr(obj)
except UnicodeError:
# Giving up
return u''
elif nonstring == 'passthru':
return obj
elif nonstring == 'empty':
@ -187,6 +204,7 @@ def to_text(obj, encoding='utf-8', errors=None, nonstring='simplerepr'):
return to_text(value, encoding, errors)
#: :py:func:`to_native`
#: Transform a variable into the native str type for the python version
#:

14
lib/ansible/module_utils/basic.py
Unescape Escape View File

@ -805,7 +805,7 @@ class AnsibleModule(object):
if not HAVE_SELINUX or not self.selinux_enabled():
return context
try:
ret = selinux.matchpathcon(to_native(path, errors='strict'), mode)
ret = selinux.matchpathcon(to_native(path, errors='surrogate_or_strict'), mode)
except OSError:
return context
if ret[0] == -1:
@ -820,7 +820,7 @@ class AnsibleModule(object):
if not HAVE_SELINUX or not self.selinux_enabled():
return context
try:
ret = selinux.lgetfilecon_raw(to_native(path, errors='strict'))
ret = selinux.lgetfilecon_raw(to_native(path, errors='surrogate_or_strict'))
except OSError:
e = get_exception()
if e.errno == errno.ENOENT:
@ -2121,10 +2121,10 @@ class AnsibleModule(object):
to_clean_args = args
if PY2:
if isinstance(args, text_type):
to_clean_args = args.encode('utf-8')
to_clean_args = to_bytes(args)
else:
if isinstance(args, binary_type):
to_clean_args = args.decode('utf-8', errors='replace')
to_clean_args = to_text(args)
if isinstance(args, (text_type, binary_type)):
to_clean_args = shlex.split(to_clean_args)
@ -2193,11 +2193,7 @@ class AnsibleModule(object):
if not binary_data:
data += '\n'
if isinstance(data, text_type):
if PY3:
errors = 'surrogateescape'
else:
errors = 'strict'
data = data.encode('utf-8', errors=errors)
data = to_bytes(data)
cmd.stdin.write(data)
cmd.stdin.close()

55
lib/ansible/parsing/dataloader.py
Unescape Escape View File

@ -27,16 +27,15 @@ import tempfile
from yaml import YAMLError
from ansible.compat.six import text_type, string_types
from ansible.errors import AnsibleFileNotFound, AnsibleParserError, AnsibleError
from ansible.errors.yaml_strings import YAML_SYNTAX_ERROR
from ansible.module_utils.basic import is_executable
from ansible.module_utils._text import to_bytes, to_native, to_text
from ansible.parsing.vault import VaultLib
from ansible.parsing.quoting import unquote
from ansible.parsing.yaml.loader import AnsibleLoader
from ansible.parsing.yaml.objects import AnsibleBaseYAMLObject, AnsibleUnicode
from ansible.module_utils.basic import is_executable
from ansible.utils.path import unfrackpath
from ansible.utils.unicode import to_unicode, to_bytes, to_str
try:
from __main__ import display
@ -44,6 +43,7 @@ except ImportError:
from ansible.utils.display import Display
display = Display()
class DataLoader():
'''
@ -127,15 +127,15 @@ class DataLoader():
def path_exists(self, path):
path = self.path_dwim(path)
return os.path.exists(to_bytes(path, errors='strict'))
return os.path.exists(to_bytes(path, errors='surrogate_or_strict'))
def is_file(self, path):
path = self.path_dwim(path)
return os.path.isfile(to_bytes(path, errors='strict')) or path == os.devnull
return os.path.isfile(to_bytes(path, errors='surrogate_or_strict')) or path == os.devnull
def is_directory(self, path):
path = self.path_dwim(path)
return os.path.isdir(to_bytes(path, errors='strict'))
return os.path.isdir(to_bytes(path, errors='surrogate_or_strict'))
def list_directory(self, path):
path = self.path_dwim(path)
@ -178,7 +178,7 @@ class DataLoader():
data = self._vault.decrypt(data, filename=b_file_name)
show_content = False
data = to_unicode(data, errors='strict')
data = to_text(data, errors='surrogate_or_strict')
return (data, show_content)
except (IOError, OSError) as e:
@ -208,7 +208,7 @@ class DataLoader():
''' sets the base directory, used to find files when a relative path is given '''
if basedir is not None:
self._basedir = to_unicode(basedir)
self._basedir = to_text(basedir)
def path_dwim(self, given):
'''
@ -216,14 +216,14 @@ class DataLoader():
'''
given = unquote(given)
given = to_unicode(given, errors='strict')
given = to_text(given, errors='surrogate_or_strict')
if given.startswith(u"/"):
return os.path.abspath(given)
elif given.startswith(u"~"):
return os.path.abspath(os.path.expanduser(given))
else:
basedir = to_unicode(self._basedir, errors='strict')
basedir = to_text(self._basedir, errors='surrogate_or_strict')
return os.path.abspath(os.path.join(basedir, given))
def path_dwim_relative(self, path, dirname, source):
@ -247,8 +247,8 @@ class DataLoader():
basedir = unfrackpath(path)
# is it a role and if so make sure you get correct base path
if path.endswith('tasks') and os.path.exists(to_bytes(os.path.join(path,'main.yml'), errors='strict')) \
or os.path.exists(to_bytes(os.path.join(path,'tasks/main.yml'), errors='strict')):
if path.endswith('tasks') and os.path.exists(to_bytes(os.path.join(path,'main.yml'), errors='surrogate_or_strict')) \
or os.path.exists(to_bytes(os.path.join(path,'tasks/main.yml'), errors='surrogate_or_strict')):
isrole = True
if path.endswith('tasks'):
basedir = unfrackpath(os.path.dirname(path))
@ -271,7 +271,7 @@ class DataLoader():
search.append(self.path_dwim(source))
for candidate in search:
if os.path.exists(to_bytes(candidate, errors='strict')):
if os.path.exists(to_bytes(candidate, errors='surrogate_or_strict')):
break
return candidate
@ -296,13 +296,13 @@ class DataLoader():
elif source.startswith('~') or source.startswith(os.path.sep):
# path is absolute, no relative needed, check existence and return source
test_path = unfrackpath(b_source)
if os.path.exists(to_bytes(test_path, errors='strict')):
if os.path.exists(to_bytes(test_path, errors='surrogate_or_strict')):
result = test_path
else:
search = []
for path in paths:
upath = unfrackpath(path)
b_upath = to_bytes(upath, errors='strict')
b_upath = to_bytes(upath, errors='surrogate_or_strict')
b_mydir = os.path.dirname(b_upath)
# if path is in role and 'tasks' not there already, add it into the search
@ -324,11 +324,11 @@ class DataLoader():
search.append(os.path.join(to_bytes(self.get_basedir()), b_dirname, b_source))
search.append(os.path.join(to_bytes(self.get_basedir()), b_source))
display.debug(u'search_path:\n\t%s' % to_unicode(b'\n\t'.join(search), errors='replace'))
display.debug(u'search_path:\n\t%s' % to_text(b'\n\t'.join(search)))
for b_candidate in search:
display.vvvvv(u'looking for "%s" at "%s"' % (source, to_unicode(b_candidate)))
display.vvvvv(u'looking for "%s" at "%s"' % (source, to_text(b_candidate)))
if os.path.exists(b_candidate):
result = to_unicode(b_candidate)
result = to_text(b_candidate)
break
return result
@ -339,8 +339,8 @@ class DataLoader():
retrieve password from STDOUT
"""
this_path = os.path.realpath(to_bytes(os.path.expanduser(vault_password_file), errors='strict'))
if not os.path.exists(to_bytes(this_path, errors='strict')):
this_path = os.path.realpath(to_bytes(os.path.expanduser(vault_password_file), errors='surrogate_or_strict'))
if not os.path.exists(to_bytes(this_path, errors='surrogate_or_strict')):
raise AnsibleFileNotFound("The vault password file %s was not found" % this_path)
if self.is_executable(this_path):
@ -348,7 +348,8 @@ class DataLoader():
# STDERR not captured to make it easier for users to prompt for input in their scripts
p = subprocess.Popen(this_path, stdout=subprocess.PIPE)
except OSError as e:
raise AnsibleError("Problem running vault password script %s (%s). If this is not a script, remove the executable bit from the file." % (' '.join(this_path), e))
raise AnsibleError("Problem running vault password script %s (%s)."
" If this is not a script, remove the executable bit from the file." % (' '.join(this_path), to_native(e)))
stdout, stderr = p.communicate()
self.set_vault_password(stdout.strip('\r\n'))
else:
@ -381,11 +382,11 @@ class DataLoader():
"""
if not file_path or not isinstance(file_path, string_types):
raise AnsibleParserError("Invalid filename: '%s'" % to_str(file_path))
raise AnsibleParserError("Invalid filename: '%s'" % to_native(file_path))
b_file_path = to_bytes(file_path, errors='strict')
b_file_path = to_bytes(file_path, errors='surrogate_or_strict')
if not self.path_exists(b_file_path) or not self.is_file(b_file_path):
raise AnsibleFileNotFound("the file_name '%s' does not exist, or is not readable" % to_str(file_path))
raise AnsibleFileNotFound("the file_name '%s' does not exist, or is not readable" % to_native(file_path))
if not self._vault:
self._vault = VaultLib(password="")
@ -410,7 +411,7 @@ class DataLoader():
return real_path
except (IOError, OSError) as e:
raise AnsibleParserError("an error occurred while trying to read the file '%s': %s" % (to_str(real_path), to_str(e)))
raise AnsibleParserError("an error occurred while trying to read the file '%s': %s" % (to_native(real_path), to_native(e)))
def cleanup_tmp_file(self, file_path):
"""
@ -420,11 +421,11 @@ class DataLoader():
"""
if file_path in self._tempfiles:
os.unlink(file_path)
self._tempfiles.remove(file_path);
self._tempfiles.remove(file_path)
def cleanup_all_tmp_files(self):
for f in self._tempfiles:
try:
self.cleanup_tmp_file(f)
except:
pass #TODO: this should at least warn
pass # TODO: this should at least warn

13
lib/ansible/parsing/splitter.py
Unescape Escape View File

@ -23,8 +23,10 @@ import re
import codecs
from ansible.errors import AnsibleParserError
from ansible.module_utils._text import to_text
from ansible.parsing.quoting import unquote
# Decode escapes adapted from rspeer's answer here:
# http://stackoverflow.com/questions/4020539/process-escape-sequences-in-a-string-in-python
_HEXCHAR = '[a-fA-F0-9]'
@ -36,12 +38,14 @@ _ESCAPE_SEQUENCE_RE = re.compile(r'''
| \\[\\'"abfnrtv] # Single-character escapes
)'''.format(_HEXCHAR*8, _HEXCHAR*4, _HEXCHAR*2), re.UNICODE | re.VERBOSE)
def _decode_escapes(s):
def decode_match(match):
return codecs.decode(match.group(0), 'unicode-escape')
return _ESCAPE_SEQUENCE_RE.sub(decode_match, s)
def parse_kv(args, check_raw=False):
'''
Convert a string of key/value items to a dict. If any free-form params
@ -50,9 +54,7 @@ def parse_kv(args, check_raw=False):
they will simply be ignored.
'''
### FIXME: args should already be a unicode string
from ansible.utils.unicode import to_unicode
args = to_unicode(args, nonstring='passthru')
args = to_text(args, nonstring='passthru')
options = {}
if args is not None:
@ -60,7 +62,7 @@ def parse_kv(args, check_raw=False):
vargs = split_args(args)
except ValueError as ve:
if 'no closing quotation' in str(ve).lower():
raise AnsibleParsingError("error parsing argument string, try quoting the entire line.")
raise AnsibleParserError("error parsing argument string, try quoting the entire line.")
else:
raise
@ -99,6 +101,7 @@ def parse_kv(args, check_raw=False):
return options
def _get_quote_state(token, quote_char):
'''
the goal of this block is to determine if the quoted string
@ -118,6 +121,7 @@ def _get_quote_state(token, quote_char):
quote_char = cur_char
return quote_char
def _count_jinja2_blocks(token, cur_depth, open_token, close_token):
'''
this function counts the number of opening/closing blocks for a
@ -132,6 +136,7 @@ def _count_jinja2_blocks(token, cur_depth, open_token, close_token):
cur_depth = 0
return cur_depth
def split_args(args):
'''
Splits args on whitespace, but intelligently reassembles

30
lib/ansible/parsing/vault/__init__.py
Unescape Escape View File

@ -29,16 +29,10 @@ from ansible.errors import AnsibleError
from hashlib import sha256
from binascii import hexlify
from binascii import unhexlify
try:
from __main__ import display
except ImportError:
from ansible.utils.display import Display
display = Display()
from hashlib import md5
# Note: Only used for loading obsolete VaultAES files. All files are written
# using the newer VaultAES256 which does not require md5
from hashlib import md5
try:
from Crypto.Hash import SHA256, HMAC
@ -67,6 +61,15 @@ try:
except ImportError:
HAS_AES = False
from ansible.compat.six import PY3
from ansible.module_utils._text import to_bytes, to_text
try:
from __main__ import display
except ImportError:
from ansible.utils.display import Display
display = Display()
# OpenSSL pbkdf2_hmac
HAS_PBKDF2HMAC = False
try:
@ -81,12 +84,11 @@ except Exception as e:
import traceback
display.debug("Traceback from import of cryptography was {0}".format(traceback.format_exc()))
from ansible.compat.six import PY3
from ansible.utils.unicode import to_unicode, to_bytes
HAS_ANY_PBKDF2HMAC = HAS_PBKDF2 or HAS_PBKDF2HMAC
CRYPTO_UPGRADE = "ansible-vault requires a newer version of pycrypto than the one installed on your platform. You may fix this with OS-specific commands such as: yum install python-devel; rpm -e --nodeps python-crypto; pip install pycrypto"
CRYPTO_UPGRADE = "ansible-vault requires a newer version of pycrypto than the one installed on your platform." \
" You may fix this with OS-specific commands such as: yum install python-devel; rpm -e --nodeps python-crypto; pip install pycrypto"
b_HEADER = b'$ANSIBLE_VAULT'
HEADER = '$ANSIBLE_VAULT'
@ -105,6 +107,7 @@ def check_prereqs():
class AnsibleVaultError(AnsibleError):
pass
def is_encrypted(b_data):
""" Test if this is vault encrypted data blob
@ -116,6 +119,7 @@ def is_encrypted(b_data):
return True
return False
def is_encrypted_file(file_obj):
"""Test if the contents of a file obj are a vault encrypted data blob.
@ -252,7 +256,7 @@ class VaultLib:
b_header = HEADER.encode('utf-8')
header = b';'.join([b_header, self.b_version,
to_bytes(self.cipher_name,'utf-8',errors='strict')])
to_bytes(self.cipher_name,'utf-8', errors='strict')])
tmpdata = [header]
tmpdata += [b_data[i:i + 80] for i in range(0, len(b_data), 80)]
tmpdata += [b'']
@ -278,7 +282,7 @@ class VaultLib:
tmpheader = tmpdata[0].strip().split(b';')
self.b_version = tmpheader[1].strip()
self.cipher_name = to_unicode(tmpheader[2].strip())
self.cipher_name = to_text(tmpheader[2].strip())
clean_data = b''.join(tmpdata[1:])
return clean_data
@ -528,6 +532,7 @@ class VaultEditor:
return editor
class VaultFile(object):
def __init__(self, password, filename):
@ -568,6 +573,7 @@ class VaultFile(object):
else:
return self.filename
########################################
# CIPHERS #
########################################

8
lib/ansible/parsing/yaml/constructor.py
Unescape Escape View File

@ -22,12 +22,11 @@ __metaclass__ = type
from yaml.constructor import Constructor, ConstructorError
from yaml.nodes import MappingNode
from ansible.module_utils._text import to_bytes
from ansible.parsing.vault import VaultLib
from ansible.parsing.yaml.objects import AnsibleMapping, AnsibleSequence, AnsibleUnicode
from ansible.parsing.yaml.objects import AnsibleVaultEncryptedUnicode
from ansible.vars.unsafe_proxy import wrap_var
from ansible.parsing.vault import VaultLib
from ansible.utils.unicode import to_bytes
try:
from __main__ import display
@ -74,7 +73,8 @@ class AnsibleConstructor(Constructor):
"found unacceptable key (%s)" % exc, key_node.start_mark)
if key in mapping:
display.warning(u'While constructing a mapping from {1}, line {2}, column {3}, found a duplicate dict key ({0}). Using last defined value only.'.format(key, *mapping.ansible_pos))
display.warning(u'While constructing a mapping from {1}, line {2}, column {3}, found a duplicate dict key ({0}).'
u' Using last defined value only.'.format(key, *mapping.ansible_pos))
value = self.construct_object(value_node, deep=deep)
mapping[key] = value

3
lib/ansible/parsing/yaml/objects.py
Unescape Escape View File

@ -22,8 +22,7 @@ __metaclass__ = type
import yaml
from ansible.compat.six import text_type
from ansible.errors import AnsibleError
from ansible.utils.unicode import to_bytes
from ansible.module_utils._text import to_bytes
class AnsibleBaseYAMLObject(object):

24
lib/ansible/playbook/base.py
Unescape Escape View File

@ -19,25 +19,22 @@
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type
import collections
import itertools
import operator
import uuid
from copy import copy as shallowcopy, deepcopy
from copy import copy as shallowcopy
from functools import partial
from inspect import getmembers
from ansible.compat.six import iteritems, string_types, with_metaclass
from jinja2.exceptions import UndefinedError
from ansible.compat.six import iteritems, string_types, with_metaclass
from ansible.errors import AnsibleParserError, AnsibleUndefinedVariable
from ansible.parsing.dataloader import DataLoader
from ansible.module_utils._text import to_text
from ansible.playbook.attribute import Attribute, FieldAttribute
from ansible.parsing.dataloader import DataLoader
from ansible.utils.boolean import boolean
from ansible.utils.vars import combine_vars, isidentifier
from ansible.utils.unicode import to_unicode
try:
from __main__ import display
@ -52,6 +49,7 @@ def _generic_g(prop_name, self):
except KeyError:
raise AttributeError("'%s' object has no attribute '%s'" % (self.__class__.__name__, prop_name))
def _generic_g_method(prop_name, self):
try:
if self._squashed:
@ -61,6 +59,7 @@ def _generic_g_method(prop_name, self):
except KeyError:
raise AttributeError("'%s' object has no attribute '%s'" % (self.__class__.__name__, prop_name))
def _generic_g_parent(prop_name, self):
try:
value = self._attributes[prop_name]
@ -74,12 +73,15 @@ def _generic_g_parent(prop_name, self):
return value
def _generic_s(prop_name, self, value):
self._attributes[prop_name] = value
def _generic_d(prop_name, self):
del self._attributes[prop_name]
class BaseMeta(type):
"""
@ -142,6 +144,7 @@ class BaseMeta(type):
return super(BaseMeta, cls).__new__(cls, name, parents, dct)
class Base(with_metaclass(BaseMeta, object)):
# connection/transport
@ -376,7 +379,7 @@ class Base(with_metaclass(BaseMeta, object)):
# and make sure the attribute is of the type it should be
if value is not None:
if attribute.isa == 'string':
value = to_unicode(value)
value = to_text(value)
elif attribute.isa == 'int':
value = int(value)
elif attribute.isa == 'float':
@ -395,8 +398,8 @@ class Base(with_metaclass(BaseMeta, object)):
elif not isinstance(value, list):
if isinstance(value, string_types) and attribute.isa == 'barelist':
display.deprecated(
"Using comma separated values for a list has been deprecated. " \
"You should instead use the correct YAML syntax for lists. " \
"Using comma separated values for a list has been deprecated. "
"You should instead use the correct YAML syntax for lists. "
)
value = value.split(',')
else:
@ -532,4 +535,3 @@ class Base(with_metaclass(BaseMeta, object)):
setattr(self, '_uuid', data.get('uuid'))
self._finalized = data.get('finalized', False)
self._squashed = data.get('squashed', False)

10
lib/ansible/playbook/task.py
Unescape Escape View File

@ -22,12 +22,10 @@ __metaclass__ = type
import os
from ansible.compat.six import iteritems, string_types
from ansible.errors import AnsibleError, AnsibleParserError
from ansible.module_utils._text import to_native
from ansible.parsing.mod_args import ModuleArgsParser
from ansible.parsing.yaml.objects import AnsibleBaseYAMLObject, AnsibleMapping, AnsibleUnicode
from ansible.plugins import lookup_loader
from ansible.playbook.attribute import FieldAttribute
from ansible.playbook.base import Base
@ -38,7 +36,6 @@ from ansible.playbook.loop_control import LoopControl
from ansible.playbook.role import Role
from ansible.playbook.taggable import Taggable
from ansible.utils.unicode import to_str
try:
from __main__ import display
@ -182,7 +179,7 @@ class Task(Base, Conditional, Taggable, Become):
try:
(action, args, delegate_to) = args_parser.parse()
except AnsibleParserError as e:
raise AnsibleParserError(to_str(e), obj=ds)
raise AnsibleParserError(to_native(e), obj=ds)
# the command/shell/script modules used to support the `cmd` arg,
# which corresponds to what we now call _raw_params, so move that
@ -233,7 +230,7 @@ class Task(Base, Conditional, Taggable, Become):
def _load_loop_control(self, attr, ds):
if not isinstance(ds, dict):
raise AnsibleParserError(
"the `loop_control` value must be specified as a dictionary and cannot " \
"the `loop_control` value must be specified as a dictionary and cannot "
"be a variable itself (though it can contain variables)",
obj=ds,
)
@ -452,4 +449,3 @@ class Task(Base, Conditional, Taggable, Become):
if self._parent:
return self._parent.all_parents_static()
return True

15
lib/ansible/plugins/__init__.py
Unescape Escape View File

@ -31,7 +31,8 @@ import warnings
from collections import defaultdict
from ansible import constants as C
from ansible.utils.unicode import to_unicode
from ansible.module_utils._text import to_text
try:
from __main__ import display
@ -44,9 +45,11 @@ MODULE_CACHE = {}
PATH_CACHE = {}
PLUGIN_PATH_CACHE = {}
def get_all_plugin_loaders():
return [(name, obj) for (name, obj) in inspect.getmembers(sys.modules[__name__]) if isinstance(obj, PluginLoader)]
class PluginLoader:
'''
@ -72,11 +75,11 @@ class PluginLoader:
self.config = config
if not class_name in MODULE_CACHE:
if class_name not in MODULE_CACHE:
MODULE_CACHE[class_name] = {}
if not class_name in PATH_CACHE:
if class_name not in PATH_CACHE:
PATH_CACHE[class_name] = None
if not class_name in PLUGIN_PATH_CACHE:
if class_name not in PLUGIN_PATH_CACHE:
PLUGIN_PATH_CACHE[class_name] = defaultdict(dict)
self._module_cache = MODULE_CACHE[class_name]
@ -250,7 +253,7 @@ class PluginLoader:
try:
full_paths = (os.path.join(path, f) for f in os.listdir(path))
except OSError as e:
display.warning("Error accessing plugin paths: %s" % to_unicode(e))
display.warning("Error accessing plugin paths: %s" % to_text(e))
for full_path in (f for f in full_paths if os.path.isfile(f) and not f.endswith('__init__.py')):
full_name = os.path.basename(full_path)
@ -389,7 +392,7 @@ class PluginLoader:
try:
obj = getattr(self._module_cache[path], self.class_name)
except AttributeError as e:
display.warning("Skipping plugin (%s) as it seems to be invalid: %s" % (path, to_unicode(e)))
display.warning("Skipping plugin (%s) as it seems to be invalid: %s" % (path, to_text(e)))
continue
if self.base_class:

48
lib/ansible/plugins/action/__init__.py
Unescape Escape View File

@ -35,9 +35,10 @@ from ansible.compat.six import binary_type, text_type, iteritems, with_metaclass
from ansible import constants as C
from ansible.errors import AnsibleError, AnsibleConnectionFailure
from ansible.executor.module_common import modify_module
from ansible.release import __version__
from ansible.module_utils._text import to_bytes, to_native, to_text
from ansible.parsing.utils.jsonify import jsonify
from ansible.utils.unicode import to_bytes, to_str, to_unicode
from ansible.release import __version__
try:
from __main__ import display
@ -146,7 +147,8 @@ class ActionBase(with_metaclass(ABCMeta, object)):
"run 'git submodule update --init --recursive' to correct this problem." % (module_name))
# insert shared code and arguments into the module
(module_data, module_style, module_shebang) = modify_module(module_name, module_path, module_args, task_vars=task_vars, module_compression=self._play_context.module_compression)
(module_data, module_style, module_shebang) = modify_module(module_name, module_path, module_args,
task_vars=task_vars, module_compression=self._play_context.module_compression)
return (module_style, module_shebang, module_data, module_path)
@ -283,10 +285,10 @@ class ActionBase(with_metaclass(ABCMeta, object)):
afd, afile = tempfile.mkstemp()
afo = os.fdopen(afd, 'wb')
try:
data = to_bytes(data, errors='strict')
data = to_bytes(data, errors='surrogate_or_strict')
afo.write(data)
except Exception as e:
raise AnsibleError("failure writing module data to temporary file for transfer: %s" % str(e))
raise AnsibleError("failure writing module data to temporary file for transfer: %s" % to_native(e))
afo.flush()
afo.close()
@ -372,17 +374,22 @@ class ActionBase(with_metaclass(ABCMeta, object)):
res = self._remote_chown(remote_paths, self._play_context.become_user)
if res['rc'] != 0 and remote_user == 'root':
# chown failed even if remove_user is root
raise AnsibleError('Failed to change ownership of the temporary files Ansible needs to create despite connecting as root. Unprivileged become user would be unable to read the file.')
raise AnsibleError('Failed to change ownership of the temporary files Ansible needs to create despite connecting as root.'
' Unprivileged become user would be unable to read the file.')
elif res['rc'] != 0:
if C.ALLOW_WORLD_READABLE_TMPFILES:
# chown and fs acls failed -- do things this insecure
# way only if the user opted in in the config file
display.warning('Using world-readable permissions for temporary files Ansible needs to create when becoming an unprivileged user which may be insecure. For information on securing this, see https://docs.ansible.com/ansible/become.html#becoming-an-unprivileged-user')
display.warning('Using world-readable permissions for temporary files Ansible needs to create when becoming an unprivileged user.'
' This may be insecure. For information on securing this, see'
' https://docs.ansible.com/ansible/become.html#becoming-an-unprivileged-user')
res = self._remote_chmod(remote_paths, 'a+%s' % mode)
if res['rc'] != 0:
raise AnsibleError('Failed to set file mode on remote files (rc: {0}, err: {1})'.format(res['rc'], res['stderr']))
else:
raise AnsibleError('Failed to set permissions on the temporary files Ansible needs to create when becoming an unprivileged user (rc: {0}, err: {1}). For information on working around this, see https://docs.ansible.com/ansible/become.html#becoming-an-unprivileged-user'.format(res['rc'], res['stderr']))
raise AnsibleError('Failed to set permissions on the temporary files Ansible needs to create when becoming an unprivileged user'
' (rc: {0}, err: {1}). For information on working around this,'
' see https://docs.ansible.com/ansible/become.html#becoming-an-unprivileged-user'.format(res['rc'], res['stderr']))
elif execute:
# Can't depend on the file being transferred with execute
# permissions. Only need user perms because no become was
@ -438,7 +445,7 @@ class ActionBase(with_metaclass(ABCMeta, object)):
mystat['stat']['checksum'] = '1'
# happens sometimes when it is a dir and not on bsd
if not 'checksum' in mystat['stat']:
if 'checksum' not in mystat['stat']:
mystat['stat']['checksum'] = ''
return mystat['stat']
@ -461,15 +468,14 @@ class ActionBase(with_metaclass(ABCMeta, object)):
else:
x = remote_stat['checksum'] # if 1, file is missing
except AnsibleError as e:
errormsg = to_unicode(e)
if errormsg.endswith('Permission denied'):
errormsg = to_text(e)
if errormsg.endswith(u'Permission denied'):
x = "2" # cannot read file
elif errormsg.endswith('MODULE FAILURE'):
elif errormsg.endswith(u'MODULE FAILURE'):
x = "4" # python not found or module uncaught exception
finally:
return x
def _remote_expand_user(self, path):
''' takes a remote path and performs tilde expansion on the remote host '''
if not path.startswith('~'): # FIXME: Windows paths may start with "~ instead of just ~
@ -681,7 +687,8 @@ class ActionBase(with_metaclass(ABCMeta, object)):
tmp_rm_res = self._low_level_execute_command(tmp_rm_cmd, sudoable=False)
tmp_rm_data = self._parse_returned_data(tmp_rm_res)
if tmp_rm_data.get('rc', 0) != 0:
display.warning('Error deleting remote temporary files (rc: {0}, stderr: {1})'.format(tmp_rm_res.get('rc'), tmp_rm_res.get('stderr', 'No error string available.')))
display.warning('Error deleting remote temporary files (rc: {0}, stderr: {1})'.format(tmp_rm_res.get('rc'),
tmp_rm_res.get('stderr', 'No error string available.')))
# parse the main result
data = self._parse_returned_data(res)
@ -709,7 +716,7 @@ class ActionBase(with_metaclass(ABCMeta, object)):
data['exception'] = res['stderr']
return data
def _low_level_execute_command(self, cmd, sudoable=True, in_data=None, executable=None, encoding_errors='replace'):
def _low_level_execute_command(self, cmd, sudoable=True, in_data=None, executable=None, encoding_errors='surrogate_or_replace'):
'''
This is the function which executes the low level shell command, which
may be commands to create/remove directories for temporary files, or to
@ -758,16 +765,16 @@ class ActionBase(with_metaclass(ABCMeta, object)):
# stdout and stderr may be either a file-like or a bytes object.
# Convert either one to a text type
if isinstance(stdout, binary_type):
out = to_unicode(stdout, errors=encoding_errors)
out = to_text(stdout, errors=encoding_errors)
elif not isinstance(stdout, text_type):
out = to_unicode(b''.join(stdout.readlines()), errors=encoding_errors)
out = to_text(b''.join(stdout.readlines()), errors=encoding_errors)
else:
out = stdout
if isinstance(stderr, binary_type):
err = to_unicode(stderr, errors=encoding_errors)
err = to_text(stderr, errors=encoding_errors)
elif not isinstance(stderr, text_type):
err = to_unicode(b''.join(stderr.readlines()), errors=encoding_errors)
err = to_text(b''.join(stderr.readlines()), errors=encoding_errors)
else:
err = stderr
@ -871,7 +878,6 @@ class ActionBase(with_metaclass(ABCMeta, object)):
result = self._loader.path_dwim_relative_stack(path_stack, dirname, needle)
if result is None:
raise AnsibleError("Unable to find '%s' in expected paths." % to_str(needle))
raise AnsibleError("Unable to find '%s' in expected paths." % to_native(needle))
return result

6
lib/ansible/plugins/action/assemble.py
Unescape Escape View File

@ -24,10 +24,10 @@ import tempfile
import re
from ansible.errors import AnsibleError
from ansible.module_utils._text import to_native, to_text
from ansible.plugins.action import ActionBase
from ansible.utils.boolean import boolean
from ansible.utils.hashing import checksum_s
from ansible.utils.unicode import to_str, to_unicode
class ActionModule(ActionBase):
@ -42,7 +42,7 @@ class ActionModule(ActionBase):
delimit_me = False
add_newline = False
for f in (to_unicode(p, errors='strict') for p in sorted(os.listdir(src_path))):
for f in (to_text(p, errors='surrogate_or_strict') for p in sorted(os.listdir(src_path))):
if compiled_regexp and not compiled_regexp.search(f):
continue
fragment = u"%s/%s" % (src_path, f)
@ -114,7 +114,7 @@ class ActionModule(ActionBase):
src = self._find_needle('files', src)
except AnsibleError as e:
result['failed'] = True
result['msg'] = to_str(e)
result['msg'] = to_native(e)
return result
if not os.path.isdir(src):

9
lib/ansible/plugins/action/async.py
Unescape Escape View File

@ -22,9 +22,10 @@ import pipes
import random
from ansible import constants as C
from ansible.plugins.action import ActionBase
from ansible.compat.six import iteritems
from ansible.utils.unicode import to_unicode
from ansible.module_utils._text import to_text
from ansible.plugins.action import ActionBase
class ActionModule(ActionBase):
@ -76,7 +77,7 @@ class ActionModule(ActionBase):
elif module_style == 'old':
args_data = ""
for k, v in iteritems(module_args):
args_data += '%s="%s" ' % (k, pipes.quote(to_unicode(v)))
args_data += '%s="%s" ' % (k, pipes.quote(to_text(v)))
argsfile = self._transfer_data(self._connection._shell.join_path(tmp, 'arguments'), args_data)
remote_paths = tmp, remote_module_path, remote_async_module_path
@ -100,7 +101,7 @@ class ActionModule(ActionBase):
if not self._should_remove_tmp_path(tmp):
async_cmd.append("-preserve_tmp")
async_cmd = " ".join([to_unicode(x) for x in async_cmd])
async_cmd = " ".join(to_text(x) for x in async_cmd)
result.update(self._low_level_execute_command(cmd=async_cmd))
result['changed'] = True

18
lib/ansible/plugins/action/copy.py
Unescape Escape View File

@ -24,10 +24,10 @@ import os
import tempfile
from ansible.errors import AnsibleError
from ansible.module_utils._text import to_bytes, to_native, to_text
from ansible.plugins.action import ActionBase
from ansible.utils.boolean import boolean
from ansible.utils.hashing import checksum
from ansible.utils.unicode import to_bytes, to_str, to_unicode
class ActionModule(ActionBase):
@ -81,7 +81,7 @@ class ActionModule(ActionBase):
source = content_tempfile
except Exception as err:
result['failed'] = True
result['msg'] = "could not write content temp file: %s" % to_str(err)
result['msg'] = "could not write content temp file: %s" % to_native(err)
return result
# if we have first_available_file in our vars
@ -96,14 +96,14 @@ class ActionModule(ActionBase):
source = self._find_needle('files', source)
except AnsibleError as e:
result['failed'] = True
result['msg'] = to_unicode(e)
result['msg'] = to_text(e)
return result
# A list of source file tuples (full_path, relative_path) which will try to copy to the destination
source_files = []
# If source is a directory populate our list else source is a file and translate it to a tuple.
if os.path.isdir(to_bytes(source, errors='strict')):
if os.path.isdir(to_bytes(source, errors='surrogate_or_strict')):
# Get the amount of spaces to remove to get the relative path.
if source_trailing_slash:
sz = len(source)
@ -113,7 +113,7 @@ class ActionModule(ActionBase):
# Walk the directory and append the file tuples to source_files.
for base_path, sub_folders, files in os.walk(to_bytes(source)):
for file in files:
full_path = to_unicode(os.path.join(base_path, file), errors='strict')
full_path = to_text(os.path.join(base_path, file), errors='surrogate_or_strict')
rel_path = full_path[sz:]
if rel_path.startswith('/'):
rel_path = rel_path[1:]
@ -247,7 +247,9 @@ class ActionModule(ActionBase):
if 'content' in new_module_args:
del new_module_args['content']
module_return = self._execute_module(module_name='copy', module_args=new_module_args, task_vars=task_vars, tmp=tmp, delete_remote_tmp=delete_remote_tmp)
module_return = self._execute_module(module_name='copy',
module_args=new_module_args, task_vars=task_vars,
tmp=tmp, delete_remote_tmp=delete_remote_tmp)
module_executed = True
else:
@ -272,7 +274,9 @@ class ActionModule(ActionBase):
)
# Execute the file module.
module_return = self._execute_module(module_name='file', module_args=new_module_args, task_vars=task_vars, tmp=tmp, delete_remote_tmp=delete_remote_tmp)
module_return = self._execute_module(module_name='file',
module_args=new_module_args, task_vars=task_vars,
tmp=tmp, delete_remote_tmp=delete_remote_tmp)
module_executed = True
if not module_return.get('checksum'):

4
lib/ansible/plugins/action/debug.py
Unescape Escape View File

@ -20,8 +20,8 @@ __metaclass__ = type
from ansible.compat.six import string_types
from ansible.errors import AnsibleUndefinedVariable
from ansible.module_utils._text import to_text
from ansible.plugins.action import ActionBase
from ansible.utils.unicode import to_unicode
class ActionModule(ActionBase):
@ -66,7 +66,7 @@ class ActionModule(ActionBase):
if isinstance(self._task.args['var'], (list, dict)):
# If var is a list or dict, use the type as key to display
result[to_unicode(type(self._task.args['var']))] = results
result[to_text(type(self._task.args['var']))] = results
else:
result[self._task.args['var']] = results
else:

4
lib/ansible/plugins/action/fetch.py
Unescape Escape View File

@ -21,11 +21,11 @@ import os
import base64
from ansible.errors import AnsibleError
from ansible.module_utils._text import to_bytes
from ansible.plugins.action import ActionBase
from ansible.utils.boolean import boolean
from ansible.utils.hashing import checksum, checksum_s, md5, secure_hash
from ansible.utils.path import makedirs_safe
from ansible.utils.unicode import to_bytes
class ActionModule(ActionBase):
@ -160,7 +160,7 @@ class ActionModule(ActionBase):
self._connection.fetch_file(source, dest)
else:
try:
f = open(to_bytes(dest, errors='strict'), 'w')
f = open(to_bytes(dest, errors='surrogate_or_strict'), 'wb')
f.write(remote_data)
f.close()
except (IOError, OSError) as e:

4
lib/ansible/plugins/action/include_vars.py
Unescape Escape View File

@ -22,8 +22,8 @@ from os import path, walk
import re
from ansible.errors import AnsibleError
from ansible.module_utils._text import to_native
from ansible.plugins.action import ActionBase
from ansible.utils.unicode import to_str
class ActionModule(ActionBase):
@ -137,7 +137,7 @@ class ActionModule(ActionBase):
results.update(updated_results)
except AnsibleError as e:
err_msg = to_str(e)
err_msg = to_native(e)
raise AnsibleError(err_msg)
if self.return_results_as_name:

9
lib/ansible/plugins/action/net_config.py
Unescape Escape View File

@ -26,10 +26,12 @@ import glob
import urlparse
from ansible.plugins.action import ActionBase
from ansible.utils.unicode import to_unicode
from ansible.module_utils._text import to_text
PRIVATE_KEYS_RE = re.compile('__.+__')
class ActionModule(ActionBase):
TRANSFERS_FILES = False
@ -56,7 +58,6 @@ class ActionModule(ActionBase):
result['__backup__'])
result['backup_path'] = filepath
# strip out any keys that have two leading and two trailing
# underscore characters
for key in result.keys():
@ -98,7 +99,7 @@ class ActionModule(ActionBase):
try:
with open(source, 'r') as f:
template_data = to_unicode(f.read())
template_data = to_text(f.read())
except IOError:
return dict(failed=True, msg='unable to load src file')
@ -114,5 +115,3 @@ class ActionModule(ActionBase):
searchpath.append(os.path.dirname(source))
self._templar.environment.loader.searchpath = searchpath
self._task.args['src'] = self._templar.template(template_data)

10
lib/ansible/plugins/action/net_template.py
Unescape Escape View File

@ -19,18 +19,18 @@
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type
import sys
import os
import time
import glob
import urlparse
from ansible.module_utils._text import to_text
from ansible.plugins.action import ActionBase
from ansible.utils.boolean import boolean
from ansible.utils.unicode import to_unicode
BOOLEANS = ('true', 'false', 'yes', 'no')
class ActionModule(ActionBase):
TRANSFERS_FILES = False
@ -92,7 +92,7 @@ class ActionModule(ActionBase):
try:
with open(source, 'r') as f:
template_data = to_unicode(f.read())
template_data = to_text(f.read())
except IOError:
return dict(failed=True, msg='unable to load src file')
@ -108,5 +108,3 @@ class ActionModule(ActionBase):
searchpath.append(os.path.dirname(source))
self._templar.environment.loader.searchpath = searchpath
self._task.args['src'] = self._templar.template(template_data)

6
lib/ansible/plugins/action/patch.py
Unescape Escape View File

@ -20,10 +20,10 @@ __metaclass__ = type
import os
from ansible.errors import AnsibleError
from ansible.module_utils._text import to_native
from ansible.plugins.action import ActionBase
from ansible.utils.boolean import boolean
from ansible.errors import AnsibleError
from ansible.utils.unicode import to_str
class ActionModule(ActionBase):
@ -52,7 +52,7 @@ class ActionModule(ActionBase):
src = self._find_needle('files', src)
except AnsibleError as e:
result['failed'] = True
result['msg'] = to_str(e)
result['msg'] = to_native(e)
return result
# create the remote tmp dir if needed, and put the source file there

7
lib/ansible/plugins/action/script.py
Unescape Escape View File

@ -19,15 +19,14 @@ __metaclass__ = type
import os
from ansible.plugins.action import ActionBase
from ansible.errors import AnsibleError
from ansible.utils.unicode import to_str
from ansible.module_utils._text import to_native
from ansible.plugins.action import ActionBase
class ActionModule(ActionBase):
TRANSFERS_FILES = True
def run(self, tmp=None, task_vars=None):
''' handler for file transfer operations '''
if task_vars is None:
@ -74,7 +73,7 @@ class ActionModule(ActionBase):
try:
source = self._loader.get_real_file(self._find_needle('files', source))
except AnsibleError as e:
return dict(failed=True, msg=to_str(e))
return dict(failed=True, msg=to_native(e))
# transfer the file to a remote tmp location
tmp_src = self._connection._shell.join_path(tmp, os.path.basename(source))

11
lib/ansible/plugins/action/template.py
Unescape Escape View File

@ -23,12 +23,11 @@ import pwd
import time
from ansible import constants as C
from ansible.errors import AnsibleError
from ansible.module_utils._text import to_bytes, to_native, to_text
from ansible.plugins.action import ActionBase
from ansible.utils.hashing import checksum_s
from ansible.utils.boolean import boolean
from ansible.utils.unicode import to_bytes, to_unicode, to_str
from ansible.errors import AnsibleError
class ActionModule(ActionBase):
@ -78,7 +77,7 @@ class ActionModule(ActionBase):
source = self._find_needle('templates', source)
except AnsibleError as e:
result['failed'] = True
result['msg'] = to_str(e)
result['msg'] = to_native(e)
if 'failed' in result:
return result
@ -96,7 +95,7 @@ class ActionModule(ActionBase):
b_source = to_bytes(source)
try:
with open(b_source, 'r') as f:
template_data = to_unicode(f.read())
template_data = to_text(f.read())
try:
template_uid = pwd.getpwuid(os.stat(b_source).st_uid).pw_name
@ -163,7 +162,7 @@ class ActionModule(ActionBase):
if self._play_context.diff:
diff = self._get_diff_data(dest, resultant, task_vars, source_file=False)
if not self._play_context.check_mode: # do actual work thorugh copy
if not self._play_context.check_mode: # do actual work through copy
xfered = self._transfer_data(self._connection._shell.join_path(tmp, 'source'), resultant)
# fix file permissions when the copy is done as a different user

7
lib/ansible/plugins/action/unarchive.py
Unescape Escape View File

@ -20,10 +20,11 @@ __metaclass__ = type
import os
from ansible.errors import AnsibleError
from ansible.module_utils._text import to_native
from ansible.plugins.action import ActionBase
from ansible.utils.boolean import boolean
from ansible.errors import AnsibleError
from ansible.utils.unicode import to_str
class ActionModule(ActionBase):
@ -82,7 +83,7 @@ class ActionModule(ActionBase):
source = self._loader.get_real_file(self._find_needle('files', source))
except AnsibleError as e:
result['failed'] = True
result['msg'] = to_str(e)
result['msg'] = to_native(e)
self._remove_tmp_path(tmp)
return result

11
lib/ansible/plugins/action/win_reboot.py
Unescape Escape View File

@ -19,26 +19,24 @@
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type
from ansible.plugins.action import ActionBase
from ansible.utils.boolean import boolean
from ansible.utils.unicode import to_unicode
from ansible.errors import AnsibleUndefinedVariable
import socket
import time
import traceback
from datetime import datetime, timedelta
from ansible.plugins.action import ActionBase
try:
from __main__ import display
except ImportError:
from ansible.utils.display import Display
display = Display()
class TimedOutException(Exception):
pass
class ActionModule(ActionBase):
TRANSFERS_FILES = False
@ -137,4 +135,3 @@ class ActionModule(ActionBase):
result['msg'] = toex.message
return result

2
lib/ansible/plugins/cache/jsonfile.py vendored
Unescape Escape View File

@ -31,9 +31,9 @@ except ImportError:
from ansible import constants as C
from ansible.errors import AnsibleError
from ansible.module_utils._text import to_bytes
from ansible.parsing.utils.jsonify import jsonify
from ansible.plugins.cache.base import BaseCacheModule
from ansible.utils.unicode import to_bytes
try:
from __main__ import display

18
lib/ansible/plugins/callback/__init__.py
Unescape Escape View File

@ -24,12 +24,10 @@ import difflib
import warnings
from copy import deepcopy
from ansible.compat.six import string_types
from ansible import constants as C
from ansible.vars import strip_internal_keys
from ansible.module_utils._text import to_text
from ansible.utils.color import stringc
from ansible.utils.unicode import to_unicode
from ansible.vars import strip_internal_keys
try:
from __main__ import display as global_display
@ -37,14 +35,15 @@ except ImportError:
from ansible.utils.display import Display
global_display = Display()
__all__ = ["CallbackBase"]
try:
from __main__ import cli
except ImportError:
# using API w/o cli
cli = False
__all__ = ["CallbackBase"]
class CallbackBase:
'''
@ -146,8 +145,8 @@ class CallbackBase:
after_header = "after: %s" % diff['after_header']
else:
after_header = 'after'
differ = difflib.unified_diff(to_unicode(diff['before']).splitlines(True),
to_unicode(diff['after']).splitlines(True),
differ = difflib.unified_diff(to_text(diff['before']).splitlines(True),
to_text(diff['after']).splitlines(True),
fromfile=before_header,
tofile=after_header,
fromfiledate='',
@ -166,7 +165,7 @@ class CallbackBase:
if has_diff:
ret.append('\n')
if 'prepared' in diff:
ret.append(to_unicode(diff['prepared']))
ret.append(to_text(diff['prepared']))
except UnicodeDecodeError:
ret.append(">> the files are different, but the diff library cannot compare unicode strings\n\n")
return u''.join(ret)
@ -362,4 +361,3 @@ class CallbackBase:
def v2_runner_retry(self, result):
pass

5
lib/ansible/plugins/callback/junit.py
Unescape Escape View File

@ -21,8 +21,8 @@ __metaclass__ = type
import os
import time
from ansible.module_utils._text import to_bytes
from ansible.plugins.callback import CallbackBase
from ansible.utils.unicode import to_bytes
try:
from junit_xml import TestSuite, TestCase
@ -40,6 +40,7 @@ except ImportError:
except ImportError:
HAS_ORDERED_DICT = False
class CallbackModule(CallbackBase):
"""
This callback writes playbook output to a JUnit formatted XML file.
@ -181,7 +182,7 @@ class CallbackModule(CallbackBase):
output_file = os.path.join(self._output_dir, '%s-%s.xml' % (self._playbook_name, time.time()))
with open(output_file, 'wb') as xml:
xml.write(to_bytes(report, errors='strict'))
xml.write(to_bytes(report, errors='surrogate_or_strict'))
def v2_playbook_on_start(self, playbook):
self._playbook_path = playbook._file_name

3
lib/ansible/plugins/callback/log_plays.py
Unescape Escape View File

@ -23,9 +23,10 @@ import os
import time
import json
from ansible.utils.unicode import to_bytes
from ansible.module_utils._text import to_bytes
from ansible.plugins.callback import CallbackBase
# NOTE: in Ansible 1.2 or later general logging is available without
# this plugin, just set ANSIBLE_LOG_PATH as an environment variable
# or log_path in the DEFAULTS section of your ansible configuration

3
lib/ansible/plugins/callback/mail.py
Unescape Escape View File

@ -25,9 +25,10 @@ import smtplib
import json
from ansible.compat.six import string_types
from ansible.utils.unicode import to_bytes
from ansible.module_utils._text import to_bytes
from ansible.plugins.callback import CallbackBase
def mail(subject='Ansible error mail', sender=None, to=None, cc=None, bcc=None, body=None, smtphost=None):
if sender is None:

5
lib/ansible/plugins/callback/tree.py
Unescape Escape View File

@ -20,10 +20,10 @@ __metaclass__ = type
import os
from ansible.constants import TREE_DIR
from ansible.module_utils._text import to_bytes
from ansible.plugins.callback import CallbackBase
from ansible.utils.path import makedirs_safe
from ansible.utils.unicode import to_bytes
from ansible.constants import TREE_DIR
class CallbackModule(CallbackBase):
@ -68,4 +68,3 @@ class CallbackModule(CallbackBase):
def v2_runner_on_unreachable(self, result):
self.result_to_tree(result)

8
lib/ansible/plugins/connection/__init__.py
Unescape Escape View File

@ -1,4 +1,3 @@
# (c) 2015 Toshio Kuratomi <tkuratomi@ansible.com>
#
# This file is part of Ansible
@ -32,8 +31,9 @@ from ansible.compat.six import with_metaclass
from ansible import constants as C
from ansible.compat.six import string_types
from ansible.errors import AnsibleError
from ansible.module_utils._text import to_bytes, to_text
from ansible.plugins import shell_loader
from ansible.utils.unicode import to_bytes, to_unicode
try:
from __main__ import display
@ -138,9 +138,9 @@ class ConnectionBase(with_metaclass(ABCMeta, object)):
# exception, it merely mangles the output:
# >>> shlex.split(u't e')
# ['t\x00\x00\x00', '\x00\x00\x00e\x00\x00\x00']
return [to_unicode(x.strip()) for x in shlex.split(to_bytes(argstring)) if x.strip()]
return [to_text(x.strip()) for x in shlex.split(to_bytes(argstring)) if x.strip()]
except AttributeError:
return [to_unicode(x.strip()) for x in shlex.split(argstring) if x.strip()]
return [to_text(x.strip()) for x in shlex.split(argstring) if x.strip()]
@abstractproperty
def transport(self):

7
lib/ansible/plugins/connection/accelerate.py
Unescape Escape View File

@ -27,10 +27,11 @@ import time
from ansible import constants as C
from ansible.errors import AnsibleError, AnsibleFileNotFound, AnsibleConnectionFailure
from ansible.module_utils._text import to_bytes
from ansible.parsing.utils.jsonify import jsonify
from ansible.plugins.connection import ConnectionBase
from ansible.utils.encrypt import key_for_hostname, keyczar_encrypt, keyczar_decrypt
from ansible.utils.unicode import to_bytes
try:
from __main__ import display
@ -211,7 +212,7 @@ class Connection(ConnectionBase):
''' transfer a file from local to remote '''
display.vvv("PUT %s TO %s" % (in_path, out_path), host=self._play_context.remote_addr)
in_path = to_bytes(in_path, errors='strict')
in_path = to_bytes(in_path, errors='surrogate_or_strict')
if not os.path.exists(in_path):
raise AnsibleFileNotFound("file or module does not exist: %s" % in_path)
@ -265,7 +266,7 @@ class Connection(ConnectionBase):
if self.send_data(data):
raise AnsibleError("failed to initiate the file fetch with %s" % self._play_context.remote_addr)
fh = open(to_bytes(out_path, errors='strict'), "w")
fh = open(to_bytes(out_path, errors='surrogate_or_strict'), "w")
try:
bytes = 0
while True:

11
lib/ansible/plugins/connection/chroot.py
Unescape Escape View File

@ -28,9 +28,10 @@ import traceback
from ansible import constants as C
from ansible.errors import AnsibleError
from ansible.plugins.connection import ConnectionBase, BUFSIZE
from ansible.module_utils.basic import is_executable
from ansible.utils.unicode import to_bytes
from ansible.module_utils._text import to_bytes
from ansible.plugins.connection import ConnectionBase, BUFSIZE
try:
from __main__ import display
@ -93,7 +94,7 @@ class Connection(ConnectionBase):
local_cmd = [self.chroot_cmd, self.chroot, executable, '-c', cmd]
display.vvv("EXEC %s" % (local_cmd), host=self.chroot)
local_cmd = [to_bytes(i, errors='strict') for i in local_cmd]
local_cmd = [to_bytes(i, errors='surrogate_or_strict') for i in local_cmd]
p = subprocess.Popen(local_cmd, shell=False, stdin=stdin,
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
@ -129,7 +130,7 @@ class Connection(ConnectionBase):
out_path = pipes.quote(self._prefix_login_path(out_path))
try:
with open(to_bytes(in_path, errors='strict'), 'rb') as in_file:
with open(to_bytes(in_path, errors='surrogate_or_strict'), 'rb') as in_file:
try:
p = self._buffered_exec_command('dd of=%s bs=%s' % (out_path, BUFSIZE), stdin=in_file)
except OSError:
@ -155,7 +156,7 @@ class Connection(ConnectionBase):
except OSError:
raise AnsibleError("chroot connection requires dd command in the chroot")
with open(to_bytes(out_path, errors='strict'), 'wb+') as out_file:
with open(to_bytes(out_path, errors='surrogate_or_strict'), 'wb+') as out_file:
try:
chunk = p.stdout.read(BUFSIZE)
while chunk:

13
lib/ansible/plugins/connection/docker.py
Unescape Escape View File

@ -35,8 +35,9 @@ from distutils.version import LooseVersion
import ansible.constants as C
from ansible.errors import AnsibleError, AnsibleFileNotFound
from ansible.module_utils._text import to_bytes
from ansible.plugins.connection import ConnectionBase, BUFSIZE
from ansible.utils.unicode import to_bytes
try:
from __main__ import display
@ -196,7 +197,7 @@ class Connection(ConnectionBase):
local_cmd = self._build_exec_cmd([self._play_context.executable, '-c', cmd])
display.vvv("EXEC %s" % (local_cmd,), host=self._play_context.remote_addr)
local_cmd = [to_bytes(i, errors='strict') for i in local_cmd]
local_cmd = [to_bytes(i, errors='surrogate_or_strict') for i in local_cmd]
p = subprocess.Popen(local_cmd, shell=False, stdin=subprocess.PIPE,
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
@ -223,7 +224,7 @@ class Connection(ConnectionBase):
display.vvv("PUT %s TO %s" % (in_path, out_path), host=self._play_context.remote_addr)
out_path = self._prefix_login_path(out_path)
if not os.path.exists(to_bytes(in_path, errors='strict')):
if not os.path.exists(to_bytes(in_path, errors='surrogate_or_strict')):
raise AnsibleFileNotFound(
"file or module does not exist: %s" % in_path)
@ -233,8 +234,8 @@ class Connection(ConnectionBase):
# Although docker version 1.8 and later provide support, the
# owner and group of the files are always set to root
args = self._build_exec_cmd([self._play_context.executable, "-c", "dd of=%s bs=%s" % (out_path, BUFSIZE)])
args = [to_bytes(i, errors='strict') for i in args]
with open(to_bytes(in_path, errors='strict'), 'rb') as in_file:
args = [to_bytes(i, errors='surrogate_or_strict') for i in args]
with open(to_bytes(in_path, errors='surrogate_or_strict'), 'rb') as in_file:
try:
p = subprocess.Popen(args, stdin=in_file,
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
@ -256,7 +257,7 @@ class Connection(ConnectionBase):
out_dir = os.path.dirname(out_path)
args = [self.docker_cmd, "cp", "%s:%s" % (self._play_context.remote_addr, in_path), out_dir]
args = [to_bytes(i, errors='strict') for i in args]
args = [to_bytes(i, errors='surrogate_or_strict') for i in args]
p = subprocess.Popen(args, stdin=subprocess.PIPE,
stdout=subprocess.PIPE, stderr=subprocess.PIPE)

9
lib/ansible/plugins/connection/jail.py
Unescape Escape View File

@ -27,10 +27,9 @@ import pipes
import subprocess
import traceback
from ansible import constants as C
from ansible.errors import AnsibleError
from ansible.module_utils._text import to_bytes
from ansible.plugins.connection import ConnectionBase, BUFSIZE
from ansible.utils.unicode import to_bytes
try:
from __main__ import display
@ -117,7 +116,7 @@ class Connection(ConnectionBase):
local_cmd += [self.jail, self._play_context.executable, '-c', set_env + cmd]
display.vvv("EXEC %s" % (local_cmd,), host=self.jail)
local_cmd = [to_bytes(i, errors='strict') for i in local_cmd]
local_cmd = [to_bytes(i, errors='surrogate_or_strict') for i in local_cmd]
p = subprocess.Popen(local_cmd, shell=False, stdin=stdin,
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
@ -153,7 +152,7 @@ class Connection(ConnectionBase):
out_path = pipes.quote(self._prefix_login_path(out_path))
try:
with open(to_bytes(in_path, errors='strict'), 'rb') as in_file:
with open(to_bytes(in_path, errors='surrogate_or_strict'), 'rb') as in_file:
try:
p = self._buffered_exec_command('dd of=%s bs=%s' % (out_path, BUFSIZE), stdin=in_file)
except OSError:
@ -179,7 +178,7 @@ class Connection(ConnectionBase):
except OSError:
raise AnsibleError("jail connection requires dd command in the jail")
with open(to_bytes(out_path, errors='strict'), 'wb+') as out_file:
with open(to_bytes(out_path, errors='surrogate_or_strict'), 'wb+') as out_file:
try:
chunk = p.stdout.read(BUFSIZE)
while chunk:

9
lib/ansible/plugins/connection/libvirt_lxc.py
Unescape Escape View File

@ -29,8 +29,9 @@ import traceback
from ansible import constants as C
from ansible.errors import AnsibleError
from ansible.module_utils._text import to_bytes
from ansible.plugins.connection import ConnectionBase, BUFSIZE
from ansible.utils.unicode import to_bytes
try:
from __main__ import display
@ -94,7 +95,7 @@ class Connection(ConnectionBase):
local_cmd += [self.lxc, '--', executable, '-c', cmd]
display.vvv("EXEC %s" % (local_cmd,), host=self.lxc)
local_cmd = [to_bytes(i, errors='strict') for i in local_cmd]
local_cmd = [to_bytes(i, errors='surrogate_or_strict') for i in local_cmd]
p = subprocess.Popen(local_cmd, shell=False, stdin=stdin,
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
@ -130,7 +131,7 @@ class Connection(ConnectionBase):
out_path = pipes.quote(self._prefix_login_path(out_path))
try:
with open(to_bytes(in_path, errors='strict'), 'rb') as in_file:
with open(to_bytes(in_path, errors='surrogate_or_strict'), 'rb') as in_file:
try:
p = self._buffered_exec_command('dd of=%s bs=%s' % (out_path, BUFSIZE), stdin=in_file)
except OSError:
@ -156,7 +157,7 @@ class Connection(ConnectionBase):
except OSError:
raise AnsibleError("chroot connection requires dd command in the chroot")
with open(to_bytes(out_path, errors='strict'), 'wb+') as out_file:
with open(to_bytes(out_path, errors='surrogate_or_strict'), 'wb+') as out_file:
try:
chunk = p.stdout.read(BUFSIZE)
while chunk:

13
lib/ansible/plugins/connection/local.py
Unescape Escape View File

@ -30,8 +30,9 @@ from ansible.compat.six import text_type, binary_type
import ansible.constants as C
from ansible.errors import AnsibleError, AnsibleFileNotFound
from ansible.module_utils._text import to_bytes, to_native
from ansible.plugins.connection import ConnectionBase
from ansible.utils.unicode import to_bytes, to_str
try:
from __main__ import display
@ -122,14 +123,14 @@ class Connection(ConnectionBase):
super(Connection, self).put_file(in_path, out_path)
display.vvv(u"PUT {0} TO {1}".format(in_path, out_path), host=self._play_context.remote_addr)
if not os.path.exists(to_bytes(in_path, errors='strict')):
raise AnsibleFileNotFound("file or module does not exist: {0}".format(to_str(in_path)))
if not os.path.exists(to_bytes(in_path, errors='surrogate_or_strict')):
raise AnsibleFileNotFound("file or module does not exist: {0}".format(to_native(in_path)))
try:
shutil.copyfile(to_bytes(in_path, errors='strict'), to_bytes(out_path, errors='strict'))
shutil.copyfile(to_bytes(in_path, errors='surrogate_or_strict'), to_bytes(out_path, errors='surrogate_or_strict'))
except shutil.Error:
raise AnsibleError("failed to copy: {0} and {1} are the same".format(to_str(in_path), to_str(out_path)))
raise AnsibleError("failed to copy: {0} and {1} are the same".format(to_native(in_path), to_native(out_path)))
except IOError as e:
raise AnsibleError("failed to transfer file to {0}: {1}".format(to_str(out_path), to_str(e)))
raise AnsibleError("failed to transfer file to {0}: {1}".format(to_native(out_path), to_native(e)))
def fetch_file(self, in_path, out_path):
''' fetch a file from local to local -- for copatibility '''

22
lib/ansible/plugins/connection/lxc.py
Unescape Escape View File

@ -24,10 +24,6 @@ import traceback
import select
import fcntl
import errno
from ansible import errors
from ansible import constants as C
from ansible.plugins.connection import ConnectionBase
from ansible.utils.unicode import to_bytes
HAS_LIBLXC = False
try:
@ -36,6 +32,12 @@ try:
except ImportError:
pass
from ansible import constants as C
from ansible import errors
from ansible.module_utils._text import to_bytes
from ansible.plugins.connection import ConnectionBase
class Connection(ConnectionBase):
''' Local lxc based connections '''
@ -102,8 +104,8 @@ class Connection(ConnectionBase):
''' run a command on the chroot '''
super(Connection, self).exec_command(cmd, in_data=in_data, sudoable=sudoable)
executable = to_bytes(self._play_context.executable, errors='strict')
local_cmd = [executable, '-c', to_bytes(cmd, errors='strict')]
executable = to_bytes(self._play_context.executable, errors='surrogate_or_strict')
local_cmd = [executable, '-c', to_bytes(cmd, errors='surrogate_or_strict')]
read_stdout, write_stdout = None, None
read_stderr, write_stderr = None, None
@ -154,8 +156,8 @@ class Connection(ConnectionBase):
''' transfer a file from local to lxc '''
super(Connection, self).put_file(in_path, out_path)
self._display.vvv("PUT %s TO %s" % (in_path, out_path), host=self.container_name)
in_path = to_bytes(in_path, errors='strict')
out_path = to_bytes(out_path, errors='strict')
in_path = to_bytes(in_path, errors='surrogate_or_strict')
out_path = to_bytes(out_path, errors='surrogate_or_strict')
if not os.path.exists(in_path):
msg = "file or module does not exist: %s" % in_path
@ -182,8 +184,8 @@ class Connection(ConnectionBase):
''' fetch a file from lxc to local '''
super(Connection, self).fetch_file(in_path, out_path)
self._display.vvv("FETCH %s TO %s" % (in_path, out_path), host=self.container_name)
in_path = to_bytes(in_path, errors='strict')
out_path = to_bytes(out_path, errors='strict')
in_path = to_bytes(in_path, errors='surrogate_or_strict')
out_path = to_bytes(out_path, errors='surrogate_or_strict')
try:
dst_file = open(out_path, "wb")

16
lib/ansible/plugins/connection/lxd.py
Unescape Escape View File

@ -23,8 +23,8 @@ from distutils.spawn import find_executable
from subprocess import call, Popen, PIPE
from ansible.errors import AnsibleError, AnsibleConnectionFailure, AnsibleFileNotFound
from ansible.module_utils._text import to_bytes, to_text
from ansible.plugins.connection import ConnectionBase
from ansible.utils.unicode import to_bytes, to_unicode
class Connection(ConnectionBase):
@ -61,14 +61,14 @@ class Connection(ConnectionBase):
local_cmd = [self._lxc_cmd, "exec", self._host, "--", self._play_context.executable, "-c", cmd]
local_cmd = [to_bytes(i, errors='strict') for i in local_cmd]
in_data = to_bytes(in_data, errors='strict', nonstring='passthru')
local_cmd = [to_bytes(i, errors='surrogate_or_strict') for i in local_cmd]
in_data = to_bytes(in_data, errors='surrogate_or_strict', nonstring='passthru')
process = Popen(local_cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE)
stdout, stderr = process.communicate(in_data)
stdout = to_unicode(stdout)
stderr = to_unicode(stderr)
stdout = to_text(stdout)
stderr = to_text(stderr)
if stderr == "error: Container is not running.\n":
raise AnsibleConnectionFailure("container not running: %s" % self._host)
@ -84,12 +84,12 @@ class Connection(ConnectionBase):
self._display.vvv(u"PUT {0} TO {1}".format(in_path, out_path), host=self._host)
if not os.path.isfile(to_bytes(in_path, errors='strict')):
if not os.path.isfile(to_bytes(in_path, errors='surrogate_or_strict')):
raise AnsibleFileNotFound("input path is not a file: %s" % in_path)
local_cmd = [self._lxc_cmd, "file", "push", in_path, self._host + "/" + out_path]
local_cmd = [to_bytes(i, errors='strict') for i in local_cmd]
local_cmd = [to_bytes(i, errors='surrogate_or_strict') for i in local_cmd]
call(local_cmd)
@ -101,7 +101,7 @@ class Connection(ConnectionBase):
local_cmd = [self._lxc_cmd, "file", "pull", self._host + "/" + in_path, out_path]
local_cmd = [to_bytes(i, errors='strict') for i in local_cmd]
local_cmd = [to_bytes(i, errors='surrogate_or_strict') for i in local_cmd]
call(local_cmd)

11
lib/ansible/plugins/connection/paramiko_ssh.py
Unescape Escape View File

@ -44,7 +44,7 @@ from ansible.compat.six.moves import input
from ansible.errors import AnsibleError, AnsibleConnectionFailure, AnsibleFileNotFound
from ansible.plugins.connection import ConnectionBase
from ansible.utils.path import makedirs_safe
from ansible.utils.unicode import to_bytes
from ansible.module_utils._text import to_bytes
try:
from __main__ import display
@ -52,6 +52,7 @@ except ImportError:
from ansible.utils.display import Display
display = Display()
AUTHENTICITY_MSG="""
paramiko: The authenticity of host '%s' can't be established.
The %s key fingerprint is %s.
@ -273,7 +274,7 @@ class Connection(ConnectionBase):
display.vvv("EXEC %s" % cmd, host=self._play_context.remote_addr)
cmd = to_bytes(cmd, errors='strict')
cmd = to_bytes(cmd, errors='surrogate_or_strict')
no_prompt_out = b''
no_prompt_err = b''
@ -330,7 +331,7 @@ class Connection(ConnectionBase):
display.vvv("PUT %s TO %s" % (in_path, out_path), host=self._play_context.remote_addr)
if not os.path.exists(to_bytes(in_path, errors='strict')):
if not os.path.exists(to_bytes(in_path, errors='surrogate_or_strict')):
raise AnsibleFileNotFound("file or module does not exist: %s" % in_path)
try:
@ -339,7 +340,7 @@ class Connection(ConnectionBase):
raise AnsibleError("failed to open a SFTP connection (%s)" % e)
try:
self.sftp.put(to_bytes(in_path, errors='strict'), to_bytes(out_path, errors='strict'))
self.sftp.put(to_bytes(in_path, errors='surrogate_or_strict'), to_bytes(out_path, errors='surrogate_or_strict'))
except IOError:
raise AnsibleError("failed to transfer file to %s" % out_path)
@ -365,7 +366,7 @@ class Connection(ConnectionBase):
raise AnsibleError("failed to open a SFTP connection (%s)", e)
try:
self.sftp.get(to_bytes(in_path, errors='strict'), to_bytes(out_path, errors='strict'))
self.sftp.get(to_bytes(in_path, errors='surrogate_or_strict'), to_bytes(out_path, errors='surrogate_or_strict'))
except IOError:
raise AnsibleError("failed to transfer file from %s" % in_path)

24
lib/ansible/plugins/connection/ssh.py
Unescape Escape View File

@ -29,11 +29,11 @@ import subprocess
import time
from ansible import constants as C
from ansible.compat.six import text_type, binary_type
from ansible.errors import AnsibleError, AnsibleConnectionFailure, AnsibleFileNotFound
from ansible.module_utils._text import to_bytes, to_native, to_text
from ansible.plugins.connection import ConnectionBase
from ansible.utils.path import unfrackpath, makedirs_safe
from ansible.utils.unicode import to_bytes, to_unicode, to_str
from ansible.compat.six import text_type, binary_type
try:
from __main__ import display
@ -107,7 +107,7 @@ class Connection(ConnectionBase):
explanation of why they were added.
"""
self._command += args
display.vvvvv('SSH: ' + explanation + ': (%s)' % ')('.join(map(to_unicode, args)), host=self._play_context.remote_addr)
display.vvvvv('SSH: ' + explanation + ': (%s)' % ')('.join(map(to_text, args)), host=self._play_context.remote_addr)
def _build_command(self, binary, *other_args):
'''
@ -222,7 +222,7 @@ class Connection(ConnectionBase):
# The directory must exist and be writable.
makedirs_safe(b_cpdir, 0o700)
if not os.access(b_cpdir, os.W_OK):
raise AnsibleError("Cannot write to ControlPath %s" % to_str(cpdir))
raise AnsibleError("Cannot write to ControlPath %s" % to_native(cpdir))
args = ("-o", "ControlPath=" + C.ANSIBLE_SSH_CONTROL_PATH % dict(directory=cpdir))
self._add_args("found only ControlPersist; added ControlPath", args)
@ -275,7 +275,7 @@ class Connection(ConnectionBase):
output = []
for b_line in b_chunk.splitlines(True):
display_line = to_unicode(b_line, errors='replace').rstrip('\r\n')
display_line = to_text(b_line).rstrip('\r\n')
suppress_output = False
#display.debug("Examining line (source=%s, state=%s): '%s'" % (source, state, display_line))
@ -314,7 +314,7 @@ class Connection(ConnectionBase):
Starts the command and communicates with it until it ends.
'''
display_cmd = list(map(pipes.quote, map(to_unicode, cmd)))
display_cmd = list(map(pipes.quote, map(to_text, cmd)))
display.vvv(u'SSH: EXEC {0}'.format(u' '.join(display_cmd)), host=self.host)
# Start the given command. If we don't need to pipeline data, we can try
@ -424,7 +424,7 @@ class Connection(ConnectionBase):
if p.poll() is not None:
break
self._terminate_process(p)
raise AnsibleError('Timeout (%ds) waiting for privilege escalation prompt: %s' % (timeout, to_str(b_stdout)))
raise AnsibleError('Timeout (%ds) waiting for privilege escalation prompt: %s' % (timeout, to_native(b_stdout)))
# Read whatever output is available on stdout and stderr, and stop
# listening to the pipe if it's been closed.
@ -434,14 +434,14 @@ class Connection(ConnectionBase):
if b_chunk == b'':
rpipes.remove(p.stdout)
b_tmp_stdout += b_chunk
display.debug("stdout chunk (state=%s):\n>>>%s<<<\n" % (state, to_unicode(b_chunk, errors='replace')))
display.debug("stdout chunk (state=%s):\n>>>%s<<<\n" % (state, to_text(b_chunk)))
if p.stderr in rfd:
b_chunk = p.stderr.read()
if b_chunk == b'':
rpipes.remove(p.stderr)
b_tmp_stderr += b_chunk
display.debug("stderr chunk (state=%s):\n>>>%s<<<\n" % (state, to_unicode(b_chunk, errors='replace')))
display.debug("stderr chunk (state=%s):\n>>>%s<<<\n" % (state, to_text(b_chunk)))
# We examine the output line-by-line until we have negotiated any
# privilege escalation prompt and subsequent success/error message.
@ -631,8 +631,8 @@ class Connection(ConnectionBase):
super(Connection, self).put_file(in_path, out_path)
display.vvv(u"PUT {0} TO {1}".format(in_path, out_path), host=self.host)
if not os.path.exists(to_bytes(in_path, errors='strict')):
raise AnsibleFileNotFound("file or module does not exist: {0}".format(to_str(in_path)))
if not os.path.exists(to_bytes(in_path, errors='surrogate_or_strict')):
raise AnsibleFileNotFound("file or module does not exist: {0}".format(to_native(in_path)))
# scp and sftp require square brackets for IPv6 addresses, but
# accept them for hostnames and IPv4 addresses too.
@ -649,7 +649,7 @@ class Connection(ConnectionBase):
(returncode, stdout, stderr) = self._run(cmd, in_data)
if returncode != 0:
raise AnsibleError("failed to transfer file to {0}:\n{1}\n{2}".format(to_str(out_path), to_str(stdout), to_str(stderr)))
raise AnsibleError("failed to transfer file to {0}:\n{1}\n{2}".format(to_native(out_path), to_native(stdout), to_native(stderr)))
def fetch_file(self, in_path, out_path):
''' fetch a file from remote to local '''

64
lib/ansible/plugins/connection/winrm.py
Unescape Escape View File

@ -26,9 +26,21 @@ import shlex
import traceback
import json
HAVE_KERBEROS = False
try:
import kerberos
HAVE_KERBEROS = True
except ImportError:
pass
from ansible.compat.six import string_types
from ansible.compat.six.moves.urllib.parse import urlunsplit
from ansible.errors import AnsibleError, AnsibleConnectionFailure
from ansible.errors import AnsibleFileNotFound
from ansible.module_utils._text import to_bytes, to_native, to_text
from ansible.plugins.connection import ConnectionBase
from ansible.utils.hashing import secure_hash
from ansible.utils.path import makedirs_safe
try:
import winrm
@ -42,25 +54,13 @@ try:
except ImportError:
raise AnsibleError("xmltodict is not installed")
HAVE_KERBEROS = False
try:
import kerberos
HAVE_KERBEROS = True
except ImportError:
pass
from ansible.errors import AnsibleFileNotFound
from ansible.plugins.connection import ConnectionBase
from ansible.utils.hashing import secure_hash
from ansible.utils.path import makedirs_safe
from ansible.utils.unicode import to_bytes, to_unicode, to_str
try:
from __main__ import display
except ImportError:
from ansible.utils.display import Display
display = Display()
class Connection(ConnectionBase):
'''WinRM connections over HTTP/HTTPS.'''
@ -156,10 +156,10 @@ class Connection(ConnectionBase):
return protocol
except Exception as e:
err_msg = to_unicode(e).strip()
if re.search(to_unicode(r'Operation\s+?timed\s+?out'), err_msg, re.I):
err_msg = to_text(e).strip()
if re.search(to_text(r'Operation\s+?timed\s+?out'), err_msg, re.I):
raise AnsibleError('the connection attempt timed out')
m = re.search(to_unicode(r'Code\s+?(\d{3})'), err_msg)
m = re.search(to_text(r'Code\s+?(\d{3})'), err_msg)
if m:
code = int(m.groups()[0])
if code == 401:
@ -167,9 +167,9 @@ class Connection(ConnectionBase):
elif code == 411:
return protocol
errors.append(u'%s: %s' % (transport, err_msg))
display.vvvvv(u'WINRM CONNECTION ERROR: %s\n%s' % (err_msg, to_unicode(traceback.format_exc())), host=self._winrm_host)
display.vvvvv(u'WINRM CONNECTION ERROR: %s\n%s' % (err_msg, to_text(traceback.format_exc())), host=self._winrm_host)
if errors:
raise AnsibleConnectionFailure(', '.join(map(to_str, errors)))
raise AnsibleConnectionFailure(', '.join(map(to_native, errors)))
else:
raise AnsibleError('No transport found for WinRM connection')
@ -220,12 +220,12 @@ class Connection(ConnectionBase):
# TODO: check result from response and set stdin_push_failed if we have nonzero
if from_exec:
display.vvvvv('WINRM RESULT %r' % to_unicode(response), host=self._winrm_host)
display.vvvvv('WINRM RESULT %r' % to_text(response), host=self._winrm_host)
else:
display.vvvvvv('WINRM RESULT %r' % to_unicode(response), host=self._winrm_host)
display.vvvvvv('WINRM RESULT %r' % to_text(response), host=self._winrm_host)
display.vvvvvv('WINRM STDOUT %s' % to_unicode(response.std_out), host=self._winrm_host)
display.vvvvvv('WINRM STDERR %s' % to_unicode(response.std_err), host=self._winrm_host)
display.vvvvvv('WINRM STDOUT %s' % to_text(response.std_out), host=self._winrm_host)
display.vvvvvv('WINRM STDERR %s' % to_text(response.std_err), host=self._winrm_host)
if stdin_push_failed:
raise AnsibleError('winrm send_input failed; \nstdout: %s\nstderr %s' % (response.std_out, response.std_err))
@ -250,7 +250,7 @@ class Connection(ConnectionBase):
def exec_command(self, cmd, in_data=None, sudoable=True):
super(Connection, self).exec_command(cmd, in_data=in_data, sudoable=sudoable)
cmd_parts = shlex.split(to_bytes(cmd), posix=False)
cmd_parts = map(to_unicode, cmd_parts)
cmd_parts = map(to_text, cmd_parts)
script = None
cmd_ext = cmd_parts and self._shell._unquote(cmd_parts[0]).lower()[-4:] or ''
# Support running .ps1 files (via script/raw).
@ -266,7 +266,7 @@ class Connection(ConnectionBase):
cmd_parts = self._shell._encode_script(script, as_list=True, strict_mode=False)
if '-EncodedCommand' in cmd_parts:
encoded_cmd = cmd_parts[cmd_parts.index('-EncodedCommand') + 1]
decoded_cmd = to_unicode(base64.b64decode(encoded_cmd).decode('utf-16-le'))
decoded_cmd = to_text(base64.b64decode(encoded_cmd).decode('utf-16-le'))
display.vvv("EXEC %s" % decoded_cmd, host=self._winrm_host)
else:
display.vvv("EXEC %s" % cmd, host=self._winrm_host)
@ -300,9 +300,9 @@ class Connection(ConnectionBase):
# FUTURE: determine buffer size at runtime via remote winrm config?
def _put_file_stdin_iterator(self, in_path, out_path, buffer_size=250000):
in_size = os.path.getsize(to_bytes(in_path, errors='strict'))
in_size = os.path.getsize(to_bytes(in_path, errors='surrogate_or_strict'))
offset = 0
with open(to_bytes(in_path, errors='strict'), 'rb') as in_file:
with open(to_bytes(in_path, errors='surrogate_or_strict'), 'rb') as in_file:
for out_data in iter((lambda:in_file.read(buffer_size)), ''):
offset += len(out_data)
self._display.vvvvv('WINRM PUT "%s" to "%s" (offset=%d size=%d)' % (in_path, out_path, offset, len(out_data)), host=self._winrm_host)
@ -318,7 +318,7 @@ class Connection(ConnectionBase):
super(Connection, self).put_file(in_path, out_path)
out_path = self._shell._unquote(out_path)
display.vvv('PUT "%s" TO "%s"' % (in_path, out_path), host=self._winrm_host)
if not os.path.exists(to_bytes(in_path, errors='strict')):
if not os.path.exists(to_bytes(in_path, errors='surrogate_or_strict')):
raise AnsibleFileNotFound('file or module does not exist: "%s"' % in_path)
script_template = u'''
@ -357,7 +357,7 @@ class Connection(ConnectionBase):
result = self._winrm_exec(cmd_parts[0], cmd_parts[1:], stdin_iterator=self._put_file_stdin_iterator(in_path, out_path))
# TODO: improve error handling
if result.status_code != 0:
raise AnsibleError(to_str(result.std_err))
raise AnsibleError(to_native(result.std_err))
put_output = json.loads(result.std_out)
remote_sha1 = put_output.get("sha1")
@ -368,7 +368,7 @@ class Connection(ConnectionBase):
local_sha1 = secure_hash(in_path)
if not remote_sha1 == local_sha1:
raise AnsibleError("Remote sha1 hash {0} does not match local hash {1}".format(to_str(remote_sha1), to_str(local_sha1)))
raise AnsibleError("Remote sha1 hash {0} does not match local hash {1}".format(to_native(remote_sha1), to_native(local_sha1)))
def fetch_file(self, in_path, out_path):
super(Connection, self).fetch_file(in_path, out_path)
@ -407,7 +407,7 @@ class Connection(ConnectionBase):
cmd_parts = self._shell._encode_script(script, as_list=True)
result = self._winrm_exec(cmd_parts[0], cmd_parts[1:])
if result.status_code != 0:
raise IOError(to_str(result.std_err))
raise IOError(to_native(result.std_err))
if result.std_out.strip() == '[DIR]':
data = None
else:
@ -418,9 +418,9 @@ class Connection(ConnectionBase):
else:
if not out_file:
# If out_path is a directory and we're expecting a file, bail out now.
if os.path.isdir(to_bytes(out_path, errors='strict')):
if os.path.isdir(to_bytes(out_path, errors='surrogate_or_strict')):
break
out_file = open(to_bytes(out_path, errors='strict'), 'wb')
out_file = open(to_bytes(out_path, errors='surrogate_or_strict'), 'wb')
out_file.write(data)
if len(data) < buffer_size:
break

3
lib/ansible/plugins/connection/zone.py
Unescape Escape View File

@ -31,7 +31,8 @@ import traceback
from ansible import constants as C
from ansible.errors import AnsibleError
from ansible.plugins.connection import ConnectionBase, BUFSIZE
from ansible.utils.unicode import to_bytes
from ansible.module_utils._text import to_bytes
try:
from __main__ import display

26
lib/ansible/plugins/filter/core.py
Unescape Escape View File

@ -19,7 +19,6 @@
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type
import sys
import base64
import itertools
@ -39,15 +38,6 @@ import uuid
import yaml
from jinja2.filters import environmentfilter
from ansible.compat.six import iteritems, string_types
from ansible import errors
from ansible.parsing.yaml.dumper import AnsibleDumper
from ansible.utils.hashing import md5s, checksum_s
from ansible.utils.unicode import unicode_wrap, to_unicode
from ansible.utils.vars import merge_hash
from ansible.vars.hostvars import HostVars
from ansible.compat.six.moves import reduce
try:
import passlib.hash
@ -55,6 +45,16 @@ try:
except:
HAS_PASSLIB = False
from ansible import errors
from ansible.compat.six import iteritems, string_types
from ansible.compat.six.moves import reduce
from ansible.module_utils._text import to_text
from ansible.parsing.yaml.dumper import AnsibleDumper
from ansible.utils.hashing import md5s, checksum_s
from ansible.utils.unicode import unicode_wrap
from ansible.utils.vars import merge_hash
from ansible.vars.hostvars import HostVars
UUID_NAMESPACE_ANSIBLE = uuid.UUID('361E6D51-FAEC-444A-9079-341386DA8E2E')
@ -72,12 +72,12 @@ class AnsibleJSONEncoder(json.JSONEncoder):
def to_yaml(a, *args, **kw):
'''Make verbose, human readable yaml'''
transformed = yaml.dump(a, Dumper=AnsibleDumper, allow_unicode=True, **kw)
return to_unicode(transformed)
return to_text(transformed)
def to_nice_yaml(a, indent=4, *args, **kw):
'''Make verbose, human readable yaml'''
transformed = yaml.dump(a, Dumper=AnsibleDumper, indent=indent, allow_unicode=True, default_flow_style=False, **kw)
return to_unicode(transformed)
return to_text(transformed)
def to_json(a, *args, **kw):
''' Convert the value to JSON '''
@ -132,7 +132,7 @@ def fileglob(pathname):
def regex_replace(value='', pattern='', replacement='', ignorecase=False):
''' Perform a `re.sub` returning a string '''
value = to_unicode(value, errors='strict', nonstring='simplerepr')
value = to_text(value, errors='surrogate_or_strict', nonstring='simplerepr')
if ignorecase:
flags = re.I

4
lib/ansible/plugins/lookup/__init__.py
Unescape Escape View File

@ -98,8 +98,8 @@ class LookupBase(with_metaclass(ABCMeta, object)):
must be converted into python's unicode type as the strings will be run
through jinja2 which has this requirement. You can use::
from ansible.utils.unicode import to_unicode
result_string = to_unicode(result_string)
from ansible.module_utils._text import to_text
result_string = to_text(result_string)
"""
pass

7
lib/ansible/plugins/lookup/csvfile.py
Unescape Escape View File

@ -22,7 +22,8 @@ import csv
from ansible.errors import AnsibleError
from ansible.plugins.lookup import LookupBase
from ansible.utils.unicode import to_bytes, to_str, to_unicode
from ansible.module_utils._text import to_bytes, to_native, to_text
class CSVRecoder:
"""
@ -49,7 +50,7 @@ class CSVReader:
def next(self):
row = self.reader.next()
return [to_unicode(s) for s in row]
return [to_text(s) for s in row]
def __iter__(self):
return self
@ -66,7 +67,7 @@ class LookupModule(LookupBase):
if row[0] == key:
return row[int(col)]
except Exception as e:
raise AnsibleError("csvfile: %s" % to_str(e))
raise AnsibleError("csvfile: %s" % to_native(e))
return dflt

7
lib/ansible/plugins/lookup/fileglob.py
Unescape Escape View File

@ -22,7 +22,8 @@ import glob
from ansible.plugins.lookup import LookupBase
from ansible.errors import AnsibleFileNotFound
from ansible.utils.unicode import to_bytes, to_unicode
from ansible.module_utils._text import to_bytes, to_text
class LookupModule(LookupBase):
@ -36,6 +37,6 @@ class LookupModule(LookupBase):
except AnsibleFileNotFound:
dwimmed_path = None
if dwimmed_path:
globbed = glob.glob(to_bytes(os.path.join(dwimmed_path, term_file), errors='strict'))
ret.extend(to_unicode(g, errors='strict') for g in globbed if os.path.isfile(g))
globbed = glob.glob(to_bytes(os.path.join(dwimmed_path, term_file), errors='surrogate_or_strict'))
ret.extend(to_text(g, errors='surrogate_or_strict') for g in globbed if os.path.isfile(g))
return ret

17
lib/ansible/plugins/lookup/filetree.py
Unescape Escape View File

@ -22,12 +22,6 @@ import pwd
import grp
import stat
from ansible.plugins.lookup import LookupBase
from ansible.utils.unicode import to_str
from __main__ import display
warning = display.warning
HAVE_SELINUX=False
try:
import selinux
@ -35,6 +29,11 @@ try:
except ImportError:
pass
from ansible.plugins.lookup import LookupBase
from ansible.module_utils._text import to_native
from __main__ import display
# If selinux fails to find a default, return an array of None
def selinux_context(path):
@ -43,7 +42,7 @@ def selinux_context(path):
try:
# note: the selinux module uses byte strings on python2 and text
# strings on python3
ret = selinux.lgetfilecon_raw(to_str(path))
ret = selinux.lgetfilecon_raw(to_native(path))
except OSError:
return context
if ret[0] != -1:
@ -60,7 +59,7 @@ def file_props(root, path):
try:
st = os.lstat(abspath)
except OSError as e:
warning('filetree: Error using stat() on path %s (%s)' % (abspath, e))
display.warning('filetree: Error using stat() on path %s (%s)' % (abspath, e))
return None
ret = dict(root=root, path=path)
@ -74,7 +73,7 @@ def file_props(root, path):
ret['state'] = 'file'
ret['src'] = abspath
else:
warning('filetree: Error file type of %s is not supported' % abspath)
display.warning('filetree: Error file type of %s is not supported' % abspath)
return None
ret['uid'] = st.st_uid

8
lib/ansible/plugins/lookup/ini.py
Unescape Escape View File

@ -30,7 +30,7 @@ except ImportError:
from ansible.errors import AnsibleError
from ansible.plugins.lookup import LookupBase
from ansible.utils.unicode import to_bytes
from ansible.module_utils._text import to_bytes, to_text
def _parse_params(term):
@ -59,13 +59,15 @@ class LookupModule(LookupBase):
def read_properties(self, filename, key, dflt, is_regexp):
config = StringIO()
config.write(u'[java_properties]\n' + open(to_bytes(filename, errors='strict')).read())
current_cfg_file = open(to_bytes(filename, errors='surrogate_or_strict'), 'rb')
config.write(u'[java_properties]\n' + to_text(current_cfg_file.read(), errors='surrogate_or_strict'))
config.seek(0, os.SEEK_SET)
self.cp.readfp(config)
return self.get_value(key, 'java_properties', dflt, is_regexp)
def read_ini(self, filename, key, section, dflt, is_regexp):
self.cp.readfp(open(to_bytes(filename, errors='strict')))
self.cp.readfp(open(to_bytes(filename, errors='surrogate_or_strict')))
return self.get_value(key, section, dflt, is_regexp)
def get_value(self, key, section, dflt, is_regexp):

6
lib/ansible/plugins/lookup/shelvefile.py
Unescape Escape View File

@ -21,10 +21,10 @@ import shelve
from ansible.errors import AnsibleError
from ansible.plugins.lookup import LookupBase
from ansible.utils.unicode import to_bytes, to_unicode
from ansible.module_utils._text import to_bytes, to_text
class LookupModule(LookupBase):
class LookupModule(LookupBase):
def read_shelve(self, shelve_filename, key):
"""
@ -66,7 +66,7 @@ class LookupModule(LookupBase):
if res is None:
raise AnsibleError("Key %s not found in shelve file %s" % (key, file))
# Convert the value read to string
ret.append(to_unicode(res))
ret.append(to_text(res))
break
else:
raise AnsibleError("Could not locate shelve file in lookup: %s" % file)

6
lib/ansible/plugins/lookup/template.py
Unescape Escape View File

@ -21,7 +21,7 @@ import os
from ansible.errors import AnsibleError
from ansible.plugins.lookup import LookupBase
from ansible.utils.unicode import to_unicode, to_bytes
from ansible.module_utils._text import to_bytes, to_text
try:
from __main__ import display
@ -43,8 +43,8 @@ class LookupModule(LookupBase):
lookupfile = self.find_file_in_search_path(variables, 'templates', term)
display.vvvv("File lookup using %s as file" % lookupfile)
if lookupfile:
with open(to_bytes(lookupfile, errors='strict'), 'r') as f:
template_data = to_unicode(f.read())
with open(to_bytes(lookupfile, errors='surrogate_or_strict'), 'rb') as f:
template_data = to_text(f.read(), errors='surrogate_or_strict')
# set jinja2 internal search path for includes
if 'ansible_search_path' in variables:

8
lib/ansible/plugins/lookup/url.py
Unescape Escape View File

@ -18,11 +18,11 @@ from __future__ import (absolute_import, division, print_function)
__metaclass__ = type
from ansible.compat.six.moves.urllib.error import HTTPError, URLError
from ansible.errors import AnsibleError
from ansible.plugins.lookup import LookupBase
from ansible.module_utils._text import to_text
from ansible.module_utils.urls import open_url, ConnectionError, SSLValidationError
from ansible.utils.unicode import to_unicode
from ansible.compat.six.moves.urllib.error import HTTPError, URLError
from ansible.plugins.lookup import LookupBase
try:
from __main__ import display
@ -52,5 +52,5 @@ class LookupModule(LookupBase):
raise AnsibleError("Error connecting to %s: %s" % (term, str(e)))
for line in response.read().splitlines():
ret.append(to_unicode(line))
ret.append(to_text(line))
return ret

13
lib/ansible/plugins/shell/powershell.py
Unescape Escape View File

@ -22,9 +22,9 @@ import os
import re
import shlex
from ansible.compat.six import text_type
from ansible.errors import AnsibleError
from ansible.utils.unicode import to_bytes, to_unicode
from ansible.module_utils._text import to_bytes, to_text
_common_args = ['PowerShell', '-NoProfile', '-NonInteractive', '-ExecutionPolicy', 'Unrestricted']
@ -34,6 +34,7 @@ _powershell_version = os.environ.get('POWERSHELL_VERSION', None)
if _powershell_version:
_common_args = ['PowerShell', '-Version', _powershell_version] + _common_args[1:]
class ShellModule(object):
# Common shell filenames that this plugin handles
@ -60,7 +61,7 @@ class ShellModule(object):
raise AnsibleError("PowerShell environment value for key '%s' exceeds 32767 characters in length" % key)
# powershell single quoted literals need single-quote doubling as their only escaping
value = value.replace("'", "''")
return text_type(value)
return to_text(value, errors='surrogate_or_strict')
def env_prefix(self, **kwargs):
env = self.env.copy()
@ -164,7 +165,7 @@ class ShellModule(object):
def build_module_command(self, env_string, shebang, cmd, arg_path=None, rm_tmp=None):
cmd_parts = shlex.split(to_bytes(cmd), posix=False)
cmd_parts = map(to_unicode, cmd_parts)
cmd_parts = map(to_text, cmd_parts)
if shebang and shebang.lower() == '#!powershell':
if not self._unquote(cmd_parts[0]).lower().endswith('.ps1'):
cmd_parts[0] = '"%s.ps1"' % self._unquote(cmd_parts[0])
@ -219,7 +220,7 @@ class ShellModule(object):
def _unquote(self, value):
'''Remove any matching quotes that wrap the given value.'''
value = to_unicode(value or '')
value = to_text(value or '')
m = re.match(r'^\s*?\'(.*?)\'\s*?$', value)
if m:
return m.group(1)
@ -244,7 +245,7 @@ class ShellModule(object):
def _encode_script(self, script, as_list=False, strict_mode=True):
'''Convert a PowerShell script to a single base64-encoded command.'''
script = to_unicode(script)
script = to_text(script)
if strict_mode:
script = u'Set-StrictMode -Version Latest\r\n%s' % script
script = '\n'.join([x.strip() for x in script.splitlines() if x.strip()])

23
lib/ansible/plugins/strategy/__init__.py
Unescape Escape View File

@ -19,18 +19,11 @@
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type
import json
import time
import zlib
from collections import defaultdict
from jinja2.exceptions import UndefinedError
from ansible.compat.six.moves import queue as Queue
from ansible.compat.six import iteritems, text_type, string_types
from ansible import constants as C
from ansible.compat.six import iteritems, string_types
from ansible.errors import AnsibleError, AnsibleParserError, AnsibleUndefinedVariable
from ansible.executor.play_iterator import PlayIterator
from ansible.executor.task_result import TaskResult
from ansible.inventory.host import Host
from ansible.inventory.group import Group
@ -38,11 +31,10 @@ from ansible.playbook.helpers import load_list_of_blocks
from ansible.playbook.included_file import IncludedFile
from ansible.playbook.task_include import TaskInclude
from ansible.playbook.role_include import IncludeRole
from ansible.plugins import action_loader, connection_loader, filter_loader, lookup_loader, module_loader, test_loader
from ansible.plugins import action_loader
from ansible.template import Templar
from ansible.utils.unicode import to_unicode
from ansible.vars.unsafe_proxy import wrap_var
from ansible.vars import combine_vars, strip_internal_keys
from ansible.module_utils._text import to_text
try:
@ -138,7 +130,7 @@ class StrategyBase:
ret_results = []
def get_original_host(host_name):
host_name = to_unicode(host_name)
host_name = to_text(host_name)
if host_name in self._inventory._hosts_cache:
return self._inventory._hosts_cache[host_name]
else:
@ -161,7 +153,7 @@ class StrategyBase:
target_handler_name = templar.template(handler_task.get_name())
if target_handler_name == handler_name:
return handler_task
except (UndefinedError, AnsibleUndefinedVariable) as e:
except (UndefinedError, AnsibleUndefinedVariable):
# We skip this handler due to the fact that it may be using
# a variable in the name that was conditionally included via
# set_fact or some other method, and we don't want to error
@ -182,7 +174,7 @@ class StrategyBase:
target_handler_name = templar.template(target_handler.get_name())
if target_handler_name == handler_name:
return True
except (UndefinedError, AnsibleUndefinedVariable) as e:
except (UndefinedError, AnsibleUndefinedVariable):
pass
return parent_handler_match(target_handler._parent, handler_name)
else:
@ -567,14 +559,13 @@ class StrategyBase:
# mark all of the hosts including this file as failed, send callbacks,
# and increment the stats for this host
for host in included_file._hosts:
tr = TaskResult(host=host, task=included_file._task, return_data=dict(failed=True, reason=to_unicode(e)))
tr = TaskResult(host=host, task=included_file._task, return_data=dict(failed=True, reason=to_text(e)))
iterator.mark_host_failed(host)
self._tqm._failed_hosts[host.name] = True
self._tqm._stats.increment('failures', host.name)
self._tqm.send_callback('v2_runner_on_failed', tr)
return []
# finally, send the callback and return the list of blocks loaded
self._tqm.send_callback('v2_playbook_on_include', included_file)
display.debug("done processing included file")

15
lib/ansible/plugins/strategy/free.py
Unescape Escape View File

@ -26,7 +26,8 @@ from ansible.playbook.included_file import IncludedFile
from ansible.plugins import action_loader
from ansible.plugins.strategy import StrategyBase
from ansible.template import Templar
from ansible.utils.unicode import to_unicode
from ansible.module_utils._text import to_text
try:
from __main__ import display
@ -66,8 +67,7 @@ class StrategyModule(StrategyBase):
break
work_to_do = False # assume we have no more work to do
starting_host = last_host # save current position so we know when we've
# looped back around and need to break
starting_host = last_host # save current position so we know when we've looped back around and need to break
# try and find an unblocked host with a task to run
host_results = []
@ -109,7 +109,7 @@ class StrategyModule(StrategyBase):
display.debug("done getting variables")
try:
task.name = to_unicode(templar.template(task.name, fail_on_undefined=False), nonstring='empty')
task.name = to_text(templar.template(task.name, fail_on_undefined=False), nonstring='empty')
display.debug("done templating")
except:
# just ignore any errors during task name templating,
@ -120,10 +120,10 @@ class StrategyModule(StrategyBase):
run_once = templar.template(task.run_once) or action and getattr(action, 'BYPASS_HOST_LOOP', False)
if run_once:
if action and getattr(action, 'BYPASS_HOST_LOOP', False):
raise AnsibleError("The '%s' module bypasses the host loop, which is currently not supported in the free strategy " \
raise AnsibleError("The '%s' module bypasses the host loop, which is currently not supported in the free strategy "
"and would instead execute for every host in the inventory list." % task.action, obj=task._ds)
else:
display.warning("Using run_once with the free strategy is not currently supported. This task will still be " \
display.warning("Using run_once with the free strategy is not currently supported. This task will still be "
"executed for every host in the inventory list.")
# check to see if this task should be skipped, due to it being a member of a
@ -143,7 +143,8 @@ class StrategyModule(StrategyBase):
# handle step if needed, skip meta actions as they are used internally
if not self._step or self._take_step(task, host_name):
if task.any_errors_fatal:
display.warning("Using any_errors_fatal with the free strategy is not supported, as tasks are executed independently on each host")
display.warning("Using any_errors_fatal with the free strategy is not supported,"
" as tasks are executed independently on each host")
self._tqm.send_callback('v2_playbook_on_task_start', task, is_conditional=False)
self._queue_task(host, task, task_vars, play_context)
del task_vars

7
lib/ansible/plugins/strategy/linear.py
Unescape Escape View File

@ -29,7 +29,8 @@ from ansible.playbook.task import Task
from ansible.plugins import action_loader
from ansible.plugins.strategy import StrategyBase
from ansible.template import Templar
from ansible.utils.unicode import to_unicode
from ansible.module_utils._text import to_text
try:
from __main__ import display
@ -243,7 +244,7 @@ class StrategyModule(StrategyBase):
saved_name = task.name
display.debug("done copying, going to template now")
try:
task.name = to_unicode(templar.template(task.name, fail_on_undefined=False), nonstring='empty')
task.name = to_text(templar.template(task.name, fail_on_undefined=False), nonstring='empty')
display.debug("done templating")
except:
# just ignore any errors during task name templating,
@ -368,7 +369,7 @@ class StrategyModule(StrategyBase):
for host in included_file._hosts:
self._tqm._failed_hosts[host.name] = True
iterator.mark_host_failed(host)
display.error(to_unicode(e), wrap_text=False)
display.error(to_text(e), wrap_text=False)
include_failure = True
continue

36
lib/ansible/template/__init__.py
Unescape Escape View File

@ -25,8 +25,8 @@ import os
import re
from io import StringIO
from numbers import Number
from ansible.compat.six import string_types, text_type, binary_type
from jinja2 import Environment
from jinja2.loaders import FileSystemLoader
from jinja2.exceptions import TemplateSyntaxError, UndefinedError
@ -34,19 +34,20 @@ from jinja2.utils import concat as j2_concat
from jinja2.runtime import StrictUndefined
from ansible import constants as C
from ansible.compat.six import string_types, text_type
from ansible.errors import AnsibleError, AnsibleFilterError, AnsibleUndefinedVariable
from ansible.plugins import filter_loader, lookup_loader, test_loader
from ansible.template.safe_eval import safe_eval
from ansible.template.template import AnsibleJ2Template
from ansible.template.vars import AnsibleJ2Vars
from ansible.utils.unicode import to_unicode, to_str
from ansible.module_utils._text import to_native, to_text
try:
from hashlib import sha1
except ImportError:
from sha import sha as sha1
from numbers import Number
try:
from __main__ import display
@ -107,6 +108,7 @@ def _escape_backslashes(data, jinja_env):
return data
def _count_newlines_from_end(in_str):
'''
Counts the number of newlines at the end of a string. This is used during
@ -255,10 +257,10 @@ class Templar:
if prev_idx is not None:
# replace the opening
data.seek(prev_idx, os.SEEK_SET)
data.write(to_unicode(self.environment.comment_start_string))
data.write(to_text(self.environment.comment_start_string))
# replace the closing
data.seek(token_start, os.SEEK_SET)
data.write(to_unicode(self.environment.comment_end_string))
data.write(to_text(self.environment.comment_end_string))
else:
raise AnsibleError("Error while cleaning data for safety: unhandled regex match")
@ -293,7 +295,7 @@ class Templar:
return self._clean_data(variable)
else:
# Do we need to convert these into text_type as well?
# return self._clean_data(to_unicode(variable._obj, nonstring='passthru'))
# return self._clean_data(to_text(variable._obj, nonstring='passthru'))
return self._clean_data(variable._obj)
try:
@ -383,7 +385,7 @@ class Templar:
returns True if the data contains a variable pattern
'''
if isinstance(data, string_types):
for marker in [self.environment.block_start_string, self.environment.variable_start_string, self.environment.comment_start_string]:
for marker in (self.environment.block_start_string, self.environment.variable_start_string, self.environment.comment_start_string):
if marker in data:
return True
return False
@ -399,7 +401,8 @@ class Templar:
first_part = variable.split("|")[0].split(".")[0].split("[")[0]
if (contains_filters or first_part in self._available_variables) and self.environment.variable_start_string not in variable:
if bare_deprecated:
display.deprecated("Using bare variables is deprecated. Update your playbooks so that the environment value uses the full variable syntax ('%s%s%s')" %
display.deprecated("Using bare variables is deprecated."
" Update your playbooks so that the environment value uses the full variable syntax ('%s%s%s')" %
(self.environment.variable_start_string, variable, self.environment.variable_end_string))
return "%s%s%s" % (self.environment.variable_start_string, variable, self.environment.variable_end_string)
@ -485,10 +488,10 @@ class Templar:
try:
t = myenv.from_string(data)
except TemplateSyntaxError as e:
raise AnsibleError("template error while templating string: %s. String: %s" % (to_str(e), to_str(data)))
raise AnsibleError("template error while templating string: %s. String: %s" % (to_native(e), to_native(data)))
except Exception as e:
if 'recursion' in to_str(e):
raise AnsibleError("recursive loop detected in template string: %s" % to_str(data))
if 'recursion' in to_native(e):
raise AnsibleError("recursive loop detected in template string: %s" % to_native(data))
else:
return data
@ -503,13 +506,13 @@ class Templar:
try:
res = j2_concat(rf)
except TypeError as te:
if 'StrictUndefined' in to_str(te):
errmsg = "Unable to look up a name or access an attribute in template string (%s).\n" % to_str(data)
errmsg += "Make sure your variable name does not contain invalid characters like '-': %s" % to_str(te)
if 'StrictUndefined' in to_native(te):
errmsg = "Unable to look up a name or access an attribute in template string (%s).\n" % to_native(data)
errmsg += "Make sure your variable name does not contain invalid characters like '-': %s" % to_native(te)
raise AnsibleUndefinedVariable(errmsg)
else:
display.debug("failing because of a type error, template data is: %s" % to_str(data))
raise AnsibleError("Unexpected templating type error occurred on (%s): %s" % (to_str(data),to_str(te)))
display.debug("failing because of a type error, template data is: %s" % to_native(data))
raise AnsibleError("Unexpected templating type error occurred on (%s): %s" % (to_native(data),to_native(te)))
if preserve_trailing_newlines:
# The low level calls above do not preserve the newline
@ -534,4 +537,3 @@ class Templar:
else:
#TODO: return warning about undefined var
return data

6
lib/ansible/template/vars.py
Unescape Escape View File

@ -21,7 +21,8 @@ __metaclass__ = type
from ansible.compat.six import iteritems
from jinja2.utils import missing
from ansible.utils.unicode import to_unicode
from ansible.module_utils._text import to_native
__all__ = ['AnsibleJ2Vars']
@ -88,7 +89,7 @@ class AnsibleJ2Vars:
try:
value = self._templar.template(variable)
except Exception as e:
raise type(e)(to_unicode(variable) + ': ' + e.message)
raise type(e)(to_native(variable) + ': ' + e.message)
return value
def add_locals(self, locals):
@ -99,4 +100,3 @@ class AnsibleJ2Vars:
if locals is None:
return self
return AnsibleJ2Vars(self._templar, self._globals, locals=locals, *self._extras)

12
lib/ansible/utils/display.py
Unescape Escape View File

@ -35,7 +35,8 @@ from termios import TIOCGWINSZ
from ansible import constants as C
from ansible.errors import AnsibleError
from ansible.utils.color import stringc
from ansible.utils.unicode import to_bytes, to_unicode
from ansible.module_utils._text import to_bytes, to_text
try:
# Python 2
@ -45,7 +46,6 @@ except NameError:
pass
logger = None
#TODO: make this a logging callback instead
if C.DEFAULT_LOG_PATH:
@ -124,7 +124,7 @@ class Display:
# Convert back to text string on python3
# We first convert to a byte string so that we get rid of
# characters that are invalid in the user's locale
msg2 = to_unicode(msg2, self._output_encoding(stderr=stderr))
msg2 = to_text(msg2, self._output_encoding(stderr=stderr))
if not stderr:
fileobj = sys.stdout
@ -149,7 +149,7 @@ class Display:
# Convert back to text string on python3
# We first convert to a byte string so that we get rid of
# characters that are invalid in the user's locale
msg2 = to_unicode(msg2, self._output_encoding(stderr=stderr))
msg2 = to_text(msg2, self._output_encoding(stderr=stderr))
if color == C.COLOR_ERROR:
logger.error(msg2)
@ -279,7 +279,7 @@ class Display:
if sys.version_info >= (3,):
# Convert back into text on python3. We do this double conversion
# to get rid of characters that are illegal in the user's locale
prompt_string = to_unicode(prompt_string)
prompt_string = to_text(prompt_string)
if private:
return getpass.getpass(msg)
@ -323,7 +323,7 @@ class Display:
result = do_encrypt(result, encrypt, salt_size, salt)
# handle utf-8 chars
result = to_unicode(result, errors='strict')
result = to_text(result, errors='surrogate_or_strict')
return result
@staticmethod

8
lib/ansible/utils/hashing.py
Unescape Escape View File

@ -39,14 +39,14 @@ except ImportError:
_md5 = None
from ansible.errors import AnsibleError
from ansible.utils.unicode import to_bytes
from ansible.module_utils._text import to_bytes
def secure_hash_s(data, hash_func=sha1):
''' Return a secure hash hex digest of data. '''
digest = hash_func()
data = to_bytes(data, errors='strict')
data = to_bytes(data, errors='surrogate_or_strict')
digest.update(data)
return digest.hexdigest()
@ -54,12 +54,12 @@ def secure_hash_s(data, hash_func=sha1):
def secure_hash(filename, hash_func=sha1):
''' Return a secure hash hex digest of local file, None if file is not present or a directory. '''
if not os.path.exists(to_bytes(filename, errors='strict')) or os.path.isdir(to_bytes(filename, errors='strict')):
if not os.path.exists(to_bytes(filename, errors='surrogate_or_strict')) or os.path.isdir(to_bytes(filename, errors='strict')):
return None
digest = hash_func()
blocksize = 64 * 1024
try:
infile = open(to_bytes(filename, errors='strict'), 'rb')
infile = open(to_bytes(filename, errors='surrogate_or_strict'), 'rb')
block = infile.read(blocksize)
while block:
digest.update(block)

14
lib/ansible/utils/path.py
Unescape Escape View File

@ -20,11 +20,12 @@ __metaclass__ = type
import os
from errno import EEXIST
from ansible.errors import AnsibleError
from ansible.utils.unicode import to_bytes, to_str, to_unicode
from ansible.compat.six import PY2
from ansible.module_utils._text import to_bytes, to_native, to_text
__all__ = ['unfrackpath', 'makedirs_safe']
def unfrackpath(path):
'''
Returns a path that is free of symlinks, environment
@ -40,10 +41,9 @@ def unfrackpath(path):
example::
'$HOME/../../var/mail' becomes '/var/spool/mail'
'''
canonical_path = os.path.normpath(os.path.realpath(os.path.expanduser(os.path.expandvars(to_bytes(path, errors='strict')))))
if PY2:
return to_unicode(canonical_path, errors='strict')
return to_unicode(canonical_path, errors='surrogateescape')
canonical_path = os.path.normpath(os.path.realpath(os.path.expanduser(os.path.expandvars(to_bytes(path, errors='surrogate_or_strict')))))
return to_text(canonical_path, errors='surrogate_or_strict')
def makedirs_safe(path, mode=None):
'''Safe way to create dirs in muliprocess/thread environments.
@ -64,4 +64,4 @@ def makedirs_safe(path, mode=None):
os.makedirs(b_rpath)
except OSError as e:
if e.errno != EEXIST:
raise AnsibleError("Unable to create local directories(%s): %s" % (to_str(rpath), to_str(e)))
raise AnsibleError("Unable to create local directories(%s): %s" % (to_native(rpath), to_native(e)))

5
lib/ansible/utils/shlex.py
Unescape Escape View File

@ -21,8 +21,7 @@ __metaclass__ = type
import shlex
from ansible.compat.six import PY3
from ansible.utils.unicode import to_bytes, to_unicode
from ansible.module_utils._text import to_bytes, to_text
if PY3:
@ -31,5 +30,5 @@ if PY3:
else:
# shlex.split() wants bytes (i.e. ``str``) input on Python 2
def shlex_split(s, comments=False, posix=True):
return map(to_unicode, shlex.split(to_bytes(s), comments, posix))
return map(to_text, shlex.split(to_bytes(s), comments, posix))
shlex_split.__doc__ = shlex.split.__doc__

276
lib/ansible/utils/unicode.py
Unescape Escape View File

@ -19,264 +19,48 @@
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type
from ansible.compat.six import string_types, text_type, binary_type, PY3
from ansible.module_utils._text import to_bytes as _to_bytes, to_text, to_native
# to_bytes and to_unicode were written by Toshio Kuratomi for the
# python-kitchen library https://pypi.python.org/pypi/kitchen
# They are licensed in kitchen under the terms of the GPLv2+
# They were copied and modified for use in ansible by Toshio in Jan 2015
# (simply removing the deprecated features)
try:
from __main__ import display
except ImportError:
from ansible.utils.display import Display
display = Display()
#: Aliases for the utf-8 codec
_UTF8_ALIASES = frozenset(('utf-8', 'UTF-8', 'utf8', 'UTF8', 'utf_8', 'UTF_8',
'utf', 'UTF', 'u8', 'U8'))
#: Aliases for the latin-1 codec
_LATIN1_ALIASES = frozenset(('latin-1', 'LATIN-1', 'latin1', 'LATIN1',
'latin', 'LATIN', 'l1', 'L1', 'cp819', 'CP819', '8859', 'iso8859-1',
'ISO8859-1', 'iso-8859-1', 'ISO-8859-1'))
# EXCEPTION_CONVERTERS is defined below due to using to_unicode
__all__ = ('to_bytes', 'to_unicode', 'to_str', 'unicode_wrap')
def to_unicode(obj, encoding='utf-8', errors='replace', nonstring=None):
'''Convert an object into a :class:`unicode` string
:arg obj: Object to convert to a :class:`unicode` string. This should
normally be a byte :class:`str`
:kwarg encoding: What encoding to try converting the byte :class:`str` as.
Defaults to :term:`utf-8`
:kwarg errors: If errors are found while decoding, perform this action.
Defaults to ``replace`` which replaces the invalid bytes with
a character that means the bytes were unable to be decoded. Other
values are the same as the error handling schemes in the `codec base
classes
<http://docs.python.org/library/codecs.html#codec-base-classes>`_.
For instance ``strict`` which raises an exception and ``ignore`` which
simply omits the non-decodable characters.
:kwarg nonstring: How to treat nonstring values. Possible values are:
###
### Backwards compat
###
:simplerepr: Attempt to call the object's "simple representation"
method and return that value. Python-2.3+ has two methods that
try to return a simple representation: :meth:`object.__unicode__`
and :meth:`object.__str__`. We first try to get a usable value
from :meth:`object.__unicode__`. If that fails we try the same
with :meth:`object.__str__`.
:empty: Return an empty :class:`unicode` string
:strict: Raise a :exc:`TypeError`
:passthru: Return the object unchanged
:repr: Attempt to return a :class:`unicode` string of the repr of the
object
def to_bytes(*args, **kwargs):
display.deprecated(u'ansible.utils.unicode.to_bytes is deprecated. Use ansible.module_utils._text.to_bytes instead', version=u'2.4')
if 'errors' not in kwargs:
kwargs['errors'] = 'replace'
return _to_bytes(*args, **kwargs)
Default is ``simplerepr``
:raises TypeError: if :attr:`nonstring` is ``strict`` and
a non-:class:`basestring` object is passed in or if :attr:`nonstring`
is set to an unknown value
:raises UnicodeDecodeError: if :attr:`errors` is ``strict`` and
:attr:`obj` is not decodable using the given encoding
:returns: :class:`unicode` string or the original object depending on the
value of :attr:`nonstring`.
def to_unicode(*args, **kwargs):
display.deprecated(u'ansible.utils.unicode.to_unicode is deprecated. Use ansible.module_utils._text.to_text instead', version=u'2.4')
if 'errors' not in kwargs:
kwargs['errors'] = 'replace'
return to_text(*args, **kwargs)
Usually this should be used on a byte :class:`str` but it can take both
byte :class:`str` and :class:`unicode` strings intelligently. Nonstring
objects are handled in different ways depending on the setting of the
:attr:`nonstring` parameter.
The default values of this function are set so as to always return
a :class:`unicode` string and never raise an error when converting from
a byte :class:`str` to a :class:`unicode` string. However, when you do
not pass validly encoded text (or a nonstring object), you may end up with
output that you don't expect. Be sure you understand the requirements of
your data, not just ignore errors by passing it through this function.
'''
# Could use isbasestring/isunicode here but we want this code to be as
# fast as possible
if isinstance(obj, text_type):
return obj
if isinstance(obj, binary_type):
if encoding in _UTF8_ALIASES:
return text_type(obj, 'utf-8', errors)
if encoding in _LATIN1_ALIASES:
return text_type(obj, 'latin-1', errors)
return obj.decode(encoding, errors)
def to_str(*args, **kwargs):
display.deprecated(u'ansible.utils.unicode.to_str is deprecated. Use ansible.module_utils._text.to_native instead', version=u'2.4')
if 'errors' not in kwargs:
kwargs['errors'] = 'replace'
return to_native(*args, **kwargs)
if not nonstring:
nonstring = 'simplerepr'
if nonstring == 'empty':
return u''
elif nonstring == 'passthru':
return obj
elif nonstring == 'simplerepr':
try:
simple = obj.__unicode__()
except (AttributeError, UnicodeError):
simple = None
if not simple:
try:
simple = text_type(obj)
except UnicodeError:
try:
simple = obj.__str__()
except (UnicodeError, AttributeError):
simple = u''
if isinstance(simple, binary_type):
return text_type(simple, encoding, errors)
return simple
elif nonstring in ('repr', 'strict'):
obj_repr = repr(obj)
if isinstance(obj_repr, binary_type):
obj_repr = text_type(obj_repr, encoding, errors)
if nonstring == 'repr':
return obj_repr
raise TypeError('to_unicode was given "%(obj)s" which is neither'
' a byte string (str) or a unicode string' %
{'obj': obj_repr.encode(encoding, 'replace')})
### End Backwards compat
raise TypeError('nonstring value, %(param)s, is not set to a valid'
' action' % {'param': nonstring})
def to_bytes(obj, encoding='utf-8', errors='replace', nonstring=None):
'''Convert an object into a byte :class:`str`
:arg obj: Object to convert to a byte :class:`str`. This should normally
be a :class:`unicode` string.
:kwarg encoding: Encoding to use to convert the :class:`unicode` string
into a byte :class:`str`. Defaults to :term:`utf-8`.
:kwarg errors: If errors are found while encoding, perform this action.
Defaults to ``replace`` which replaces the invalid bytes with
a character that means the bytes were unable to be encoded. Other
values are the same as the error handling schemes in the `codec base
classes
<http://docs.python.org/library/codecs.html#codec-base-classes>`_.
For instance ``strict`` which raises an exception and ``ignore`` which
simply omits the non-encodable characters.
:kwarg nonstring: How to treat nonstring values. Possible values are:
:simplerepr: Attempt to call the object's "simple representation"
method and return that value. Python-2.3+ has two methods that
try to return a simple representation: :meth:`object.__unicode__`
and :meth:`object.__str__`. We first try to get a usable value
from :meth:`object.__str__`. If that fails we try the same
with :meth:`object.__unicode__`.
:empty: Return an empty byte :class:`str`
:strict: Raise a :exc:`TypeError`
:passthru: Return the object unchanged
:repr: Attempt to return a byte :class:`str` of the :func:`repr` of the
object
Default is ``simplerepr``.
:raises TypeError: if :attr:`nonstring` is ``strict`` and
a non-:class:`basestring` object is passed in or if :attr:`nonstring`
is set to an unknown value.
:raises UnicodeEncodeError: if :attr:`errors` is ``strict`` and all of the
bytes of :attr:`obj` are unable to be encoded using :attr:`encoding`.
:returns: byte :class:`str` or the original object depending on the value
of :attr:`nonstring`.
.. warning::
If you pass a byte :class:`str` into this function the byte
:class:`str` is returned unmodified. It is **not** re-encoded with
the specified :attr:`encoding`. The easiest way to achieve that is::
to_bytes(to_unicode(text), encoding='utf-8')
The initial :func:`to_unicode` call will ensure text is
a :class:`unicode` string. Then, :func:`to_bytes` will turn that into
a byte :class:`str` with the specified encoding.
Usually, this should be used on a :class:`unicode` string but it can take
either a byte :class:`str` or a :class:`unicode` string intelligently.
Nonstring objects are handled in different ways depending on the setting
of the :attr:`nonstring` parameter.
The default values of this function are set so as to always return a byte
:class:`str` and never raise an error when converting from unicode to
bytes. However, when you do not pass an encoding that can validly encode
the object (or a non-string object), you may end up with output that you
don't expect. Be sure you understand the requirements of your data, not
just ignore errors by passing it through this function.
'''
# Could use isbasestring, isbytestring here but we want this to be as fast
# as possible
if isinstance(obj, binary_type):
return obj
if isinstance(obj, text_type):
return obj.encode(encoding, errors)
if not nonstring:
nonstring = 'simplerepr'
if nonstring == 'empty':
return b''
elif nonstring == 'passthru':
return obj
elif nonstring == 'simplerepr':
try:
simple = str(obj)
except UnicodeError:
try:
simple = obj.__str__()
except (AttributeError, UnicodeError):
simple = None
if not simple:
try:
simple = obj.__unicode__()
except (AttributeError, UnicodeError):
simple = b''
if isinstance(simple, text_type):
simple = simple.encode(encoding, 'replace')
return simple
elif nonstring in ('repr', 'strict'):
try:
obj_repr = obj.__repr__()
except (AttributeError, UnicodeError):
obj_repr = b''
if isinstance(obj_repr, text_type):
obj_repr = obj_repr.encode(encoding, errors)
else:
obj_repr = binary_type(obj_repr)
if nonstring == 'repr':
return obj_repr
raise TypeError('to_bytes was given "%(obj)s" which is neither'
' a unicode string or a byte string (str)' % {'obj': obj_repr})
raise TypeError('nonstring value, %(param)s, is not set to a valid'
' action' % {'param': nonstring})
# force the return value of a function to be unicode. Use with partial to
# ensure that a filter will return unicode values.
def unicode_wrap(func, *args, **kwargs):
return to_unicode(func(*args, **kwargs), nonstring='passthru')
"""If a function returns a string, force it to be a text string.
# Alias for converting to native strings.
# Native strings are the default string type for the particular version of
# python. The objects are called "str" in both py2 and py3 but they mean
# different things. In py2, it's a byte string like in C. In py3 it's an
# abstract text type (like py2's unicode type).
#
# Use this when raising exceptions and wanting to get the string
# representation of an object for the exception message. For example:
#
# try:
# do_something()
# except Exception as e:
# raise AnsibleError(to_str(e))
#
# Note that this is because python's exception handling expects native strings
# and doe the wrong thing if given the other sort of string (in py2, if given
# unicode strings, it could traceback or omit the message. in py3, if given
# byte strings it prints their repr (so the message ends up as b'message').
#
# If you use ansible's API instead of re-raising an exception, use to_unicode
# instead:
#
# try:
# do_something()
# except Exception as e:
# display.warn(to_unicode(e))
if PY3:
to_str = to_unicode
else:
to_str = to_bytes
Use with partial to ensure that filter plugins will return text values.
"""
return to_text(func(*args, **kwargs), nonstring='passthru')

12
lib/ansible/utils/vars.py
Unescape Escape View File

@ -28,7 +28,7 @@ from ansible.compat.six import iteritems, string_types
from ansible import constants as C
from ansible.errors import AnsibleError
from ansible.parsing.splitter import parse_kv
from ansible.utils.unicode import to_unicode, to_str
from ansible.module_utils._text import to_native, to_text
def _validate_mutable_mappings(a, b):
@ -49,11 +49,12 @@ def _validate_mutable_mappings(a, b):
try:
myvars.append(dumps(x))
except:
myvars.append(to_str(x))
myvars.append(to_native(x))
raise AnsibleError("failed to combine variables, expected dicts but got a '{0}' and a '{1}': \n{2}\n{3}".format(
a.__class__.__name__, b.__class__.__name__, myvars[0], myvars[1])
)
def combine_vars(a, b):
"""
Return a copy of dictionaries of variables based on configured hash behavior
@ -68,6 +69,7 @@ def combine_vars(a, b):
result.update(b)
return result
def merge_hash(a, b):
"""
Recursively merges hash b into a so that keys from b take precedence over keys from a
@ -95,10 +97,11 @@ def merge_hash(a, b):
return result
def load_extra_vars(loader, options):
extra_vars = {}
for extra_vars_opt in options.extra_vars:
extra_vars_opt = to_unicode(extra_vars_opt, errors='strict')
extra_vars_opt = to_text(extra_vars_opt, errors='surrogate_or_strict')
if extra_vars_opt.startswith(u"@"):
# Argument is a YAML file (JSON is a subset of YAML)
data = loader.load_from_file(extra_vars_opt[1:])
@ -111,6 +114,7 @@ def load_extra_vars(loader, options):
extra_vars = combine_vars(extra_vars, data)
return extra_vars
def load_options_vars(options):
options_vars = {}
# For now only return check mode, but we can easily return more
@ -118,6 +122,7 @@ def load_options_vars(options):
options_vars['ansible_check_mode'] = options.check
return options_vars
def isidentifier(ident):
"""
Determines, if string is valid Python identifier using the ast module.
@ -148,4 +153,3 @@ def isidentifier(ident):
return False
return True

13
lib/ansible/vars/unsafe_proxy.py
Unescape Escape View File

@ -54,18 +54,21 @@ from __future__ import (absolute_import, division, print_function)
__metaclass__ = type
import json
from ansible.utils.unicode import to_unicode
from ansible.compat.six import string_types, text_type
from ansible.module_utils._text import to_text
__all__ = ['UnsafeProxy', 'AnsibleUnsafe', 'AnsibleJSONUnsafeEncoder', 'AnsibleJSONUnsafeDecoder', 'wrap_var']
class AnsibleUnsafe(object):
__UNSAFE__ = True
class AnsibleUnsafeText(text_type, AnsibleUnsafe):
pass
class UnsafeProxy(object):
def __new__(cls, obj, *args, **kwargs):
# In our usage we should only receive unicode strings.
@ -73,10 +76,11 @@ class UnsafeProxy(object):
# we're given but we may want to take it out for testing and sanitize
# our input instead.
if isinstance(obj, string_types):
obj = to_unicode(obj, errors='strict')
obj = to_text(obj, errors='surrogate_or_strict')
return AnsibleUnsafeText(obj)
return obj
class AnsibleJSONUnsafeEncoder(json.JSONEncoder):
def encode(self, obj):
if isinstance(obj, AnsibleUnsafe):
@ -84,6 +88,7 @@ class AnsibleJSONUnsafeEncoder(json.JSONEncoder):
else:
return super(AnsibleJSONUnsafeEncoder, self).encode(obj)
class AnsibleJSONUnsafeDecoder(json.JSONDecoder):
def decode(self, obj):
value = super(AnsibleJSONUnsafeDecoder, self).decode(obj)
@ -92,6 +97,7 @@ class AnsibleJSONUnsafeDecoder(json.JSONDecoder):
else:
return value
def _wrap_dict(v):
for k in v.keys():
if v[k] is not None:
@ -115,4 +121,3 @@ def wrap_var(v):
if v is not None and not isinstance(v, AnsibleUnsafe):
v = UnsafeProxy(v)
return v

5
test/units/mock/procenv.py
Unescape Escape View File

@ -27,7 +27,8 @@ from contextlib import contextmanager
from io import BytesIO, StringIO
from ansible.compat.six import PY3
from ansible.compat.tests import unittest
from ansible.utils.unicode import to_bytes
from ansible.module_utils._text import to_bytes
@contextmanager
def swap_stdin_and_argv(stdin_data='', argv_data=tuple()):
@ -48,6 +49,7 @@ def swap_stdin_and_argv(stdin_data='', argv_data=tuple()):
sys.stdin = real_stdin
sys.argv = real_argv
@contextmanager
def swap_stdout():
"""
@ -62,6 +64,7 @@ def swap_stdout():
yield fake_stream
sys.stdout = old_stdout
class ModuleTestCase(unittest.TestCase):
def setUp(self, module_args=None):
if module_args is None:

3
test/units/parsing/vault/test_vault.py
Unescape Escape View File

@ -30,11 +30,12 @@ from binascii import hexlify
from nose.plugins.skip import SkipTest
from ansible.compat.tests import unittest
from ansible.utils.unicode import to_bytes, to_unicode
from ansible import errors
from ansible.parsing.vault import VaultLib
from ansible.parsing import vault
from ansible.module_utils._text import to_bytes
# Counter import fails for 2.0.1, requires >= 2.6.1 from pip
try:

28
test/units/parsing/vault/test_vault_editor.py
Unescape Escape View File

@ -27,11 +27,12 @@ from nose.plugins.skip import SkipTest
from ansible.compat.tests import unittest
from ansible.compat.tests.mock import patch
from ansible.utils.unicode import to_bytes, to_unicode
from ansible import errors
from ansible.parsing.vault import VaultLib
from ansible.parsing.vault import VaultEditor
from ansible.module_utils._text import to_bytes, to_text
# Counter import fails for 2.0.1, requires >= 2.6.1 from pip
try:
@ -66,6 +67,7 @@ v11_data = """$ANSIBLE_VAULT;1.1;AES256
3631633031323837340a396530313963373030343933616133393566366137363761373930663833
3739"""
class TestVaultEditor(unittest.TestCase):
def setUp(self):
@ -121,20 +123,19 @@ class TestVaultEditor(unittest.TestCase):
error_hit = False
try:
ve.decrypt_file(v10_file.name)
except errors.AnsibleError as e:
except errors.AnsibleError:
error_hit = True
# verify decrypted content
f = open(v10_file.name, "rb")
fdata = to_unicode(f.read())
fdata = to_text(f.read())
f.close()
os.unlink(v10_file.name)
assert error_hit == False, "error decrypting 1.0 file"
assert error_hit is False, "error decrypting 1.0 file"
assert fdata.strip() == "foo", "incorrect decryption of 1.0 file: %s" % fdata.strip()
def test_decrypt_1_1(self):
if not HAS_AES or not HAS_COUNTER or not HAS_PBKDF2:
raise SkipTest
@ -149,20 +150,19 @@ class TestVaultEditor(unittest.TestCase):
error_hit = False
try:
ve.decrypt_file(v11_file.name)
except errors.AnsibleError as e:
except errors.AnsibleError:
error_hit = True
# verify decrypted content
f = open(v11_file.name, "rb")
fdata = to_unicode(f.read())
fdata = to_text(f.read())
f.close()
os.unlink(v11_file.name)
assert error_hit == False, "error decrypting 1.0 file"
assert error_hit is False, "error decrypting 1.0 file"
assert fdata.strip() == "foo", "incorrect decryption of 1.0 file: %s" % fdata.strip()
@unittest.skipIf(sys.version_info[0] >= 3, "VaultAES still needs to be ported to Python 3")
def test_rekey_migration(self):
"""
@ -182,7 +182,7 @@ class TestVaultEditor(unittest.TestCase):
error_hit = False
try:
ve.rekey_file(v10_file.name, 'ansible2')
except errors.AnsibleError as e:
except errors.AnsibleError:
error_hit = True
# verify decrypted content
@ -190,7 +190,7 @@ class TestVaultEditor(unittest.TestCase):
fdata = f.read()
f.close()
assert error_hit == False, "error rekeying 1.0 file to 1.1"
assert error_hit is False, "error rekeying 1.0 file to 1.1"
# ensure filedata can be decrypted, is 1.1 and is AES256
vl = VaultLib("ansible2")
@ -198,13 +198,11 @@ class TestVaultEditor(unittest.TestCase):
error_hit = False
try:
dec_data = vl.decrypt(fdata)
except errors.AnsibleError as e:
except errors.AnsibleError:
error_hit = True
os.unlink(v10_file.name)
assert vl.cipher_name == "AES256", "wrong cipher name set after rekey: %s" % vl.cipher_name
assert error_hit == False, "error decrypting migrated 1.0 file"
assert error_hit is False, "error decrypting migrated 1.0 file"
assert dec_data.strip() == "foo", "incorrect decryption of rekeyed/migrated file: %s" % dec_data

4
test/units/parsing/yaml/test_loader.py
Unescape Escape View File

@ -32,7 +32,6 @@ from ansible.parsing.yaml.loader import AnsibleLoader
from ansible.parsing import vault
from ansible.parsing.yaml.objects import AnsibleVaultEncryptedUnicode
from ansible.parsing.yaml.dumper import AnsibleDumper
from ansible.utils.unicode import to_bytes
from units.mock.yaml_helper import YamlTestUtils
@ -41,6 +40,7 @@ try:
except ImportError:
from yaml.parser import ParserError
class NameStringIO(StringIO):
"""In py2.6, StringIO doesn't let you set name because a baseclass has it
as readonly property"""
@ -49,6 +49,7 @@ class NameStringIO(StringIO):
def __init__(self, *args, **kwargs):
super(NameStringIO, self).__init__(*args, **kwargs)
class TestAnsibleLoaderBasic(unittest.TestCase):
def setUp(self):
@ -283,6 +284,7 @@ class TestAnsibleLoaderVault(unittest.TestCase, YamlTestUtils):
self.assertFalse(plaintext_var != vault_string)
self.assertFalse(vault_string != plaintext_var)
class TestAnsibleLoaderPlay(unittest.TestCase):
def setUp(self):

18
test/units/plugins/action/test_action.py
Unescape Escape View File

@ -21,8 +21,6 @@
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type
import ast
import json
import pipes
import os
@ -33,7 +31,6 @@ except ImportError:
from nose.tools import eq_, raises
from ansible.release import __version__ as ansible_version
from ansible import constants as C
from ansible.compat.six import text_type
from ansible.compat.tests import unittest
@ -41,12 +38,12 @@ from ansible.compat.tests.mock import patch, MagicMock, mock_open
from ansible.errors import AnsibleError
from ansible.playbook.play_context import PlayContext
from ansible.plugins import PluginLoader
from ansible.plugins.action import ActionBase
from ansible.template import Templar
from ansible.utils.unicode import to_bytes
from units.mock.loader import DictDataLoader
from ansible.module_utils._text import to_bytes
python_module_replacers = b"""
#!/usr/bin/python
@ -67,11 +64,13 @@ WINDOWS_ARGS = "<<INCLUDE_ANSIBLE_MODULE_JSON_ARGS>>"
class DerivedActionBase(ActionBase):
TRANSFERS_FILES = False
def run(self, tmp=None, task_vars=None):
# We're not testing the plugin run() method, just the helper
# methods ActionBase defines
return super(DerivedActionBase, self).run(tmp=tmp, task_vars=task_vars)
class TestActionBase(unittest.TestCase):
def test_action_base_run(self):
@ -144,7 +143,7 @@ class TestActionBase(unittest.TestCase):
self.assertRaises(AnsibleError, action_base._configure_module, 'badmodule', mock_task.args)
# test powershell module formatting
with patch.object(builtins, 'open', mock_open(read_data=to_bytes(powershell_module_replacers.strip(), encoding='utf-8'))) as m:
with patch.object(builtins, 'open', mock_open(read_data=to_bytes(powershell_module_replacers.strip(), encoding='utf-8'))):
mock_task.action = 'win_copy'
mock_task.args = dict(b=2)
mock_connection.module_implementation_preferences = ('.ps1',)
@ -497,7 +496,10 @@ class TestActionBase(unittest.TestCase):
action_base._connection.has_pipelining = True
action_base._low_level_execute_command.return_value = dict(stdout='{"rc": 0, "stdout": "ok"}')
self.assertEqual(action_base._execute_module(module_name=None, module_args=None), dict(_ansible_parsed=True, rc=0, stdout="ok", stdout_lines=['ok']))
self.assertEqual(action_base._execute_module(module_name='foo', module_args=dict(z=9, y=8, x=7), task_vars=dict(a=1)), dict(_ansible_parsed=True, rc=0, stdout="ok", stdout_lines=['ok']))
self.assertEqual(action_base._execute_module(module_name='foo',
module_args=dict(z=9, y=8, x=7), task_vars=dict(a=1)),
dict(_ansible_parsed=True, rc=0, stdout="ok",
stdout_lines=['ok']))
# test with needing/removing a remote tmp path
action_base._configure_module.return_value = ('old', '#!/usr/bin/python', 'this is the module data', 'path')
@ -555,6 +557,7 @@ class TestActionBase(unittest.TestCase):
finally:
C.BECOME_ALLOW_SAME_USER = become_allow_same_user
# Note: Using nose's generator test cases here so we can't inherit from
# unittest.TestCase
class TestFilterNonJsonLines(object):
@ -592,4 +595,3 @@ class TestFilterNonJsonLines(object):
def test_unparsable_filter_non_json_lines(self):
for stdout_line in self.unparsable_cases:
yield self.check_unparsable_filter_non_json_lines, stdout_line

41
test/units/plugins/connections/test_connection_ssh.py
Unescape Escape View File

@ -22,17 +22,17 @@ from __future__ import (absolute_import, division, print_function)
__metaclass__ = type
import pipes
import sys
from io import StringIO
from ansible.compat.tests import unittest
from ansible.compat.tests.mock import patch, MagicMock, mock_open
from ansible.compat.tests.mock import patch, MagicMock
from ansible import constants as C
from ansible.errors import AnsibleError, AnsibleConnectionFailure, AnsibleFileNotFound
from ansible.playbook.play_context import PlayContext
from ansible.plugins.connection import ssh
from ansible.utils.unicode import to_bytes, to_unicode
from ansible.module_utils._text import to_bytes
class TestConnectionBaseClass(unittest.TestCase):
@ -277,20 +277,22 @@ class TestConnectionBaseClass(unittest.TestCase):
C.ANSIBLE_SSH_RETRIES = 9
# test a regular, successful execution
conn._exec_command.return_value = (0, 'stdout', '')
conn._exec_command.return_value = (0, b'stdout', b'')
res = conn.exec_command('ssh', 'some data')
self.assertEquals(res, (0, b'stdout', b''), msg='exec_command did not return what the _exec_command helper returned')
# test a retry, followed by success
conn._exec_command.return_value = None
conn._exec_command.side_effect = [(255, '', ''), (0, 'stdout', '')]
conn._exec_command.side_effect = [(255, '', ''), (0, b'stdout', b'')]
res = conn.exec_command('ssh', 'some data')
self.assertEquals(res, (0, b'stdout', b''), msg='exec_command did not return what the _exec_command helper returned')
# test multiple failures
conn._exec_command.side_effect = [(255, '', '')]*10
conn._exec_command.side_effect = [(255, b'', b'')] * 10
self.assertRaises(AnsibleConnectionFailure, conn.exec_command, 'ssh', 'some data')
# test other failure from exec_command
conn._exec_command.side_effect = [Exception('bad')]*10
conn._exec_command.side_effect = [Exception('bad')] * 10
self.assertRaises(Exception, conn.exec_command, 'ssh', 'some data')
@patch('os.path.exists')
@ -308,20 +310,22 @@ class TestConnectionBaseClass(unittest.TestCase):
# test with C.DEFAULT_SCP_IF_SSH enabled
C.DEFAULT_SCP_IF_SSH = True
res = conn.put_file('/path/to/in/file', '/path/to/dest/file')
conn.put_file('/path/to/in/file', '/path/to/dest/file')
conn._run.assert_called_with('some command to run', None)
res = conn.put_file(u'/path/to/in/file/with/unicode-fö〩', u'/path/to/dest/file/with/unicode-fö〩')
conn.put_file(u'/path/to/in/file/with/unicode-fö〩', u'/path/to/dest/file/with/unicode-fö〩')
conn._run.assert_called_with('some command to run', None)
# test with C.DEFAULT_SCP_IF_SSH disabled
C.DEFAULT_SCP_IF_SSH = False
expected_in_data = b' '.join((b'put', to_bytes(pipes.quote('/path/to/in/file')), to_bytes(pipes.quote('/path/to/dest/file')))) + b'\n'
res = conn.put_file('/path/to/in/file', '/path/to/dest/file')
conn.put_file('/path/to/in/file', '/path/to/dest/file')
conn._run.assert_called_with('some command to run', expected_in_data)
expected_in_data = b' '.join((b'put', to_bytes(pipes.quote('/path/to/in/file/with/unicode-fö〩')), to_bytes(pipes.quote('/path/to/dest/file/with/unicode-fö〩')))) + b'\n'
res = conn.put_file(u'/path/to/in/file/with/unicode-fö〩', u'/path/to/dest/file/with/unicode-fö〩')
expected_in_data = b' '.join((b'put',
to_bytes(pipes.quote('/path/to/in/file/with/unicode-fö〩')),
to_bytes(pipes.quote('/path/to/dest/file/with/unicode-fö〩')))) + b'\n'
conn.put_file(u'/path/to/in/file/with/unicode-fö〩', u'/path/to/dest/file/with/unicode-fö〩')
conn._run.assert_called_with('some command to run', expected_in_data)
# test that a non-zero rc raises an error
@ -346,23 +350,24 @@ class TestConnectionBaseClass(unittest.TestCase):
# test with C.DEFAULT_SCP_IF_SSH enabled
C.DEFAULT_SCP_IF_SSH = True
res = conn.fetch_file('/path/to/in/file', '/path/to/dest/file')
conn.fetch_file('/path/to/in/file', '/path/to/dest/file')
conn._run.assert_called_with('some command to run', None)
res = conn.fetch_file(u'/path/to/in/file/with/unicode-fö〩', u'/path/to/dest/file/with/unicode-fö〩')
conn.fetch_file(u'/path/to/in/file/with/unicode-fö〩', u'/path/to/dest/file/with/unicode-fö〩')
conn._run.assert_called_with('some command to run', None)
# test with C.DEFAULT_SCP_IF_SSH disabled
C.DEFAULT_SCP_IF_SSH = False
expected_in_data = b' '.join((b'get', to_bytes(pipes.quote('/path/to/in/file')), to_bytes(pipes.quote('/path/to/dest/file')))) + b'\n'
res = conn.fetch_file('/path/to/in/file', '/path/to/dest/file')
conn.fetch_file('/path/to/in/file', '/path/to/dest/file')
conn._run.assert_called_with('some command to run', expected_in_data)
expected_in_data = b' '.join((b'get', to_bytes(pipes.quote('/path/to/in/file/with/unicode-fö〩')), to_bytes(pipes.quote('/path/to/dest/file/with/unicode-fö〩')))) + b'\n'
res = conn.fetch_file(u'/path/to/in/file/with/unicode-fö〩', u'/path/to/dest/file/with/unicode-fö〩')
expected_in_data = b' '.join((b'get',
to_bytes(pipes.quote('/path/to/in/file/with/unicode-fö〩')),
to_bytes(pipes.quote('/path/to/dest/file/with/unicode-fö〩')))) + b'\n'
conn.fetch_file(u'/path/to/in/file/with/unicode-fö〩', u'/path/to/dest/file/with/unicode-fö〩')
conn._run.assert_called_with('some command to run', expected_in_data)
# test that a non-zero rc raises an error
conn._run.return_value = (1, 'stdout', 'some errors')
self.assertRaises(AnsibleError, conn.fetch_file, '/path/to/bad/file', '/remote/path/to/file')

Write Preview
Loading…
Cancel
Save