Move a path being passed around as a byte string to being passed around as a text string. (#17190)

This is enough to get minimal copy module working on python3

We have t omodify dataloader's path_dwim_relative_stack and everything
that calls it to use text paths instead of byte string paths
pull/17191/head
Toshio Kuratomi 8 years ago committed by GitHub
parent 20bde8f549
commit 313d4b2c9e

@ -36,7 +36,7 @@ from ansible.parsing.yaml.loader import AnsibleLoader
from ansible.parsing.yaml.objects import AnsibleBaseYAMLObject, AnsibleUnicode
from ansible.module_utils.basic import is_executable
from ansible.utils.path import unfrackpath
from ansible.utils.unicode import to_unicode, to_bytes
from ansible.utils.unicode import to_unicode, to_bytes, to_str
try:
from __main__ import display
@ -279,45 +279,56 @@ class DataLoader():
def path_dwim_relative_stack(self, paths, dirname, source):
'''
find one file in first path in stack taking roles into account and adding play basedir as fallback
:arg paths: A list of text strings which are the paths to look for the filename in.
:arg dirname: A text string representing a directory. The directory
is prepended to the source to form the path to search for.
:arg source: A text string which is the filename to search for
:rtype: A text string
:returns: An absolute path to the filename ``source``
'''
b_dirname = to_bytes(dirname)
b_source = to_bytes(source)
result = None
if not source:
display.warning('Invalid request to find a file that matches an empty string or "null" value')
elif source.startswith('~') or source.startswith(os.path.sep):
# path is absolute, no relative needed, check existence and return source
test_path = to_bytes(unfrackpath(source),errors='strict')
if os.path.exists(test_path):
test_path = unfrackpath(b_source)
if os.path.exists(to_bytes(test_path, errors='strict')):
result = test_path
else:
search = []
for path in paths:
upath = unfrackpath(path)
mydir = os.path.dirname(upath)
b_upath = to_bytes(upath, errors='strict')
b_mydir = os.path.dirname(b_upath)
# if path is in role and 'tasks' not there already, add it into the search
if upath.endswith('tasks') and os.path.exists(to_bytes(os.path.join(upath,'main.yml'), errors='strict')) \
or os.path.exists(to_bytes(os.path.join(upath,'tasks/main.yml'), errors='strict')) \
or os.path.exists(to_bytes(os.path.join(os.path.dirname(upath),'tasks/main.yml'), errors='strict')):
if mydir.endswith('tasks'):
search.append(os.path.join(os.path.dirname(mydir), dirname, source))
search.append(os.path.join(mydir, source))
if b_upath.endswith(b'tasks') and os.path.exists(os.path.join(b_upath, b'main.yml')) \
or os.path.exists(os.path.join(b_upath, b'tasks/main.yml')) \
or os.path.exists(os.path.join(b_mydir, b'tasks/main.yml')):
if b_mydir.endswith(b'tasks'):
search.append(os.path.join(os.path.dirname(b_mydir), b_dirname, b_source))
search.append(os.path.join(b_mydir, b_source))
else:
search.append(os.path.join(upath, dirname, source))
search.append(os.path.join(upath, 'tasks', source))
elif dirname not in source.split('/'):
search.append(os.path.join(b_upath, b_dirname, b_source))
search.append(os.path.join(b_upath, b'tasks', b_source))
elif b_dirname not in b_source.split(b'/'):
# don't add dirname if user already is using it in source
search.append(os.path.join(upath, dirname, source))
search.append(os.path.join(upath, source))
search.append(os.path.join(b_upath, b_dirname, b_source))
search.append(os.path.join(b_upath, b_source))
# always append basedir as last resort
search.append(os.path.join(self.get_basedir(), dirname, source))
search.append(os.path.join(self.get_basedir(), source))
display.debug('search_path:\n\t' + '\n\t'.join(search))
for candidate in search:
display.vvvvv('looking for "%s" at "%s"' % (source, candidate))
if os.path.exists(to_bytes(candidate, errors='strict')):
result = candidate
search.append(os.path.join(to_bytes(self.get_basedir()), b_dirname, b_source))
search.append(os.path.join(to_bytes(self.get_basedir()), b_source))
display.debug(u'search_path:\n\t%s' % to_unicode(b'\n\t'.join(search), errors='replace'))
for b_candidate in search:
display.vvvvv(u'looking for "%s" at "%s"' % (source, to_unicode(b_candidate)))
if os.path.exists(b_candidate):
result = to_unicode(b_candidate)
break
return result
@ -370,10 +381,11 @@ class DataLoader():
"""
if not file_path or not isinstance(file_path, string_types):
raise AnsibleParserError("Invalid filename: '%s'" % str(file_path))
raise AnsibleParserError("Invalid filename: '%s'" % to_str(file_path))
if not self.path_exists(file_path) or not self.is_file(file_path):
raise AnsibleFileNotFound("the file_name '%s' does not exist, or is not readable" % file_path)
b_file_path = to_bytes(file_path, errors='strict')
if not self.path_exists(b_file_path) or not self.is_file(b_file_path):
raise AnsibleFileNotFound("the file_name '%s' does not exist, or is not readable" % to_str(file_path))
if not self._vault:
self._vault = VaultLib(password="")
@ -398,7 +410,7 @@ class DataLoader():
return real_path
except (IOError, OSError) as e:
raise AnsibleParserError("an error occurred while trying to read the file '%s': %s" % (real_path, str(e)))
raise AnsibleParserError("an error occurred while trying to read the file '%s': %s" % (to_str(real_path), to_str(e)))
def cleanup_tmp_file(self, file_path):
"""

@ -37,7 +37,7 @@ from ansible.errors import AnsibleError, AnsibleConnectionFailure
from ansible.executor.module_common import modify_module
from ansible.release import __version__
from ansible.parsing.utils.jsonify import jsonify
from ansible.utils.unicode import to_bytes, to_unicode
from ansible.utils.unicode import to_bytes, to_str, to_unicode
try:
from __main__ import display
@ -844,7 +844,7 @@ class ActionBase(with_metaclass(ABCMeta, object)):
result = self._loader.path_dwim_relative_stack(path_stack, dirname, needle)
if result is None:
raise AnsibleError("Unable to find '%s' in expected paths." % needle)
raise AnsibleError("Unable to find '%s' in expected paths." % to_str(needle))
return result

@ -27,7 +27,7 @@ from ansible.errors import AnsibleError
from ansible.plugins.action import ActionBase
from ansible.utils.boolean import boolean
from ansible.utils.hashing import checksum_s
from ansible.utils.unicode import to_str
from ansible.utils.unicode import to_str, to_unicode
class ActionModule(ActionBase):
@ -42,10 +42,10 @@ class ActionModule(ActionBase):
delimit_me = False
add_newline = False
for f in sorted(os.listdir(src_path)):
for f in (to_unicode(p, errors='strict') for p in sorted(os.listdir(src_path))):
if compiled_regexp and not compiled_regexp.search(f):
continue
fragment = "%s/%s" % (src_path, f)
fragment = u"%s/%s" % (src_path, f)
if not os.path.isfile(fragment) or (ignore_hidden and os.path.basename(fragment).startswith('.')):
continue
@ -119,7 +119,7 @@ class ActionModule(ActionBase):
if not os.path.isdir(src):
result['failed'] = True
result['msg'] = "Source (%s) is not a directory" % src
result['msg'] = u"Source (%s) is not a directory" % src
return result
_re = None

@ -18,10 +18,12 @@ from __future__ import (absolute_import, division, print_function)
__metaclass__ = type
import json
import pipes
import random
from ansible import constants as C
from ansible.plugins.action import ActionBase
from ansible.compat.six import iteritems
from ansible.utils.unicode import to_unicode
class ActionModule(ActionBase):

@ -27,7 +27,7 @@ from ansible.errors import AnsibleError
from ansible.plugins.action import ActionBase
from ansible.utils.boolean import boolean
from ansible.utils.hashing import checksum
from ansible.utils.unicode import to_bytes, to_str
from ansible.utils.unicode import to_bytes, to_str, to_unicode
class ActionModule(ActionBase):
@ -96,7 +96,7 @@ class ActionModule(ActionBase):
source = self._find_needle('files', source)
except AnsibleError as e:
result['failed'] = True
result['msg'] = to_str(e)
result['msg'] = to_unicode(e)
return result
# A list of source file tuples (full_path, relative_path) which will try to copy to the destination
@ -111,7 +111,7 @@ class ActionModule(ActionBase):
sz = len(source.rsplit('/', 1)[0]) + 1
# Walk the directory and append the file tuples to source_files.
for base_path, sub_folders, files in os.walk(source):
for base_path, sub_folders, files in os.walk(to_bytes(source)):
for file in files:
full_path = os.path.join(base_path, file)
rel_path = full_path[sz:]

@ -93,19 +93,20 @@ class ActionModule(ActionBase):
dest = os.path.join(dest, base)
# template the source data locally & get ready to transfer
b_source = to_bytes(source)
try:
with open(source, 'r') as f:
with open(b_source, 'r') as f:
template_data = to_unicode(f.read())
try:
template_uid = pwd.getpwuid(os.stat(source).st_uid).pw_name
template_uid = pwd.getpwuid(os.stat(b_source).st_uid).pw_name
except:
template_uid = os.stat(source).st_uid
template_uid = os.stat(b_source).st_uid
temp_vars = task_vars.copy()
temp_vars['template_host'] = os.uname()[1]
temp_vars['template_path'] = source
temp_vars['template_mtime'] = datetime.datetime.fromtimestamp(os.path.getmtime(source))
temp_vars['template_mtime'] = datetime.datetime.fromtimestamp(os.path.getmtime(b_source))
temp_vars['template_uid'] = template_uid
temp_vars['template_fullpath'] = os.path.abspath(source)
temp_vars['template_run_date'] = datetime.datetime.now()
@ -118,7 +119,7 @@ class ActionModule(ActionBase):
)
temp_vars['ansible_managed'] = time.strftime(
managed_str,
time.localtime(os.path.getmtime(source))
time.localtime(os.path.getmtime(b_source))
)
# Create a new searchpath list to assign to the templar environment's file

@ -38,7 +38,7 @@ class LookupModule(LookupBase):
# Find the file in the expected search path
lookupfile = self.find_file_in_search_path(variables, 'files', term)
display.vvvv("File lookup using %s as file" % lookupfile)
display.vvvv(u"File lookup using %s as file" % lookupfile)
try:
if lookupfile:
contents, show_data = self._loader._get_file_contents(lookupfile)

@ -22,6 +22,7 @@ import glob
from ansible.plugins.lookup import LookupBase
from ansible.errors import AnsibleFileNotFound
from ansible.module_utils.unicode import to_bytes
class LookupModule(LookupBase):
@ -35,6 +36,6 @@ class LookupModule(LookupBase):
except AnsibleFileNotFound:
dwimmed_path = None
if dwimmed_path:
globbed = glob.glob(os.path.join(dwimmed_path, term_file))
ret.extend(g for g in globbed if os.path.isfile(g))
globbed = glob.glob(to_bytes(os.path.join(dwimmed_path, term_file), errors='strict'))
ret.extend(to_unicode(g, errors='strict') for g in globbed if os.path.isfile(g))
return ret

@ -58,13 +58,13 @@ class LookupModule(LookupBase):
def read_properties(self, filename, key, dflt, is_regexp):
config = StringIO()
config.write(u'[java_properties]\n' + open(filename).read())
config.write(u'[java_properties]\n' + open(to_bytes(filename, errors='strict')).read())
config.seek(0, os.SEEK_SET)
self.cp.readfp(config)
return self.get_value(key, 'java_properties', dflt, is_regexp)
def read_ini(self, filename, key, section, dflt, is_regexp):
self.cp.readfp(open(filename))
self.cp.readfp(open(to_bytes(filename, errors='strict')))
return self.get_value(key, section, dflt, is_regexp)
def get_value(self, key, section, dflt, is_regexp):

@ -21,6 +21,7 @@ import shelve
from ansible.errors import AnsibleError
from ansible.plugins.lookup import LookupBase
from ansible.utils.unicode import to_bytes, to_unicode
class LookupModule(LookupBase):
@ -29,7 +30,7 @@ class LookupModule(LookupBase):
"""
Read the value of "key" from a shelve file
"""
d = shelve.open(shelve_filename)
d = shelve.open(to_bytes(shelve_filename))
res = d.get(key, None)
d.close()
return res
@ -65,7 +66,7 @@ class LookupModule(LookupBase):
if res is None:
raise AnsibleError("Key %s not found in shelve file %s" % (key, file))
# Convert the value read to string
ret.append(str(res))
ret.append(to_unicode(res))
break
else:
raise AnsibleError("Could not locate shelve file in lookup: %s" % file)

@ -43,7 +43,7 @@ class LookupModule(LookupBase):
lookupfile = self.find_file_in_search_path(variables, 'templates', term)
display.vvvv("File lookup using %s as file" % lookupfile)
if lookupfile:
with open(lookupfile, 'r') as f:
with open(to_bytes(lookupfile, errors='strict'), 'r') as f:
template_data = to_unicode(f.read())
# set jinja2 internal search path for includes

@ -20,29 +20,48 @@ __metaclass__ = type
import os
from errno import EEXIST
from ansible.errors import AnsibleError
from ansible.utils.unicode import to_bytes, to_str
from ansible.utils.unicode import to_bytes, to_str, to_unicode
from ansible.compat.six import PY2
__all__ = ['unfrackpath', 'makedirs_safe']
def unfrackpath(path):
'''
returns a path that is free of symlinks, environment
Returns a path that is free of symlinks, environment
variables, relative path traversals and symbols (~)
example:
:arg path: A byte or text string representing a path to be canonicalized
:raises UnicodeDecodeError: If the canonicalized version of the path
contains non-utf8 byte sequences.
:rtype: A text string (unicode on pyyhon2, str on python3).
:returns: An absolute path with symlinks, environment variables, and tilde
expanded. Note that this does not check whether a path exists.
example::
'$HOME/../../var/mail' becomes '/var/spool/mail'
'''
return os.path.normpath(os.path.realpath(os.path.expanduser(os.path.expandvars(to_bytes(path, errors='strict')))))
canonical_path = os.path.normpath(os.path.realpath(os.path.expanduser(os.path.expandvars(to_bytes(path, errors='strict')))))
if PY2:
return to_unicode(canonical_path, errors='strict')
return to_unicode(canonical_path, errors='surrogateescape')
def makedirs_safe(path, mode=None):
'''Safe way to create dirs in muliprocess/thread environments'''
'''Safe way to create dirs in muliprocess/thread environments.
:arg path: A byte or text string representing a directory to be created
:kwarg mode: If given, the mode to set the directory to
:raises AnsibleError: If the directory cannot be created and does not already exists.
:raises UnicodeDecodeError: if the path is not decodable in the utf-8 encoding.
'''
rpath = unfrackpath(path)
if not os.path.exists(rpath):
b_rpath = to_bytes(rpath)
if not os.path.exists(b_rpath):
try:
if mode:
os.makedirs(rpath, mode)
os.makedirs(b_rpath, mode)
else:
os.makedirs(rpath)
os.makedirs(b_rpath)
except OSError as e:
if e.errno != EEXIST:
raise AnsibleError("Unable to create local directories(%s): %s" % (rpath, to_str(e)))
raise AnsibleError("Unable to create local directories(%s): %s" % (to_str(rpath), to_str(e)))

Loading…
Cancel
Save