From f7571cb37fcfb854abc8174fc721f8b474191ff2 Mon Sep 17 00:00:00 2001 From: Marius Gedminas Date: Wed, 30 Sep 2015 09:01:09 +0300 Subject: [PATCH 1/7] Python 3: there's no __builtin__ module This caused an ImportError in a test module and showed up as one test failure. Now the test module can get imported and many more tests fail (on Python 3). Such is life. ;-) --- test/units/module_utils/test_basic.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/test/units/module_utils/test_basic.py b/test/units/module_utils/test_basic.py index 1a2fbefd438..faa14dbbb0a 100644 --- a/test/units/module_utils/test_basic.py +++ b/test/units/module_utils/test_basic.py @@ -20,10 +20,9 @@ from __future__ import (absolute_import, division) __metaclass__ = type -import __builtin__ import errno -from nose.tools import timed +from six.moves import builtins from ansible.compat.tests import unittest from ansible.compat.tests.mock import patch, MagicMock, mock_open, Mock @@ -37,16 +36,16 @@ class TestModuleUtilsBasic(unittest.TestCase): pass def test_module_utils_basic_imports(self): - realimport = __builtin__.__import__ + realimport = builtins.__import__ def _mock_import(name, *args, **kwargs): if name == 'json': raise ImportError() realimport(name, *args, **kwargs) - with patch.object(__builtin__, '__import__', _mock_import, create=True) as m: + with patch.object(builtins, '__import__', _mock_import, create=True) as m: m('ansible.module_utils.basic') - __builtin__.__import__('ansible.module_utils.basic') + builtins.__import__('ansible.module_utils.basic') def test_module_utils_basic_get_platform(self): with patch('platform.system', return_value='foo'): @@ -510,7 +509,7 @@ class TestModuleUtilsBasic(unittest.TestCase): m = mock_open() m.side_effect = OSError - with patch('__builtin__.open', m, create=True): + with patch.object(builtins, 'open', m, create=True): self.assertEqual(am.is_special_selinux_path('/some/path/that/should/be/nfs'), (False, None)) mount_data = [ @@ -524,7 +523,7 @@ class TestModuleUtilsBasic(unittest.TestCase): m = mock_open(read_data=''.join(mount_data)) m.return_value.readlines.return_value = mount_data - with patch('__builtin__.open', m, create=True): + with patch.object(builtins, 'open', m, create=True): self.assertEqual(am.is_special_selinux_path('/some/random/path'), (False, None)) self.assertEqual(am.is_special_selinux_path('/some/path/that/should/be/nfs'), (True, ['foo_u', 'foo_r', 'foo_t', 's0'])) self.assertEqual(am.is_special_selinux_path('/weird/random/fstype/path'), (True, ['foo_u', 'foo_r', 'foo_t', 's0'])) From 9f3e5ceb14cbec68561433a820de87a8954adc7b Mon Sep 17 00:00:00 2001 From: Marius Gedminas Date: Wed, 30 Sep 2015 09:08:41 +0300 Subject: [PATCH 2/7] Make sure 'basestring', 'bytes' and 'unicode' are defined Python 3 doesn't have 'basestring' and 'unicode'. Python 2.4 doesn't have 'bytes' --- lib/ansible/module_utils/basic.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/lib/ansible/module_utils/basic.py b/lib/ansible/module_utils/basic.py index 582aa35e3b2..0995fdd7582 100644 --- a/lib/ansible/module_utils/basic.py +++ b/lib/ansible/module_utils/basic.py @@ -73,6 +73,22 @@ try: except ImportError: imap = map # Python 3 +try: + basestring +except NameError: + basestring = str # Python 3 + +try: + unicode +except NameError: + unicode = str # Python 3 + +try: + bytes +except NameError: + bytes = str # Python 2 + + try: import json # Detect the python-json library which is incompatible From c7be004c197ca8545635245ed955759d319a447b Mon Sep 17 00:00:00 2001 From: Marius Gedminas Date: Wed, 30 Sep 2015 09:09:25 +0300 Subject: [PATCH 3/7] Python 3 has no dict.iteritems() (Reminder: cannot use six here, module_utils get shipped to remote machines that may not have six installed -- besides six doens't support Python 2.4.) --- lib/ansible/module_utils/basic.py | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/lib/ansible/module_utils/basic.py b/lib/ansible/module_utils/basic.py index 0995fdd7582..e5c53345049 100644 --- a/lib/ansible/module_utils/basic.py +++ b/lib/ansible/module_utils/basic.py @@ -88,6 +88,15 @@ try: except NameError: bytes = str # Python 2 +try: + dict.iteritems +except AttributeError: + def iteritems(d): # Python 3 + return d.items() +else: + def iteritems(d): # Python 2 + return d.iteritems() + try: import json @@ -299,7 +308,7 @@ def json_dict_unicode_to_bytes(d, encoding='utf-8'): if isinstance(d, unicode): return d.encode(encoding) elif isinstance(d, dict): - return dict(imap(json_dict_unicode_to_bytes, d.iteritems(), repeat(encoding))) + return dict(imap(json_dict_unicode_to_bytes, iteritems(d), repeat(encoding))) elif isinstance(d, list): return list(imap(json_dict_unicode_to_bytes, d, repeat(encoding))) elif isinstance(d, tuple): @@ -317,7 +326,7 @@ def json_dict_bytes_to_unicode(d, encoding='utf-8'): if isinstance(d, str): return unicode(d, encoding) elif isinstance(d, dict): - return dict(imap(json_dict_bytes_to_unicode, d.iteritems(), repeat(encoding))) + return dict(imap(json_dict_bytes_to_unicode, iteritems(d), repeat(encoding))) elif isinstance(d, list): return list(imap(json_dict_bytes_to_unicode, d, repeat(encoding))) elif isinstance(d, tuple): @@ -414,7 +423,7 @@ class AnsibleModule(object): self.aliases = {} if add_file_common_args: - for k, v in FILE_COMMON_ARGUMENTS.iteritems(): + for k, v in FILE_COMMON_ARGUMENTS.items(): if k not in self.argument_spec: self.argument_spec[k] = v @@ -950,7 +959,7 @@ class AnsibleModule(object): def _handle_aliases(self): aliases_results = {} #alias:canon - for (k,v) in self.argument_spec.iteritems(): + for (k,v) in self.argument_spec.items(): self._legal_inputs.append(k) aliases = v.get('aliases', None) default = v.get('default', None) @@ -971,7 +980,7 @@ class AnsibleModule(object): return aliases_results def _check_arguments(self, check_invalid_arguments): - for (k,v) in self.params.iteritems(): + for (k,v) in self.params.items(): if k == '_ansible_check_mode' and v: if not self.supports_check_mode: @@ -1023,7 +1032,7 @@ class AnsibleModule(object): def _check_required_arguments(self): ''' ensure all required arguments are present ''' missing = [] - for (k,v) in self.argument_spec.iteritems(): + for (k,v) in self.argument_spec.items(): required = v.get('required', False) if required and k not in self.params: missing.append(k) @@ -1046,7 +1055,7 @@ class AnsibleModule(object): def _check_argument_values(self): ''' ensure all arguments have the requested values, and there are no stray arguments ''' - for (k,v) in self.argument_spec.iteritems(): + for (k,v) in self.argument_spec.items(): choices = v.get('choices',None) if choices is None: continue @@ -1189,7 +1198,7 @@ class AnsibleModule(object): def _check_argument_types(self): ''' ensure all arguments have the requested type ''' - for (k, v) in self.argument_spec.iteritems(): + for (k, v) in self.argument_spec.items(): wanted = v.get('type', None) if wanted is None: continue @@ -1208,7 +1217,7 @@ class AnsibleModule(object): self.fail_json(msg="argument %s is of type %s and we were unable to convert to %s" % (k, type(value), wanted)) def _set_defaults(self, pre=True): - for (k,v) in self.argument_spec.iteritems(): + for (k,v) in self.argument_spec.items(): default = v.get('default', None) if pre == True: # this prevents setting defaults on required items From 213029a21e88887217607bb8dff127f37d879970 Mon Sep 17 00:00:00 2001 From: Marius Gedminas Date: Wed, 30 Sep 2015 09:10:18 +0300 Subject: [PATCH 4/7] Python 3: don't convert unicode to unicode --- lib/ansible/module_utils/basic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ansible/module_utils/basic.py b/lib/ansible/module_utils/basic.py index e5c53345049..9727cf14a03 100644 --- a/lib/ansible/module_utils/basic.py +++ b/lib/ansible/module_utils/basic.py @@ -323,7 +323,7 @@ def json_dict_bytes_to_unicode(d, encoding='utf-8'): and dict container types (the containers that the json module returns) ''' - if isinstance(d, str): + if isinstance(d, bytes): return unicode(d, encoding) elif isinstance(d, dict): return dict(imap(json_dict_bytes_to_unicode, iteritems(d), repeat(encoding))) From 500e7a1263269d59631ff5ff976bed996bbf628f Mon Sep 17 00:00:00 2001 From: Marius Gedminas Date: Wed, 30 Sep 2015 09:13:26 +0300 Subject: [PATCH 5/7] Fix test: _to_filesystem_str produces byte strings --- test/units/module_utils/test_basic.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/units/module_utils/test_basic.py b/test/units/module_utils/test_basic.py index faa14dbbb0a..60decd8dac7 100644 --- a/test/units/module_utils/test_basic.py +++ b/test/units/module_utils/test_basic.py @@ -536,8 +536,8 @@ class TestModuleUtilsBasic(unittest.TestCase): argument_spec = dict(), ) - self.assertEqual(am._to_filesystem_str(u'foo'), 'foo') - self.assertEqual(am._to_filesystem_str(u'föö'), 'f\xc3\xb6\xc3\xb6') + self.assertEqual(am._to_filesystem_str(u'foo'), b'foo') + self.assertEqual(am._to_filesystem_str(u'föö'), b'f\xc3\xb6\xc3\xb6') def test_module_utils_basic_ansible_module_user_and_group(self): from ansible.module_utils import basic From 0371b145679bf5fb18f8e9d8f8a4f2faed62ff52 Mon Sep 17 00:00:00 2001 From: Marius Gedminas Date: Wed, 30 Sep 2015 09:16:45 +0300 Subject: [PATCH 6/7] Fix test: selinux gets passed byte strings (I don't have a system with selinux to test if the module still wants byte strings even on Python 3.) --- test/units/module_utils/test_basic.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/units/module_utils/test_basic.py b/test/units/module_utils/test_basic.py index 60decd8dac7..78116847de5 100644 --- a/test/units/module_utils/test_basic.py +++ b/test/units/module_utils/test_basic.py @@ -602,7 +602,7 @@ class TestModuleUtilsBasic(unittest.TestCase): with patch.dict('sys.modules', {'selinux': basic.selinux}): with patch('selinux.lsetfilecon', return_value=0) as m: self.assertEqual(am.set_context_if_different('/path/to/file', ['foo_u', 'foo_r', 'foo_t', 's0'], False), True) - m.assert_called_with('/path/to/file', 'foo_u:foo_r:foo_t:s0') + m.assert_called_with(b'/path/to/file', 'foo_u:foo_r:foo_t:s0') m.reset_mock() am.check_mode = True self.assertEqual(am.set_context_if_different('/path/to/file', ['foo_u', 'foo_r', 'foo_t', 's0'], False), True) @@ -619,7 +619,7 @@ class TestModuleUtilsBasic(unittest.TestCase): with patch('selinux.lsetfilecon', return_value=0) as m: self.assertEqual(am.set_context_if_different('/path/to/file', ['foo_u', 'foo_r', 'foo_t', 's0'], False), True) - m.assert_called_with('/path/to/file', 'sp_u:sp_r:sp_t:s0') + m.assert_called_with(b'/path/to/file', 'sp_u:sp_r:sp_t:s0') delattr(basic, 'selinux') From cc1e1648f5ffeb9ae86e21b462b3aca4a369ffb9 Mon Sep 17 00:00:00 2001 From: Marius Gedminas Date: Wed, 30 Sep 2015 18:22:02 +0300 Subject: [PATCH 7/7] Tweak comment style --- lib/ansible/module_utils/basic.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/lib/ansible/module_utils/basic.py b/lib/ansible/module_utils/basic.py index 9727cf14a03..b1af1164b42 100644 --- a/lib/ansible/module_utils/basic.py +++ b/lib/ansible/module_utils/basic.py @@ -69,31 +69,41 @@ import errno from itertools import repeat try: - from itertools import imap # Python 2 + # Python 2 + from itertools import imap except ImportError: - imap = map # Python 3 + # Python 3 + imap = map try: + # Python 2 basestring except NameError: - basestring = str # Python 3 + # Python 3 + basestring = str try: + # Python 2 unicode except NameError: - unicode = str # Python 3 + # Python 3 + unicode = str try: + # Python 2.6+ bytes except NameError: - bytes = str # Python 2 + # Python 2.4 + bytes = str try: dict.iteritems except AttributeError: - def iteritems(d): # Python 3 + # Python 3 + def iteritems(d): return d.items() else: + # Python 2 def iteritems(d): # Python 2 return d.iteritems()