From 27b0f3241b2b8397eb626f1770ba34439e121347 Mon Sep 17 00:00:00 2001 From: Yannig Date: Wed, 24 Aug 2016 21:04:20 +0200 Subject: [PATCH] new filter human_bytes: convert a string (ex: 1Mo, 1K) into bytes (#12074) * Rework human_readable and human_to_bytes. New filter human_to_bytes. * Fix for python 3. --- lib/ansible/module_utils/basic.py | 116 +++++++++++------- lib/ansible/plugins/filter/mathstuff.py | 36 ++---- .../roles/test_filters/tasks/main.yml | 27 ++++ 3 files changed, 112 insertions(+), 67 deletions(-) diff --git a/lib/ansible/module_utils/basic.py b/lib/ansible/module_utils/basic.py index b0896a104d0..cb969b9ceed 100644 --- a/lib/ansible/module_utils/basic.py +++ b/lib/ansible/module_utils/basic.py @@ -31,6 +31,8 @@ BOOLEANS_TRUE = ['y', 'yes', 'on', '1', 'true', 1, True] BOOLEANS_FALSE = ['n', 'no', 'off', '0', 'false', 0, False] BOOLEANS = BOOLEANS_TRUE + BOOLEANS_FALSE +SIZE_RANGES = { 'Y': 1<<80, 'Z': 1<<70, 'E': 1<<60, 'P': 1<<50, 'T': 1<<40, 'G': 1<<30, 'M': 1<<20, 'K': 1<<10, 'B': 1 } + # ansible modules can be written in any language. To simplify # development of Python modules, the functions available here can # be used to do many common tasks @@ -466,6 +468,72 @@ def heuristic_log_sanitize(data, no_log_values=None): output = remove_values(output, no_log_values) return output +def bytes_to_human(size, isbits=False, unit=None): + + base = 'Bytes' + if isbits: + base = 'bits' + suffix = '' + + for suffix, limit in sorted(iteritems(SIZE_RANGES), key=lambda item: -item[1]): + if (unit is None and size >= limit) or unit is not None and unit.upper() == suffix[0]: + break + + if limit != 1: + suffix += base[0] + else: + suffix = base + + return '%.2f %s' % (float(size)/ limit, suffix) + +def human_to_bytes(number, default_unit=None, isbits=False): + + ''' + Convert number in string format into bytes (ex: '2K' => 2048) or using unit argument + ex: + human_to_bytes('10M') <=> human_to_bytes(10, 'M') + ''' + m = re.search('^\s*(\d*\.?\d*)\s*([A-Za-z]+)?', str(number), flags=re.IGNORECASE) + if m is None: + raise ValueError("human_to_bytes() can't interpret following string: %s" % str(number)) + try: + num = float(m.group(1)) + except: + raise ValueError("human_to_bytes() can't interpret following number: %s (original input string: %s)" % (m.group(1), number)) + + unit = m.group(2) + if unit is None: + unit = default_unit + + if unit is None: + ''' No unit given, returning raw number ''' + return int(round(num)) + range_key = unit[0].upper() + try: + limit = SIZE_RANGES[range_key] + except: + raise ValueError("human_to_bytes() failed to convert %s (unit = %s). The suffix must be one of %s" % (number, unit, ", ".join(SIZE_RANGES.keys()))) + + # default value + unit_class = 'B' + unit_class_name = 'byte' + # handling bits case + if isbits: + unit_class = 'b' + unit_class_name = 'bit' + # check unit value if more than one character (KB, MB) + if len(unit) > 1: + expect_message = 'expect %s%s or %s' % (range_key, unit_class, range_key) + if range_key == 'B': + expect_message = 'expect %s or %s' % (unit_class, unit_class_name) + + if unit_class_name in unit.lower(): + pass + elif unit[1] != unit_class: + raise ValueError("human_to_bytes() failed to convert %s. Value is not a valid string (%s)" % (number, expect_message)) + + return int(round(num * limit)) + def is_executable(path): '''is the given path executable? @@ -1468,7 +1536,7 @@ class AnsibleModule(object): def _check_type_bits(self, value): try: - self.human_to_bytes(value, bits=True) + self.human_to_bytes(value, isbits=True) except ValueError: raise TypeError('%s cannot be converted to a Bit value' % type(value)) @@ -2181,53 +2249,13 @@ class AnsibleModule(object): fh.close() def bytes_to_human(self, size): - - ranges = ( - (1 << 70, 'ZB'), - (1 << 60, 'EB'), - (1 << 50, 'PB'), - (1 << 40, 'TB'), - (1 << 30, 'GB'), - (1 << 20, 'MB'), - (1 << 10, 'KB'), - (1, 'Bytes') - ) - for limit, suffix in ranges: - if size >= limit: - break - return '%.2f %s' % (float(size)/ limit, suffix) + return bytes_to_human(size) # for backwards compatibility pretty_bytes = bytes_to_human - def human_to_bytes(number, bits=False): - - result = None - suffixes = ['B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB'] - full = 'Bytes' - - if bits: - suffixes = [ x.replace('B', 'b') for x in suffixes ] - full = 'Bits' - - if number is None: - result = 0 - elif isinstance(number, int): - result = number - elif number.isdigit(): - result = int(number) - elif full in number: - result = int(number.replace(full,'')) - else: - for i, suffix in enumerate(suffixes): - if suffix in number: - result = int(number.replace(suffix ,'')) * (1024 ** i) - break - - if result is None: - raise ValueError("Failed to convert %s. The suffix must be one of %s or %s" % (number, full, ', '.join(suffixes))) - - return result + def human_to_bytes(self, number, isbits=False): + return human_to_bytes(number, isbits) # # Backwards compat diff --git a/lib/ansible/plugins/filter/mathstuff.py b/lib/ansible/plugins/filter/mathstuff.py index f18e3baf0e0..0800f317e1d 100644 --- a/lib/ansible/plugins/filter/mathstuff.py +++ b/lib/ansible/plugins/filter/mathstuff.py @@ -23,6 +23,7 @@ __metaclass__ = type import math import collections from ansible import errors +from ansible.module_utils import basic def unique(a): if isinstance(a,collections.Hashable): @@ -99,30 +100,18 @@ def inversepower(x, base=2): def human_readable(size, isbits=False, unit=None): + ''' Return a human readable string ''' + try: + return basic.bytes_to_human(size, isbits, unit) + except: + raise errors.AnsibleFilterError("human_readable() can't interpret following string: %s" % size) - base = 'bits' if isbits else 'Bytes' - suffix = '' - - ranges = ( - (1<<70, 'Z'), - (1<<60, 'E'), - (1<<50, 'P'), - (1<<40, 'T'), - (1<<30, 'G'), - (1<<20, 'M'), - (1<<10, 'K'), - (1, base) - ) - - for limit, suffix in ranges: - if (unit is None and size >= limit) or \ - unit is not None and unit.upper() == suffix: - break - - if limit != 1: - suffix += base[0] - - return '%.2f %s' % (float(size)/ limit, suffix) +def human_to_bytes(size, default_unit=None, isbits=False): + ''' Return bytes count from a human readable string ''' + try: + return basic.human_to_bytes(size, default_unit, isbits) + except: + raise errors.AnsibleFilterError("human_to_bytes() can't interpret following string: %s" % size) class FilterModule(object): ''' Ansible math jinja2 filters ''' @@ -147,5 +136,6 @@ class FilterModule(object): # computer theory 'human_readable' : human_readable, + 'human_to_bytes' : human_to_bytes, } diff --git a/test/integration/roles/test_filters/tasks/main.yml b/test/integration/roles/test_filters/tasks/main.yml index 3e500ae6042..cfb51ce2790 100644 --- a/test/integration/roles/test_filters/tasks/main.yml +++ b/test/integration/roles/test_filters/tasks/main.yml @@ -61,13 +61,40 @@ - 'diff_result.stdout == ""' - name: Verify human_readable + tags: "human_readable" assert: that: + - '"1.00 Bytes" == 1|human_readable' + - '"1.00 bits" == 1|human_readable(isbits=True)' - '"10.00 KB" == 10240|human_readable' - '"97.66 MB" == 102400000|human_readable' - '"0.10 GB" == 102400000|human_readable(unit="G")' - '"0.10 Gb" == 102400000|human_readable(isbits=True, unit="G")' +- name: Verify human_to_bytes + tags: "human_to_bytes" + assert: + that: + - "{{'0'|human_to_bytes}} == 0" + - "{{'0.1'|human_to_bytes}} == 0" + - "{{'0.9'|human_to_bytes}} == 1" + - "{{'1'|human_to_bytes}} == 1" + - "{{'10.00 KB'|human_to_bytes}} == 10240" + - "{{ '11 MB'|human_to_bytes}} == 11534336" + - "{{ '1.1 GB'|human_to_bytes}} == 1181116006" + - "{{'10.00 Kb'|human_to_bytes(isbits=True)}} == 10240" + +- name: Verify human_to_bytes (bad string) + tags: "human_to_bytes" + set_fact: bad_string="{{'10.00 foo'|human_to_bytes}}" + ignore_errors: yes + register: _ + +- name: Verify human_to_bytes (bad string) + tags: "human_to_bytes" + assert: + that: "{{_.failed}}" + - name: Container lookups with extract assert: that: