From 93f593f568ec399489ca6c24bceebc4dbb5f6179 Mon Sep 17 00:00:00 2001 From: Phil Sphicas Date: Wed, 27 May 2020 22:30:40 +0000 Subject: [PATCH] Support binary prefixes for node storage size This change allows node storage sizes to be specified using binary prefixes (MiB, GiB, TiB) in addition to the existing supported formats (MB, GB, TB). Change-Id: Idef88b648a75bad87625acf1d73af011480cc0b9 --- doc/source/topology.rst | 3 + .../drivers/node/maasdriver/actions/node.py | 28 ++- .../unit/test_maasdriver_calculate_bytes.py | 191 +++++++++++++++--- 3 files changed, 184 insertions(+), 38 deletions(-) diff --git a/doc/source/topology.rst b/doc/source/topology.rst index dccc53e5..8e61874c 100644 --- a/doc/source/topology.rst +++ b/doc/source/topology.rst @@ -599,6 +599,9 @@ parts: * m|M|mb|MB: Megabytes or 10^6 * the numeric * g|G|gb|GB: Gigabytes or 10^9 * the numeric * t|T|tb|TB: Terabytes or 10^12 * the numeric + * mi|Mi|mib|MiB: Mebibytes or 2^20 * the numeric + * gi|Gi|gib|GiB: Gibibytes or 2^30 * the numeric + * ti|Ti|tib|TiB: Tibibytes or 2^40 * the numeric * %: The percentage of total device or volume group space Volume Groups and Logical Volumes diff --git a/python/drydock_provisioner/drivers/node/maasdriver/actions/node.py b/python/drydock_provisioner/drivers/node/maasdriver/actions/node.py index 6c52c840..67020b9d 100644 --- a/python/drydock_provisioner/drivers/node/maasdriver/actions/node.py +++ b/python/drydock_provisioner/drivers/node/maasdriver/actions/node.py @@ -1986,7 +1986,8 @@ class ApplyNodeStorage(BaseMaasAction): storage_layout['layout_type'] = 'flat' storage_layout['root_device'] = n.get_logicalname( root_dev.name) - storage_layout['root_size'] = root_block.size + storage_layout['root_size'] = ApplyNodeStorage.calculate_bytes( + root_block.size) elif isinstance(root_block, hostprofile.HostVolume): storage_layout['layout_type'] = 'lvm' if len(root_dev.physical_devices) != 1: @@ -1999,12 +2000,14 @@ class ApplyNodeStorage(BaseMaasAction): continue storage_layout['root_device'] = n.get_logicalname( root_dev.physical_devices[0]) - storage_layout['root_lv_size'] = root_block.size + storage_layout['root_lv_size'] = ApplyNodeStorage.calculate_bytes( + root_block.size) storage_layout['root_lv_name'] = root_block.name storage_layout['root_vg_name'] = root_dev.name if boot_block is not None: - storage_layout['boot_size'] = boot_block.size + storage_layout['boot_size'] = ApplyNodeStorage.calculate_bytes( + boot_block.size) msg = "Setting node %s root storage layout: %s" % ( n.name, str(storage_layout)) @@ -2190,9 +2193,12 @@ class ApplyNodeStorage(BaseMaasAction): Calculate the size as specified in size_str in the context of the provided blockdev or vg. Valid size_str format below. - #m or #M or #mb or #MB = # * 1024 * 1024 - #g or #G or #gb or #GB = # * 1024 * 1024 * 1024 - #t or #T or #tb or #TB = # * 1024 * 1024 * 1024 * 1024 + #m or #M or #mb or #MB = # * 1000 * 1000 + #g or #G or #gb or #GB = # * 1000 * 1000 * 1000 + #t or #T or #tb or #TB = # * 1000 * 1000 * 1000 * 1000 + #mi or #Mi or #mib or #MiB = # * 1024 * 1024 + #gi or #Gi or #gib or #GiB = # * 1024 * 1024 * 1024 + #ti or #Ti or #tib or #TiB = # * 1024 * 1024 * 1024 * 1024 #% = Percentage of the total storage in the context Prepend '>' to the above to note the size as a minimum and the calculated size being the @@ -2207,7 +2213,7 @@ class ApplyNodeStorage(BaseMaasAction): size_str is interpreted in the context of this device :return size: The calculated size in bytes """ - pattern = r'(>?)(\d+)([mMbBgGtT%]{1,2})' + pattern = r'(>?)(\d+)([mMbBgGtTi%]{1,3})' regex = re.compile(pattern) match = regex.match(size_str) @@ -2228,10 +2234,16 @@ class ApplyNodeStorage(BaseMaasAction): computed_size = base_size * (1000 * 1000 * 1000) elif match.group(3) in ['t', 'T', 'tb', 'TB']: computed_size = base_size * (1000 * 1000 * 1000 * 1000) + elif match.group(3) in ['mi', 'Mi', 'mib', 'MiB']: + computed_size = base_size * (1024 * 1024) + elif match.group(3) in ['gi', 'Gi', 'gib', 'GiB']: + computed_size = base_size * (1024 * 1024 * 1024) + elif match.group(3) in ['ti', 'Ti', 'tib', 'TiB']: + computed_size = base_size * (1024 * 1024 * 1024 * 1024) elif match.group(3) == '%': computed_size = math.floor((base_size / 100) * int(context.size)) - if computed_size > int(context.available_size): + if context and computed_size > int(context.available_size): raise errors.NotEnoughStorage() if match.group(1) == '>': diff --git a/python/tests/unit/test_maasdriver_calculate_bytes.py b/python/tests/unit/test_maasdriver_calculate_bytes.py index 7ffd5cdf..3f720d0d 100644 --- a/python/tests/unit/test_maasdriver_calculate_bytes.py +++ b/python/tests/unit/test_maasdriver_calculate_bytes.py @@ -27,139 +27,270 @@ class TestCalculateBytes(): def test_calculate_m_label(self): '''Convert megabyte labels to x * 10^6 bytes.''' size_str = '15m' - drive_size = 20 * 1000 * 1000 - + drive_size = 20 * 10**6 drive = BlockDevice(None, size=drive_size, available_size=drive_size) calc_size = ApplyNodeStorage.calculate_bytes( size_str=size_str, context=drive) - assert calc_size == 15 * 1000 * 1000 + assert calc_size == 15 * 10**6 def test_calculate_mb_label(self): '''Convert megabyte labels to x * 10^6 bytes.''' size_str = '15mb' - drive_size = 20 * 1000 * 1000 + drive_size = 20 * 10**6 drive = BlockDevice(None, size=drive_size, available_size=drive_size) calc_size = ApplyNodeStorage.calculate_bytes( size_str=size_str, context=drive) - assert calc_size == 15 * 1000 * 1000 + assert calc_size == 15 * 10**6 def test_calculate_M_label(self): '''Convert megabyte labels to x * 10^6 bytes.''' size_str = '15M' - drive_size = 20 * 1000 * 1000 + drive_size = 20 * 10**6 drive = BlockDevice(None, size=drive_size, available_size=drive_size) calc_size = ApplyNodeStorage.calculate_bytes( size_str=size_str, context=drive) - assert calc_size == 15 * 1000 * 1000 + assert calc_size == 15 * 10**6 def test_calculate_MB_label(self): '''Convert megabyte labels to x * 10^6 bytes.''' size_str = '15MB' - drive_size = 20 * 1000 * 1000 + drive_size = 20 * 10**6 drive = BlockDevice(None, size=drive_size, available_size=drive_size) calc_size = ApplyNodeStorage.calculate_bytes( size_str=size_str, context=drive) - assert calc_size == 15 * 1000 * 1000 + assert calc_size == 15 * 10**6 def test_calculate_g_label(self): '''Convert gigabyte labels to x * 10^9 bytes.''' size_str = '15g' - drive_size = 20 * 1000 * 1000 * 1000 + drive_size = 20 * 10**9 drive = BlockDevice(None, size=drive_size, available_size=drive_size) calc_size = ApplyNodeStorage.calculate_bytes( size_str=size_str, context=drive) - assert calc_size == 15 * 1000 * 1000 * 1000 + assert calc_size == 15 * 10**9 def test_calculate_gb_label(self): '''Convert gigabyte labels to x * 10^9 bytes.''' size_str = '15gb' - drive_size = 20 * 1000 * 1000 * 1000 + drive_size = 20 * 10**9 drive = BlockDevice(None, size=drive_size, available_size=drive_size) calc_size = ApplyNodeStorage.calculate_bytes( size_str=size_str, context=drive) - assert calc_size == 15 * 1000 * 1000 * 1000 + assert calc_size == 15 * 10**9 def test_calculate_G_label(self): '''Convert gigabyte labels to x * 10^9 bytes.''' size_str = '15G' - drive_size = 20 * 1000 * 1000 * 1000 + drive_size = 20 * 10**9 drive = BlockDevice(None, size=drive_size, available_size=drive_size) calc_size = ApplyNodeStorage.calculate_bytes( size_str=size_str, context=drive) - assert calc_size == 15 * 1000 * 1000 * 1000 + assert calc_size == 15 * 10**9 def test_calculate_GB_label(self): '''Convert gigabyte labels to x * 10^9 bytes.''' size_str = '15GB' - drive_size = 20 * 1000 * 1000 * 1000 + drive_size = 20 * 10**9 drive = BlockDevice(None, size=drive_size, available_size=drive_size) calc_size = ApplyNodeStorage.calculate_bytes( size_str=size_str, context=drive) - assert calc_size == 15 * 1000 * 1000 * 1000 + assert calc_size == 15 * 10**9 def test_calculate_t_label(self): '''Convert terabyte labels to x * 10^12 bytes.''' size_str = '15t' - drive_size = 20 * 1000 * 1000 * 1000 * 1000 + drive_size = 20 * 10**12 drive = BlockDevice(None, size=drive_size, available_size=drive_size) calc_size = ApplyNodeStorage.calculate_bytes( size_str=size_str, context=drive) - assert calc_size == 15 * 1000 * 1000 * 1000 * 1000 + assert calc_size == 15 * 10**12 def test_calculate_tb_label(self): '''Convert terabyte labels to x * 10^12 bytes.''' size_str = '15tb' - drive_size = 20 * 1000 * 1000 * 1000 * 1000 + drive_size = 20 * 10**12 drive = BlockDevice(None, size=drive_size, available_size=drive_size) calc_size = ApplyNodeStorage.calculate_bytes( size_str=size_str, context=drive) - assert calc_size == 15 * 1000 * 1000 * 1000 * 1000 + assert calc_size == 15 * 10**12 def test_calculate_T_label(self): '''Convert terabyte labels to x * 10^12 bytes.''' size_str = '15T' - drive_size = 20 * 1000 * 1000 * 1000 * 1000 + drive_size = 20 * 10**12 drive = BlockDevice(None, size=drive_size, available_size=drive_size) calc_size = ApplyNodeStorage.calculate_bytes( size_str=size_str, context=drive) - assert calc_size == 15 * 1000 * 1000 * 1000 * 1000 + assert calc_size == 15 * 10**12 def test_calculate_TB_label(self): '''Convert terabyte labels to x * 10^12 bytes.''' size_str = '15TB' - drive_size = 20 * 1000 * 1000 * 1000 * 1000 + drive_size = 20 * 10**12 drive = BlockDevice(None, size=drive_size, available_size=drive_size) calc_size = ApplyNodeStorage.calculate_bytes( size_str=size_str, context=drive) - assert calc_size == 15 * 1000 * 1000 * 1000 * 1000 + assert calc_size == 15 * 10**12 + + def test_calculate_mi_label(self): + '''Convert mebibyte labels to x * 2^20 bytes.''' + size_str = '15mi' + drive_size = 20 * 2**20 + drive = BlockDevice(None, size=drive_size, available_size=drive_size) + + calc_size = ApplyNodeStorage.calculate_bytes( + size_str=size_str, context=drive) + + assert calc_size == 15 * 2**20 + + def test_calculate_mib_label(self): + '''Convert mebibyte labels to x * 2^20 bytes.''' + size_str = '15mib' + drive_size = 20 * 2**20 + drive = BlockDevice(None, size=drive_size, available_size=drive_size) + + calc_size = ApplyNodeStorage.calculate_bytes( + size_str=size_str, context=drive) + + assert calc_size == 15 * 2**20 + + def test_calculate_Mi_label(self): + '''Convert mebibyte labels to x * 2^20 bytes.''' + size_str = '15Mi' + drive_size = 20 * 2**20 + drive = BlockDevice(None, size=drive_size, available_size=drive_size) + + calc_size = ApplyNodeStorage.calculate_bytes( + size_str=size_str, context=drive) + + assert calc_size == 15 * 2**20 + + def test_calculate_MiB_label(self): + '''Convert mebibyte labels to x * 2^20 bytes.''' + size_str = '15MiB' + drive_size = 20 * 2**20 + drive = BlockDevice(None, size=drive_size, available_size=drive_size) + + calc_size = ApplyNodeStorage.calculate_bytes( + size_str=size_str, context=drive) + + assert calc_size == 15 * 2**20 + + def test_calculate_gi_label(self): + '''Convert gibibyte labels to x * 2^30 bytes.''' + size_str = '15gi' + drive_size = 20 * 2**30 + drive = BlockDevice(None, size=drive_size, available_size=drive_size) + + calc_size = ApplyNodeStorage.calculate_bytes( + size_str=size_str, context=drive) + + assert calc_size == 15 * 2**30 + + def test_calculate_gib_label(self): + '''Convert gibibyte labels to x * 2^30 bytes.''' + size_str = '15gib' + drive_size = 20 * 2**30 + drive = BlockDevice(None, size=drive_size, available_size=drive_size) + + calc_size = ApplyNodeStorage.calculate_bytes( + size_str=size_str, context=drive) + + assert calc_size == 15 * 2**30 + + def test_calculate_Gi_label(self): + '''Convert gibibyte labels to x * 2^30 bytes.''' + size_str = '15Gi' + drive_size = 20 * 2**30 + drive = BlockDevice(None, size=drive_size, available_size=drive_size) + + calc_size = ApplyNodeStorage.calculate_bytes( + size_str=size_str, context=drive) + + assert calc_size == 15 * 2**30 + + def test_calculate_GiB_label(self): + '''Convert gibibyte labels to x * 2^30 bytes.''' + size_str = '15GiB' + drive_size = 20 * 2**30 + drive = BlockDevice(None, size=drive_size, available_size=drive_size) + + calc_size = ApplyNodeStorage.calculate_bytes( + size_str=size_str, context=drive) + + assert calc_size == 15 * 2**30 + + def test_calculate_ti_label(self): + '''Convert tebibyte labels to x * 2^40 bytes.''' + size_str = '15ti' + drive_size = 20 * 2**40 + drive = BlockDevice(None, size=drive_size, available_size=drive_size) + + calc_size = ApplyNodeStorage.calculate_bytes( + size_str=size_str, context=drive) + + assert calc_size == 15 * 2**40 + + def test_calculate_tib_label(self): + '''Convert tebibyte labels to x * 2^40 bytes.''' + size_str = '15tib' + drive_size = 20 * 2**40 + drive = BlockDevice(None, size=drive_size, available_size=drive_size) + + calc_size = ApplyNodeStorage.calculate_bytes( + size_str=size_str, context=drive) + + assert calc_size == 15 * 2**40 + + def test_calculate_Ti_label(self): + '''Convert tebibyte labels to x * 2^40 bytes.''' + size_str = '15Ti' + drive_size = 20 * 2**40 + drive = BlockDevice(None, size=drive_size, available_size=drive_size) + + calc_size = ApplyNodeStorage.calculate_bytes( + size_str=size_str, context=drive) + + assert calc_size == 15 * 2**40 + + def test_calculate_TiB_label(self): + '''Convert tebibyte labels to x * 2^40 bytes.''' + size_str = '15TiB' + drive_size = 20 * 2**40 + drive = BlockDevice(None, size=drive_size, available_size=drive_size) + + calc_size = ApplyNodeStorage.calculate_bytes( + size_str=size_str, context=drive) + + assert calc_size == 15 * 2**40 def test_calculate_percent_blockdev(self): '''Convert a percent of total blockdev space to explicit byte count.''' - drive_size = 20 * 1000 * 1000 # 20 mb drive + drive_size = 20 * 10**6 # 20 mb drive part_size = math.floor(.2 * drive_size) # calculate 20% of drive size size_str = '20%' @@ -172,7 +303,7 @@ class TestCalculateBytes(): def test_calculate_percent_vg(self): '''Convert a percent of total blockdev space to explicit byte count.''' - vg_size = 20 * 1000 * 1000 # 20 mb drive + vg_size = 20 * 10**6 # 20 mb drive lv_size = math.floor(.2 * vg_size) # calculate 20% of drive size size_str = '20%' @@ -185,7 +316,7 @@ class TestCalculateBytes(): def test_calculate_overprovision(self): '''When calculated space is higher than available space, raise an exception.''' - vg_size = 20 * 1000 * 1000 # 20 mb drive + vg_size = 20 * 10**6 # 20 mb drive vg_available = 10 # 10 bytes available size_str = '80%' @@ -196,8 +327,8 @@ class TestCalculateBytes(): def test_calculate_min_label(self): '''Adding the min marker '>' should provision all available space.''' - vg_size = 20 * 1000 * 1000 # 20 mb drive - vg_available = 15 * 1000 * 1000 + vg_size = 20 * 10**6 # 20 mb drive + vg_available = 15 * 10**6 size_str = '>10%' vg = VolumeGroup(None, size=vg_size, available_size=vg_available)