sig_cloud/diskimage-builder
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add thin provisioning support to growvols

Browse Source 
This change enhances the growvols script to support all volumes being
backed by one thin provisioning pool.

If a pool is detected, the following occurs:
- validation to confirm every volume is backed by the pool
- only the pool is extended into the new partition
- volumes are extended by the same amount as the non thin-provisioned
  case

This results in no volumes being over-provisioned, so
out-of-space behaviour will be the same as the non thin-provisioned
case.

This change also switches to using /dev/mapper device mapper paths for
volume block devices, since that is the only path the thin pool is
mapped to.

Change-Id: I96085fc889e72c942cfef7e3acb6f6cd73f606dd
This commit is contained in:
Steve Baker 2022-07-05 16:05:37 +12:00
parent 15430098a0
commit f61548d863
2 changed files with 187 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

68
diskimage_builder/elements/growvols/static/usr/local/sbin/growvols
View File

@ -105,7 +105,7 @@ growvols --device sda --group vg img-rootfs=20% fs_home=20% fs_var=60%
default=os.environ.get('GROWVOLS_ARGS', '').split(),
help='A label or mountpoint, and the proportion to '
'grow it by. Defaults to $GROWVOLS_ARGS. '
'For example: /home=80% img-rootfs=20%')
'For example: /home=80%% img-rootfs=20%%')
parser.add_argument('--group', metavar='GROUP',
default=os.environ.get('GROWVOLS_GROUP'),
help='The name of the LVM group to extend. Defaults '
@ -417,9 +417,7 @@ def find_grow_vols(opts, devices, group, total_size_bytes):
if size_bytes == 0:
continue
# remove the group- prefix from the name
name = device['NAME'][len(group) + 1:]
volume_path = '/dev/%s/%s' % (group, name)
volume_path = '/dev/mapper/%s' % device['NAME']
grow_vols[volume_path] = size_bytes
@ -437,6 +435,43 @@ def find_sector_size(disk_name):
return size
def find_thin_pool(devices, group):
LOG.info('Finding LVM thin pool')
lvs = execute(['lvs', '--noheadings', '--options',
'lv_name,lv_dm_path,lv_attr,pool_lv'])
lvs_devices = [lvsd.strip().split() for lvsd in lvs.split('\n')
if lvsd.strip()]
thin_pool_device = None
thin_pool_name = None
# find any thin pool
for d in lvs_devices:
lv_name, lv_dm_path, lv_attr = d[:3]
if lv_attr.startswith('t'):
# this device is a thin pool
thin_pool_device = lv_dm_path
thin_pool_name = lv_name
break
if not thin_pool_device:
return
# ensure every volume uses the pool
for d in lvs_devices:
lv_name, lv_dm_path, lv_attr = d[:3]
if not lv_attr.startswith('V'):
# Ignore device not a volume
continue
pool_lv = None
if len(d) == 4:
pool_lv = d[3]
if pool_lv != thin_pool_name:
raise Exception('All volumes need to be in pool %s. '
'%s is in pool %s' %
(thin_pool_name, lv_name, pool_lv))
return thin_pool_device
def main(argv):
opts = parse_opts(argv)
configure_logger(opts.verbose, opts.debug)
@ -466,6 +501,7 @@ def main(argv):
devname = find_next_device_name(devices, disk_name, partnum)
dev_path = '/dev/%s' % devname
grow_vols = find_grow_vols(opts, devices, group, size_bytes)
thin_pool = find_thin_pool(devices, group)
commands = []
@ -491,16 +527,30 @@ def main(argv):
dev_path
], 'Add physical volume %s to group %s' % (devname, group)))
if thin_pool:
# total size available, rounded down to whole extents
pool_size = size_bytes - size_bytes % PHYSICAL_EXTENT_BYTES
commands.append(Command([
'lvextend',
'-L+%sB' % pool_size,
thin_pool,
dev_path
], 'Add %s to thin pool %s' % (convert_bytes(pool_size),
thin_pool)))
for volume_path, size_bytes in grow_vols.items():
if size_bytes > 0:
commands.append(Command([
extend_args = [
'lvextend',
'--size',
'+%sB' % size_bytes,
volume_path,
dev_path
], 'Add %s to logical volume %s' % (convert_bytes(size_bytes),
volume_path)))
volume_path
]
if not thin_pool:
extend_args.append(dev_path)
commands.append(Command(
extend_args, 'Add %s to logical volume %s' % (
convert_bytes(size_bytes), volume_path)))
for volume_path, size_bytes in grow_vols.items():
if size_bytes > 0:

146
diskimage_builder/elements/growvols/tests/test_growvols.py
View File

@ -114,6 +114,28 @@ SGDISK_LARGEST = "%s\n%s\n" % (SECTOR_START, SECTOR_END)
# output of vgs --noheadings --options vg_name
VGS = " vg\n"
# output of lvs --noheadings --options lv_name,lv_dm_path,lv_attr,pool_lv
LVS = '''
lv_audit /dev/mapper/vg-lv_audit Vwi-aotz--
lv_home /dev/mapper/vg-lv_home Vwi-aotz--
lv_log /dev/mapper/vg-lv_log Vwi-aotz--
lv_root /dev/mapper/vg-lv_root Vwi-aotz--
lv_srv /dev/mapper/vg-lv_srv Vwi-aotz--
lv_tmp /dev/mapper/vg-lv_tmp Vwi-aotz--
lv_var /dev/mapper/vg-lv_var Vwi-aotz--
'''
LVS_THIN = '''
lv_audit /dev/mapper/vg-lv_audit Vwi-aotz-- lv_thinpool
lv_home /dev/mapper/vg-lv_home Vwi-aotz-- lv_thinpool
lv_log /dev/mapper/vg-lv_log Vwi-aotz-- lv_thinpool
lv_root /dev/mapper/vg-lv_root Vwi-aotz-- lv_thinpool
lv_srv /dev/mapper/vg-lv_srv Vwi-aotz-- lv_thinpool
lv_thinpool /dev/mapper/vg-lv_thinpool twi-aotz--
lv_tmp /dev/mapper/vg-lv_tmp Vwi-aotz-- lv_thinpool
lv_var /dev/mapper/vg-lv_var Vwi-aotz-- lv_thinpool
'''
class TestGrowvols(base.BaseTestCase):
@ -360,30 +382,30 @@ class TestGrowvols(base.BaseTestCase):
# buy default, assign all to /
opts.grow_vols = ['']
self.assertEqual(
{'/dev/lv/lv_root': fidy_g},
growvols.find_grow_vols(opts, DEVICES, 'lv', fidy_g)
{'/dev/mapper/vg-lv_root': fidy_g},
growvols.find_grow_vols(opts, DEVICES, 'vg', fidy_g)
)
# assign to /home, /var, remainder to /
opts.grow_vols = ['/home=20%', 'fs_var=40%']
self.assertEqual(
{
'/dev/lv/lv_home': ten_g,
'/dev/lv/lv_var': ten_g * 2,
'/dev/lv/lv_root': ten_g * 2
'/dev/mapper/vg-lv_home': ten_g,
'/dev/mapper/vg-lv_var': ten_g * 2,
'/dev/mapper/vg-lv_root': ten_g * 2
},
growvols.find_grow_vols(opts, DEVICES, 'lv', fidy_g)
growvols.find_grow_vols(opts, DEVICES, 'vg', fidy_g)
)
# assign to /home, /var, /tmp by amount
opts.grow_vols = ['/home=19GiB', 'fs_var=30GiB', '/tmp=1GiB']
self.assertEqual(
{
'/dev/lv/lv_home': one_g * 19,
'/dev/lv/lv_var': one_g * 30,
'/dev/lv/lv_tmp': one_g
'/dev/mapper/vg-lv_home': one_g * 19,
'/dev/mapper/vg-lv_var': one_g * 30,
'/dev/mapper/vg-lv_tmp': one_g
},
growvols.find_grow_vols(opts, DEVICES, 'lv', fidy_g)
growvols.find_grow_vols(opts, DEVICES, 'vg', fidy_g)
)
@mock.patch('builtins.open', autospec=True)
@ -400,6 +422,41 @@ class TestGrowvols(base.BaseTestCase):
mock_open.side_effect = FileNotFoundError
self.assertRaises(FileNotFoundError, growvols.find_sector_size, 'sdx')
@mock.patch('growvols.execute')
def test_find_thin_pool(self, mock_execute):
# No thin pool
mock_execute.return_value = LVS
self.assertIsNone(growvols.find_thin_pool(DEVICES, 'vg'))
mock_execute.assert_called_once_with([
'lvs', '--noheadings', '--options',
'lv_name,lv_dm_path,lv_attr,pool_lv'])
# One thin pool, all volumes use it
mock_execute.return_value = LVS_THIN
self.assertEqual('/dev/mapper/vg-lv_thinpool',
growvols.find_thin_pool(DEVICES, 'vg'))
# One pool, not used by all volumes
mock_execute.return_value = '''
lv_thinpool /dev/mapper/vg-lv_thinpool twi-aotz--
lv_home /dev/mapper/vg-lv_home Vwi-aotz--
lv_root /dev/mapper/vg-lv_root Vwi-aotz-- lv_thinpool'''
e = self.assertRaises(Exception, growvols.find_thin_pool,
DEVICES, 'vg')
self.assertEqual('All volumes need to be in pool lv_thinpool. '
'lv_home is in pool None', str(e))
# Two pools, volumes use both
mock_execute.return_value = '''
lv_thin1 /dev/mapper/vg-lv_thin1 twi-aotz--
lv_thin2 /dev/mapper/vg-lv_thin2 twi-aotz--
lv_home /dev/mapper/vg-lv_home Vwi-aotz-- lv_thin2
lv_root /dev/mapper/vg-lv_root Vwi-aotz-- lv_thin1'''
e = self.assertRaises(Exception, growvols.find_thin_pool,
DEVICES, 'vg')
self.assertEqual('All volumes need to be in pool lv_thin1. '
'lv_home is in pool lv_thin2', str(e))
@mock.patch('growvols.find_sector_size')
@mock.patch('growvols.execute')
def test_main(self, mock_execute, mock_sector_size):
@ -410,6 +467,7 @@ class TestGrowvols(base.BaseTestCase):
LSBLK,
SGDISK_LARGEST,
VGS,
LVS,
]
growvols.main(['growvols', '--noop'])
mock_execute.assert_has_calls([
@ -418,6 +476,8 @@ class TestGrowvols(base.BaseTestCase):
mock.call(['sgdisk', '--first-aligned-in-largest',
'--end-of-largest', '/dev/sda']),
mock.call(['vgs', '--noheadings', '--options', 'vg_name']),
mock.call(['lvs', '--noheadings', '--options',
'lv_name,lv_dm_path,lv_attr,pool_lv'])
])
# no arguments, assign all to /
@ -426,6 +486,7 @@ class TestGrowvols(base.BaseTestCase):
LSBLK,
SGDISK_LARGEST,
VGS,
LVS,
'', '', '', '', '', ''
]
growvols.main(['growvols', '--yes'])
@ -435,14 +496,16 @@ class TestGrowvols(base.BaseTestCase):
mock.call(['sgdisk', '--first-aligned-in-largest',
'--end-of-largest', '/dev/sda']),
mock.call(['vgs', '--noheadings', '--options', 'vg_name']),
mock.call(['lvs', '--noheadings', '--options',
'lv_name,lv_dm_path,lv_attr,pool_lv']),
mock.call(['sgdisk', '--new=5:79267840:488265727',
'--change-name=5:growvols', '/dev/sda']),
mock.call(['partprobe']),
mock.call(['pvcreate', '/dev/sda5']),
mock.call(['vgextend', 'vg', '/dev/sda5']),
mock.call(['lvextend', '--size', '+209404821504B',
'/dev/vg/lv_root', '/dev/sda5']),
mock.call(['xfs_growfs', '/dev/vg/lv_root'])
'/dev/mapper/vg-lv_root', '/dev/sda5']),
mock.call(['xfs_growfs', '/dev/mapper/vg-lv_root'])
])
# assign to /home, /var, remainder to /
@ -451,6 +514,7 @@ class TestGrowvols(base.BaseTestCase):
LSBLK,
SGDISK_LARGEST,
VGS,
LVS,
'', '', '', '', '', '', '', '', '', ''
]
growvols.main(['growvols', '--yes', '--group', 'vg',
@ -461,20 +525,22 @@ class TestGrowvols(base.BaseTestCase):
mock.call(['sgdisk', '--first-aligned-in-largest',
'--end-of-largest', '/dev/sda']),
mock.call(['vgs', '--noheadings', '--options', 'vg_name']),
mock.call(['lvs', '--noheadings', '--options',
'lv_name,lv_dm_path,lv_attr,pool_lv']),
mock.call(['sgdisk', '--new=5:79267840:488265727',
'--change-name=5:growvols', '/dev/sda']),
mock.call(['partprobe']),
mock.call(['pvcreate', '/dev/sda5']),
mock.call(['vgextend', 'vg', '/dev/sda5']),
mock.call(['lvextend', '--size', '+41880125440B',
'/dev/vg/lv_home', '/dev/sda5']),
'/dev/mapper/vg-lv_home', '/dev/sda5']),
mock.call(['lvextend', '--size', '+83760250880B',
'/dev/vg/lv_var', '/dev/sda5']),
'/dev/mapper/vg-lv_var', '/dev/sda5']),
mock.call(['lvextend', '--size', '+83764445184B',
'/dev/vg/lv_root', '/dev/sda5']),
mock.call(['xfs_growfs', '/dev/vg/lv_home']),
mock.call(['xfs_growfs', '/dev/vg/lv_var']),
mock.call(['xfs_growfs', '/dev/vg/lv_root']),
'/dev/mapper/vg-lv_root', '/dev/sda5']),
mock.call(['xfs_growfs', '/dev/mapper/vg-lv_home']),
mock.call(['xfs_growfs', '/dev/mapper/vg-lv_var']),
mock.call(['xfs_growfs', '/dev/mapper/vg-lv_root']),
])
# no space to grow, failed
@ -485,6 +551,7 @@ class TestGrowvols(base.BaseTestCase):
LSBLK,
sgdisk_largest,
VGS,
LVS,
]
self.assertEqual(
2,
@ -496,8 +563,51 @@ class TestGrowvols(base.BaseTestCase):
LSBLK,
sgdisk_largest,
VGS,
LVS,
]
self.assertEqual(
0,
growvols.main(['growvols'])
)
@mock.patch('growvols.find_sector_size')
@mock.patch('growvols.execute')
def test_main_thin_provision(self, mock_execute, mock_sector_size):
mock_sector_size.return_value = 512
# assign to /home, /var, remainder to /
mock_execute.reset_mock()
mock_execute.side_effect = [
LSBLK,
SGDISK_LARGEST,
VGS,
LVS_THIN,
'', '', '', '', '', '', '', '', '', '', ''
]
growvols.main(['growvols', '--yes', '--group', 'vg',
'/home=20%', 'fs_var=40%'])
mock_execute.assert_has_calls([
mock.call(['lsblk', '-Po',
'kname,pkname,name,label,type,fstype,mountpoint']),
mock.call(['sgdisk', '--first-aligned-in-largest',
'--end-of-largest', '/dev/sda']),
mock.call(['vgs', '--noheadings', '--options', 'vg_name']),
mock.call(['lvs', '--noheadings', '--options',
'lv_name,lv_dm_path,lv_attr,pool_lv']),
mock.call(['sgdisk', '--new=5:79267840:488265727',
'--change-name=5:growvols', '/dev/sda']),
mock.call(['partprobe']),
mock.call(['pvcreate', '/dev/sda5']),
mock.call(['vgextend', 'vg', '/dev/sda5']),
mock.call(['lvextend', '-L+209404821504B',
'/dev/mapper/vg-lv_thinpool', '/dev/sda5']),
mock.call(['lvextend', '--size', '+41880125440B',
'/dev/mapper/vg-lv_home']),
mock.call(['lvextend', '--size', '+83760250880B',
'/dev/mapper/vg-lv_var']),
mock.call(['lvextend', '--size', '+83764445184B',
'/dev/mapper/vg-lv_root']),
mock.call(['xfs_growfs', '/dev/mapper/vg-lv_home']),
mock.call(['xfs_growfs', '/dev/mapper/vg-lv_var']),
mock.call(['xfs_growfs', '/dev/mapper/vg-lv_root']),
])