Add thin provisioning support to growvols

This change enhances the growvols script to support all volumes being
backed by one thin provisioning pool.

If a pool is detected, the following occurs:
- validation to confirm every volume is backed by the pool
- only the pool is extended into the new partition
- volumes are extended by the same amount as the non thin-provisioned
  case

This results in no volumes being over-provisioned, so
out-of-space behaviour will be the same as the non thin-provisioned
case.

This change also switches to using /dev/mapper device mapper paths for
volume block devices, since that is the only path the thin pool is
mapped to.

Change-Id: I96085fc889e72c942cfef7e3acb6f6cd73f606dd
This commit is contained in:
Steve Baker 2022-07-05 16:05:37 +12:00
parent 15430098a0
commit f61548d863
2 changed files with 187 additions and 27 deletions

View File

@ -105,7 +105,7 @@ growvols --device sda --group vg img-rootfs=20% fs_home=20% fs_var=60%
default=os.environ.get('GROWVOLS_ARGS', '').split(), default=os.environ.get('GROWVOLS_ARGS', '').split(),
help='A label or mountpoint, and the proportion to ' help='A label or mountpoint, and the proportion to '
'grow it by. Defaults to $GROWVOLS_ARGS. ' 'grow it by. Defaults to $GROWVOLS_ARGS. '
'For example: /home=80% img-rootfs=20%') 'For example: /home=80%% img-rootfs=20%%')
parser.add_argument('--group', metavar='GROUP', parser.add_argument('--group', metavar='GROUP',
default=os.environ.get('GROWVOLS_GROUP'), default=os.environ.get('GROWVOLS_GROUP'),
help='The name of the LVM group to extend. Defaults ' help='The name of the LVM group to extend. Defaults '
@ -417,9 +417,7 @@ def find_grow_vols(opts, devices, group, total_size_bytes):
if size_bytes == 0: if size_bytes == 0:
continue continue
# remove the group- prefix from the name volume_path = '/dev/mapper/%s' % device['NAME']
name = device['NAME'][len(group) + 1:]
volume_path = '/dev/%s/%s' % (group, name)
grow_vols[volume_path] = size_bytes grow_vols[volume_path] = size_bytes
@ -437,6 +435,43 @@ def find_sector_size(disk_name):
return size return size
def find_thin_pool(devices, group):
LOG.info('Finding LVM thin pool')
lvs = execute(['lvs', '--noheadings', '--options',
'lv_name,lv_dm_path,lv_attr,pool_lv'])
lvs_devices = [lvsd.strip().split() for lvsd in lvs.split('\n')
if lvsd.strip()]
thin_pool_device = None
thin_pool_name = None
# find any thin pool
for d in lvs_devices:
lv_name, lv_dm_path, lv_attr = d[:3]
if lv_attr.startswith('t'):
# this device is a thin pool
thin_pool_device = lv_dm_path
thin_pool_name = lv_name
break
if not thin_pool_device:
return
# ensure every volume uses the pool
for d in lvs_devices:
lv_name, lv_dm_path, lv_attr = d[:3]
if not lv_attr.startswith('V'):
# Ignore device not a volume
continue
pool_lv = None
if len(d) == 4:
pool_lv = d[3]
if pool_lv != thin_pool_name:
raise Exception('All volumes need to be in pool %s. '
'%s is in pool %s' %
(thin_pool_name, lv_name, pool_lv))
return thin_pool_device
def main(argv): def main(argv):
opts = parse_opts(argv) opts = parse_opts(argv)
configure_logger(opts.verbose, opts.debug) configure_logger(opts.verbose, opts.debug)
@ -466,6 +501,7 @@ def main(argv):
devname = find_next_device_name(devices, disk_name, partnum) devname = find_next_device_name(devices, disk_name, partnum)
dev_path = '/dev/%s' % devname dev_path = '/dev/%s' % devname
grow_vols = find_grow_vols(opts, devices, group, size_bytes) grow_vols = find_grow_vols(opts, devices, group, size_bytes)
thin_pool = find_thin_pool(devices, group)
commands = [] commands = []
@ -491,16 +527,30 @@ def main(argv):
dev_path dev_path
], 'Add physical volume %s to group %s' % (devname, group))) ], 'Add physical volume %s to group %s' % (devname, group)))
if thin_pool:
# total size available, rounded down to whole extents
pool_size = size_bytes - size_bytes % PHYSICAL_EXTENT_BYTES
commands.append(Command([
'lvextend',
'-L+%sB' % pool_size,
thin_pool,
dev_path
], 'Add %s to thin pool %s' % (convert_bytes(pool_size),
thin_pool)))
for volume_path, size_bytes in grow_vols.items(): for volume_path, size_bytes in grow_vols.items():
if size_bytes > 0: if size_bytes > 0:
commands.append(Command([ extend_args = [
'lvextend', 'lvextend',
'--size', '--size',
'+%sB' % size_bytes, '+%sB' % size_bytes,
volume_path, volume_path
dev_path ]
], 'Add %s to logical volume %s' % (convert_bytes(size_bytes), if not thin_pool:
volume_path))) extend_args.append(dev_path)
commands.append(Command(
extend_args, 'Add %s to logical volume %s' % (
convert_bytes(size_bytes), volume_path)))
for volume_path, size_bytes in grow_vols.items(): for volume_path, size_bytes in grow_vols.items():
if size_bytes > 0: if size_bytes > 0:

View File

@ -114,6 +114,28 @@ SGDISK_LARGEST = "%s\n%s\n" % (SECTOR_START, SECTOR_END)
# output of vgs --noheadings --options vg_name # output of vgs --noheadings --options vg_name
VGS = " vg\n" VGS = " vg\n"
# output of lvs --noheadings --options lv_name,lv_dm_path,lv_attr,pool_lv
LVS = '''
lv_audit /dev/mapper/vg-lv_audit Vwi-aotz--
lv_home /dev/mapper/vg-lv_home Vwi-aotz--
lv_log /dev/mapper/vg-lv_log Vwi-aotz--
lv_root /dev/mapper/vg-lv_root Vwi-aotz--
lv_srv /dev/mapper/vg-lv_srv Vwi-aotz--
lv_tmp /dev/mapper/vg-lv_tmp Vwi-aotz--
lv_var /dev/mapper/vg-lv_var Vwi-aotz--
'''
LVS_THIN = '''
lv_audit /dev/mapper/vg-lv_audit Vwi-aotz-- lv_thinpool
lv_home /dev/mapper/vg-lv_home Vwi-aotz-- lv_thinpool
lv_log /dev/mapper/vg-lv_log Vwi-aotz-- lv_thinpool
lv_root /dev/mapper/vg-lv_root Vwi-aotz-- lv_thinpool
lv_srv /dev/mapper/vg-lv_srv Vwi-aotz-- lv_thinpool
lv_thinpool /dev/mapper/vg-lv_thinpool twi-aotz--
lv_tmp /dev/mapper/vg-lv_tmp Vwi-aotz-- lv_thinpool
lv_var /dev/mapper/vg-lv_var Vwi-aotz-- lv_thinpool
'''
class TestGrowvols(base.BaseTestCase): class TestGrowvols(base.BaseTestCase):
@ -360,30 +382,30 @@ class TestGrowvols(base.BaseTestCase):
# buy default, assign all to / # buy default, assign all to /
opts.grow_vols = [''] opts.grow_vols = ['']
self.assertEqual( self.assertEqual(
{'/dev/lv/lv_root': fidy_g}, {'/dev/mapper/vg-lv_root': fidy_g},
growvols.find_grow_vols(opts, DEVICES, 'lv', fidy_g) growvols.find_grow_vols(opts, DEVICES, 'vg', fidy_g)
) )
# assign to /home, /var, remainder to / # assign to /home, /var, remainder to /
opts.grow_vols = ['/home=20%', 'fs_var=40%'] opts.grow_vols = ['/home=20%', 'fs_var=40%']
self.assertEqual( self.assertEqual(
{ {
'/dev/lv/lv_home': ten_g, '/dev/mapper/vg-lv_home': ten_g,
'/dev/lv/lv_var': ten_g * 2, '/dev/mapper/vg-lv_var': ten_g * 2,
'/dev/lv/lv_root': ten_g * 2 '/dev/mapper/vg-lv_root': ten_g * 2
}, },
growvols.find_grow_vols(opts, DEVICES, 'lv', fidy_g) growvols.find_grow_vols(opts, DEVICES, 'vg', fidy_g)
) )
# assign to /home, /var, /tmp by amount # assign to /home, /var, /tmp by amount
opts.grow_vols = ['/home=19GiB', 'fs_var=30GiB', '/tmp=1GiB'] opts.grow_vols = ['/home=19GiB', 'fs_var=30GiB', '/tmp=1GiB']
self.assertEqual( self.assertEqual(
{ {
'/dev/lv/lv_home': one_g * 19, '/dev/mapper/vg-lv_home': one_g * 19,
'/dev/lv/lv_var': one_g * 30, '/dev/mapper/vg-lv_var': one_g * 30,
'/dev/lv/lv_tmp': one_g '/dev/mapper/vg-lv_tmp': one_g
}, },
growvols.find_grow_vols(opts, DEVICES, 'lv', fidy_g) growvols.find_grow_vols(opts, DEVICES, 'vg', fidy_g)
) )
@mock.patch('builtins.open', autospec=True) @mock.patch('builtins.open', autospec=True)
@ -400,6 +422,41 @@ class TestGrowvols(base.BaseTestCase):
mock_open.side_effect = FileNotFoundError mock_open.side_effect = FileNotFoundError
self.assertRaises(FileNotFoundError, growvols.find_sector_size, 'sdx') self.assertRaises(FileNotFoundError, growvols.find_sector_size, 'sdx')
@mock.patch('growvols.execute')
def test_find_thin_pool(self, mock_execute):
# No thin pool
mock_execute.return_value = LVS
self.assertIsNone(growvols.find_thin_pool(DEVICES, 'vg'))
mock_execute.assert_called_once_with([
'lvs', '--noheadings', '--options',
'lv_name,lv_dm_path,lv_attr,pool_lv'])
# One thin pool, all volumes use it
mock_execute.return_value = LVS_THIN
self.assertEqual('/dev/mapper/vg-lv_thinpool',
growvols.find_thin_pool(DEVICES, 'vg'))
# One pool, not used by all volumes
mock_execute.return_value = '''
lv_thinpool /dev/mapper/vg-lv_thinpool twi-aotz--
lv_home /dev/mapper/vg-lv_home Vwi-aotz--
lv_root /dev/mapper/vg-lv_root Vwi-aotz-- lv_thinpool'''
e = self.assertRaises(Exception, growvols.find_thin_pool,
DEVICES, 'vg')
self.assertEqual('All volumes need to be in pool lv_thinpool. '
'lv_home is in pool None', str(e))
# Two pools, volumes use both
mock_execute.return_value = '''
lv_thin1 /dev/mapper/vg-lv_thin1 twi-aotz--
lv_thin2 /dev/mapper/vg-lv_thin2 twi-aotz--
lv_home /dev/mapper/vg-lv_home Vwi-aotz-- lv_thin2
lv_root /dev/mapper/vg-lv_root Vwi-aotz-- lv_thin1'''
e = self.assertRaises(Exception, growvols.find_thin_pool,
DEVICES, 'vg')
self.assertEqual('All volumes need to be in pool lv_thin1. '
'lv_home is in pool lv_thin2', str(e))
@mock.patch('growvols.find_sector_size') @mock.patch('growvols.find_sector_size')
@mock.patch('growvols.execute') @mock.patch('growvols.execute')
def test_main(self, mock_execute, mock_sector_size): def test_main(self, mock_execute, mock_sector_size):
@ -410,6 +467,7 @@ class TestGrowvols(base.BaseTestCase):
LSBLK, LSBLK,
SGDISK_LARGEST, SGDISK_LARGEST,
VGS, VGS,
LVS,
] ]
growvols.main(['growvols', '--noop']) growvols.main(['growvols', '--noop'])
mock_execute.assert_has_calls([ mock_execute.assert_has_calls([
@ -418,6 +476,8 @@ class TestGrowvols(base.BaseTestCase):
mock.call(['sgdisk', '--first-aligned-in-largest', mock.call(['sgdisk', '--first-aligned-in-largest',
'--end-of-largest', '/dev/sda']), '--end-of-largest', '/dev/sda']),
mock.call(['vgs', '--noheadings', '--options', 'vg_name']), mock.call(['vgs', '--noheadings', '--options', 'vg_name']),
mock.call(['lvs', '--noheadings', '--options',
'lv_name,lv_dm_path,lv_attr,pool_lv'])
]) ])
# no arguments, assign all to / # no arguments, assign all to /
@ -426,6 +486,7 @@ class TestGrowvols(base.BaseTestCase):
LSBLK, LSBLK,
SGDISK_LARGEST, SGDISK_LARGEST,
VGS, VGS,
LVS,
'', '', '', '', '', '' '', '', '', '', '', ''
] ]
growvols.main(['growvols', '--yes']) growvols.main(['growvols', '--yes'])
@ -435,14 +496,16 @@ class TestGrowvols(base.BaseTestCase):
mock.call(['sgdisk', '--first-aligned-in-largest', mock.call(['sgdisk', '--first-aligned-in-largest',
'--end-of-largest', '/dev/sda']), '--end-of-largest', '/dev/sda']),
mock.call(['vgs', '--noheadings', '--options', 'vg_name']), mock.call(['vgs', '--noheadings', '--options', 'vg_name']),
mock.call(['lvs', '--noheadings', '--options',
'lv_name,lv_dm_path,lv_attr,pool_lv']),
mock.call(['sgdisk', '--new=5:79267840:488265727', mock.call(['sgdisk', '--new=5:79267840:488265727',
'--change-name=5:growvols', '/dev/sda']), '--change-name=5:growvols', '/dev/sda']),
mock.call(['partprobe']), mock.call(['partprobe']),
mock.call(['pvcreate', '/dev/sda5']), mock.call(['pvcreate', '/dev/sda5']),
mock.call(['vgextend', 'vg', '/dev/sda5']), mock.call(['vgextend', 'vg', '/dev/sda5']),
mock.call(['lvextend', '--size', '+209404821504B', mock.call(['lvextend', '--size', '+209404821504B',
'/dev/vg/lv_root', '/dev/sda5']), '/dev/mapper/vg-lv_root', '/dev/sda5']),
mock.call(['xfs_growfs', '/dev/vg/lv_root']) mock.call(['xfs_growfs', '/dev/mapper/vg-lv_root'])
]) ])
# assign to /home, /var, remainder to / # assign to /home, /var, remainder to /
@ -451,6 +514,7 @@ class TestGrowvols(base.BaseTestCase):
LSBLK, LSBLK,
SGDISK_LARGEST, SGDISK_LARGEST,
VGS, VGS,
LVS,
'', '', '', '', '', '', '', '', '', '' '', '', '', '', '', '', '', '', '', ''
] ]
growvols.main(['growvols', '--yes', '--group', 'vg', growvols.main(['growvols', '--yes', '--group', 'vg',
@ -461,20 +525,22 @@ class TestGrowvols(base.BaseTestCase):
mock.call(['sgdisk', '--first-aligned-in-largest', mock.call(['sgdisk', '--first-aligned-in-largest',
'--end-of-largest', '/dev/sda']), '--end-of-largest', '/dev/sda']),
mock.call(['vgs', '--noheadings', '--options', 'vg_name']), mock.call(['vgs', '--noheadings', '--options', 'vg_name']),
mock.call(['lvs', '--noheadings', '--options',
'lv_name,lv_dm_path,lv_attr,pool_lv']),
mock.call(['sgdisk', '--new=5:79267840:488265727', mock.call(['sgdisk', '--new=5:79267840:488265727',
'--change-name=5:growvols', '/dev/sda']), '--change-name=5:growvols', '/dev/sda']),
mock.call(['partprobe']), mock.call(['partprobe']),
mock.call(['pvcreate', '/dev/sda5']), mock.call(['pvcreate', '/dev/sda5']),
mock.call(['vgextend', 'vg', '/dev/sda5']), mock.call(['vgextend', 'vg', '/dev/sda5']),
mock.call(['lvextend', '--size', '+41880125440B', mock.call(['lvextend', '--size', '+41880125440B',
'/dev/vg/lv_home', '/dev/sda5']), '/dev/mapper/vg-lv_home', '/dev/sda5']),
mock.call(['lvextend', '--size', '+83760250880B', mock.call(['lvextend', '--size', '+83760250880B',
'/dev/vg/lv_var', '/dev/sda5']), '/dev/mapper/vg-lv_var', '/dev/sda5']),
mock.call(['lvextend', '--size', '+83764445184B', mock.call(['lvextend', '--size', '+83764445184B',
'/dev/vg/lv_root', '/dev/sda5']), '/dev/mapper/vg-lv_root', '/dev/sda5']),
mock.call(['xfs_growfs', '/dev/vg/lv_home']), mock.call(['xfs_growfs', '/dev/mapper/vg-lv_home']),
mock.call(['xfs_growfs', '/dev/vg/lv_var']), mock.call(['xfs_growfs', '/dev/mapper/vg-lv_var']),
mock.call(['xfs_growfs', '/dev/vg/lv_root']), mock.call(['xfs_growfs', '/dev/mapper/vg-lv_root']),
]) ])
# no space to grow, failed # no space to grow, failed
@ -485,6 +551,7 @@ class TestGrowvols(base.BaseTestCase):
LSBLK, LSBLK,
sgdisk_largest, sgdisk_largest,
VGS, VGS,
LVS,
] ]
self.assertEqual( self.assertEqual(
2, 2,
@ -496,8 +563,51 @@ class TestGrowvols(base.BaseTestCase):
LSBLK, LSBLK,
sgdisk_largest, sgdisk_largest,
VGS, VGS,
LVS,
] ]
self.assertEqual( self.assertEqual(
0, 0,
growvols.main(['growvols']) growvols.main(['growvols'])
) )
@mock.patch('growvols.find_sector_size')
@mock.patch('growvols.execute')
def test_main_thin_provision(self, mock_execute, mock_sector_size):
mock_sector_size.return_value = 512
# assign to /home, /var, remainder to /
mock_execute.reset_mock()
mock_execute.side_effect = [
LSBLK,
SGDISK_LARGEST,
VGS,
LVS_THIN,
'', '', '', '', '', '', '', '', '', '', ''
]
growvols.main(['growvols', '--yes', '--group', 'vg',
'/home=20%', 'fs_var=40%'])
mock_execute.assert_has_calls([
mock.call(['lsblk', '-Po',
'kname,pkname,name,label,type,fstype,mountpoint']),
mock.call(['sgdisk', '--first-aligned-in-largest',
'--end-of-largest', '/dev/sda']),
mock.call(['vgs', '--noheadings', '--options', 'vg_name']),
mock.call(['lvs', '--noheadings', '--options',
'lv_name,lv_dm_path,lv_attr,pool_lv']),
mock.call(['sgdisk', '--new=5:79267840:488265727',
'--change-name=5:growvols', '/dev/sda']),
mock.call(['partprobe']),
mock.call(['pvcreate', '/dev/sda5']),
mock.call(['vgextend', 'vg', '/dev/sda5']),
mock.call(['lvextend', '-L+209404821504B',
'/dev/mapper/vg-lv_thinpool', '/dev/sda5']),
mock.call(['lvextend', '--size', '+41880125440B',
'/dev/mapper/vg-lv_home']),
mock.call(['lvextend', '--size', '+83760250880B',
'/dev/mapper/vg-lv_var']),
mock.call(['lvextend', '--size', '+83764445184B',
'/dev/mapper/vg-lv_root']),
mock.call(['xfs_growfs', '/dev/mapper/vg-lv_home']),
mock.call(['xfs_growfs', '/dev/mapper/vg-lv_var']),
mock.call(['xfs_growfs', '/dev/mapper/vg-lv_root']),
])