Add thin provisioning support to growvols

This change enhances the growvols script to support all volumes being
backed by one thin provisioning pool.

If a pool is detected, the following occurs:
- validation to confirm every volume is backed by the pool
- only the pool is extended into the new partition
- volumes are extended by the same amount as the non thin-provisioned
  case

This results in no volumes being over-provisioned, so
out-of-space behaviour will be the same as the non thin-provisioned
case.

This change also switches to using /dev/mapper device mapper paths for
volume block devices, since that is the only path the thin pool is
mapped to.

Change-Id: I96085fc889e72c942cfef7e3acb6f6cd73f606dd

diskimage_builder/elements/growvols/static/usr/local/sbin/growvols

@@ -105,7 +105,7 @@ growvols --device sda --group vg img-rootfs=20% fs_home=20% fs_var=60%
                         default=os.environ.get('GROWVOLS_ARGS', '').split(),
                         help='A label or mountpoint, and the proportion to '
                              'grow it by. Defaults to $GROWVOLS_ARGS. '
-                             'For example: /home=80% img-rootfs=20%')
+                             'For example: /home=80%% img-rootfs=20%%')
     parser.add_argument('--group', metavar='GROUP',
                         default=os.environ.get('GROWVOLS_GROUP'),
                         help='The name of the LVM group to extend. Defaults '
@@ -417,9 +417,7 @@ def find_grow_vols(opts, devices, group, total_size_bytes):
         if size_bytes == 0:
             continue
 
-        # remove the group- prefix from the name
+        volume_path = '/dev/mapper/%s' % device['NAME']
-        name = device['NAME'][len(group) + 1:]
-        volume_path = '/dev/%s/%s' % (group, name)
 
         grow_vols[volume_path] = size_bytes
 
@@ -437,6 +435,43 @@ def find_sector_size(disk_name):
         return size
 
 
+def find_thin_pool(devices, group):
+    LOG.info('Finding LVM thin pool')
+    lvs = execute(['lvs', '--noheadings', '--options',
+                   'lv_name,lv_dm_path,lv_attr,pool_lv'])
+    lvs_devices = [lvsd.strip().split() for lvsd in lvs.split('\n')
+                   if lvsd.strip()]
+    thin_pool_device = None
+    thin_pool_name = None
+
+    # find any thin pool
+    for d in lvs_devices:
+        lv_name, lv_dm_path, lv_attr = d[:3]
+        if lv_attr.startswith('t'):
+            # this device is a thin pool
+            thin_pool_device = lv_dm_path
+            thin_pool_name = lv_name
+            break
+
+    if not thin_pool_device:
+        return
+
+    # ensure every volume uses the pool
+    for d in lvs_devices:
+        lv_name, lv_dm_path, lv_attr = d[:3]
+        if not lv_attr.startswith('V'):
+            # Ignore device not a volume
+            continue
+        pool_lv = None
+        if len(d) == 4:
+            pool_lv = d[3]
+        if pool_lv != thin_pool_name:
+            raise Exception('All volumes need to be in pool %s. '
+                            '%s is in pool %s' %
+                            (thin_pool_name, lv_name, pool_lv))
+    return thin_pool_device
+
+
 def main(argv):
     opts = parse_opts(argv)
     configure_logger(opts.verbose, opts.debug)
@@ -466,6 +501,7 @@ def main(argv):
     devname = find_next_device_name(devices, disk_name, partnum)
     dev_path = '/dev/%s' % devname
     grow_vols = find_grow_vols(opts, devices, group, size_bytes)
+    thin_pool = find_thin_pool(devices, group)
 
     commands = []
 
@@ -491,16 +527,30 @@ def main(argv):
         dev_path
     ], 'Add physical volume %s to group %s' % (devname, group)))
 
+    if thin_pool:
+        # total size available, rounded down to whole extents
+        pool_size = size_bytes - size_bytes % PHYSICAL_EXTENT_BYTES
+        commands.append(Command([
+            'lvextend',
+            '-L+%sB' % pool_size,
+            thin_pool,
+            dev_path
+        ], 'Add %s to thin pool %s' % (convert_bytes(pool_size),
+                                                     thin_pool)))
+
     for volume_path, size_bytes in grow_vols.items():
         if size_bytes > 0:
-            commands.append(Command([
+            extend_args = [
                 'lvextend',
                 '--size',
                 '+%sB' % size_bytes,
-                volume_path,
+                volume_path
-                dev_path
+            ]
-            ], 'Add %s to logical volume %s' % (convert_bytes(size_bytes),
+            if not thin_pool:
-                                                volume_path)))
+                extend_args.append(dev_path)
+            commands.append(Command(
+                extend_args, 'Add %s to logical volume %s' % (
+                    convert_bytes(size_bytes), volume_path)))
 
     for volume_path, size_bytes in grow_vols.items():
         if size_bytes > 0:

diskimage_builder/elements/growvols/tests/test_growvols.py

@@ -114,6 +114,28 @@ SGDISK_LARGEST = "%s\n%s\n" % (SECTOR_START, SECTOR_END)
 # output of vgs --noheadings --options vg_name
 VGS = "  vg\n"
 
+# output of lvs --noheadings --options lv_name,lv_dm_path,lv_attr,pool_lv
+LVS = '''
+  lv_audit    /dev/mapper/vg-lv_audit    Vwi-aotz--
+  lv_home     /dev/mapper/vg-lv_home     Vwi-aotz--
+  lv_log      /dev/mapper/vg-lv_log      Vwi-aotz--
+  lv_root     /dev/mapper/vg-lv_root     Vwi-aotz--
+  lv_srv      /dev/mapper/vg-lv_srv      Vwi-aotz--
+  lv_tmp      /dev/mapper/vg-lv_tmp      Vwi-aotz--
+  lv_var      /dev/mapper/vg-lv_var      Vwi-aotz--
+'''
+
+LVS_THIN = '''
+  lv_audit    /dev/mapper/vg-lv_audit    Vwi-aotz-- lv_thinpool
+  lv_home     /dev/mapper/vg-lv_home     Vwi-aotz-- lv_thinpool
+  lv_log      /dev/mapper/vg-lv_log      Vwi-aotz-- lv_thinpool
+  lv_root     /dev/mapper/vg-lv_root     Vwi-aotz-- lv_thinpool
+  lv_srv      /dev/mapper/vg-lv_srv      Vwi-aotz-- lv_thinpool
+  lv_thinpool /dev/mapper/vg-lv_thinpool twi-aotz--
+  lv_tmp      /dev/mapper/vg-lv_tmp      Vwi-aotz-- lv_thinpool
+  lv_var      /dev/mapper/vg-lv_var      Vwi-aotz-- lv_thinpool
+'''
+
 
 class TestGrowvols(base.BaseTestCase):
 
@@ -360,30 +382,30 @@ class TestGrowvols(base.BaseTestCase):
         # buy default, assign all to /
         opts.grow_vols = ['']
         self.assertEqual(
-            {'/dev/lv/lv_root': fidy_g},
+            {'/dev/mapper/vg-lv_root': fidy_g},
-            growvols.find_grow_vols(opts, DEVICES, 'lv', fidy_g)
+            growvols.find_grow_vols(opts, DEVICES, 'vg', fidy_g)
         )
 
         # assign to /home, /var, remainder to /
         opts.grow_vols = ['/home=20%', 'fs_var=40%']
         self.assertEqual(
             {
-                '/dev/lv/lv_home': ten_g,
+                '/dev/mapper/vg-lv_home': ten_g,
-                '/dev/lv/lv_var': ten_g * 2,
+                '/dev/mapper/vg-lv_var': ten_g * 2,
-                '/dev/lv/lv_root': ten_g * 2
+                '/dev/mapper/vg-lv_root': ten_g * 2
             },
-            growvols.find_grow_vols(opts, DEVICES, 'lv', fidy_g)
+            growvols.find_grow_vols(opts, DEVICES, 'vg', fidy_g)
         )
 
         # assign to /home, /var, /tmp by amount
         opts.grow_vols = ['/home=19GiB', 'fs_var=30GiB', '/tmp=1GiB']
         self.assertEqual(
             {
-                '/dev/lv/lv_home': one_g * 19,
+                '/dev/mapper/vg-lv_home': one_g * 19,
-                '/dev/lv/lv_var': one_g * 30,
+                '/dev/mapper/vg-lv_var': one_g * 30,
-                '/dev/lv/lv_tmp': one_g
+                '/dev/mapper/vg-lv_tmp': one_g
             },
-            growvols.find_grow_vols(opts, DEVICES, 'lv', fidy_g)
+            growvols.find_grow_vols(opts, DEVICES, 'vg', fidy_g)
         )
 
     @mock.patch('builtins.open', autospec=True)
@@ -400,6 +422,41 @@ class TestGrowvols(base.BaseTestCase):
         mock_open.side_effect = FileNotFoundError
         self.assertRaises(FileNotFoundError, growvols.find_sector_size, 'sdx')
 
+    @mock.patch('growvols.execute')
+    def test_find_thin_pool(self, mock_execute):
+        # No thin pool
+        mock_execute.return_value = LVS
+        self.assertIsNone(growvols.find_thin_pool(DEVICES, 'vg'))
+        mock_execute.assert_called_once_with([
+            'lvs', '--noheadings', '--options',
+            'lv_name,lv_dm_path,lv_attr,pool_lv'])
+
+        # One thin pool, all volumes use it
+        mock_execute.return_value = LVS_THIN
+        self.assertEqual('/dev/mapper/vg-lv_thinpool',
+                         growvols.find_thin_pool(DEVICES, 'vg'))
+
+        # One pool, not used by all volumes
+        mock_execute.return_value = '''
+  lv_thinpool /dev/mapper/vg-lv_thinpool twi-aotz--
+  lv_home     /dev/mapper/vg-lv_home     Vwi-aotz--
+  lv_root     /dev/mapper/vg-lv_root     Vwi-aotz-- lv_thinpool'''
+        e = self.assertRaises(Exception, growvols.find_thin_pool,
+                              DEVICES, 'vg')
+        self.assertEqual('All volumes need to be in pool lv_thinpool. '
+                         'lv_home is in pool None', str(e))
+
+        # Two pools, volumes use both
+        mock_execute.return_value = '''
+  lv_thin1 /dev/mapper/vg-lv_thin1 twi-aotz--
+  lv_thin2 /dev/mapper/vg-lv_thin2 twi-aotz--
+  lv_home  /dev/mapper/vg-lv_home  Vwi-aotz-- lv_thin2
+  lv_root  /dev/mapper/vg-lv_root  Vwi-aotz-- lv_thin1'''
+        e = self.assertRaises(Exception, growvols.find_thin_pool,
+                              DEVICES, 'vg')
+        self.assertEqual('All volumes need to be in pool lv_thin1. '
+                         'lv_home is in pool lv_thin2', str(e))
+
     @mock.patch('growvols.find_sector_size')
     @mock.patch('growvols.execute')
     def test_main(self, mock_execute, mock_sector_size):
@@ -410,6 +467,7 @@ class TestGrowvols(base.BaseTestCase):
             LSBLK,
             SGDISK_LARGEST,
             VGS,
+            LVS,
         ]
         growvols.main(['growvols', '--noop'])
         mock_execute.assert_has_calls([
@@ -418,6 +476,8 @@ class TestGrowvols(base.BaseTestCase):
             mock.call(['sgdisk', '--first-aligned-in-largest',
                        '--end-of-largest', '/dev/sda']),
             mock.call(['vgs', '--noheadings', '--options', 'vg_name']),
+            mock.call(['lvs', '--noheadings', '--options',
+                       'lv_name,lv_dm_path,lv_attr,pool_lv'])
         ])
 
         # no arguments, assign all to /
@@ -426,6 +486,7 @@ class TestGrowvols(base.BaseTestCase):
             LSBLK,
             SGDISK_LARGEST,
             VGS,
+            LVS,
             '', '', '', '', '', ''
         ]
         growvols.main(['growvols', '--yes'])
@@ -435,14 +496,16 @@ class TestGrowvols(base.BaseTestCase):
             mock.call(['sgdisk', '--first-aligned-in-largest',
                        '--end-of-largest', '/dev/sda']),
             mock.call(['vgs', '--noheadings', '--options', 'vg_name']),
+            mock.call(['lvs', '--noheadings', '--options',
+                       'lv_name,lv_dm_path,lv_attr,pool_lv']),
             mock.call(['sgdisk', '--new=5:79267840:488265727',
                        '--change-name=5:growvols', '/dev/sda']),
             mock.call(['partprobe']),
             mock.call(['pvcreate', '/dev/sda5']),
             mock.call(['vgextend', 'vg', '/dev/sda5']),
             mock.call(['lvextend', '--size', '+209404821504B',
-                       '/dev/vg/lv_root', '/dev/sda5']),
+                       '/dev/mapper/vg-lv_root', '/dev/sda5']),
-            mock.call(['xfs_growfs', '/dev/vg/lv_root'])
+            mock.call(['xfs_growfs', '/dev/mapper/vg-lv_root'])
         ])
 
         # assign to /home, /var, remainder to /
@@ -451,6 +514,7 @@ class TestGrowvols(base.BaseTestCase):
             LSBLK,
             SGDISK_LARGEST,
             VGS,
+            LVS,
             '', '', '', '', '', '', '', '', '', ''
         ]
         growvols.main(['growvols', '--yes', '--group', 'vg',
@@ -461,20 +525,22 @@ class TestGrowvols(base.BaseTestCase):
             mock.call(['sgdisk', '--first-aligned-in-largest',
                        '--end-of-largest', '/dev/sda']),
             mock.call(['vgs', '--noheadings', '--options', 'vg_name']),
+            mock.call(['lvs', '--noheadings', '--options',
+                       'lv_name,lv_dm_path,lv_attr,pool_lv']),
             mock.call(['sgdisk', '--new=5:79267840:488265727',
                        '--change-name=5:growvols', '/dev/sda']),
             mock.call(['partprobe']),
             mock.call(['pvcreate', '/dev/sda5']),
             mock.call(['vgextend', 'vg', '/dev/sda5']),
             mock.call(['lvextend', '--size', '+41880125440B',
-                       '/dev/vg/lv_home', '/dev/sda5']),
+                       '/dev/mapper/vg-lv_home', '/dev/sda5']),
             mock.call(['lvextend', '--size', '+83760250880B',
-                       '/dev/vg/lv_var', '/dev/sda5']),
+                       '/dev/mapper/vg-lv_var', '/dev/sda5']),
             mock.call(['lvextend', '--size', '+83764445184B',
-                       '/dev/vg/lv_root', '/dev/sda5']),
+                       '/dev/mapper/vg-lv_root', '/dev/sda5']),
-            mock.call(['xfs_growfs', '/dev/vg/lv_home']),
+            mock.call(['xfs_growfs', '/dev/mapper/vg-lv_home']),
-            mock.call(['xfs_growfs', '/dev/vg/lv_var']),
+            mock.call(['xfs_growfs', '/dev/mapper/vg-lv_var']),
-            mock.call(['xfs_growfs', '/dev/vg/lv_root']),
+            mock.call(['xfs_growfs', '/dev/mapper/vg-lv_root']),
         ])
 
         # no space to grow, failed
@@ -485,6 +551,7 @@ class TestGrowvols(base.BaseTestCase):
             LSBLK,
             sgdisk_largest,
             VGS,
+            LVS,
         ]
         self.assertEqual(
             2,
@@ -496,8 +563,51 @@ class TestGrowvols(base.BaseTestCase):
             LSBLK,
             sgdisk_largest,
             VGS,
+            LVS,
         ]
         self.assertEqual(
             0,
             growvols.main(['growvols'])
         )
+
+    @mock.patch('growvols.find_sector_size')
+    @mock.patch('growvols.execute')
+    def test_main_thin_provision(self, mock_execute, mock_sector_size):
+        mock_sector_size.return_value = 512
+
+        # assign to /home, /var, remainder to /
+        mock_execute.reset_mock()
+        mock_execute.side_effect = [
+            LSBLK,
+            SGDISK_LARGEST,
+            VGS,
+            LVS_THIN,
+            '', '', '', '', '', '', '', '', '', '', ''
+        ]
+        growvols.main(['growvols', '--yes', '--group', 'vg',
+                       '/home=20%', 'fs_var=40%'])
+        mock_execute.assert_has_calls([
+            mock.call(['lsblk', '-Po',
+                       'kname,pkname,name,label,type,fstype,mountpoint']),
+            mock.call(['sgdisk', '--first-aligned-in-largest',
+                       '--end-of-largest', '/dev/sda']),
+            mock.call(['vgs', '--noheadings', '--options', 'vg_name']),
+            mock.call(['lvs', '--noheadings', '--options',
+                       'lv_name,lv_dm_path,lv_attr,pool_lv']),
+            mock.call(['sgdisk', '--new=5:79267840:488265727',
+                       '--change-name=5:growvols', '/dev/sda']),
+            mock.call(['partprobe']),
+            mock.call(['pvcreate', '/dev/sda5']),
+            mock.call(['vgextend', 'vg', '/dev/sda5']),
+            mock.call(['lvextend', '-L+209404821504B',
+                       '/dev/mapper/vg-lv_thinpool', '/dev/sda5']),
+            mock.call(['lvextend', '--size', '+41880125440B',
+                       '/dev/mapper/vg-lv_home']),
+            mock.call(['lvextend', '--size', '+83760250880B',
+                       '/dev/mapper/vg-lv_var']),
+            mock.call(['lvextend', '--size', '+83764445184B',
+                       '/dev/mapper/vg-lv_root']),
+            mock.call(['xfs_growfs', '/dev/mapper/vg-lv_home']),
+            mock.call(['xfs_growfs', '/dev/mapper/vg-lv_var']),
+            mock.call(['xfs_growfs', '/dev/mapper/vg-lv_root']),
+        ])