diff --git a/diskimage_builder/elements/growvols/README.rst b/diskimage_builder/elements/growvols/README.rst new file mode 100644 index 00000000..da44eca6 --- /dev/null +++ b/diskimage_builder/elements/growvols/README.rst @@ -0,0 +1,60 @@ +======== +growvols +======== + +Grow one or more LVM volumes on first boot. + +This installs utility `growvols` which grows the logical volumes in an LVM group +to take available device space. + +The positional arguments specify how available space is allocated. They +have the format = where: + + is the label or the mountpoint of the logical volume + is an integer growth amount in the specified unit + is one of the supported units + +Supported units are: + +% percentage of available device space before any changes are made +MiB mebibyte (1048576 bytes) +GiB gibibyte (1073741824 bytes) +MB megabyte (1000000 bytes) +GB gigabyte (1000000000 bytes) + +Each argument is processed in order and the requested amount is allocated +to each volume until the disk is full. This means that if space is +overallocated, the last volumes may only grow by the remaining space, or +not grow at all, and a warning will be printed. When space is underallocated +the remaining space will be given to the root volume (mounted at /). + +The currently supported partition layout is: +- Exactly one of the partitions containing an LVM group +- The disk having unpartitioned space to grow with +- The LVM logical volumes being formatted with XFS filesystems + +Example usage: + +growvols /var=80% /home=20GB + +growvols --device sda --group vg img-rootfs=20% fs_home=20GiB fs_var=60% + +Environment variables can be set during image build to enable a systemd unit +which will run growvols on boot: + +# DIB_GROWVOLS_TRIGGER defaults to 'manual'. When set to 'systemd' a systemd +# unit will run using the following arguments +export DIB_GROWVOLS_TRIGGER=systemd + +# DIB_GROWVOLS_ARGS contains the positional arguments for which volumes to grow +# by what amount. If omitted the volume mounted at / will grow by all available +# space +export DIB_GROWVOLS_ARGS="img-rootfs=20% fs_home=20GiB fs_var=60%" + +# DIB_GROWVOLS_GROUP contains the name of the LVM group to extend. Defaults the +# discovered group if only one exists. +export DIB_GROWVOLS_GROUP=vg + +# DIB_GROWVOLS_DEVICE is the name of the disk block device to grow the +# volumes in (such as "sda"). Defaults to the disk containing the root mount. +export DIB_GROWVOLS_DEVICE=sda diff --git a/diskimage_builder/elements/growvols/__init__.py b/diskimage_builder/elements/growvols/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/diskimage_builder/elements/growvols/element-deps b/diskimage_builder/elements/growvols/element-deps new file mode 100644 index 00000000..69a71fde --- /dev/null +++ b/diskimage_builder/elements/growvols/element-deps @@ -0,0 +1,2 @@ +dib-init-system +install-static diff --git a/diskimage_builder/elements/growvols/environment.d/15-growvols b/diskimage_builder/elements/growvols/environment.d/15-growvols new file mode 100644 index 00000000..b015cb0f --- /dev/null +++ b/diskimage_builder/elements/growvols/environment.d/15-growvols @@ -0,0 +1,18 @@ +# Copyright 2021 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +export DIB_GROWVOLS_TRIGGER=${DIB_GROWVOLS_TRIGGER:-manual} +export DIB_GROWVOLS_ARGS=${DIB_GROWVOLS_ARGS:-} +export DIB_GROWVOLS_GROUP=${DIB_GROWVOLS_GROUP:-} +export DIB_GROWVOLS_DEVICE=${DIB_GROWVOLS_DEVICE:-} diff --git a/diskimage_builder/elements/growvols/init-scripts/systemd/growvols.service b/diskimage_builder/elements/growvols/init-scripts/systemd/growvols.service new file mode 100644 index 00000000..1b17ecb1 --- /dev/null +++ b/diskimage_builder/elements/growvols/init-scripts/systemd/growvols.service @@ -0,0 +1,14 @@ +[Unit] +Description=Grow LVM volumes +Wants=systemd-udev-settle.service +After=systemd-udev-settle.service + +[Service] +Type=oneshot +EnvironmentFile=-/etc/sysconfig/growvols +User=root +ExecStart=/usr/local/sbin/growvols --verbose --yes +RemainAfterExit=true + +[Install] +WantedBy=multi-user.target diff --git a/diskimage_builder/elements/growvols/package-installs.yaml b/diskimage_builder/elements/growvols/package-installs.yaml new file mode 100644 index 00000000..510db0c6 --- /dev/null +++ b/diskimage_builder/elements/growvols/package-installs.yaml @@ -0,0 +1,5 @@ +gdisk: +util-linux: +lvm2: +parted: +xfsprogs: \ No newline at end of file diff --git a/diskimage_builder/elements/growvols/post-install.d/80-growvols b/diskimage_builder/elements/growvols/post-install.d/80-growvols new file mode 100755 index 00000000..8a29c0c6 --- /dev/null +++ b/diskimage_builder/elements/growvols/post-install.d/80-growvols @@ -0,0 +1,29 @@ +#!/bin/bash + +if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then + set -x +fi +set -eu +set -o pipefail + +case ${DIB_GROWVOLS_TRIGGER} in + manual) echo "growvols triggered manually after boot"; exit ;; + systemd) echo "growvols triggered by systemd" ;; + *) echo "Unsupported DIB_GROWVOLS_TRIGGER: ${DIB_GROWVOLS_TRIGGER}"; exit 1 ;; +esac + +cat << EOF | sudo tee /etc/sysconfig/growvols > /dev/null +GROWVOLS_ARGS="${DIB_GROWVOLS_ARGS}" +GROWVOLS_GROUP="${DIB_GROWVOLS_GROUP}" +GROWVOLS_DEVICE="${DIB_GROWVOLS_DEVICE}" +EOF + +case "$DIB_INIT_SYSTEM" in + systemd) + systemctl enable growvols.service + ;; + *) + echo "Unsupported init system" + exit 1 + ;; +esac diff --git a/diskimage_builder/elements/growvols/static/usr/local/sbin/growvols b/diskimage_builder/elements/growvols/static/usr/local/sbin/growvols new file mode 100755 index 00000000..ed1e116e --- /dev/null +++ b/diskimage_builder/elements/growvols/static/usr/local/sbin/growvols @@ -0,0 +1,536 @@ +#!/usr/bin/env python3 + +# Copyright 2021 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +import argparse +import collections +import logging +import os +import re +import shlex +import subprocess +import sys + +LOG = logging.getLogger(__name__) + +UNIT_BYTES = { + 'MB': 1000000, + 'MiB': 1048576, + 'GB': 1000000000, + 'GiB': 1073741824 +} +UNITS = ['%'] +UNITS.extend(UNIT_BYTES.keys()) +AMOUNT_UNIT_RE = re.compile('^([0-9]+)(%s)$' % '|'.join(UNITS)) + +# Only create growth partition if there is at least 1GiB available +MIN_DISK_SPACE_BYTES = UNIT_BYTES['GiB'] + +# Default LVM physical extent size is 4MiB +PHYSICAL_EXTENT_BYTES = 4 * UNIT_BYTES['MiB'] + + +class Command(object): + """ An object to represent a command to run with associated comment """ + + cmd = None + + comment = None + + def __init__(self, cmd, comment=None): + self.cmd = cmd + self.comment = comment + + def __repr__(self): + if self.comment: + return "\n# %s\n%s" % (self.comment, printable_cmd(self.cmd)) + return printable_cmd(self.cmd) + + +def parse_opts(argv): + parser = argparse.ArgumentParser( + formatter_class=argparse.RawDescriptionHelpFormatter, + description=''' +Grow the logical volumes in an LVM group to take available device space. + +The positional arguments specify how available space is allocated. They +have the format = where: + + is the label or the mountpoint of the logical volume + is an integer growth amount in the specified unit + is one of the supported units + +Supported units are: + +% percentage of available device space before any changes are made +MiB mebibyte (1048576 bytes) +GiB gibibyte (1073741824 bytes) +MB megabyte (1000000 bytes) +GB gigabyte (1000000000 bytes) + +Each argument is processed in order and the requested amount is allocated +to each volume until the disk is full. This means that if space is +overallocated, the last volumes may only grow by the remaining space, or +not grow at all, and a warning will be printed. When space is underallocated +the remaining space will be given to the root volume (mounted at /). + +The currently supported partition layout is: +- Exactly one of the partitions containing an LVM group +- The disk having unpartitioned space to grow with +- The LVM logical volumes being formatted with XFS filesystems + +Example usage: + +growvols /var=80% /home=20GB + +growvols --device sda --group vg img-rootfs=20% fs_home=20% fs_var=60% + +''') + parser.add_argument('grow_vols', + nargs="*", + metavar="=", + default=os.environ.get('GROWVOLS_ARGS', '').split(), + help='A label or mountpoint, and the proportion to ' + 'grow it by. Defaults to $GROWVOLS_ARGS. ' + 'For example: /home=80% img-rootfs=20%') + parser.add_argument('--group', metavar='GROUP', + default=os.environ.get('GROWVOLS_GROUP'), + help='The name of the LVM group to extend. Defaults ' + 'to $GROWVOLS_GROUP or the discovered group ' + 'if only one exists.') + parser.add_argument('--device', metavar='DEVICE', + default=os.environ.get('GROWVOLS_DEVICE'), + help='The name of the disk block device to grow the ' + 'volumes in (such as "sda"). Defaults to the ' + 'disk containing the root mount.') + + parser.add_argument( + '--exit-on-no-grow', + action='store_true', + help="Exit with error code 2 if no volume could be grown. ", + default=False) + + parser.add_argument( + '-d', '--debug', + dest="debug", + action='store_true', + help="Print debugging output.", + required=False) + parser.add_argument( + '-v', '--verbose', + dest="verbose", + action='store_true', + help="Print verbose output.", + required=False) + parser.add_argument( + '--noop', + dest="noop", + action='store_true', + help="Return the configuration commands, without applying them.", + required=False) + parser.add_argument( + '-y', '--yes', + help='Skip yes/no prompt (assume yes).', + default=False, + action="store_true") + + opts = parser.parse_args(argv[1:]) + + return opts + + +def configure_logger(verbose=False, debug=False): + LOG_FORMAT = '[%(levelname)s] %(message)s' + log_level = logging.WARN + + if debug: + log_level = logging.DEBUG + elif verbose: + log_level = logging.INFO + + logging.basicConfig(format=LOG_FORMAT, + level=log_level) + + +def printable_cmd(cmd): + """Convert a command list to a log printable string""" + cmd_quoted = [shlex.quote(c) for c in cmd] + return ' '.join(cmd_quoted) + + +def convert_bytes(num): + """Format a bytes amount with units MB, GB etc""" + step_unit = 1000.0 + + for x in ['B', 'KB', 'MB', 'GB', 'TB']: + if num < step_unit: + return "%d%s" % (num, x) + num /= step_unit + + +def execute(cmd): + """Run a command""" + LOG.info('Running: %s', printable_cmd(cmd)) + + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True + ) + out, err = process.communicate() + if process.returncode != 0: + error_msg = ( + 'Running command failed: cmd "{}", stdout "{}",' + ' stderr "{}"'.format( + ' '.join(cmd), + out, + err + ) + ) + LOG.error(error_msg) + raise Exception(error_msg) + LOG.debug('Result: %s', out) + return out + + +def parse_shell_vars(output): + """Parse tags from the lsblk/blkid output. + + Parses format KEY="VALUE" KEY2="VALUE2". + + :return: a generator yielding dicts with information from each line. + """ + for line in output.strip().split('\n'): + if line.strip(): + yield {key: value for key, value in + (v.split('=', 1) for v in shlex.split(line))} + + +def find_device(devices, keys, value): + """Search devices list for one device that has a key matching the value""" + if isinstance(keys, str): + keys = [keys] + for device in devices: + for key in keys: + if device.get(key) == value: + return device + + +def find_disk(opts, devices): + """Find the disk device given a device option, or searching from mount /""" + if opts.device: + device = find_device(devices, ['KNAME', 'NAME'], opts.device) + if not device: + raise Exception('Could not find specified --device: %s' + % opts.device) + else: + device = find_device(devices, 'MOUNTPOINT', '/') + if not device: + raise Exception('Could not find mountpoint for /') + + while True: + device = find_device(devices, 'KNAME', device['PKNAME']) + if not device: + break + if device['TYPE'] == 'disk': + break + if not device['PKNAME']: + break + if not device: + raise Exception('Could not detect disk device') + + if device['TYPE'] != 'disk': + raise Exception('Expected a device with TYPE="disk", got: %s' + % device['TYPE']) + return device + + +def find_space(disk_name): + LOG.info('Finding spare space to grow into') + dev_path = '/dev/%s' % disk_name + largest = execute([ + 'sgdisk', + '--first-aligned-in-largest', + '--end-of-largest', + dev_path]) + start_end = largest.strip().split('\n') + sector_start = int(start_end[0]) + sector_end = int(start_end[1]) + size_sectors = sector_end - sector_start + return sector_start, sector_end, size_sectors + + +def find_devices(): + LOG.info('Finding all block devices') + lsblk = execute([ + 'lsblk', + '-Po', + 'kname,pkname,name,label,type,fstype,mountpoint']) + return list(parse_shell_vars(lsblk)) + + +def find_group(opts): + LOG.info('Finding LVM volume group') + vgs = execute(['vgs', '--noheadings', '--options', 'vg_name']) + vg_names = set([vg.strip() for vg in vgs.split('\n') if vg.strip()]) + if opts.group: + if opts.group not in vg_names: + raise Exception('Could not find specified --group: %s' + % opts.group) + return opts.group + if len(vg_names) > 1: + raise Exception('More than one volume group, specify one to ' + 'use with --group: %s' % ', '.join(sorted(vg_names))) + if len(vg_names) < 1: + raise Exception('No volume groups found') + return vg_names.pop() + + +def find_next_partnum(devices, disk_name): + return len([d for d in devices if d['PKNAME'] == disk_name]) + 1 + + +def find_next_device_name(devices, disk_name, partnum): + existing_partnum = partnum - 1 + + # try partition scheme for SATA etc, then NVMe + for part_template in '%s%d', '%sp%d': + part = part_template % (disk_name, existing_partnum) + LOG.debug('Looking for device %s' % part) + if find_device(devices, 'KNAME', part): + return part_template % (disk_name, partnum) + + raise Exception('Could not find partition naming scheme for %s' + % disk_name) + + +def prompt_user_for_confirmation(message): + """Prompt user for a y/N confirmation + + Use this function to prompt the user for a y/N confirmation + with the provided message. The [y/N] should be included in + the provided message to this function to indicate the expected + input for confirmation. You can customize the positive response if + y/N is not a desired input. + + :param message: Confirmation string prompt + :param positive_response: Beginning character for a positive user input + :return: boolean true for valid confirmation, false for all others + """ + try: + if not sys.stdin.isatty(): + LOG.error('User interaction required, cannot confirm.') + return False + else: + sys.stdout.write(message) + sys.stdout.flush() + prompt_response = sys.stdin.readline().lower() + if not prompt_response.startswith('y'): + print('Taking no action.') + return False + LOG.info('User confirmed action.') + return True + except KeyboardInterrupt: # ctrl-c + print('User did not confirm action (ctrl-c) so taking no action.') + except EOFError: # ctrl-d + print('User did not confirm action (ctrl-d) so taking no action.') + return False + + +def amount_unit_to_extent(amount_unit, total_size_bytes, remaining_bytes): + m = AMOUNT_UNIT_RE.match(amount_unit) + if not m: + raise Exception('Value for not valid: %s' + % amount_unit) + + amount = int(m.group(1)) + unit = m.group(2) + if unit in UNIT_BYTES: + bytes = amount * UNIT_BYTES[unit] + LOG.debug('%s is %s' % (amount_unit, convert_bytes(bytes))) + else: + bytes = int((amount / 100.0) * total_size_bytes) + LOG.debug('%s of %s is %s' + % (amount_unit, + convert_bytes(total_size_bytes), + convert_bytes(bytes))) + + if remaining_bytes <= 0: + LOG.debug('No remaining space available to allocate %s' % amount_unit) + bytes = 0 + elif bytes > remaining_bytes: + LOG.debug('Cannot allocate all of %s, only %s remaining space' + % (convert_bytes(bytes), convert_bytes(remaining_bytes))) + bytes = remaining_bytes + + # reduce to align on physical extent size + bytes = bytes - bytes % PHYSICAL_EXTENT_BYTES + + return bytes, remaining_bytes - bytes + + +def find_grow_vols(opts, devices, group, total_size_bytes): + grow_vols = collections.OrderedDict() + remaining_bytes = total_size_bytes + + arg_error = (' must be of the format ' + '=, ' + 'for example: /home=100% or fs_home=100%') + grow_vols_opt = list(gv for gv in opts.grow_vols if gv) + # append an implicit /=100% to grow root by any underallocation + grow_vols_opt.append('/=100%') + + for gv in grow_vols_opt: + if '=' not in gv: + raise Exception(arg_error) + devname, amount_unit = gv.split('=', 1) + device = find_device(devices, ['LABEL', 'MOUNTPOINT'], devname) + if not device: + raise Exception('Could not find device %s for argument: %s' + % (devname, gv)) + + if device['TYPE'] != 'lvm': + raise Exception('Device %(NAME)s is not of type lvm: %(TYPE)s' + % device) + + if device['FSTYPE'] != 'xfs': + raise Exception('Device %(NAME)s is not of fstype xfs: %(FSTYPE)s' + % device) + + size_bytes, remaining_bytes = amount_unit_to_extent( + amount_unit, total_size_bytes, remaining_bytes) + + if size_bytes == 0: + continue + + # remove the group- prefix from the name + name = device['NAME'][len(group) + 1:] + volume_path = '/dev/%s/%s' % (group, name) + + grow_vols[volume_path] = size_bytes + + return grow_vols + + +def find_sector_size(disk_name): + sector_path = '/sys/block/%s/queue/logical_block_size' % disk_name + + LOG.info('Reading sector size from %s' % sector_path) + with open(sector_path) as f: + size = int(f.read().strip()) + LOG.info('Sector size of %s is %s' + % (disk_name, convert_bytes(size))) + return size + + +def main(argv): + opts = parse_opts(argv) + configure_logger(opts.verbose, opts.debug) + + devices = find_devices() + disk = find_disk(opts, devices) + disk_name = disk['KNAME'] + + sector_bytes = find_sector_size(disk_name) + sector_start, sector_end, size_sectors = find_space(disk_name) + + min_required_sectors = MIN_DISK_SPACE_BYTES // sector_bytes + + size_bytes = size_sectors * sector_bytes + if size_sectors < min_required_sectors: + msg = ('Need at least %s sectors to expand into, there ' + 'is only: %s' % (min_required_sectors, size_sectors)) + if opts.exit_on_no_grow: + LOG.error(msg) + return 2 + + LOG.info(msg) + return 0 + + group = find_group(opts) + partnum = find_next_partnum(devices, disk_name) + devname = find_next_device_name(devices, disk_name, partnum) + dev_path = '/dev/%s' % devname + grow_vols = find_grow_vols(opts, devices, group, size_bytes) + + commands = [] + + commands.append(Command([ + 'sgdisk', + '--new=%d:%d:%d' % (partnum, sector_start, sector_end), + '--change-name=%d:growvols' % partnum, + '/dev/%s' % disk_name + ], 'Create new partition %s' % devname)) + + commands.append(Command( + ['partprobe'], + 'Inform the OS of partition table changes')) + + commands.append(Command([ + 'pvcreate', + dev_path + ], 'Initialize %s for use by LVM' % devname)) + + commands.append(Command([ + 'vgextend', + group, + dev_path + ], 'Add physical volume %s to group %s' % (devname, group))) + + for volume_path, size_bytes in grow_vols.items(): + if size_bytes > 0: + commands.append(Command([ + 'lvextend', + '--size', + '+%sB' % size_bytes, + volume_path, + dev_path + ], 'Add %s to logical volume %s' % (convert_bytes(size_bytes), + volume_path))) + + for volume_path, size_bytes in grow_vols.items(): + if size_bytes > 0: + commands.append(Command([ + 'xfs_growfs', + volume_path + ], 'Grow XFS filesystem for %s' % volume_path)) + + if opts.noop: + print('The following commands would have run:') + else: + print('The following commands will be run:') + + for cmd in commands: + print(cmd) + sys.stdout.flush() + + if opts.noop: + return 0 + + yes = opts.yes + if not yes: + yes = prompt_user_for_confirmation( + '\nAre you sure you want to run these commands? [y/N] ') + if yes: + for cmd in commands: + execute(cmd.cmd) + + return 0 + + +if __name__ == '__main__': + sys.exit(main(sys.argv)) diff --git a/diskimage_builder/elements/growvols/tests/__init__.py b/diskimage_builder/elements/growvols/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/diskimage_builder/elements/growvols/tests/test_growvols.py b/diskimage_builder/elements/growvols/tests/test_growvols.py new file mode 100644 index 00000000..c1ad50f0 --- /dev/null +++ b/diskimage_builder/elements/growvols/tests/test_growvols.py @@ -0,0 +1,503 @@ +# Copyright 2014 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +import imp +import mock +import os +from oslotest import base + +module_path = (os.path.dirname(os.path.realpath(__file__)) + + '/../static/usr/local/sbin/growvols') +growvols = imp.load_source('growvols', module_path) + +# output of lsblk -Po kname,pkname,name,label,type,fstype,mountpoint +LSBLK = """KNAME="sda" PKNAME="" NAME="sda" LABEL="" TYPE="disk" FSTYPE="" MOUNTPOINT="" +KNAME="sda1" PKNAME="sda" NAME="sda1" LABEL="MKFS_ESP" TYPE="part" FSTYPE="vfat" MOUNTPOINT="/boot/efi" +KNAME="sda2" PKNAME="sda" NAME="sda2" LABEL="" TYPE="part" FSTYPE="" MOUNTPOINT="" +KNAME="sda3" PKNAME="sda" NAME="sda3" LABEL="" TYPE="part" FSTYPE="LVM2_member" MOUNTPOINT="" +KNAME="sda4" PKNAME="sda" NAME="sda4" LABEL="config-2" TYPE="part" FSTYPE="iso9660" MOUNTPOINT="" +KNAME="dm-0" PKNAME="sda3" NAME="vg-lv_root" LABEL="img-rootfs" TYPE="lvm" FSTYPE="xfs" MOUNTPOINT="/" +KNAME="dm-1" PKNAME="sda3" NAME="vg-lv_tmp" LABEL="fs_tmp" TYPE="lvm" FSTYPE="xfs" MOUNTPOINT="/tmp" +KNAME="dm-2" PKNAME="sda3" NAME="vg-lv_var" LABEL="fs_var" TYPE="lvm" FSTYPE="xfs" MOUNTPOINT="/var" +KNAME="dm-3" PKNAME="sda3" NAME="vg-lv_home" LABEL="fs_home" TYPE="lvm" FSTYPE="xfs" MOUNTPOINT="/home" +""" # noqa + +DEVICES = [{ + "FSTYPE": "", + "KNAME": "sda", + "LABEL": "", + "MOUNTPOINT": "", + "NAME": "sda", + "PKNAME": "", + "TYPE": "disk", +}, { + "FSTYPE": "vfat", + "KNAME": "sda1", + "LABEL": "MKFS_ESP", + "MOUNTPOINT": "/boot/efi", + "NAME": "sda1", + "PKNAME": "sda", + "TYPE": "part", +}, { + "FSTYPE": "", + "KNAME": "sda2", + "LABEL": "", + "MOUNTPOINT": "", + "NAME": "sda2", + "PKNAME": "sda", + "TYPE": "part", +}, { + "FSTYPE": "LVM2_member", + "KNAME": "sda3", + "LABEL": "", + "MOUNTPOINT": "", + "NAME": "sda3", + "PKNAME": "sda", + "TYPE": "part", +}, { + "FSTYPE": "iso9660", + "KNAME": "sda4", + "LABEL": "config-2", + "MOUNTPOINT": "", + "NAME": "sda4", + "PKNAME": "sda", + "TYPE": "part", +}, { + "FSTYPE": "xfs", + "KNAME": "dm-0", + "LABEL": "img-rootfs", + "MOUNTPOINT": "/", + "NAME": "vg-lv_root", + "PKNAME": "sda3", + "TYPE": "lvm", +}, { + "FSTYPE": "xfs", + "KNAME": "dm-1", + "LABEL": "fs_tmp", + "MOUNTPOINT": "/tmp", + "NAME": "vg-lv_tmp", + "PKNAME": "sda3", + "TYPE": "lvm", +}, { + "FSTYPE": "xfs", + "KNAME": "dm-2", + "LABEL": "fs_var", + "MOUNTPOINT": "/var", + "NAME": "vg-lv_var", + "PKNAME": "sda3", + "TYPE": "lvm", +}, { + "FSTYPE": "xfs", + "KNAME": "dm-3", + "LABEL": "fs_home", + "MOUNTPOINT": "/home", + "NAME": "vg-lv_home", + "PKNAME": "sda3", + "TYPE": "lvm", +}] + +# output of sgdisk --first-aligned-in-largest --end-of-largest /dev/sda +SECTOR_START = 79267840 +SECTOR_END = 488265727 +SGDISK_LARGEST = "%s\n%s\n" % (SECTOR_START, SECTOR_END) + +# output of vgs --noheadings --options vg_name +VGS = " vg\n" + + +class TestGrowvols(base.BaseTestCase): + + def test_printable_cmd(self): + self.assertEqual( + "foo --thing 'bar baz'", + growvols.printable_cmd(['foo', '--thing', "bar baz"]) + ) + + def test_convert_bytes(self): + self.assertEqual('100B', growvols.convert_bytes(100)) + self.assertEqual('1KB', growvols.convert_bytes(1000)) + self.assertEqual('2MB', growvols.convert_bytes(2000000)) + self.assertEqual('3GB', growvols.convert_bytes(3000000000)) + self.assertEqual('4TB', growvols.convert_bytes(4000000000000)) + + @mock.patch('subprocess.Popen') + def test_execute(self, mock_popen): + mock_process = mock.Mock() + mock_process.returncode = 0 + mock_process.communicate.return_value = ('did the thing', '') + mock_popen.return_value = mock_process + + result = growvols.execute(['do', 'the', 'thing']) + self.assertEqual('did the thing', result) + + mock_process.returncode = 1 + mock_process.communicate.return_value = ('', 'ouch') + + e = self.assertRaises(Exception, growvols.execute, + ['do', 'the', 'thing']) + self.assertIn('ouch', str(e)) + + def test_parse_shell_vars(self): + devices = list(growvols.parse_shell_vars(LSBLK)) + self.assertEqual(DEVICES, devices) + + def test_find_device(self): + sda = { + "FSTYPE": "", + "KNAME": "sda", + "LABEL": "", + "MOUNTPOINT": "", + "NAME": "sda", + "PKNAME": "", + "TYPE": "disk", + } + fs_home = { + "FSTYPE": "xfs", + "KNAME": "dm-3", + "LABEL": "fs_home", + "MOUNTPOINT": "/home", + "NAME": "vg-lv_home", + "PKNAME": "sda3", + "TYPE": "lvm", + } + self.assertEqual( + sda, growvols.find_device(DEVICES, 'NAME', 'sda')) + self.assertEqual( + fs_home, + growvols.find_device( + DEVICES, ['KNAME', 'NAME'], 'vg-lv_home')) + self.assertEqual( + fs_home, + growvols.find_device( + DEVICES, ['KNAME', 'NAME'], 'dm-3')) + self.assertIsNone( + growvols.find_device( + DEVICES, ['KNAME', 'NAME'], 'asdf')) + + def test_find_disk(self): + devices = list(growvols.parse_shell_vars(LSBLK)) + opts = mock.Mock() + opts.device = None + sda = growvols.find_device(devices, 'NAME', 'sda') + + # discover via MOUNTPOINT / + self.assertEqual(sda, growvols.find_disk(opts, devices)) + + # fetch sda + opts.device = 'sda' + self.assertEqual(sda, growvols.find_disk(opts, devices)) + + # delete sda3, so can't find relationship + # from MOUNTPOINT / to sda + opts.device = None + devices = [d for d in devices if d['NAME'] != 'sda3'] + e = self.assertRaises(Exception, growvols.find_disk, opts, devices) + self.assertEqual('Could not detect disk device', str(e)) + + # no sdb + opts.device = 'sdb' + e = self.assertRaises(Exception, growvols.find_disk, opts, devices) + self.assertEqual('Could not find specified --device: sdb', str(e)) + + # sda is not TYPE disk + sda['TYPE'] = 'dissed' + opts.device = 'sda' + e = self.assertRaises(Exception, growvols.find_disk, opts, devices) + self.assertEqual('Expected a device with TYPE="disk", got: dissed', + str(e)) + + @mock.patch('growvols.execute') + def test_find_space(self, mock_execute): + mock_execute.return_value = SGDISK_LARGEST + sector_start, sector_end, size_sectors = growvols.find_space( + 'sda') + self.assertEqual(SECTOR_START, sector_start) + self.assertEqual(SECTOR_END, sector_end) + self.assertEqual(SECTOR_END - SECTOR_START, size_sectors) + mock_execute.assert_called_once_with([ + 'sgdisk', + '--first-aligned-in-largest', + '--end-of-largest', + '/dev/sda']) + + @mock.patch('growvols.execute') + def test_find_devices(self, mock_execute): + mock_execute.return_value = LSBLK + self.assertEqual(DEVICES, growvols.find_devices()) + mock_execute.assert_called_once_with([ + 'lsblk', + '-Po', + 'kname,pkname,name,label,type,fstype,mountpoint']) + + @mock.patch('growvols.execute') + def test_find_group(self, mock_execute): + mock_execute.return_value = VGS + opts = mock.Mock() + opts.group = None + self.assertEqual('vg', growvols.find_group(opts)) + mock_execute.assert_called_once_with([ + 'vgs', '--noheadings', '--options', 'vg_name']) + + # no volume groups + mock_execute.return_value = "\n" + e = self.assertRaises(Exception, growvols.find_group, opts) + self.assertEqual('No volume groups found', str(e)) + + # multiple volume groups + mock_execute.return_value = " vg\nvg2\nvg3" + e = self.assertRaises(Exception, growvols.find_group, opts) + self.assertEqual('More than one volume group, specify one to ' + 'use with --group: vg, vg2, vg3', str(e)) + + # multiple volume groups with explicit group argument + opts.group = 'vg' + self.assertEqual('vg', growvols.find_group(opts)) + + # no such group + opts.group = 'novg' + e = self.assertRaises(Exception, growvols.find_group, opts) + self.assertEqual('Could not find specified --group: novg', str(e)) + + def test_find_next_partnum(self): + self.assertEqual(5, growvols.find_next_partnum(DEVICES, 'sda')) + self.assertEqual(1, growvols.find_next_partnum(DEVICES, 'sdb')) + + def test_find_next_device_name(self): + devices = list(growvols.parse_shell_vars(LSBLK)) + + # Use SATA etc device naming + self.assertEqual( + 'sda5', + growvols.find_next_device_name(devices, 'sda', 5)) + + # No partitions + e = self.assertRaises(Exception, growvols.find_next_device_name, + devices, 'sdb', 1) + self.assertEquals( + 'Could not find partition naming scheme for sdb', str(e)) + + # Use NVMe device naming + for i in (1, 2, 3, 4): + d = growvols.find_device(devices, 'KNAME', 'sda%s' % i) + d['KNAME'] = 'nvme0p%s' % i + self.assertEqual( + 'nvme0p5', + growvols.find_next_device_name(devices, 'nvme0', 5)) + + def test_amount_unit_to_extent(self): + one_m = growvols.UNIT_BYTES['MiB'] + four_m = one_m * 4 + one_g = growvols.UNIT_BYTES['GiB'] + ten_g = one_g * 10 + forty_g = ten_g * 4 + fidy_g = ten_g * 5 + + # invalid amounts + self.assertRaises(Exception, growvols.amount_unit_to_extent, + '100', one_g, one_g) + self.assertRaises(Exception, growvols.amount_unit_to_extent, + '100B', one_g, one_g) + self.assertRaises(Exception, growvols.amount_unit_to_extent, + '100%%', one_g, one_g) + self.assertRaises(Exception, growvols.amount_unit_to_extent, + '100TiB', one_g, one_g) + self.assertRaises(Exception, growvols.amount_unit_to_extent, + 'i100MB', one_g, one_g) + + # GiB amount + self.assertEqual( + (ten_g, forty_g), + growvols.amount_unit_to_extent('10GiB', fidy_g, fidy_g) + ) + + # percentage amount + self.assertEqual( + (ten_g, forty_g), + growvols.amount_unit_to_extent('20%', fidy_g, fidy_g) + ) + + # not enough space left + self.assertEqual( + (0, one_m), + growvols.amount_unit_to_extent('20%', fidy_g, one_m) + ) + + # exactly one extent + self.assertEqual( + (four_m, fidy_g - four_m), + growvols.amount_unit_to_extent('4MiB', fidy_g, fidy_g) + ) + + # under one extent is zero + self.assertEqual( + (0, fidy_g), + growvols.amount_unit_to_extent('3MiB', fidy_g, fidy_g) + ) + + # just over one extent is one extent + self.assertEqual( + (four_m, fidy_g - four_m), + growvols.amount_unit_to_extent('5MiB', fidy_g, fidy_g) + ) + + def test_find_grow_vols(self): + one_g = growvols.UNIT_BYTES['GiB'] + ten_g = one_g * 10 + fidy_g = ten_g * 5 + + opts = mock.Mock() + + # buy default, assign all to / + opts.grow_vols = [''] + self.assertEqual( + {'/dev/lv/lv_root': fidy_g}, + growvols.find_grow_vols(opts, DEVICES, 'lv', fidy_g) + ) + + # assign to /home, /var, remainder to / + opts.grow_vols = ['/home=20%', 'fs_var=40%'] + self.assertEqual( + { + '/dev/lv/lv_home': ten_g, + '/dev/lv/lv_var': ten_g * 2, + '/dev/lv/lv_root': ten_g * 2 + }, + growvols.find_grow_vols(opts, DEVICES, 'lv', fidy_g) + ) + + # assign to /home, /var, /tmp by amount + opts.grow_vols = ['/home=19GiB', 'fs_var=30GiB', '/tmp=1GiB'] + self.assertEqual( + { + '/dev/lv/lv_home': one_g * 19, + '/dev/lv/lv_var': one_g * 30, + '/dev/lv/lv_tmp': one_g + }, + growvols.find_grow_vols(opts, DEVICES, 'lv', fidy_g) + ) + + @mock.patch('builtins.open', autospec=True) + def test_find_sector_size(self, mock_open): + mock_open.return_value.__enter__ = lambda s: s + mock_open.return_value.__exit__ = mock.Mock() + read_mock = mock_open.return_value.read + read_mock.side_effect = ['512'] + + # disk sdx exists + self.assertEqual(512, growvols.find_sector_size('sdx')) + + # disk sdx doesn't exist + mock_open.side_effect = FileNotFoundError + self.assertRaises(FileNotFoundError, growvols.find_sector_size, 'sdx') + + @mock.patch('growvols.find_sector_size') + @mock.patch('growvols.execute') + def test_main(self, mock_execute, mock_sector_size): + mock_sector_size.return_value = 512 + + # noop, only discover block device info + mock_execute.side_effect = [ + LSBLK, + SGDISK_LARGEST, + VGS, + ] + growvols.main(['growvols', '--noop']) + mock_execute.assert_has_calls([ + mock.call(['lsblk', '-Po', + 'kname,pkname,name,label,type,fstype,mountpoint']), + mock.call(['sgdisk', '--first-aligned-in-largest', + '--end-of-largest', '/dev/sda']), + mock.call(['vgs', '--noheadings', '--options', 'vg_name']), + ]) + + # no arguments, assign all to / + mock_execute.reset_mock() + mock_execute.side_effect = [ + LSBLK, + SGDISK_LARGEST, + VGS, + '', '', '', '', '', '' + ] + growvols.main(['growvols', '--yes']) + mock_execute.assert_has_calls([ + mock.call(['lsblk', '-Po', + 'kname,pkname,name,label,type,fstype,mountpoint']), + mock.call(['sgdisk', '--first-aligned-in-largest', + '--end-of-largest', '/dev/sda']), + mock.call(['vgs', '--noheadings', '--options', 'vg_name']), + mock.call(['sgdisk', '--new=5:79267840:488265727', + '--change-name=5:growvols', '/dev/sda']), + mock.call(['partprobe']), + mock.call(['pvcreate', '/dev/sda5']), + mock.call(['vgextend', 'vg', '/dev/sda5']), + mock.call(['lvextend', '--size', '+209404821504B', + '/dev/vg/lv_root', '/dev/sda5']), + mock.call(['xfs_growfs', '/dev/vg/lv_root']) + ]) + + # assign to /home, /var, remainder to / + mock_execute.reset_mock() + mock_execute.side_effect = [ + LSBLK, + SGDISK_LARGEST, + VGS, + '', '', '', '', '', '', '', '', '', '' + ] + growvols.main(['growvols', '--yes', '--group', 'vg', + '/home=20%', 'fs_var=40%']) + mock_execute.assert_has_calls([ + mock.call(['lsblk', '-Po', + 'kname,pkname,name,label,type,fstype,mountpoint']), + mock.call(['sgdisk', '--first-aligned-in-largest', + '--end-of-largest', '/dev/sda']), + mock.call(['vgs', '--noheadings', '--options', 'vg_name']), + mock.call(['sgdisk', '--new=5:79267840:488265727', + '--change-name=5:growvols', '/dev/sda']), + mock.call(['partprobe']), + mock.call(['pvcreate', '/dev/sda5']), + mock.call(['vgextend', 'vg', '/dev/sda5']), + mock.call(['lvextend', '--size', '+41880125440B', + '/dev/vg/lv_home', '/dev/sda5']), + mock.call(['lvextend', '--size', '+83760250880B', + '/dev/vg/lv_var', '/dev/sda5']), + mock.call(['lvextend', '--size', '+83764445184B', + '/dev/vg/lv_root', '/dev/sda5']), + mock.call(['xfs_growfs', '/dev/vg/lv_home']), + mock.call(['xfs_growfs', '/dev/vg/lv_var']), + mock.call(['xfs_growfs', '/dev/vg/lv_root']), + ]) + + # no space to grow, failed + sector_start = 79267840 + sector_end = sector_start + 1024 + sgdisk_largest = "%s\n%s\n" % (sector_start, sector_end) + mock_execute.side_effect = [ + LSBLK, + sgdisk_largest, + VGS, + ] + self.assertEqual( + 2, + growvols.main(['growvols', '--exit-on-no-grow']) + ) + + # no space to grow, success + mock_execute.side_effect = [ + LSBLK, + sgdisk_largest, + VGS, + ] + self.assertEqual( + 0, + growvols.main(['growvols']) + )