From 230cbc507bef801b075fbffbec6697bceced4abb Mon Sep 17 00:00:00 2001 From: Peter Ajamian Date: Thu, 11 Nov 2021 22:16:13 +1300 Subject: [PATCH 01/28] Fix broken stream conversion logic. Stream system repo packages were ignored if there were no stream non-system packages present. --- migrate2rocky/migrate2rocky.sh | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/migrate2rocky/migrate2rocky.sh b/migrate2rocky/migrate2rocky.sh index cdf3616..afad509 100644 --- a/migrate2rocky/migrate2rocky.sh +++ b/migrate2rocky/migrate2rocky.sh @@ -732,7 +732,8 @@ package_swaps() { done # CentOS Stream specific processing - if (( ${#installed_stream_repos_pkgs[@]} )); then + if (( ${#installed_stream_repos_pkgs[@]} || + ${#installed_sys_stream_repos_pkgs[@]} )); then # Get a list of the repo files. local -a repos_files readarray -t repos_files < <( @@ -741,10 +742,13 @@ package_swaps() { grep '^/etc/yum\.repos\.d/.\+\.repo$' ) - # Remove the package from the rpm db. - saferpm -e --justdb --nodeps -a "${installed_sys_stream_repos_pkgs[@]}" || + if (( ${#installed_sys_stream_repos_pkgs[@]} )); then + # Remove the package from the rpm db. + saferpm -e --justdb --nodeps -a \ + "${installed_sys_stream_repos_pkgs[@]}" || exit_message \ "Could not remove packages from the rpm db: ${installed_sys_stream_repos_pkgs[@]}" + fi # Rename the stream repos with a prefix. sed -i 's/^\[/['"$stream_prefix"'/' "${repos_files[@]}" From fbac8d07a2a0e31126ddb8a96f92d9a9490999cd Mon Sep 17 00:00:00 2001 From: Peter Ajamian Date: Wed, 17 Nov 2021 15:51:57 +1300 Subject: [PATCH 02/28] Remove Secure Boot check. Rocky Linux now has secure boot support, so we no longer need to check for and bail on systems with secure boot enabled. --- migrate2rocky/migrate2rocky.sh | 9 --------- 1 file changed, 9 deletions(-) diff --git a/migrate2rocky/migrate2rocky.sh b/migrate2rocky/migrate2rocky.sh index cdf3616..b95bb9f 100644 --- a/migrate2rocky/migrate2rocky.sh +++ b/migrate2rocky/migrate2rocky.sh @@ -420,15 +420,6 @@ collect_system_info () { fi fi - # check if EFI secure boot is enabled - if [[ $update_efi ]]; then - if mokutil --sb-state 2>&1 | grep -q "SecureBoot enabled"; then - exit_message \ -"EFI Secure Boot is enabled but Rocky Linux doesn't provide a signed shim yet."\ -" Disable EFI Secure Boot and reboot." - fi - fi - # Don't enable these module streams, even if they are enabled in the source # distro. declare -g -a module_excludes From 6c3453041c3b485e4fd59facc1082c1d9c009182 Mon Sep 17 00:00:00 2001 From: Peter Ajamian Date: Thu, 18 Nov 2021 14:11:15 +1300 Subject: [PATCH 03/28] Fix broken mirrorlist URLs from CentOS Stream repos. The mirrorlist URLs rely on variable replacements from CentOS Stream that are not valid once we have switched to Rocky Linux. To fix this we change to the baseurl instead and fix it up to not rely on those variables that have changed. --- migrate2rocky/migrate2rocky.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/migrate2rocky/migrate2rocky.sh b/migrate2rocky/migrate2rocky.sh index c6915e8..8b3917b 100644 --- a/migrate2rocky/migrate2rocky.sh +++ b/migrate2rocky/migrate2rocky.sh @@ -741,8 +741,12 @@ package_swaps() { "Could not remove packages from the rpm db: ${installed_sys_stream_repos_pkgs[@]}" fi - # Rename the stream repos with a prefix. - sed -i 's/^\[/['"$stream_prefix"'/' "${repos_files[@]}" + # Rename the stream repos with a prefix and fix the baseurl. + sed -i \ + -e 's/^\[/['"$stream_prefix"'/' \ + -e 's|^mirrorlist=|#mirrorlist=|g' \ + -e 's|^#baseurl=http://mirror.centos.org/$contentdir/$stream/|baseurl=|http://mirror.centos.org/centos/8-stream/' \ + "${repos_files[@]}" fi # Use dnf shell to swap the system packages out. From 1abcdd13af4770fff1f50e99a126e59ad3bd29e2 Mon Sep 17 00:00:00 2001 From: Peter Ajamian Date: Thu, 18 Nov 2021 14:22:40 +1300 Subject: [PATCH 04/28] Fix up vault baseurls Fix the source repository (which are in vault.centos.org) URLs as well. --- migrate2rocky/migrate2rocky.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/migrate2rocky/migrate2rocky.sh b/migrate2rocky/migrate2rocky.sh index 8b3917b..3177da6 100644 --- a/migrate2rocky/migrate2rocky.sh +++ b/migrate2rocky/migrate2rocky.sh @@ -746,6 +746,7 @@ package_swaps() { -e 's/^\[/['"$stream_prefix"'/' \ -e 's|^mirrorlist=|#mirrorlist=|g' \ -e 's|^#baseurl=http://mirror.centos.org/$contentdir/$stream/|baseurl=|http://mirror.centos.org/centos/8-stream/' \ + -e 's|^baseurl=http://vault.centos.org/$contentdir/$stream/|baseurl=|https://vault.centos.org/centos/8-stream/' \ "${repos_files[@]}" fi From aff688f30c092289e841af911c731d38a95feea6 Mon Sep 17 00:00:00 2001 From: Peter Ajamian Date: Thu, 18 Nov 2021 14:31:33 +1300 Subject: [PATCH 05/28] grep is needed for all migrations. Move the check for grep to the main list of binaries instead of the ones just needed for EFI boot. --- migrate2rocky/migrate2rocky.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/migrate2rocky/migrate2rocky.sh b/migrate2rocky/migrate2rocky.sh index b95bb9f..4d6f3bc 100644 --- a/migrate2rocky/migrate2rocky.sh +++ b/migrate2rocky/migrate2rocky.sh @@ -230,10 +230,10 @@ bin_check() { local -a missing bins bins=( rpm dnf awk column tee tput mkdir cat arch sort uniq rmdir - rm head curl sha512sum mktemp systemd-detect-virt sed + rm head curl sha512sum mktemp systemd-detect-virt sed grep ) if [[ $update_efi ]]; then - bins+=(findmnt grub2-mkconfig efibootmgr grep mokutil lsblk) + bins+=(findmnt grub2-mkconfig efibootmgr mokutil lsblk) fi for bin in "${bins[@]}"; do if ! type "$bin" >/dev/null 2>&1; then From 22455ba2213f97dad28ed6dee094a29d69da8445 Mon Sep 17 00:00:00 2001 From: Peter Ajamian Date: Thu, 18 Nov 2021 14:38:29 +1300 Subject: [PATCH 06/28] Fix config-manager dnf plugin name. See RHBZ #1980712. --- migrate2rocky/migrate2rocky.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/migrate2rocky/migrate2rocky.sh b/migrate2rocky/migrate2rocky.sh index 4d6f3bc..79796c5 100644 --- a/migrate2rocky/migrate2rocky.sh +++ b/migrate2rocky/migrate2rocky.sh @@ -838,7 +838,7 @@ EOF # Distrosync infomsg $'Ensuring repos are enabled before the package swap\n' - safednf -y --enableplugin=config-manager config-manager \ + safednf -y --enableplugin=config_manager config-manager \ --set-enabled "${!repo_map[@]}" || { printf '%s\n' 'Repo name missing?' exit 25 @@ -853,7 +853,7 @@ EOF if (( ${#managed_repos[@]} )); then infomsg $'\nDisabling subscription managed repos\n' - safednf -y --enableplugin=config-manager config-manager \ + safednf -y --enableplugin=config_manager config-manager \ --disable "${managed_repos[@]}" fi fi From 28be74a123d1c6c9dd0234522693bf53c2722c2e Mon Sep 17 00:00:00 2001 From: Peter Ajamian Date: Thu, 18 Nov 2021 14:50:00 +1300 Subject: [PATCH 07/28] Shellcheck fixes. Various fixes reported by shellcheck. --- migrate2rocky/migrate2rocky.sh | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/migrate2rocky/migrate2rocky.sh b/migrate2rocky/migrate2rocky.sh index 79796c5..026012c 100644 --- a/migrate2rocky/migrate2rocky.sh +++ b/migrate2rocky/migrate2rocky.sh @@ -204,11 +204,13 @@ exit_clean () { pre_check () { if [[ -e /etc/rhsm/ca/katello-server-ca.pem ]]; then +# shellcheck disable=SC2026 exit_message \ 'Migration from Katello-modified systems is not supported by migrate2rocky. '\ 'See the README file for details.' fi if [[ -e /etc/salt/minion.d/susemanager.conf ]]; then +# shellcheck disable=SC2026 exit_message \ 'Migration from Uyuni/SUSE Manager-modified systems is not supported by '\ 'migrate2rocky. See the README file for details.' @@ -222,6 +224,7 @@ pre_check () { bin_check() { # Check the platform. if [[ $(os-release PLATFORM_ID) != "$SUPPORTED_PLATFORM" ]]; then +# shellcheck disable=SC2026 exit_message \ 'This script must be run on an EL8 distribution. Migration from other '\ 'distributions is not supported.' @@ -258,6 +261,7 @@ bin_check() { done; if (( ${#missing[@]} )); then +# shellcheck disable=SC2140 exit_message \ "Commands not found: ${missing[*]}. Possible bad PATH setting or corrupt "\ "installation." @@ -536,6 +540,7 @@ $'because continuing with the migration could cause further damage to system.' addl_pkg_removes+=("$pkg") done +# shellcheck disable=SC2140 printf '%s\n' '' '' \ "Found the following system packages which map from $PRETTY_NAME to Rocky "\ "Linux 8:" @@ -576,6 +581,7 @@ $'because continuing with the migration could cause further damage to system.' fi done +# shellcheck disable=SC2140 printf '%s\n' '' \ "We will replace the following $PRETTY_NAME packages with their Rocky Linux 8 "\ "equivalents" @@ -586,6 +592,7 @@ $'because continuing with the migration could cause further damage to system.' ) if (( ${#installed_sys_stream_repos_pkgs[@]} )); then +# shellcheck disable=SC2026 printf '%s\n' '' \ 'Also to aid the transition from CentOS Stream the following packages will be '\ 'removed from the rpm database but the included repos will be renamed and '\ @@ -594,6 +601,7 @@ $'because continuing with the migration could cause further damage to system.' fi if (( ${#installed_stream_repos_pkgs[@]} )); then +# shellcheck disable=SC2026 printf '%s\n' '' \ 'Also to aid the transition from CentOS Stream the repos included in the '\ 'following packages will be renamed and retained but disabled:' \ @@ -661,6 +669,7 @@ $'because continuing with the migration could cause further damage to system.' "${enabled_modules[@]}" '' if (( ${#managed_repos[@]} )); then +# shellcheck disable=SC2026 printf '%s\n' '' \ 'In addition, since this system uses subscription-manager the following '\ 'managed repos will be disabled:' \ @@ -687,6 +696,7 @@ usage() { generate_rpm_info() { mkdir /root/convert infomsg "Creating a list of RPMs installed: $1"$'\n' +# shellcheck disable=SC2140 rpm -qa --qf \ "%{NAME}|%{VERSION}|%{RELEASE}|%{INSTALLTIME}|%{VENDOR}|%{BUILDTIME}|"\ "%{BUILDHOST}|%{SOURCERPM}|%{LICENSE}|%{PACKAGER}\n" | @@ -735,7 +745,7 @@ package_swaps() { # Remove the package from the rpm db. saferpm -e --justdb --nodeps -a "${installed_sys_stream_repos_pkgs[@]}" || exit_message \ -"Could not remove packages from the rpm db: ${installed_sys_stream_repos_pkgs[@]}" +"Could not remove packages from the rpm db: ${installed_sys_stream_repos_pkgs[*]}" # Rename the stream repos with a prefix. sed -i 's/^\[/['"$stream_prefix"'/' "${repos_files[@]}" From f28f6daa0e4caa29e5b1052dca6daddc005116d4 Mon Sep 17 00:00:00 2001 From: Peter Ajamian Date: Wed, 24 Nov 2021 00:21:41 +1300 Subject: [PATCH 08/28] Certain Stream packages have to be replaced by their Rocky Linux equivalents. --- migrate2rocky/migrate2rocky.sh | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/migrate2rocky/migrate2rocky.sh b/migrate2rocky/migrate2rocky.sh index 8bfc1bc..e1aa221 100644 --- a/migrate2rocky/migrate2rocky.sh +++ b/migrate2rocky/migrate2rocky.sh @@ -133,9 +133,18 @@ stream_repos_pkgs=( [rocky-repos]=centos-stream-repos [epel-release]=epel-next-release ) + # Prefix to add to CentOS stream repo names when renaming them. stream_prefix=stream- +# Always replace these stream packages with their Rocky Linux equivalents. +stream_always_replace=( + fwupdate\* + grub2-\* + kernel + kernel-\* +) + unset CDPATH exit_message() { @@ -743,13 +752,13 @@ package_swaps() { grep '^/etc/yum\.repos\.d/.\+\.repo$' ) - if (( ${#installed_sys_stream_repos_pkgs[@]} )); then + if (( ${#installed_sys_stream_repos_pkgs[@]} )); then # Remove the package from the rpm db. saferpm -e --justdb --nodeps -a \ "${installed_sys_stream_repos_pkgs[@]}" || exit_message \ "Could not remove packages from the rpm db: ${installed_sys_stream_repos_pkgs[*]}" - fi + fi # Rename the stream repos with a prefix and fix the baseurl. sed -i \ @@ -757,7 +766,7 @@ package_swaps() { -e 's|^mirrorlist=|#mirrorlist=|g' \ -e 's|^#baseurl=http://mirror.centos.org/$contentdir/$stream/|baseurl=|http://mirror.centos.org/centos/8-stream/' \ -e 's|^baseurl=http://vault.centos.org/$contentdir/$stream/|baseurl=|https://vault.centos.org/centos/8-stream/' \ - "${repos_files[@]}" + "${repos_files[@]}" fi # Use dnf shell to swap the system packages out. @@ -905,6 +914,12 @@ EOF errmsg \ $'Failed to disable CentOS Stream repos, please check and disable manually.\n' + if (( ${#stream_always_replace[@]} )) && + [[ $(saferpm -qa "${stream_always_replace[@]}") ]]; then + safednf -y distro-sync "${stream_always_replace[@]}" || + exit_message "Error during distro-sync." + fi + infomsg $'\nCentOS Stream Migration Notes:\n\n' cat < Date: Wed, 24 Nov 2021 04:48:51 +1300 Subject: [PATCH 09/28] Fix syntax error in sed regexes. --- migrate2rocky/migrate2rocky.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/migrate2rocky/migrate2rocky.sh b/migrate2rocky/migrate2rocky.sh index e1aa221..2e4a75a 100644 --- a/migrate2rocky/migrate2rocky.sh +++ b/migrate2rocky/migrate2rocky.sh @@ -763,9 +763,9 @@ package_swaps() { # Rename the stream repos with a prefix and fix the baseurl. sed -i \ -e 's/^\[/['"$stream_prefix"'/' \ - -e 's|^mirrorlist=|#mirrorlist=|g' \ - -e 's|^#baseurl=http://mirror.centos.org/$contentdir/$stream/|baseurl=|http://mirror.centos.org/centos/8-stream/' \ - -e 's|^baseurl=http://vault.centos.org/$contentdir/$stream/|baseurl=|https://vault.centos.org/centos/8-stream/' \ + -e 's|^mirrorlist=|#mirrorlist=|' \ + -e 's|^#baseurl=http://mirror.centos.org/$contentdir/$stream/|baseurl=http://mirror.centos.org/centos/8-stream/|' \ + -e 's|^baseurl=http://vault.centos.org/$contentdir/$stream/|baseurl=https://vault.centos.org/centos/8-stream/|' \ "${repos_files[@]}" fi From 24eb7740e28a6272d3ab971ee26a9156c46eca51 Mon Sep 17 00:00:00 2001 From: Zane Williams Date: Tue, 23 Nov 2021 17:52:40 +0100 Subject: [PATCH 10/28] Re-wording this paragraph after participating in a converstaion about the sync of the mirrors on 15-NOV-2021: (https://chat.rockylinux.org/rocky-linux/pl/x7nembesc78kpr3wpgkszzrygw) Inserting this in a unique commit in case this particular piece is not acceptable. --- mirror/mirrorsync.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/mirror/mirrorsync.sh b/mirror/mirrorsync.sh index ce11bed..31db4c3 100644 --- a/mirror/mirrorsync.sh +++ b/mirror/mirrorsync.sh @@ -35,10 +35,12 @@ # You can change v to q if you do not want detailed logging opts=(-vrlptDSH --exclude="*.~tmp~" --delete-delay --delay-updates) -# Please use a mirror next to you for initial sync -# or if you are hosting a private rather than a pulic mirror. -# Local mirrors might be faster. -# Also we might restrict access to the master in the future. +# Please use a mirror geographically close to you for initial sync, +# or if you are hosting a private mirror(not publicly available). +# +# Note that local mirrors may be faster, and we might restrict +# access to the master in the future. +# # A complete list of mirrors can be found at # https://mirrors.rockylinux.org/mirrormanager/mirrors/Rocky src="msync.rockylinux.org::rocky/mirror/pub/rocky" From 595cc86748916db27abe869087157357164ee884 Mon Sep 17 00:00:00 2001 From: Zane Williams Date: Tue, 23 Nov 2021 17:53:49 +0100 Subject: [PATCH 11/28] Exit cleanly if file list is unchanged at mirrorsync run init. --- mirror/mirrorsync.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mirror/mirrorsync.sh b/mirror/mirrorsync.sh index 31db4c3..55f8a39 100644 --- a/mirror/mirrorsync.sh +++ b/mirror/mirrorsync.sh @@ -55,12 +55,12 @@ lockfile="$0.lockfile" logfile="$0.log" # Check if the filelistfile has changed on upstream mirror -# and exit if it is still the same +# and exit cleanly if it is still the same checkresult=$(rsync --no-motd --dry-run --out-format="%n" "${src}/${filelistfile}" "${dst}/${filelistfile}") if [[ -z "$checkresult" ]]; then printf "%s unchanged. Not updating at %(%c)T\n" "$filelistfile" -1 >> "$logfile" 2>&1 logger -t rsync "Not updating ${mirrormodule}: ${filelistfile} unchanged." - exit 1 + exit 0 fi # Check for existing lockfile to avoid multiple simultaneously running syncs From 1c99b76c8c3c4387630cf563f05c7cfc55ed1fd9 Mon Sep 17 00:00:00 2001 From: Zane Williams Date: Tue, 23 Nov 2021 17:56:47 +0100 Subject: [PATCH 12/28] Add variable for rsync path, in case rsync not in $PATH. --- mirror/mirrorsync.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/mirror/mirrorsync.sh b/mirror/mirrorsync.sh index 55f8a39..782b4bc 100644 --- a/mirror/mirrorsync.sh +++ b/mirror/mirrorsync.sh @@ -31,7 +31,8 @@ # SOFTWARE. # - +# Find rsync in path or use "/usr/bin/rsync". +rsync=$(which rsync || /usr/bin/rsync) # You can change v to q if you do not want detailed logging opts=(-vrlptDSH --exclude="*.~tmp~" --delete-delay --delay-updates) @@ -56,7 +57,7 @@ logfile="$0.log" # Check if the filelistfile has changed on upstream mirror # and exit cleanly if it is still the same -checkresult=$(rsync --no-motd --dry-run --out-format="%n" "${src}/${filelistfile}" "${dst}/${filelistfile}") +checkresult=$(${rsync} --no-motd --dry-run --out-format="%n" "${src}/${filelistfile}" "${dst}/${filelistfile}") if [[ -z "$checkresult" ]]; then printf "%s unchanged. Not updating at %(%c)T\n" "$filelistfile" -1 >> "$logfile" 2>&1 logger -t rsync "Not updating ${mirrormodule}: ${filelistfile} unchanged." @@ -79,7 +80,7 @@ fi printf '%s\n' "$$" > "$lockfile" printf "Started update at %(%c)T\n" -1 >> "$logfile" 2>&1 logger -t rsync "Updating ${mirrormodule}" -rsync "${opts[@]}" "${src}/" "${dst}/" >> "$logfile" 2>&1 +${rsync} "${opts[@]}" "${src}/" "${dst}/" >> "$logfile" 2>&1 logger -t rsync "Finished updating ${mirrormodule}" printf "End: %(%c)T\n" -1 >> "$logfile" 2>&1 rm -f "$lockfile" From 2995c40c8326128484e35fe9ae0f660dd7118a4a Mon Sep 17 00:00:00 2001 From: Peter Ajamian Date: Thu, 25 Nov 2021 00:29:46 +1300 Subject: [PATCH 13/28] Always replace the shim with the one for Rocky Linux. --- migrate2rocky/migrate2rocky.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/migrate2rocky/migrate2rocky.sh b/migrate2rocky/migrate2rocky.sh index 2e4a75a..24e58e9 100644 --- a/migrate2rocky/migrate2rocky.sh +++ b/migrate2rocky/migrate2rocky.sh @@ -141,6 +141,7 @@ stream_prefix=stream- stream_always_replace=( fwupdate\* grub2-\* + shim-\* kernel kernel-\* ) From 540327a1cbaf08188921f60ca3629dfd9d417f4d Mon Sep 17 00:00:00 2001 From: Peter Ajamian Date: Thu, 25 Nov 2021 01:19:09 +1300 Subject: [PATCH 14/28] Use shimx64.efi for EFI boot. We shouldn't be setting EFI boot to grubx64.efi, but rather shimx64.efi instead. This commit also makes sure that both grub2-x64 and shim-x64 are properly installed in an EFI boot system. --- migrate2rocky/migrate2rocky.sh | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/migrate2rocky/migrate2rocky.sh b/migrate2rocky/migrate2rocky.sh index 24e58e9..6e01108 100644 --- a/migrate2rocky/migrate2rocky.sh +++ b/migrate2rocky/migrate2rocky.sh @@ -127,6 +127,10 @@ repo_urls=( [rockyappstream]="https://dl.rockylinux.org/pub/rocky/${SUPPORTED_MAJOR}/AppStream/$ARCH/os/" ) +# These are additional packages that should always be installed. +# (currently blank, but we add to it for an EFI boot system). +always_install=() + # The repos package for CentOS stream requires special handling. declare -g -A stream_repos_pkgs stream_repos_pkgs=( @@ -432,6 +436,13 @@ collect_system_info () { done cd - fi + + # We need to make sure that these packages are always installed in an + # EFI system. + always_install+=( + shim-x64 + grub2-efi-x64 + ) fi # Don't enable these module streams, even if they are enabled in the source @@ -962,6 +973,12 @@ Subscription Management. If no longer desired, you can use behavior. EOF fi + + if (( ${#always_install[@]} )); then + safednf -y install "${always_install[@]}" || exit_message \ + "Error installing required packages: ${always_install[*]}" + fi + if [[ $tmp_sm_ca_dir ]]; then # Check to see if there's Subscription Manager certs which have been # removed @@ -1019,7 +1036,7 @@ fix_efi () ( exit_message "Error updating the grub config." for i in "${!efi_disk[@]}"; do efibootmgr -c -d "/dev/${efi_disk[$i]}" -p "${efi_partition[$i]}" \ - -L "Rocky Linux" -l /EFI/rocky/grubx64.efi || + -L "Rocky Linux" -l /EFI/rocky/shimx64.efi || exit_message "Error updating uEFI firmware." done ) From e560f32668ca715dc76a97826a23b53cd8ead2fe Mon Sep 17 00:00:00 2001 From: Zane Williams Date: Thu, 25 Nov 2021 13:51:45 +0100 Subject: [PATCH 15/28] After further discussion in the Infrastructure channel, the check for rsync in path has been modified to become a rsync_run function instead. It will check to see if rsync exists in $PATH, including a fallback path via the builtin bash "command", and exit accordingly. --- mirror/mirrorsync.sh | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/mirror/mirrorsync.sh b/mirror/mirrorsync.sh index 782b4bc..46238c7 100644 --- a/mirror/mirrorsync.sh +++ b/mirror/mirrorsync.sh @@ -31,8 +31,15 @@ # SOFTWARE. # -# Find rsync in path or use "/usr/bin/rsync". -rsync=$(which rsync || /usr/bin/rsync) +# Find rsync in default path +rsync_run(){ + if command -v rsync >/dev/null; then + command rsync "$@"; + else + command -p rsync "$@"; + fi; +} + # You can change v to q if you do not want detailed logging opts=(-vrlptDSH --exclude="*.~tmp~" --delete-delay --delay-updates) @@ -57,7 +64,7 @@ logfile="$0.log" # Check if the filelistfile has changed on upstream mirror # and exit cleanly if it is still the same -checkresult=$(${rsync} --no-motd --dry-run --out-format="%n" "${src}/${filelistfile}" "${dst}/${filelistfile}") +checkresult=$(rsync_run --no-motd --dry-run --out-format="%n" "${src}/${filelistfile}" "${dst}/${filelistfile}") if [[ -z "$checkresult" ]]; then printf "%s unchanged. Not updating at %(%c)T\n" "$filelistfile" -1 >> "$logfile" 2>&1 logger -t rsync "Not updating ${mirrormodule}: ${filelistfile} unchanged." @@ -80,7 +87,7 @@ fi printf '%s\n' "$$" > "$lockfile" printf "Started update at %(%c)T\n" -1 >> "$logfile" 2>&1 logger -t rsync "Updating ${mirrormodule}" -${rsync} "${opts[@]}" "${src}/" "${dst}/" >> "$logfile" 2>&1 +rsync_run "${opts[@]}" "${src}/" "${dst}/" >> "$logfile" 2>&1 logger -t rsync "Finished updating ${mirrormodule}" printf "End: %(%c)T\n" -1 >> "$logfile" 2>&1 rm -f "$lockfile" From c039add1329ec1d239157ff06210d58379587a0f Mon Sep 17 00:00:00 2001 From: Peter Ajamian Date: Thu, 9 Dec 2021 15:40:20 +1300 Subject: [PATCH 16/28] List disk space requirements in README file (#134) * Add disk space requirements to README. * Fix markdown table. * Additional table fixes * Give up on table. --- migrate2rocky/README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/migrate2rocky/README.md b/migrate2rocky/README.md index 25530d7..bc67b06 100644 --- a/migrate2rocky/README.md +++ b/migrate2rocky/README.md @@ -14,6 +14,18 @@ Running this script will convert an existing CentOS 8 system to Rocky Linux 8. [!! USE WITH CAUTION !!] ``` +### Disk Space Requirements + +Please note the following disk space requirements. These requirements may vary +from one system to another. Failure to have adequate disk space available may +result in migrate2rocky leaving the system in an unstable state: + +``` +/usr 250M +/var 1.5G +/boot 50M +``` + ### Known Issues #### Custom replacements of default repositories From da2de96c9c1332f8ec97ba4103a5fb3660673473 Mon Sep 17 00:00:00 2001 From: Peter Ajamian Date: Sat, 18 Dec 2021 17:12:11 +1300 Subject: [PATCH 17/28] Add log rotation. --- migrate2rocky/migrate2rocky.sh | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/migrate2rocky/migrate2rocky.sh b/migrate2rocky/migrate2rocky.sh index 6e01108..3b139ab 100644 --- a/migrate2rocky/migrate2rocky.sh +++ b/migrate2rocky/migrate2rocky.sh @@ -55,6 +55,26 @@ fi # Path to logfile logfile=/var/log/migrate2rocky.log +# Rotate old logs +numlogs=5 +if [[ -e $logfile ]]; then + # Here we use mv before bin_check, so simply check the exit status to see if + # it worked. + if ! mv -f "$logfile" "$logfile.0"; then + printf '%s\n' "Unable to rotate logfiles, continuing without rotation." >&2 + else + for ((i=numlogs;i>0;i--)); do + if [[ -e "$logfile.$((i-1))" ]]; then + if ! mv -f "$logfile.$((i-1))" "$logfile.$i"; then + printf '%s\n' \ +"Unable to rotate logfiles, continuing without rotation." + break + fi + fi + done + fi +fi + # Send all output to the logfile as well as stdout. # After the following we get: # Output to 1 goes to stdout and the logfile. @@ -62,7 +82,7 @@ logfile=/var/log/migrate2rocky.log # Output to 3 just goes to stdout. # Output to 4 just goes to stderr. # Output to 5 just goes to the logfile. -truncate -s0 "$logfile" + # shellcheck disable=SC2094 exec \ 3>&1 \ @@ -106,6 +126,8 @@ errmsg () { printf '%s%s%s' "$errcolor" "$msg" "$nocolor" >&4 } +infomsg 'migrate2rocky - Begin logging at %(%c)T.\n\n' -1 + export LC_ALL=C.UTF-8 unset LANGUAGE shopt -s nullglob From a208a1ce0bcafe3411321d6db4d7906197bfbecc Mon Sep 17 00:00:00 2001 From: Peter Ajamian Date: Sun, 19 Dec 2021 01:53:55 +1300 Subject: [PATCH 18/28] Check available disk space before starting migration. (#136) * Add disk space check --- migrate2rocky/migrate2rocky.sh | 42 +++++++++++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/migrate2rocky/migrate2rocky.sh b/migrate2rocky/migrate2rocky.sh index 3b139ab..0aaf534 100644 --- a/migrate2rocky/migrate2rocky.sh +++ b/migrate2rocky/migrate2rocky.sh @@ -172,6 +172,14 @@ stream_always_replace=( kernel-\* ) +# Directory to required space in MiB +declare -A dir_space_map +dir_space_map=( + [/usr]=250 + [/var]=1536 + [/boot]=50 +) + unset CDPATH exit_message() { @@ -251,6 +259,38 @@ pre_check () { 'Migration from Uyuni/SUSE Manager-modified systems is not supported by '\ 'migrate2rocky. See the README file for details.' fi + + # Get available space to compare to requirements. + # If the stock kernel is not installed we don't require space in /boot + if ! rpm -q --quiet kernel; then + dir_space_map[/boot]=0 + fi + local -a errs dirs=("${!dir_space_map[@]}") + local dir mount avail i=0 + local -A mount_avail_map dir_mount_map mount_space_map + while read -r mount avail; do + if [[ $mount == 'Filesystem' ]]; then + continue + fi + + dir=${dirs[$((i++))]} + + dir_mount_map[$dir]=$mount + mount_avail_map[$mount]=${avail%M} + (( mount_space_map[$mount]+=dir_space_map[$dir] )) + done < <(df -BM --output=source,avail "${dirs[@]}") + + for mount in "${!mount_space_map[@]}"; do + (( avail = mount_avail_map[$mount]*95/100 )) + if (( avail < mount_space_map[$mount] )); then + errs+=("Not enough space in $mount, ${mount_space_map[$mount]}M required, ${avail}M available.") + fi + done + + if (( ${#errs[@]} )); then + IFS=$'\n' + exit_message "${errs[*]}" + fi } # All of the binaries used by this script are available in a EL8 minimal install @@ -268,7 +308,7 @@ bin_check() { local -a missing bins bins=( - rpm dnf awk column tee tput mkdir cat arch sort uniq rmdir + rpm dnf awk column tee tput mkdir cat arch sort uniq rmdir df rm head curl sha512sum mktemp systemd-detect-virt sed grep ) if [[ $update_efi ]]; then From 68c77adbb3aeb7745be4629d4446352596cf49bc Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 20 Dec 2021 21:06:55 +0100 Subject: [PATCH 19/28] fix migrate2rocky script for aarch64 systems by using correct suffix for grub2-efi* and shim* packages (#138) * fix migrate2rocky script for aarch64 systems by using correct suffix for grub2-efi* and shim* packages * wrap value passed to -l option of efibootmgr command in quotes --- migrate2rocky/migrate2rocky.sh | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/migrate2rocky/migrate2rocky.sh b/migrate2rocky/migrate2rocky.sh index 0aaf534..39f95bd 100644 --- a/migrate2rocky/migrate2rocky.sh +++ b/migrate2rocky/migrate2rocky.sh @@ -160,6 +160,14 @@ stream_repos_pkgs=( [epel-release]=epel-next-release ) +# Map for package name suffix for shim/grub2-efi +# on x86_64: grub2-efi-x64, shim-x64 +# on aarch64: grub2-efi-aa64, shim-aa64 +declare -A cpu_arch_suffix_map=( + [x86_64]=x64 + [aarch64]=aa64 +) + # Prefix to add to CentOS stream repo names when renaming them. stream_prefix=stream- @@ -502,8 +510,8 @@ collect_system_info () { # We need to make sure that these packages are always installed in an # EFI system. always_install+=( - shim-x64 - grub2-efi-x64 + "shim-${cpu_arch_suffix_map[$ARCH]}" + "grub2-efi-${cpu_arch_suffix_map[$ARCH]}" ) fi @@ -1098,7 +1106,7 @@ fix_efi () ( exit_message "Error updating the grub config." for i in "${!efi_disk[@]}"; do efibootmgr -c -d "/dev/${efi_disk[$i]}" -p "${efi_partition[$i]}" \ - -L "Rocky Linux" -l /EFI/rocky/shimx64.efi || + -L "Rocky Linux" -l "/EFI/rocky/shim${cpu_arch_suffix_map[$ARCH]}.efi" || exit_message "Error updating uEFI firmware." done ) From d3d4896e39b745259eddaf3ca3d5a102d761d366 Mon Sep 17 00:00:00 2001 From: Todd Zullinger Date: Sat, 1 Jan 2022 00:51:07 -0500 Subject: [PATCH 20/28] migrate2rocky minor improvements (#140) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * migrate2rocky: use $convert_info_dir consistently We set convert_info_dir=/root/convert. Use it consistently. * migrate2rocky: avoid mkdir error with -rV options Both -r and -V call `generate_rpm_info()` which run `mkdir` to create the convert directory. The second call results in a warning: mkdir: cannot create directory ‘/root/convert’: File exists Use `mkdir -p` to avoid the spurious warning when the convert directory exists. * migrate2rocky: fix listing of rpm info files When the conversion completes, the script prints: You may review the following files: However, there are no files displayed because the find pattern does not match what `generate_rpm_info()` writes. It looks for '*-rpms-*' while the files are named '*-rpm-list-*' instead. Fix the file pattern used. While here, replace the use of `find` with a simpler `printf` to achieve the same result. --- migrate2rocky/migrate2rocky.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/migrate2rocky/migrate2rocky.sh b/migrate2rocky/migrate2rocky.sh index 39f95bd..1e05fef 100644 --- a/migrate2rocky/migrate2rocky.sh +++ b/migrate2rocky/migrate2rocky.sh @@ -785,7 +785,7 @@ usage() { } >&2 generate_rpm_info() { - mkdir /root/convert + mkdir -p "$convert_info_dir" infomsg "Creating a list of RPMs installed: $1"$'\n' # shellcheck disable=SC2140 rpm -qa --qf \ @@ -1185,7 +1185,7 @@ fi if [[ $verify_all_rpms && $convert_to_rocky ]]; then generate_rpm_info finish infomsg $'You may review the following files:\n' - find /root/convert -type f -name "$HOSTNAME-rpms-*.log" + printf '%s\n' "$convert_info_dir/$HOSTNAME-rpm-list-"*.log fi if [[ $update_efi && $convert_to_rocky ]]; then From 3c32595b23026f5c3046384fb2751d958d07160a Mon Sep 17 00:00:00 2001 From: Amin Vakil Date: Mon, 10 Jan 2022 17:31:50 +0330 Subject: [PATCH 21/28] Add github actions shellcheck --- .github/workflows/shellcheck.yml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 .github/workflows/shellcheck.yml diff --git a/.github/workflows/shellcheck.yml b/.github/workflows/shellcheck.yml new file mode 100644 index 0000000..6af9401 --- /dev/null +++ b/.github/workflows/shellcheck.yml @@ -0,0 +1,22 @@ +--- +name: shellcheck +'on': + push: + pull_request: + schedule: + - cron: '0 1 * * *' + +jobs: + + shellcheck: + name: shellcheck + runs-on: ubuntu-latest + + steps: + - name: Check out the codebase. + uses: actions/checkout@v2 + + - name: Shellcheck! + run: | + shellcheck migrate2rocky/migrate2rocky.sh + shellcheck mirror/mirrorsync.sh From d2b2b61afc1b6b8ccafe566b1a990e94c140cb5b Mon Sep 17 00:00:00 2001 From: Amin Vakil Date: Tue, 11 Jan 2022 12:26:51 +0330 Subject: [PATCH 22/28] Fix shellcheck issues (#146) * Quote to prevent splitting * Remove dir_mount_map variable * Define efi_disk and efi_partition as array * Expanding an array without an index only gives the first element * Ignore shellchecking /etc/os-release * Expressions do not need to expand --- migrate2rocky/migrate2rocky.sh | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/migrate2rocky/migrate2rocky.sh b/migrate2rocky/migrate2rocky.sh index 1e05fef..a7c4689 100644 --- a/migrate2rocky/migrate2rocky.sh +++ b/migrate2rocky/migrate2rocky.sh @@ -212,6 +212,7 @@ logmessage(){ # This just grabs a field from os-release and returns it. os-release () ( + # shellcheck source=/dev/null . /etc/os-release if ! [[ ${!1} ]]; then return 1 @@ -275,7 +276,7 @@ pre_check () { fi local -a errs dirs=("${!dir_space_map[@]}") local dir mount avail i=0 - local -A mount_avail_map dir_mount_map mount_space_map + local -A mount_avail_map mount_space_map while read -r mount avail; do if [[ $mount == 'Filesystem' ]]; then continue @@ -283,7 +284,6 @@ pre_check () { dir=${dirs[$((i++))]} - dir_mount_map[$dir]=$mount mount_avail_map[$mount]=${avail%M} (( mount_space_map[$mount]+=dir_space_map[$dir] )) done < <(df -BM --output=source,avail "${dirs[@]}") @@ -480,10 +480,10 @@ collect_system_info () { --noheadings) || exit_message "Can't find EFI mount. No EFI boot detected." kname=$(lsblk -dno kname "$efi_mount") - efi_disk=$(lsblk -dno pkname "/dev/$kname") + efi_disk=("$(lsblk -dno pkname "/dev/$kname")") - if [[ $efi_disk ]]; then - efi_partition=$(<"/sys/block/$efi_disk/$kname/partition") + if [[ ${efi_disk[0]} ]]; then + efi_partition=("$(<"/sys/block/${efi_disk[0]}/$kname/partition")") else # This is likely an md-raid or other type of virtual disk, we need # to dig a little deeper to find the actual physical disks and @@ -665,10 +665,10 @@ $'because continuing with the migration could cause further damage to system.' then # System package that needs to be swapped / disabled installed_pkg_map[$p]= - installed_sys_stream_repos_pkgs+=( ${stream_repos_pkgs[$p]} ) + installed_sys_stream_repos_pkgs+=( "${stream_repos_pkgs[$p]}" ) elif rpm --quiet -q "${stream_repos_pkgs[$p]}"; then # Non-system package, repos just need to be disabled. - installed_stream_repos_pkgs+=( ${stream_repos_pkgs[$p]} ) + installed_stream_repos_pkgs+=( "${stream_repos_pkgs[$p]}" ) fi done @@ -735,7 +735,7 @@ $'because continuing with the migration could cause further damage to system.' mod=${mod/$gl/$repl} done if [[ $mod != "${enabled_modules[$i]}" ]]; then - disable_modules+=(${enabled_modules[$i]}) + disable_modules+=("${enabled_modules[$i]}") enabled_modules[$i]=$mod fi done @@ -843,6 +843,7 @@ package_swaps() { fi # Rename the stream repos with a prefix and fix the baseurl. + # shellcheck disable=SC2016 sed -i \ -e 's/^\[/['"$stream_prefix"'/' \ -e 's|^mirrorlist=|#mirrorlist=|' \ From c770ec4191799eaa74f11a7a8c9f8293cc245be5 Mon Sep 17 00:00:00 2001 From: Amin Vakil Date: Tue, 11 Jan 2022 14:13:50 +0330 Subject: [PATCH 23/28] Install shellcheck from github releases (#147) * Install shellcheck from github releases --- .github/workflows/shellcheck.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/shellcheck.yml b/.github/workflows/shellcheck.yml index 6af9401..b9bfc12 100644 --- a/.github/workflows/shellcheck.yml +++ b/.github/workflows/shellcheck.yml @@ -16,6 +16,16 @@ jobs: - name: Check out the codebase. uses: actions/checkout@v2 + - name: Install shellcheck from GitHub releases + run: | + sudo curl -L https://github.com/koalaman/shellcheck/releases/download/v${SHELLCHECK_VERSION}/shellcheck-v${SHELLCHECK_VERSION}.$(uname -s).$(uname -m).tar.xz -o shellcheck.tar.xz + sudo tar -xvf shellcheck.tar.xz + sudo mv shellcheck-v${SHELLCHECK_VERSION}/shellcheck /usr/local/bin/shellcheck + sudo chmod 755 /usr/local/bin/shellcheck + shellcheck --version + env: + SHELLCHECK_VERSION: 0.7.2 + - name: Shellcheck! run: | shellcheck migrate2rocky/migrate2rocky.sh From 967ae48ef0d0aa5e781d5d6dee511b832cc3635c Mon Sep 17 00:00:00 2001 From: gah242s Date: Tue, 25 Jan 2022 08:45:32 -0500 Subject: [PATCH 24/28] Add recommendation for tmux or screen session Here's a potential start to a blurb about running the script in a tmux or screen session. Please feel free to change up, reword, scrap the whole thing and start afresh, etc. --- migrate2rocky/README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/migrate2rocky/README.md b/migrate2rocky/README.md index bc67b06..cd17ab9 100644 --- a/migrate2rocky/README.md +++ b/migrate2rocky/README.md @@ -26,6 +26,10 @@ result in migrate2rocky leaving the system in an unstable state: /boot 50M ``` +### Recommended Practice + +When running this script, especially via a remote session, it is highly recommended to enter a screen or tmux session before running. If a standard ssh or terminal session, such as the Cockpit Terminal window, is disrupted, the script will die and leave the system in a potential unrecoverable state. For more on tmux sessions, please see: https://github.com/tmux/tmux/wiki + ### Known Issues #### Custom replacements of default repositories From ac17f28e0ef110102bc86f3e00d57bb3ed998031 Mon Sep 17 00:00:00 2001 From: gah242s Date: Tue, 25 Jan 2022 11:55:50 -0500 Subject: [PATCH 25/28] Add Cockpit note to Known Issues I've added the blurb to the top of the known issues, since it's something that needs to be addressed before any of the other issues I saw listed. Again, feel free to do what's necessary to make it viable. --- migrate2rocky/README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/migrate2rocky/README.md b/migrate2rocky/README.md index cd17ab9..3cfdef9 100644 --- a/migrate2rocky/README.md +++ b/migrate2rocky/README.md @@ -32,6 +32,10 @@ When running this script, especially via a remote session, it is highly recommen ### Known Issues +#### Running the script in Cockpit's Terminal Screen will be interrupted + +Do not run this script through the Terminal screen built into Cockpit. As the script runs the upgrades, Cockpit will be restarted and Terminal connection will disconnect, thus stopping the script and leaving the system in an unrecoverable state. It may be possible to launch a screen or tmux session from the Cockpit Terminal, but USE AT YOUR OWN RISK. + #### Custom replacements of default repositories This script expects the **original repository configuration being present, as well From 9703480c4dddc7e6268da0a326103d9a7de3dbd0 Mon Sep 17 00:00:00 2001 From: Peter Ajamian Date: Tue, 1 Feb 2022 23:54:27 +1300 Subject: [PATCH 26/28] Vault (#154) * Add devel repo to list of repositories to migrate. * Use baseurl from our CentOS vault mirror. CentOS killed off all the CentOS 8 repos and moved them to vault. As a result migrate2rocky was no longer working to migrate CentOS 8 installs. To fix this we've mirrored the CentOS 8 vault and point migrate2rocky to that instead. --- migrate2rocky/migrate2rocky.sh | 144 +++++++++++++++++++++++++++++---- 1 file changed, 130 insertions(+), 14 deletions(-) diff --git a/migrate2rocky/migrate2rocky.sh b/migrate2rocky/migrate2rocky.sh index a7c4689..8589d24 100644 --- a/migrate2rocky/migrate2rocky.sh +++ b/migrate2rocky/migrate2rocky.sh @@ -358,7 +358,8 @@ bin_check() { repoquery () { local name val prev result result=$( - dnf -y -q --setopt=epel.excludepkgs=epel-release repoquery -i "$1" || + safednf -y -q "${dist_repourl_swaps[@]}" \ + --setopt=epel.excludepkgs=epel-release repoquery -i "$1" || exit_message "Failed to fetch info for package $1." ) if ! [[ $result ]]; then @@ -378,10 +379,11 @@ repoquery () { # This function will overwrite the repoinfo_results associative array with the # info for the resulting repository. -repoinfo () { +_repoinfo () { local name val result - result=$(dnf -y -q repoinfo "$1") || - exit_message "Failed to fetch info for repository $1." + result=$( + safednf -y -q --repo="$1" "${dist_repourl_swaps[@]}" repoinfo "$1" + ) || return if [[ $result == 'Total packages: 0' ]]; then # We didn't match this repo. return 1 @@ -399,6 +401,11 @@ repoinfo () { fi done <<<"$result" + # Set the enabled state + if [[ ! ${enabled_repo_check[$1]} ]]; then + repoinfo_results[Repo-status]=disabled + fi + # dnf repoinfo doesn't return the gpgkey, but we need that so we have to get # it from the repo file itself. # "end_of_file" is a hack here. Since it is not a valid dnf setting we know @@ -428,18 +435,45 @@ repoinfo () { ) } +# We now store the repoinfo results in a cache. +declare -g -A repoinfo_results_cache=() +repoinfo () { + if [[ ! ${repoinfo_results_cache[$1]} ]]; then + _repoinfo "$@" || return + repoinfo_results_cache[$1]=1 + for k in "${!repoinfo_results[@]}"; do + repoinfo_results_cache[$1:$k]=${repoinfo_results[$k]} + done + else + repoinfo_results=() + for k in "${!repoinfo_results_cache[@]}"; do + local repo=${k%%:*} key=${k#*:} + if [[ $repo != "$1" ]]; then + continue + fi + + repoinfo_results[$key]=${repoinfo_results_cache[$k]} + done + fi +} + provides_pkg () ( if [[ ! $1 ]]; then return 0 fi set -o pipefail - provides=$(dnf -y -q provides "$1" | awk '{print $1; nextfile}') || + provides=$( + safednf -y -q "${dist_repourl_swaps[@]}" provides "$1" | + awk '{print $1; nextfile}' + ) || return 1 set +o pipefail pkg=$(rpm -q --queryformat '%{NAME}\n' "$provides") || - pkg=$(dnf -y -q repoquery --queryformat '%{NAME}\n' "$provides") || - exit_message "Can't get package name for $provides." + pkg=$( + safednf -y -q "${dist_repourl_swaps[@]}" repoquery \ + --queryformat '%{NAME}\n' "$provides" + ) || exit_message "Can't get package name for $provides." printf '%s\n' "$pkg" ) @@ -467,6 +501,24 @@ safednf () ( dnf "${args[@]}" ) +# +# Three ways we check the repourl. If dnf repoinfo fails then we assume the URL +# is bad. A missing URL is also considered bad. Lastly we check to see if we +# can fetch the repomd.xml file from the repository, and if not then the repourl +# is considered bad. In any of these cases we'll end up replacing the repourl +# with a good one from our mirror of CentOS vault. +# +check_repourl () { + repoinfo "$1" || return + if [[ ! ${repoinfo_results[Repo-baseurl]} ]]; then + return 1 + fi + + curl -sfLI "${repoinfo_results[Repo-baseurl]%% *}repodata/repomd.xml" \ + > /dev/null + return +} + collect_system_info () { # Dump the DNF cache first so we start with a clean slate. infomsg $'\nRemoving dnf cache\n' @@ -543,13 +595,71 @@ collect_system_info () { [ha]=pacemaker-doc.noarch [powertools]=libaec-devel.$ARCH [extras]=epel-release.noarch + [devel]=quota-devel.$ARCH ) -# [devel]=quota-devel.$ARCH + + dist_id=$(os-release ID) + # We need a different dist ID for CentOS Linux vs CentOS Stream + if [[ $dist_id == centos ]] && rpm --quiet -q centos-stream-release; then + dist_id+=-stream + fi PRETTY_NAME=$(os-release PRETTY_NAME) infomsg '%s' \ - "Preparing to migrate $PRETTY_NAME to Rocky Linux 8."$'\n\n' \ - "Determining repository names for $PRETTY_NAME" + "Preparing to migrate $PRETTY_NAME to Rocky Linux 8."$'\n\n' + + # Check to see if we need to change the repourl on any system repositories + # (CentOS 8) + local -A dist_repourl_map + dist_repourl_map=( + [centos:baseos]=https://dl.rockylinux.org/vault/centos/8.5.2111/BaseOS/$ARCH/os/ + [centos:appstream]=https://dl.rockylinux.org/vault/centos/8.5.2111/AppStream/$ARCH/os/ + [centos:ha]=https://dl.rockylinux.org/vault/centos/8.5.2111/HighAvailability/$ARCH/os/ + [centos:powertools]=https://dl.rockylinux.org/vault/centos/8.5.2111/PowerTools/$ARCH/os/ + [centos:extras]=https://dl.rockylinux.org/vault/centos/8.5.2111/extras/$ARCH/os/ + [centos:devel]=https://dl.rockylinux.org/vault/centos/8.5.2111/Devel/$ARCH/os/ + ) + + # In case migration is attempted from very old CentOS (before the repository + # names were lowercased) + for name in BaseOS AppStream PowerTools Devel; do + dist_repourl_map["centos:$name"]=${dist_repourl_map["centos:${name,,}"]} + done + + # HighAvailability is different again + dist_repourl_map[centos:HighAvailability]=${dist_repourl_map[centos:ha]} + + # We need a list of enabled repositories + local -a enabled_repos=() + declare -g -A enabled_repo_check=() + declare -g -a dist_repourl_swaps=() + readarray -s 1 -t enabled_repos < <(dnf -q -y repolist --enabled) + for r in "${enabled_repos[@]}"; do + enabled_repo_check[${r%% *}]=1 + done + + + # ...and finally set a number of dnf options to replace the baseurl of these + # repos + for k in "${!dist_repourl_map[@]}"; do + local d=${k%%:*} r=${k#*:} + if [[ $d != "$dist_id" || ! ${enabled_repo_check[$r]} ]] || + check_repourl "$r"; then + continue + fi + + dist_repourl_swaps+=( + "--setopt=$r.mirrorlist=" + "--setopt=$r.metalink=" + "--setopt=$r.baseurl=" + "--setopt=$r.baseurl=${dist_repourl_map[$k]}" + ) + + infomsg 'Baseurl for %s is invalid, setting to %s.\n' \ + "$r" "${dist_repourl_map[$k]}" + done + + infomsg '%s' "Determining repository names for $PRETTY_NAME" for r in "${!pkg_repo_map[@]}"; do printf '.' @@ -578,7 +688,9 @@ collect_system_info () { # system-release here is a bit of a hack, but it ensures that the # rocky-repos package will get installed. for r in "${!repo_map[@]}"; do - repoinfo "${repo_map[$r]}" + repoinfo "${repo_map[$r]}" || + exit_message "Failed to fetch info for repository ${repo_map[$r]}." + if [[ $r == "baseos" ]]; then local baseos_filename=system-release if [[ ! ${repoinfo_results[Repo-managed]} ]]; then @@ -592,7 +704,9 @@ collect_system_info () { done # First get info for the baseos repo - repoinfo "${repo_map[baseos]}" + repoinfo "${repo_map[baseos]}" || + exit_message "Failed to fetch info for repository ${repo_map[baseos]}." + declare -g -A pkg_map provides_pkg_map declare -g -a addl_provide_removes addl_pkg_removes provides_pkg_map=( @@ -717,7 +831,8 @@ $'because continuing with the migration could cause further damage to system.' # Get a list of system enabled modules. readarray -t enabled_modules < <( set -e -o pipefail - safednf -y -q "${repo_map[@]/#/--repo=}" module list --enabled | + safednf -y -q "${repo_map[@]/#/--repo=}" "${dist_repourl_swaps[@]}" \ + module list --enabled | awk ' $1 == "@modulefailsafe", /^$/ {next} $1 == "Name", /^$/ {if ($1!="Name" && !/^$/) print $1":"$2} @@ -800,7 +915,7 @@ generate_rpm_info() { # Run a dnf update before the actual migration. pre_update() { infomsg '%s\n' "Running dnf update before we attempt the migration." - dnf -y update || exit_message \ + safednf -y "${dist_repourl_swaps[@]}" update || exit_message \ $'Error running pre-update. Stopping now to avoid putting the system in an\n'\ $'unstable state. Please correct the issues shown here and try again.' } @@ -854,6 +969,7 @@ package_swaps() { # Use dnf shell to swap the system packages out. safednf -y shell --disablerepo=\* --noautoremove \ + "${dist_repourl_swaps[@]}" \ --setopt=protected_packages= --setopt=keepcache=True \ "${dnfparameters[@]}" \ < Date: Wed, 2 Feb 2022 08:06:24 +1300 Subject: [PATCH 27/28] Add notes about migrating from EL8.0. Formatting. * Add notes about the need to run `dnf update` before migrating from EL8.0. * Add instructions to manually fix the repository URLs when migrating from CentOS 8.0 * Wrap long lines from earlier commit. --- migrate2rocky/README.md | 45 +++++++++++++++++++++++++++++++---------- 1 file changed, 34 insertions(+), 11 deletions(-) diff --git a/migrate2rocky/README.md b/migrate2rocky/README.md index 3cfdef9..4ff5168 100644 --- a/migrate2rocky/README.md +++ b/migrate2rocky/README.md @@ -28,30 +28,53 @@ result in migrate2rocky leaving the system in an unstable state: ### Recommended Practice -When running this script, especially via a remote session, it is highly recommended to enter a screen or tmux session before running. If a standard ssh or terminal session, such as the Cockpit Terminal window, is disrupted, the script will die and leave the system in a potential unrecoverable state. For more on tmux sessions, please see: https://github.com/tmux/tmux/wiki +When running this script, especially via a remote session, it is highly +recommended to enter a screen or tmux session before running. If a standard +ssh or terminal session, such as the Cockpit Terminal window, is disrupted, the +script will die and leave the system in a potential unrecoverable state. For +more on tmux sessions, please see: https://github.com/tmux/tmux/wiki ### Known Issues #### Running the script in Cockpit's Terminal Screen will be interrupted -Do not run this script through the Terminal screen built into Cockpit. As the script runs the upgrades, Cockpit will be restarted and Terminal connection will disconnect, thus stopping the script and leaving the system in an unrecoverable state. It may be possible to launch a screen or tmux session from the Cockpit Terminal, but USE AT YOUR OWN RISK. +Do not run this script through the Terminal screen built into Cockpit. As the +script runs the upgrades, Cockpit will be restarted and Terminal connection will +disconnect, thus stopping the script and leaving the system in an unrecoverable +state. It may be possible to launch a screen or tmux session from the Cockpit +Terminal, but USE AT YOUR OWN RISK. + +#### EL8.0 Migrations + +If you are attempting to migrate a system that has not been updated since 8.0 +then you must run `dnf update` before attempting the migration. + +If you are migrating from CentOS 8.0 then you must manually fix the baseurls of +the CentOS repositories before running `dnf update`: +``` +sed -i -r \ + -e 's!^mirrorlist=!#mirrorlist=!' \ + -e 's!^#?baseurl=http://(mirror|vault).centos.org/\$contentdir/\$releasever/!baseurl=https://dl.rockylinux.org/vault/centos/8.5.2111/!i' \ + /etc/yum.repos.d/CentOS-*.repo +``` #### Custom replacements of default repositories -This script expects the **original repository configuration being present, as well -as enabled** (i.e. for CentOS the `baseos` repo configuration in the +This script expects the **original repository configuration being present, as +well as enabled** (i.e. for CentOS the `baseos` repo configuration in the `/etc/yum.repos.d/CentOS-Linux-BaseOS.repo` file has to be present and enabled). -Also make sure that there are **no other repositories** which could interfere with the -original configuration. +Also make sure that there are **no other repositories** which could interfere +with the original configuration. Any distribution that has had its core repositories altered, removed, duplicated or overridden may cause migrate2rocky to break or corrupt the system when run. -Any attempt to migrate such systems, even after reversing the changes made by such -software, is not supported in any way. In all cases you should backup your system -before using migrate2rocky and USE AT YOUR OWN RISK. +Any attempt to migrate such systems, even after reversing the changes made by +such software, is not supported in any way. In all cases you should backup your +system before using migrate2rocky and USE AT YOUR OWN RISK. -This especially happens on systems configured with a centralized package management -like Katello (RedHat Satellite 6) or Uyuni (RedHat Satellite 5, SUSE Manager). +This especially happens on systems configured with a centralized package +management like Katello (RedHat Satellite 6) or Uyuni (RedHat Satellite 5, SUSE +Manager). #### RHEL migrations show error messages during conversion From f3fe6d768a6f2f36bcb4a390e17a4f31eff2e499 Mon Sep 17 00:00:00 2001 From: Peter Ajamian Date: Fri, 11 Feb 2022 14:46:51 +1300 Subject: [PATCH 28/28] Fix issue where we didn't exit when we were supposed to. Exiting from a subshell doesn't actually exit the script. Move the exit to outside of the subshell. --- migrate2rocky/migrate2rocky.sh | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/migrate2rocky/migrate2rocky.sh b/migrate2rocky/migrate2rocky.sh index 8589d24..1618fba 100644 --- a/migrate2rocky/migrate2rocky.sh +++ b/migrate2rocky/migrate2rocky.sh @@ -357,11 +357,9 @@ bin_check() { # as a special-case below to avoid having the extras repository map to epel. repoquery () { local name val prev result - result=$( - safednf -y -q "${dist_repourl_swaps[@]}" \ - --setopt=epel.excludepkgs=epel-release repoquery -i "$1" || - exit_message "Failed to fetch info for package $1." - ) + result=$(safednf -y -q "${dist_repourl_swaps[@]}" \ + --setopt=epel.excludepkgs=epel-release repoquery -i "$1") || + exit_message "Failed to fetch info for package $1." if ! [[ $result ]]; then # We didn't match this package, the repo could be disabled. return 1