From 76ea9657cc6f4c16aa8094b18306249d3a6a41b4 Mon Sep 17 00:00:00 2001 From: Louis Abel Date: Wed, 17 May 2023 14:51:12 -0700 Subject: [PATCH] make sync scripts friendly for next push --- sync/beta-release-sync-to-staging.sh | 57 ++++++--------------------- sync/common | 24 ++++++++++- sync/common_8 | 2 +- sync/lh-release-sync-to-staging.sh | 57 ++++++--------------------- sync/minor-release-sync-to-staging.sh | 10 +++++ sync/sync-to-prod.sh | 2 +- sync/sync-to-staging.sh | 23 ++++++++++- 7 files changed, 79 insertions(+), 96 deletions(-) diff --git a/sync/beta-release-sync-to-staging.sh b/sync/beta-release-sync-to-staging.sh index 08c429e..4034863 100644 --- a/sync/beta-release-sync-to-staging.sh +++ b/sync/beta-release-sync-to-staging.sh @@ -14,7 +14,8 @@ MAJ=${RLVER} # sync all pieces of a release, including extras, nfv, etc for COMPOSE in "${NONSIG_COMPOSE[@]}"; do echo "${COMPOSE}: Syncing" - pushd "${RELEASE_COMPOSE_ROOT}/compose" || { echo "${COMPOSE}: Failed to change directory"; break; } + SYNCSRC="/mnt/compose/${MAJ}-BETA/latest-${COMPOSE}-${MAJ}" + pushd "${SYNCSRC}/compose" || { echo "${COMPOSE}: Failed to change directory"; break; } if [[ "${COMPOSE}" == "Rocky" ]]; then # ISO Work before syncing @@ -40,30 +41,20 @@ for COMPOSE in "${NONSIG_COMPOSE[@]}"; do done popd || { echo "Could not change directory"; break; } done + mkdir -p live/x86_64 + ln -s live Live fi + popd || { echo "${COMPOSE}: Failed to change directory"; break; } TARGET="${STAGING_ROOT}/${CATEGORY_STUB}/${REV}" mkdir -p "${TARGET}" - # disabling because none of our files should be starting with dashes. If they - # are something is *seriously* wrong here. - # shellcheck disable=SC2035 - sudo -l && find **/* -maxdepth 0 -type d | parallel --will-cite -j 18 sudo rsync -av --chown=10004:10005 --progress --relative --human-readable \ - {} "${TARGET}" - - if [[ "${COMPOSE}" == "Rocky" ]]; then - cp COMPOSE_ID "${TARGET}" - chown 10004:10005 "${TARGET}/COMPOSE_ID" - rsync -av --chown=10004:10005 --progress --relative --human-readable metadata "${TARGET}" + pushd "${SYNCSRC}" || { echo "${COMPOSE}: Failed to change directory"; break; } + if [[ "${COMPOSE}" != "Rocky" ]]; then + rsync_no_delete_staging_with_excludes "${TARGET}" "metadata" + else + rsync_no_delete_staging "${TARGET}" fi - - # Return back to where we started - popd || { echo "${COMPOSE}: Failed to change back"; break; } - - # Create extra stuff -# pushd "${TARGET}" || { echo "${COMPOSE}: Failed to change directory"; break; } -# mkdir -p Live/x86_64 -# ln -s Live live -# popd || { echo "${COMPOSE}: Failed to change back"; break; } + popd || { echo "${COMPOSE}: Failed to change directory"; break; } done @@ -73,36 +64,10 @@ for LINK in "${LINK_REPOS[@]}"; do "${STAGING_ROOT}/${CATEGORY_STUB}/${REV}/${LINK_REPOS[$LINK]}" done -# make a kickstart directory -#for ARCH in "${ARCHES[@]}"; do -# for REPO in "${MODS_REPOS[@]}"; do -# SOURCE="${STAGING_ROOT}/${CATEGORY_STUB}/${REV}/${REPO}/${ARCH}/os" -# TARGET="${STAGING_ROOT}/${CATEGORY_STUB}/${REV}/${REPO}/${ARCH}/kickstart" -# echo "Making golden kickstart directory" -# cp -na "${SOURCE}" "${TARGET}" -# done -#done # fix treeinfo for ARCH in "${ARCHES[@]}"; do echo "Ensuring treeinfo is correct" treeinfoModder "${ARCH}" -# treeinfoModderKickstart "${ARCH}" treeinfoSaver "${ARCH}" done - -# sign all repos -#echo "Signing all repositories" -#test -f $(dirname "$0")/sign-repos-only.sh -#ret_val=$? - -#if [ "$ret_val" -eq 0 ]; then -# $(dirname "$0")/sign-repos-only.sh -#fi - -# Change Symlink if required -#echo "Setting symlink to ${REV}" -#pushd "${STAGING_ROOT}/${CATEGORY_STUB}" || exit -#/bin/rm "${STAGING_ROOT}/${CATEGORY_STUB}/latest-8" -#ln -sr "${STAGING_ROOT}/${CATEGORY_STUB}/${REV}" latest-8 -#popd || exit diff --git a/sync/common b/sync/common index da2ddd1..afc6341 100644 --- a/sync/common +++ b/sync/common @@ -97,10 +97,32 @@ function rsync_no_delete_staging() { sudo -l && rsync -vrlptDSH --chown=10004:10005 --progress --human-readable compose/ "${TARGET}" } +function rsync_no_delete_staging_pungi() { + local TARGET="${1}" + sudo -l && rsync -vrlptDSH --chown=10004:10005 --progress --human-readable \ + --exclude ppc \ + --exclude images \ + --exclude boot \ + --exclude .discinfo \ + --exclude .treeinfo \ + --exclude EFI \ + --exclude isolinux \ + --exclude media.repo \ + --exclude EULA \ + --exclude LICENSE \ + --exclude COMMUNITY-CHARTER \ + --exclude extra_files.json \ + --exclude Minimal \ + compose/ "${TARGET}" +} + function rsync_no_delete_staging_with_excludes() { local TARGET="${1}" local EXCLUDE="${2}" - sudo -l && rsync -vrlptDSH --chown=10004:10005 --progress --human-readable --exclude "${EXCLUDE}" compose/ "${TARGET}" + sudo -l && rsync -vrlptDSH --chown=10004:10005 --progress --human-readable \ + --exclude "${EXCLUDE}" \ + --exclude Minimal \ + compose/ "${TARGET}" } function rsync_no_delete_prod() { diff --git a/sync/common_8 b/sync/common_8 index 98900fc..760cc68 100644 --- a/sync/common_8 +++ b/sync/common_8 @@ -36,7 +36,7 @@ RELEASE_COMPOSE_ROOT="${COMPOSE_ROOT}/${MAJOR}${COMPOSE_APPEND}/latest-${SHORT}- # consistency. NONMODS_REPOS=( extras - Devel + devel plus ) diff --git a/sync/lh-release-sync-to-staging.sh b/sync/lh-release-sync-to-staging.sh index 08c429e..ea3106f 100644 --- a/sync/lh-release-sync-to-staging.sh +++ b/sync/lh-release-sync-to-staging.sh @@ -14,7 +14,8 @@ MAJ=${RLVER} # sync all pieces of a release, including extras, nfv, etc for COMPOSE in "${NONSIG_COMPOSE[@]}"; do echo "${COMPOSE}: Syncing" - pushd "${RELEASE_COMPOSE_ROOT}/compose" || { echo "${COMPOSE}: Failed to change directory"; break; } + SYNCSRC="/mnt/compose/${MAJ}-LookAhead/latest-${COMPOSE}-${MAJ}" + pushd "${SYNCSRC}/compose" || { echo "${COMPOSE}: Failed to change directory"; break; } if [[ "${COMPOSE}" == "Rocky" ]]; then # ISO Work before syncing @@ -40,30 +41,20 @@ for COMPOSE in "${NONSIG_COMPOSE[@]}"; do done popd || { echo "Could not change directory"; break; } done + mkdir -p live/x86_64 + ln -s live Live fi + popd || { echo "${COMPOSE}: Failed to change directory"; break; } TARGET="${STAGING_ROOT}/${CATEGORY_STUB}/${REV}" mkdir -p "${TARGET}" - # disabling because none of our files should be starting with dashes. If they - # are something is *seriously* wrong here. - # shellcheck disable=SC2035 - sudo -l && find **/* -maxdepth 0 -type d | parallel --will-cite -j 18 sudo rsync -av --chown=10004:10005 --progress --relative --human-readable \ - {} "${TARGET}" - - if [[ "${COMPOSE}" == "Rocky" ]]; then - cp COMPOSE_ID "${TARGET}" - chown 10004:10005 "${TARGET}/COMPOSE_ID" - rsync -av --chown=10004:10005 --progress --relative --human-readable metadata "${TARGET}" + pushd "${SYNCSRC}" || { echo "${COMPOSE}: Failed to change directory"; break; } + if [[ "${COMPOSE}" != "Rocky" ]]; then + rsync_no_delete_staging_with_excludes "${TARGET}" "metadata" + else + rsync_no_delete_staging "${TARGET}" fi - - # Return back to where we started - popd || { echo "${COMPOSE}: Failed to change back"; break; } - - # Create extra stuff -# pushd "${TARGET}" || { echo "${COMPOSE}: Failed to change directory"; break; } -# mkdir -p Live/x86_64 -# ln -s Live live -# popd || { echo "${COMPOSE}: Failed to change back"; break; } + popd || { echo "${COMPOSE}: Failed to change directory"; break; } done @@ -73,36 +64,10 @@ for LINK in "${LINK_REPOS[@]}"; do "${STAGING_ROOT}/${CATEGORY_STUB}/${REV}/${LINK_REPOS[$LINK]}" done -# make a kickstart directory -#for ARCH in "${ARCHES[@]}"; do -# for REPO in "${MODS_REPOS[@]}"; do -# SOURCE="${STAGING_ROOT}/${CATEGORY_STUB}/${REV}/${REPO}/${ARCH}/os" -# TARGET="${STAGING_ROOT}/${CATEGORY_STUB}/${REV}/${REPO}/${ARCH}/kickstart" -# echo "Making golden kickstart directory" -# cp -na "${SOURCE}" "${TARGET}" -# done -#done # fix treeinfo for ARCH in "${ARCHES[@]}"; do echo "Ensuring treeinfo is correct" treeinfoModder "${ARCH}" -# treeinfoModderKickstart "${ARCH}" treeinfoSaver "${ARCH}" done - -# sign all repos -#echo "Signing all repositories" -#test -f $(dirname "$0")/sign-repos-only.sh -#ret_val=$? - -#if [ "$ret_val" -eq 0 ]; then -# $(dirname "$0")/sign-repos-only.sh -#fi - -# Change Symlink if required -#echo "Setting symlink to ${REV}" -#pushd "${STAGING_ROOT}/${CATEGORY_STUB}" || exit -#/bin/rm "${STAGING_ROOT}/${CATEGORY_STUB}/latest-8" -#ln -sr "${STAGING_ROOT}/${CATEGORY_STUB}/${REV}" latest-8 -#popd || exit diff --git a/sync/minor-release-sync-to-staging.sh b/sync/minor-release-sync-to-staging.sh index b7b15dd..b681eb9 100644 --- a/sync/minor-release-sync-to-staging.sh +++ b/sync/minor-release-sync-to-staging.sh @@ -42,9 +42,17 @@ for COMPOSE in "${NONSIG_COMPOSE[@]}"; do "$(stat -c %s ${file})" \ "$(sha256sum --tag ${file})" \ | sudo tee -a CHECKSUM; + + printf "# %s: %s bytes\n%s\n" \ + "${file}" \ + "$(stat -c %s ${file})" \ + "$(sha256sum --tag ${file})" \ + | sudo tee -a "${file}.CHECKSUM" + done popd || { echo "Could not change directory"; break; } done + rm -rf Minimal mkdir -p live/x86_64 ln -s live Live fi @@ -100,4 +108,6 @@ echo "Setting symlink to ${REV}" pushd "${STAGING_ROOT}/${CATEGORY_STUB}" || exit /bin/rm "${STAGING_ROOT}/${CATEGORY_STUB}/latest-8" ln -sr "${STAGING_ROOT}/${CATEGORY_STUB}/${REV}" latest-8 +echo "Attempting hard link" +perform_hardlink "${STAGING_ROOT}/${CATEGORY_STUB}/${REV}" popd || exit diff --git a/sync/sync-to-prod.sh b/sync/sync-to-prod.sh index 2faa360..1268c7a 100644 --- a/sync/sync-to-prod.sh +++ b/sync/sync-to-prod.sh @@ -17,7 +17,7 @@ ret_val=$? if [ $ret_val -eq "0" ]; then TARGET="${PRODUCTION_ROOT}/${CATEGORY_STUB}/${REV:0:3}" mkdir -p "${TARGET}" - rsync_no_delete_prod "${REV}" "${TARGET}" + rsync_delete_prod "${REV}" "${TARGET}" echo "Syncing to prod completed. Please run the file list script." fi diff --git a/sync/sync-to-staging.sh b/sync/sync-to-staging.sh index c13f405..b42fc5c 100644 --- a/sync/sync-to-staging.sh +++ b/sync/sync-to-staging.sh @@ -12,6 +12,10 @@ if [[ $# -eq 0 ]]; then exit 1 fi +if [[ "${RLVER}" -ne "8" ]]; then + echo "This is only used for Rocky Linux 8 releases." +fi + # Major Version (eg, 8) MAJ=${RLVER} @@ -21,8 +25,25 @@ ret_val=$? if [ $ret_val -eq "0" ]; then TARGET="${STAGING_ROOT}/${CATEGORY_STUB}/${REV}" + # The target should already exist as this is used to do simple syncs. mkdir -p "${TARGET}" - rsync_no_delete_staging "${TARGET}" + + # Find all directories for this compose + repo_dirs=( $(find compose -name repodata -type d | sed 's/compose\///g') ) + + # Delete all repodata for this compose + for x in "${repo_dirs[@]}"; do + test -d "${TARGET}/${x}" + ret_val=$? + if [ $ret_val -eq "0" ]; then + /bin/rm "${TARGET:?}/${x}/"* + else + echo "${x} not found" + fi + done + + # We need to delete the old repodata + rsync_no_delete_staging_pungi "${TARGET}" echo "Hardlinking staging directory (${TARGET})" perform_hardlink "${TARGET}" fi