make sync scripts friendly for next push

sync/beta-release-sync-to-staging.sh

@@ -14,7 +14,8 @@ MAJ=${RLVER}
 # sync all pieces of a release, including extras, nfv, etc
 for COMPOSE in "${NONSIG_COMPOSE[@]}"; do
   echo "${COMPOSE}: Syncing"
-  pushd "${RELEASE_COMPOSE_ROOT}/compose" || { echo "${COMPOSE}: Failed to change directory"; break; }
+  SYNCSRC="/mnt/compose/${MAJ}-BETA/latest-${COMPOSE}-${MAJ}"
+  pushd "${SYNCSRC}/compose" || { echo "${COMPOSE}: Failed to change directory"; break; }
 
   if [[ "${COMPOSE}" == "Rocky" ]]; then
     # ISO Work before syncing
@@ -40,30 +41,20 @@ for COMPOSE in "${NONSIG_COMPOSE[@]}"; do
       done
       popd || { echo "Could not change directory"; break; }
     done
+    mkdir -p live/x86_64
+    ln -s live Live
   fi
+  popd || { echo "${COMPOSE}: Failed to change directory"; break; }
 
   TARGET="${STAGING_ROOT}/${CATEGORY_STUB}/${REV}"
   mkdir -p "${TARGET}"
-  # disabling because none of our files should be starting with dashes. If they
+  pushd "${SYNCSRC}" || { echo "${COMPOSE}: Failed to change directory"; break; }
-  # are something is *seriously* wrong here.
+  if [[ "${COMPOSE}" != "Rocky" ]]; then
-  # shellcheck disable=SC2035
+    rsync_no_delete_staging_with_excludes "${TARGET}" "metadata"
-  sudo -l && find **/* -maxdepth 0 -type d | parallel --will-cite -j 18 sudo rsync -av --chown=10004:10005 --progress --relative --human-readable \
+  else
-      {} "${TARGET}"
+    rsync_no_delete_staging "${TARGET}"
-
-  if [[ "${COMPOSE}" == "Rocky" ]]; then
-    cp COMPOSE_ID "${TARGET}"
-    chown 10004:10005 "${TARGET}/COMPOSE_ID"
-    rsync -av --chown=10004:10005 --progress --relative --human-readable metadata "${TARGET}"
   fi
-
+  popd || { echo "${COMPOSE}: Failed to change directory"; break; }
-  # Return back to where we started
-  popd || { echo "${COMPOSE}: Failed to change back"; break; }
-
-  # Create extra stuff
-#  pushd "${TARGET}" || { echo "${COMPOSE}: Failed to change directory"; break;  }
-#  mkdir -p Live/x86_64
-#  ln -s Live live
-#  popd || { echo "${COMPOSE}: Failed to change back"; break;  }
 done
 
 
@@ -73,36 +64,10 @@ for LINK in "${LINK_REPOS[@]}"; do
     "${STAGING_ROOT}/${CATEGORY_STUB}/${REV}/${LINK_REPOS[$LINK]}"
 done
 
-# make a kickstart directory
-#for ARCH in "${ARCHES[@]}"; do
-#  for REPO in "${MODS_REPOS[@]}"; do
-#    SOURCE="${STAGING_ROOT}/${CATEGORY_STUB}/${REV}/${REPO}/${ARCH}/os"
-#    TARGET="${STAGING_ROOT}/${CATEGORY_STUB}/${REV}/${REPO}/${ARCH}/kickstart"
-#    echo "Making golden kickstart directory"
-#    cp -na "${SOURCE}" "${TARGET}"
-#  done
-#done
 
 # fix treeinfo
 for ARCH in "${ARCHES[@]}"; do
   echo "Ensuring treeinfo is correct"
   treeinfoModder "${ARCH}"
-#  treeinfoModderKickstart "${ARCH}"
   treeinfoSaver "${ARCH}"
 done
-
-# sign all repos
-#echo "Signing all repositories"
-#test -f $(dirname "$0")/sign-repos-only.sh
-#ret_val=$?
-
-#if [ "$ret_val" -eq 0 ]; then
-#  $(dirname "$0")/sign-repos-only.sh
-#fi
-
-# Change Symlink if required
-#echo "Setting symlink to ${REV}"
-#pushd "${STAGING_ROOT}/${CATEGORY_STUB}" || exit
-#/bin/rm "${STAGING_ROOT}/${CATEGORY_STUB}/latest-8"
-#ln -sr "${STAGING_ROOT}/${CATEGORY_STUB}/${REV}" latest-8
-#popd || exit

sync/common

@@ -97,10 +97,32 @@ function rsync_no_delete_staging() {
   sudo -l && rsync -vrlptDSH --chown=10004:10005 --progress --human-readable compose/ "${TARGET}"
 }
 
+function rsync_no_delete_staging_pungi() {
+  local TARGET="${1}"
+  sudo -l && rsync -vrlptDSH --chown=10004:10005 --progress --human-readable \
+    --exclude ppc \
+    --exclude images \
+    --exclude boot \
+    --exclude .discinfo \
+    --exclude .treeinfo \
+    --exclude EFI \
+    --exclude isolinux \
+    --exclude media.repo \
+    --exclude EULA \
+    --exclude LICENSE \
+    --exclude COMMUNITY-CHARTER \
+    --exclude extra_files.json \
+    --exclude Minimal \
+    compose/ "${TARGET}"
+}
+
 function rsync_no_delete_staging_with_excludes() {
   local TARGET="${1}"
   local EXCLUDE="${2}"
-  sudo -l && rsync -vrlptDSH --chown=10004:10005 --progress --human-readable --exclude "${EXCLUDE}" compose/ "${TARGET}"
+  sudo -l && rsync -vrlptDSH --chown=10004:10005 --progress --human-readable \
+    --exclude "${EXCLUDE}" \
+    --exclude Minimal \
+    compose/ "${TARGET}"
 }
 
 function rsync_no_delete_prod() {

sync/common_8

@@ -36,7 +36,7 @@ RELEASE_COMPOSE_ROOT="${COMPOSE_ROOT}/${MAJOR}${COMPOSE_APPEND}/latest-${SHORT}-
 # consistency.
 NONMODS_REPOS=(
   extras
-  Devel
+  devel
   plus
 )
 

sync/lh-release-sync-to-staging.sh

@@ -14,7 +14,8 @@ MAJ=${RLVER}
 # sync all pieces of a release, including extras, nfv, etc
 for COMPOSE in "${NONSIG_COMPOSE[@]}"; do
   echo "${COMPOSE}: Syncing"
-  pushd "${RELEASE_COMPOSE_ROOT}/compose" || { echo "${COMPOSE}: Failed to change directory"; break; }
+  SYNCSRC="/mnt/compose/${MAJ}-LookAhead/latest-${COMPOSE}-${MAJ}"
+  pushd "${SYNCSRC}/compose" || { echo "${COMPOSE}: Failed to change directory"; break; }
 
   if [[ "${COMPOSE}" == "Rocky" ]]; then
     # ISO Work before syncing
@@ -40,30 +41,20 @@ for COMPOSE in "${NONSIG_COMPOSE[@]}"; do
       done
       popd || { echo "Could not change directory"; break; }
     done
+    mkdir -p live/x86_64
+    ln -s live Live
   fi
+  popd || { echo "${COMPOSE}: Failed to change directory"; break; }
 
   TARGET="${STAGING_ROOT}/${CATEGORY_STUB}/${REV}"
   mkdir -p "${TARGET}"
-  # disabling because none of our files should be starting with dashes. If they
+  pushd "${SYNCSRC}" || { echo "${COMPOSE}: Failed to change directory"; break; }
-  # are something is *seriously* wrong here.
+  if [[ "${COMPOSE}" != "Rocky" ]]; then
-  # shellcheck disable=SC2035
+    rsync_no_delete_staging_with_excludes "${TARGET}" "metadata"
-  sudo -l && find **/* -maxdepth 0 -type d | parallel --will-cite -j 18 sudo rsync -av --chown=10004:10005 --progress --relative --human-readable \
+  else
-      {} "${TARGET}"
+    rsync_no_delete_staging "${TARGET}"
-
-  if [[ "${COMPOSE}" == "Rocky" ]]; then
-    cp COMPOSE_ID "${TARGET}"
-    chown 10004:10005 "${TARGET}/COMPOSE_ID"
-    rsync -av --chown=10004:10005 --progress --relative --human-readable metadata "${TARGET}"
   fi
-
+  popd || { echo "${COMPOSE}: Failed to change directory"; break; }
-  # Return back to where we started
-  popd || { echo "${COMPOSE}: Failed to change back"; break; }
-
-  # Create extra stuff
-#  pushd "${TARGET}" || { echo "${COMPOSE}: Failed to change directory"; break;  }
-#  mkdir -p Live/x86_64
-#  ln -s Live live
-#  popd || { echo "${COMPOSE}: Failed to change back"; break;  }
 done
 
 
@@ -73,36 +64,10 @@ for LINK in "${LINK_REPOS[@]}"; do
     "${STAGING_ROOT}/${CATEGORY_STUB}/${REV}/${LINK_REPOS[$LINK]}"
 done
 
-# make a kickstart directory
-#for ARCH in "${ARCHES[@]}"; do
-#  for REPO in "${MODS_REPOS[@]}"; do
-#    SOURCE="${STAGING_ROOT}/${CATEGORY_STUB}/${REV}/${REPO}/${ARCH}/os"
-#    TARGET="${STAGING_ROOT}/${CATEGORY_STUB}/${REV}/${REPO}/${ARCH}/kickstart"
-#    echo "Making golden kickstart directory"
-#    cp -na "${SOURCE}" "${TARGET}"
-#  done
-#done
 
 # fix treeinfo
 for ARCH in "${ARCHES[@]}"; do
   echo "Ensuring treeinfo is correct"
   treeinfoModder "${ARCH}"
-#  treeinfoModderKickstart "${ARCH}"
   treeinfoSaver "${ARCH}"
 done
-
-# sign all repos
-#echo "Signing all repositories"
-#test -f $(dirname "$0")/sign-repos-only.sh
-#ret_val=$?
-
-#if [ "$ret_val" -eq 0 ]; then
-#  $(dirname "$0")/sign-repos-only.sh
-#fi
-
-# Change Symlink if required
-#echo "Setting symlink to ${REV}"
-#pushd "${STAGING_ROOT}/${CATEGORY_STUB}" || exit
-#/bin/rm "${STAGING_ROOT}/${CATEGORY_STUB}/latest-8"
-#ln -sr "${STAGING_ROOT}/${CATEGORY_STUB}/${REV}" latest-8
-#popd || exit

sync/minor-release-sync-to-staging.sh

@@ -42,9 +42,17 @@ for COMPOSE in "${NONSIG_COMPOSE[@]}"; do
           "$(stat -c %s ${file})" \
           "$(sha256sum --tag ${file})" \
         | sudo tee -a CHECKSUM;
+
+        printf "# %s: %s bytes\n%s\n" \
+          "${file}" \
+          "$(stat -c %s ${file})" \
+          "$(sha256sum --tag ${file})" \
+        | sudo tee -a "${file}.CHECKSUM"
+
       done
       popd || { echo "Could not change directory"; break; }
     done
+    rm -rf Minimal
     mkdir -p live/x86_64
     ln -s live Live
   fi
@@ -100,4 +108,6 @@ echo "Setting symlink to ${REV}"
 pushd "${STAGING_ROOT}/${CATEGORY_STUB}" || exit
 /bin/rm "${STAGING_ROOT}/${CATEGORY_STUB}/latest-8"
 ln -sr "${STAGING_ROOT}/${CATEGORY_STUB}/${REV}" latest-8
+echo "Attempting hard link"
+perform_hardlink "${STAGING_ROOT}/${CATEGORY_STUB}/${REV}"
 popd || exit

sync/sync-to-prod.sh

@@ -17,7 +17,7 @@ ret_val=$?
 if [ $ret_val -eq "0" ]; then
   TARGET="${PRODUCTION_ROOT}/${CATEGORY_STUB}/${REV:0:3}"
   mkdir -p "${TARGET}"
-  rsync_no_delete_prod "${REV}" "${TARGET}"
+  rsync_delete_prod "${REV}" "${TARGET}"
   echo "Syncing to prod completed. Please run the file list script."
 fi
 

sync/sync-to-staging.sh

@@ -12,6 +12,10 @@ if [[ $# -eq 0 ]]; then
   exit 1
 fi
 
+if [[ "${RLVER}" -ne "8" ]]; then
+  echo "This is only used for Rocky Linux 8 releases."
+fi
+
 # Major Version (eg, 8)
 MAJ=${RLVER}
 
@@ -21,8 +25,25 @@ ret_val=$?
 
 if [ $ret_val -eq "0" ]; then
   TARGET="${STAGING_ROOT}/${CATEGORY_STUB}/${REV}"
+  # The target should already exist as this is used to do simple syncs.
   mkdir -p "${TARGET}"
-  rsync_no_delete_staging "${TARGET}"
+
+  # Find all directories for this compose
+  repo_dirs=( $(find compose -name repodata -type d | sed 's/compose\///g') )
+
+  # Delete all repodata for this compose
+  for x in "${repo_dirs[@]}"; do
+    test -d "${TARGET}/${x}"
+    ret_val=$?
+    if [ $ret_val -eq "0" ]; then
+      /bin/rm "${TARGET:?}/${x}/"*
+    else
+      echo "${x} not found"
+    fi
+  done
+
+  # We need to delete the old repodata
+  rsync_no_delete_staging_pungi "${TARGET}"
   echo "Hardlinking staging directory (${TARGET})"
   perform_hardlink "${TARGET}"
 fi