make sync scripts friendly for next push

2023-05-17 14:51:12 -07:00 · 2023-05-17 14:51:12 -07:00 · 76ea9657cc
commit 76ea9657cc
parent 8392927586
7 changed files with 79 additions and 96 deletions
--- a/sync/beta-release-sync-to-staging.sh
+++ b/sync/beta-release-sync-to-staging.sh
@ -14,7 +14,8 @@ MAJ=${RLVER}
 # sync all pieces of a release, including extras, nfv, etc
 for COMPOSE in "${NONSIG_COMPOSE[@]}"; do
  echo "${COMPOSE}: Syncing"
-  pushd "${RELEASE_COMPOSE_ROOT}/compose" || { echo "${COMPOSE}: Failed to change directory"; break; }
+  SYNCSRC="/mnt/compose/${MAJ}-BETA/latest-${COMPOSE}-${MAJ}"
+  pushd "${SYNCSRC}/compose" || { echo "${COMPOSE}: Failed to change directory"; break; }

  if [[ "${COMPOSE}" == "Rocky" ]]; then
    # ISO Work before syncing
@ -40,30 +41,20 @@ for COMPOSE in "${NONSIG_COMPOSE[@]}"; do
      done
      popd || { echo "Could not change directory"; break; }
    done
+    mkdir -p live/x86_64
+    ln -s live Live
  fi
+  popd || { echo "${COMPOSE}: Failed to change directory"; break; }

  TARGET="${STAGING_ROOT}/${CATEGORY_STUB}/${REV}"
  mkdir -p "${TARGET}"
-  # disabling because none of our files should be starting with dashes. If they
-  # are something is *seriously* wrong here.
-  # shellcheck disable=SC2035
-  sudo -l && find **/* -maxdepth 0 -type d | parallel --will-cite -j 18 sudo rsync -av --chown=10004:10005 --progress --relative --human-readable \
-      {} "${TARGET}"
-
-  if [[ "${COMPOSE}" == "Rocky" ]]; then
-    cp COMPOSE_ID "${TARGET}"
-    chown 10004:10005 "${TARGET}/COMPOSE_ID"
-    rsync -av --chown=10004:10005 --progress --relative --human-readable metadata "${TARGET}"
+  pushd "${SYNCSRC}" || { echo "${COMPOSE}: Failed to change directory"; break; }
+  if [[ "${COMPOSE}" != "Rocky" ]]; then
+    rsync_no_delete_staging_with_excludes "${TARGET}" "metadata"
+  else
+    rsync_no_delete_staging "${TARGET}"
  fi
-
-  # Return back to where we started
-  popd || { echo "${COMPOSE}: Failed to change back"; break; }
-
-  # Create extra stuff
-#  pushd "${TARGET}" || { echo "${COMPOSE}: Failed to change directory"; break;  }
-#  mkdir -p Live/x86_64
-#  ln -s Live live
-#  popd || { echo "${COMPOSE}: Failed to change back"; break;  }
+  popd || { echo "${COMPOSE}: Failed to change directory"; break; }
 done


@ -73,36 +64,10 @@ for LINK in "${LINK_REPOS[@]}"; do
    "${STAGING_ROOT}/${CATEGORY_STUB}/${REV}/${LINK_REPOS[$LINK]}"
 done

-# make a kickstart directory
-#for ARCH in "${ARCHES[@]}"; do
-#  for REPO in "${MODS_REPOS[@]}"; do
-#    SOURCE="${STAGING_ROOT}/${CATEGORY_STUB}/${REV}/${REPO}/${ARCH}/os"
-#    TARGET="${STAGING_ROOT}/${CATEGORY_STUB}/${REV}/${REPO}/${ARCH}/kickstart"
-#    echo "Making golden kickstart directory"
-#    cp -na "${SOURCE}" "${TARGET}"
-#  done
-#done

 # fix treeinfo
 for ARCH in "${ARCHES[@]}"; do
  echo "Ensuring treeinfo is correct"
  treeinfoModder "${ARCH}"
-#  treeinfoModderKickstart "${ARCH}"
  treeinfoSaver "${ARCH}"
 done
-
-# sign all repos
-#echo "Signing all repositories"
-#test -f $(dirname "$0")/sign-repos-only.sh
-#ret_val=$?
-
-#if [ "$ret_val" -eq 0 ]; then
-#  $(dirname "$0")/sign-repos-only.sh
-#fi
-
-# Change Symlink if required
-#echo "Setting symlink to ${REV}"
-#pushd "${STAGING_ROOT}/${CATEGORY_STUB}" || exit
-#/bin/rm "${STAGING_ROOT}/${CATEGORY_STUB}/latest-8"
-#ln -sr "${STAGING_ROOT}/${CATEGORY_STUB}/${REV}" latest-8
-#popd || exit
--- a/sync/common
+++ b/sync/common
@ -97,10 +97,32 @@ function rsync_no_delete_staging() {
  sudo -l && rsync -vrlptDSH --chown=10004:10005 --progress --human-readable compose/ "${TARGET}"
 }

+function rsync_no_delete_staging_pungi() {
+  local TARGET="${1}"
+  sudo -l && rsync -vrlptDSH --chown=10004:10005 --progress --human-readable \
+    --exclude ppc \
+    --exclude images \
+    --exclude boot \
+    --exclude .discinfo \
+    --exclude .treeinfo \
+    --exclude EFI \
+    --exclude isolinux \
+    --exclude media.repo \
+    --exclude EULA \
+    --exclude LICENSE \
+    --exclude COMMUNITY-CHARTER \
+    --exclude extra_files.json \
+    --exclude Minimal \
+    compose/ "${TARGET}"
+}
+
 function rsync_no_delete_staging_with_excludes() {
  local TARGET="${1}"
  local EXCLUDE="${2}"
-  sudo -l && rsync -vrlptDSH --chown=10004:10005 --progress --human-readable --exclude "${EXCLUDE}" compose/ "${TARGET}"
+  sudo -l && rsync -vrlptDSH --chown=10004:10005 --progress --human-readable \
+    --exclude "${EXCLUDE}" \
+    --exclude Minimal \
+    compose/ "${TARGET}"
 }

 function rsync_no_delete_prod() {
--- a/sync/common_8
+++ b/sync/common_8
@ -36,7 +36,7 @@ RELEASE_COMPOSE_ROOT="${COMPOSE_ROOT}/${MAJOR}${COMPOSE_APPEND}/latest-${SHORT}-
 # consistency.
 NONMODS_REPOS=(
  extras
-  Devel
+  devel
  plus
 )

--- a/sync/lh-release-sync-to-staging.sh
+++ b/sync/lh-release-sync-to-staging.sh
@ -14,7 +14,8 @@ MAJ=${RLVER}
 # sync all pieces of a release, including extras, nfv, etc
 for COMPOSE in "${NONSIG_COMPOSE[@]}"; do
  echo "${COMPOSE}: Syncing"
-  pushd "${RELEASE_COMPOSE_ROOT}/compose" || { echo "${COMPOSE}: Failed to change directory"; break; }
+  SYNCSRC="/mnt/compose/${MAJ}-LookAhead/latest-${COMPOSE}-${MAJ}"
+  pushd "${SYNCSRC}/compose" || { echo "${COMPOSE}: Failed to change directory"; break; }

  if [[ "${COMPOSE}" == "Rocky" ]]; then
    # ISO Work before syncing
@ -40,30 +41,20 @@ for COMPOSE in "${NONSIG_COMPOSE[@]}"; do
      done
      popd || { echo "Could not change directory"; break; }
    done
+    mkdir -p live/x86_64
+    ln -s live Live
  fi
+  popd || { echo "${COMPOSE}: Failed to change directory"; break; }

  TARGET="${STAGING_ROOT}/${CATEGORY_STUB}/${REV}"
  mkdir -p "${TARGET}"
-  # disabling because none of our files should be starting with dashes. If they
-  # are something is *seriously* wrong here.
-  # shellcheck disable=SC2035
-  sudo -l && find **/* -maxdepth 0 -type d | parallel --will-cite -j 18 sudo rsync -av --chown=10004:10005 --progress --relative --human-readable \
-      {} "${TARGET}"
-
-  if [[ "${COMPOSE}" == "Rocky" ]]; then
-    cp COMPOSE_ID "${TARGET}"
-    chown 10004:10005 "${TARGET}/COMPOSE_ID"
-    rsync -av --chown=10004:10005 --progress --relative --human-readable metadata "${TARGET}"
+  pushd "${SYNCSRC}" || { echo "${COMPOSE}: Failed to change directory"; break; }
+  if [[ "${COMPOSE}" != "Rocky" ]]; then
+    rsync_no_delete_staging_with_excludes "${TARGET}" "metadata"
+  else
+    rsync_no_delete_staging "${TARGET}"
  fi
-
-  # Return back to where we started
-  popd || { echo "${COMPOSE}: Failed to change back"; break; }
-
-  # Create extra stuff
-#  pushd "${TARGET}" || { echo "${COMPOSE}: Failed to change directory"; break;  }
-#  mkdir -p Live/x86_64
-#  ln -s Live live
-#  popd || { echo "${COMPOSE}: Failed to change back"; break;  }
+  popd || { echo "${COMPOSE}: Failed to change directory"; break; }
 done


@ -73,36 +64,10 @@ for LINK in "${LINK_REPOS[@]}"; do
    "${STAGING_ROOT}/${CATEGORY_STUB}/${REV}/${LINK_REPOS[$LINK]}"
 done

-# make a kickstart directory
-#for ARCH in "${ARCHES[@]}"; do
-#  for REPO in "${MODS_REPOS[@]}"; do
-#    SOURCE="${STAGING_ROOT}/${CATEGORY_STUB}/${REV}/${REPO}/${ARCH}/os"
-#    TARGET="${STAGING_ROOT}/${CATEGORY_STUB}/${REV}/${REPO}/${ARCH}/kickstart"
-#    echo "Making golden kickstart directory"
-#    cp -na "${SOURCE}" "${TARGET}"
-#  done
-#done

 # fix treeinfo
 for ARCH in "${ARCHES[@]}"; do
  echo "Ensuring treeinfo is correct"
  treeinfoModder "${ARCH}"
-#  treeinfoModderKickstart "${ARCH}"
  treeinfoSaver "${ARCH}"
 done
-
-# sign all repos
-#echo "Signing all repositories"
-#test -f $(dirname "$0")/sign-repos-only.sh
-#ret_val=$?
-
-#if [ "$ret_val" -eq 0 ]; then
-#  $(dirname "$0")/sign-repos-only.sh
-#fi
-
-# Change Symlink if required
-#echo "Setting symlink to ${REV}"
-#pushd "${STAGING_ROOT}/${CATEGORY_STUB}" || exit
-#/bin/rm "${STAGING_ROOT}/${CATEGORY_STUB}/latest-8"
-#ln -sr "${STAGING_ROOT}/${CATEGORY_STUB}/${REV}" latest-8
-#popd || exit
--- a/sync/minor-release-sync-to-staging.sh
+++ b/sync/minor-release-sync-to-staging.sh
@ -42,9 +42,17 @@ for COMPOSE in "${NONSIG_COMPOSE[@]}"; do
          "$(stat -c %s ${file})" \
          "$(sha256sum --tag ${file})" \
        | sudo tee -a CHECKSUM;
+
+        printf "# %s: %s bytes\n%s\n" \
+          "${file}" \
+          "$(stat -c %s ${file})" \
+          "$(sha256sum --tag ${file})" \
+        | sudo tee -a "${file}.CHECKSUM"
+
      done
      popd || { echo "Could not change directory"; break; }
    done
+    rm -rf Minimal
    mkdir -p live/x86_64
    ln -s live Live
  fi
@ -100,4 +108,6 @@ echo "Setting symlink to ${REV}"
 pushd "${STAGING_ROOT}/${CATEGORY_STUB}" || exit
 /bin/rm "${STAGING_ROOT}/${CATEGORY_STUB}/latest-8"
 ln -sr "${STAGING_ROOT}/${CATEGORY_STUB}/${REV}" latest-8
+echo "Attempting hard link"
+perform_hardlink "${STAGING_ROOT}/${CATEGORY_STUB}/${REV}"
 popd || exit
--- a/sync/sync-to-prod.sh
+++ b/sync/sync-to-prod.sh
@ -17,7 +17,7 @@ ret_val=$?
 if [ $ret_val -eq "0" ]; then
  TARGET="${PRODUCTION_ROOT}/${CATEGORY_STUB}/${REV:0:3}"
  mkdir -p "${TARGET}"
-  rsync_no_delete_prod "${REV}" "${TARGET}"
+  rsync_delete_prod "${REV}" "${TARGET}"
  echo "Syncing to prod completed. Please run the file list script."
 fi

--- a/sync/sync-to-staging.sh
+++ b/sync/sync-to-staging.sh
@ -12,6 +12,10 @@ if [[ $# -eq 0 ]]; then
  exit 1
 fi

+if [[ "${RLVER}" -ne "8" ]]; then
+  echo "This is only used for Rocky Linux 8 releases."
+fi
+
 # Major Version (eg, 8)
 MAJ=${RLVER}

@ -21,8 +25,25 @@ ret_val=$?

 if [ $ret_val -eq "0" ]; then
  TARGET="${STAGING_ROOT}/${CATEGORY_STUB}/${REV}"
+  # The target should already exist as this is used to do simple syncs.
  mkdir -p "${TARGET}"
-  rsync_no_delete_staging "${TARGET}"
+
+  # Find all directories for this compose
+  repo_dirs=( $(find compose -name repodata -type d | sed 's/compose\///g') )
+
+  # Delete all repodata for this compose
+  for x in "${repo_dirs[@]}"; do
+    test -d "${TARGET}/${x}"
+    ret_val=$?
+    if [ $ret_val -eq "0" ]; then
+      /bin/rm "${TARGET:?}/${x}/"*
+    else
+      echo "${x} not found"
+    fi
+  done
+
+  # We need to delete the old repodata
+  rsync_no_delete_staging_pungi "${TARGET}"
  echo "Hardlinking staging directory (${TARGET})"
  perform_hardlink "${TARGET}"
 fi