revamp sync

sync/common

@@ -147,6 +147,54 @@ function perform_hardlink() {
   hardlink -x '.*\.xml.*' "${TARGET}"
 }
 
+function createrepo_only() {
+  local TARGET="${1}"
+  local TARGET_REV="${2}"
+  createrepo "${TARGET}" \
+    "--distro=cpe:/o:rocky:rocky:${TARGET_REV:0:1},Rocky Linux ${TARGET_REV:0:1}" \
+    --workers 12
+}
+
+function createrepo_comps() {
+  local TARGET="${1}"
+  local TARGET_REV="${2}"
+  local TARGET_COMPS="${3}"
+  createrepo --update "${TARGET}" \
+    --groupfile="${TARGET_COMPS}" \
+    --xz --revision="${TARGET_REV}" \
+    "--distro=cpe:/o:rocky:rocky:${TARGET_REV:0:1},Rocky Linux ${TARGET_REV:0:1}" \
+    --workers 12 --checksum=sha256
+}
+
+function modifyrepo_module() {
+  local TARGET="${1}"
+  local TARGET_YAML="${2}"
+  /bin/cp "${TARGET_YAML}" /tmp/modules.yaml
+  modifyrepo --mdtype=modules /tmp/modules.yaml \
+    "${TARGET}" --compress --compress-type=xz
+
+  /bin/rm /tmp/modules.yaml
+}
+
+function sign_data() {
+  local TARGET="${1}"
+  test -f /root/bin/sign-repo.sh && /root/bin/sign-repo.sh \
+    "${TARGET}"
+}
+
+function fix_metadata() {
+  local TARGET="${1}"
+  sed -i '/<open-size><\/open-size>/d' "${TARGET}"
+}
+
+function createrepo_updates() {
+  local TARGET="${1}"
+  local TARGET_REV="${2}"
+  createrepo "${TARGET}" \
+    "--distro=cpe:/o:rocky:rocky:${TARGET_REV:0:1},Rocky Linux ${TARGET_REV:0:1}" \
+    --workers 12 --update
+}
+
 export -f parallel_rsync_no_delete_staging
 export -f parallel_rsync_no_delete_prod
 export -f parallel_rsync_delete_staging
@@ -157,3 +205,8 @@ export -f rsync_no_delete_prod
 export -f rsync_delete_staging
 export -f rsync_delete_prod
 export -f perform_hardlink
+export -f createrepo_only
+export -f createrepo_comps
+export -f modifyrepo_module
+export -f sign_data
+export -f createrepo_updates

sync/sync-to-staging.sh

@@ -17,7 +17,7 @@ if [[ "${RLVER}" -ne "8" ]]; then
 fi
 
 # Major Version (eg, 8)
-MAJ=${RLVER}
+#MAJ=${RLVER}
 
 #cd "${RELEASE_COMPOSE_ROOT}/compose" || { echo "Failed to change directory"; ret_val=1; exit 1; }
 cd "${RELEASE_COMPOSE_ROOT}/" || { echo "Failed to change directory"; ret_val=1; exit 1; }
@@ -29,9 +29,13 @@ if [ $ret_val -eq "0" ]; then
   mkdir -p "${TARGET}"
 
   # Find all directories for this compose
-  repo_dirs=( $(find compose -name repodata -type d | sed 's/compose\///g') )
+  mapfile -t repo_dirs < <(find compose -name repodata -type d | sed 's/compose\///g')
+  mapfile -t src_dirs < <(find compose -name repodata -type d | sed 's/compose\///g ; s/\/repodata//g' | grep source)
+  mapfile -t arch_dirs < <(find compose -name repodata -type d | sed 's/compose\///g ; s/\/repodata//g' | grep -v source)
+  mapfile -t debug_dirs < <(find compose -name repodata -type d | sed 's/compose\///g ; s/\/repodata//g' | grep debug)
 
   # Delete all repodata for this compose
+  echo "** Removing all current repo data"
   for x in "${repo_dirs[@]}"; do
     test -d "${TARGET}/${x}"
     ret_val=$?
@@ -42,8 +46,101 @@ if [ $ret_val -eq "0" ]; then
     fi
   done
 
-  # We need to delete the old repodata
+  # Now that we've deleted the repo data, we need to sync
+  echo "** Syncing all new content"
   rsync_no_delete_staging_pungi "${TARGET}"
-  echo "Hardlinking staging directory (${TARGET})"
+
+  # Now we need to createrepo
+  echo "** Running createrepo on source repos"
+  for src_repo in "${src_dirs[@]}"; do
+    echo "Trying ${src_repo}..."
+    test -d "${TARGET}/${src_repo}"
+    ret_val=$?
+    if [ $ret_val -eq "0" ]; then
+      createrepo_only "${TARGET}/${src_repo}" "${REVISION}"
+      fix_metadata "${TARGET}/${src_repo}/repodata/repomd.xml"
+      sign_data "${TARGET}/${src_repo}/repodata/repomd.xml"
+    else
+      echo "${src_repo} not found"
+    fi
+  done
+
+  # We need to be specific here. If the short name is "Rocky" we have extra
+  # work. Otherwise, normal createrepo is fine.
+  echo "** Running createrepo on arch repos"
+  if [[ "${SHORT}" == "Rocky" ]]; then
+    echo "** Updating all debug repos"
+    for debug_repo in "${debug_dirs[@]}"; do
+      echo "Trying ${debug_repo}..."
+      test -d "${TARGET}/${debug_repo}"
+      ret_val=$?
+      if [ $ret_val -eq "0" ]; then
+        createrepo_only "${TARGET}/${debug_repo}" "${REVISION}"
+        fix_metadata "${TARGET}/${debug_repo}/repodata/repomd.xml"
+        sign_data "${TARGET}/${debug_repo}/repodata/repomd.xml"
+      else
+        echo "${debug_repo} not found"
+      fi
+    done
+
+    echo "** Updating all repos with comps/groups"
+    for arch in "${ARCHES[@]}"; do
+      for comp_repo in "${MODS_REPOS[@]}"; do
+        echo "Trying ${arch} ${comp_repo}..."
+        REPO_PATH="${TARGET}/${comp_repo}/${arch}/os"
+        COMP_PATH="${RELEASE_COMPOSE_ROOT}/work/${arch}/comps/comps-${comp_repo}.${arch}.xml"
+        test -d "${REPO_PATH}"
+        ret_val=$?
+        if [ $ret_val -eq "0" ]; then
+          createrepo_comps "${REPO_PATH}" "${REVISION}" "${COMP_PATH}"
+          fix_metadata "${REPO_PATH}/repodata/repomd.xml"
+          sign_data "${REPO_PATH}/repodata/repomd.xml"
+        else
+          echo "${comp_repo} not found"
+        fi
+      done
+    done
+
+    echo "** Updating modules"
+    for arch in "${ARCHES[@]}"; do
+      for mod_repo in "${MODS_REPOS[@]}"; do
+        echo "Trying ${arch} ${mod_repo}..."
+        MOD_PATH="${TARGET}/${mod_repo}/${arch}/os/repodata"
+        MOD_YAML="/mnt/compose/${RLVER}_metadata/${arch}/${mod_repo}-modules.yaml"
+        test -f "${MOD_YAML}"
+        ret_val=$?
+        if [ $ret_val -ne 0 ]; then
+          echo "Module yaml not found"
+          continue
+        fi
+
+        test -d "${MOD_PATH}"
+        ret_val=$?
+
+        if [ $ret_val -eq 0 ]; then
+          modifyrepo_module "${MOD_PATH}" "${MOD_YAML}"
+          fix_metadata "${MOD_PATH}/repomd.xml"
+          sign_data "${MOD_PATH}/repomd.xml"
+        else
+          echo "${mod_repo} not found"
+        fi
+      done
+    done
+  else
+    for arch_repo in "${arch_dirs[@]}"; do
+      echo "Trying ${arch_repo}..."
+      test -d "${TARGET}/${arch_repo}"
+      ret_val=$?
+      if [ $ret_val -eq "0" ]; then
+        createrepo_only "${TARGET}/${arch_repo}" "${REVISION}"
+        fix_metadata "${TARGET}/${arch_repo}/repodata/repomd.xml"
+        sign_data "${TARGET}/${arch_repo}/repodata/repomd.xml"
+      else
+        echo "${arch_repo} not found"
+      fi
+    done
+  fi
+
+  echo "** Hardlinking staging directory (${TARGET})"
   perform_hardlink "${TARGET}"
 fi