diff --git a/sync/common b/sync/common index afc6341..9985876 100644 --- a/sync/common +++ b/sync/common @@ -147,6 +147,54 @@ function perform_hardlink() { hardlink -x '.*\.xml.*' "${TARGET}" } +function createrepo_only() { + local TARGET="${1}" + local TARGET_REV="${2}" + createrepo "${TARGET}" \ + "--distro=cpe:/o:rocky:rocky:${TARGET_REV:0:1},Rocky Linux ${TARGET_REV:0:1}" \ + --workers 12 +} + +function createrepo_comps() { + local TARGET="${1}" + local TARGET_REV="${2}" + local TARGET_COMPS="${3}" + createrepo --update "${TARGET}" \ + --groupfile="${TARGET_COMPS}" \ + --xz --revision="${TARGET_REV}" \ + "--distro=cpe:/o:rocky:rocky:${TARGET_REV:0:1},Rocky Linux ${TARGET_REV:0:1}" \ + --workers 12 --checksum=sha256 +} + +function modifyrepo_module() { + local TARGET="${1}" + local TARGET_YAML="${2}" + /bin/cp "${TARGET_YAML}" /tmp/modules.yaml + modifyrepo --mdtype=modules /tmp/modules.yaml \ + "${TARGET}" --compress --compress-type=xz + + /bin/rm /tmp/modules.yaml +} + +function sign_data() { + local TARGET="${1}" + test -f /root/bin/sign-repo.sh && /root/bin/sign-repo.sh \ + "${TARGET}" +} + +function fix_metadata() { + local TARGET="${1}" + sed -i '/<\/open-size>/d' "${TARGET}" +} + +function createrepo_updates() { + local TARGET="${1}" + local TARGET_REV="${2}" + createrepo "${TARGET}" \ + "--distro=cpe:/o:rocky:rocky:${TARGET_REV:0:1},Rocky Linux ${TARGET_REV:0:1}" \ + --workers 12 --update +} + export -f parallel_rsync_no_delete_staging export -f parallel_rsync_no_delete_prod export -f parallel_rsync_delete_staging @@ -157,3 +205,8 @@ export -f rsync_no_delete_prod export -f rsync_delete_staging export -f rsync_delete_prod export -f perform_hardlink +export -f createrepo_only +export -f createrepo_comps +export -f modifyrepo_module +export -f sign_data +export -f createrepo_updates diff --git a/sync/sync-to-staging.sh b/sync/sync-to-staging.sh index b42fc5c..e8923ad 100644 --- a/sync/sync-to-staging.sh +++ b/sync/sync-to-staging.sh @@ -17,7 +17,7 @@ if [[ "${RLVER}" -ne "8" ]]; then fi # Major Version (eg, 8) -MAJ=${RLVER} +#MAJ=${RLVER} #cd "${RELEASE_COMPOSE_ROOT}/compose" || { echo "Failed to change directory"; ret_val=1; exit 1; } cd "${RELEASE_COMPOSE_ROOT}/" || { echo "Failed to change directory"; ret_val=1; exit 1; } @@ -29,9 +29,13 @@ if [ $ret_val -eq "0" ]; then mkdir -p "${TARGET}" # Find all directories for this compose - repo_dirs=( $(find compose -name repodata -type d | sed 's/compose\///g') ) + mapfile -t repo_dirs < <(find compose -name repodata -type d | sed 's/compose\///g') + mapfile -t src_dirs < <(find compose -name repodata -type d | sed 's/compose\///g ; s/\/repodata//g' | grep source) + mapfile -t arch_dirs < <(find compose -name repodata -type d | sed 's/compose\///g ; s/\/repodata//g' | grep -v source) + mapfile -t debug_dirs < <(find compose -name repodata -type d | sed 's/compose\///g ; s/\/repodata//g' | grep debug) # Delete all repodata for this compose + echo "** Removing all current repo data" for x in "${repo_dirs[@]}"; do test -d "${TARGET}/${x}" ret_val=$? @@ -42,8 +46,101 @@ if [ $ret_val -eq "0" ]; then fi done - # We need to delete the old repodata + # Now that we've deleted the repo data, we need to sync + echo "** Syncing all new content" rsync_no_delete_staging_pungi "${TARGET}" - echo "Hardlinking staging directory (${TARGET})" + + # Now we need to createrepo + echo "** Running createrepo on source repos" + for src_repo in "${src_dirs[@]}"; do + echo "Trying ${src_repo}..." + test -d "${TARGET}/${src_repo}" + ret_val=$? + if [ $ret_val -eq "0" ]; then + createrepo_only "${TARGET}/${src_repo}" "${REVISION}" + fix_metadata "${TARGET}/${src_repo}/repodata/repomd.xml" + sign_data "${TARGET}/${src_repo}/repodata/repomd.xml" + else + echo "${src_repo} not found" + fi + done + + # We need to be specific here. If the short name is "Rocky" we have extra + # work. Otherwise, normal createrepo is fine. + echo "** Running createrepo on arch repos" + if [[ "${SHORT}" == "Rocky" ]]; then + echo "** Updating all debug repos" + for debug_repo in "${debug_dirs[@]}"; do + echo "Trying ${debug_repo}..." + test -d "${TARGET}/${debug_repo}" + ret_val=$? + if [ $ret_val -eq "0" ]; then + createrepo_only "${TARGET}/${debug_repo}" "${REVISION}" + fix_metadata "${TARGET}/${debug_repo}/repodata/repomd.xml" + sign_data "${TARGET}/${debug_repo}/repodata/repomd.xml" + else + echo "${debug_repo} not found" + fi + done + + echo "** Updating all repos with comps/groups" + for arch in "${ARCHES[@]}"; do + for comp_repo in "${MODS_REPOS[@]}"; do + echo "Trying ${arch} ${comp_repo}..." + REPO_PATH="${TARGET}/${comp_repo}/${arch}/os" + COMP_PATH="${RELEASE_COMPOSE_ROOT}/work/${arch}/comps/comps-${comp_repo}.${arch}.xml" + test -d "${REPO_PATH}" + ret_val=$? + if [ $ret_val -eq "0" ]; then + createrepo_comps "${REPO_PATH}" "${REVISION}" "${COMP_PATH}" + fix_metadata "${REPO_PATH}/repodata/repomd.xml" + sign_data "${REPO_PATH}/repodata/repomd.xml" + else + echo "${comp_repo} not found" + fi + done + done + + echo "** Updating modules" + for arch in "${ARCHES[@]}"; do + for mod_repo in "${MODS_REPOS[@]}"; do + echo "Trying ${arch} ${mod_repo}..." + MOD_PATH="${TARGET}/${mod_repo}/${arch}/os/repodata" + MOD_YAML="/mnt/compose/${RLVER}_metadata/${arch}/${mod_repo}-modules.yaml" + test -f "${MOD_YAML}" + ret_val=$? + if [ $ret_val -ne 0 ]; then + echo "Module yaml not found" + continue + fi + + test -d "${MOD_PATH}" + ret_val=$? + + if [ $ret_val -eq 0 ]; then + modifyrepo_module "${MOD_PATH}" "${MOD_YAML}" + fix_metadata "${MOD_PATH}/repomd.xml" + sign_data "${MOD_PATH}/repomd.xml" + else + echo "${mod_repo} not found" + fi + done + done + else + for arch_repo in "${arch_dirs[@]}"; do + echo "Trying ${arch_repo}..." + test -d "${TARGET}/${arch_repo}" + ret_val=$? + if [ $ret_val -eq "0" ]; then + createrepo_only "${TARGET}/${arch_repo}" "${REVISION}" + fix_metadata "${TARGET}/${arch_repo}/repodata/repomd.xml" + sign_data "${TARGET}/${arch_repo}/repodata/repomd.xml" + else + echo "${arch_repo} not found" + fi + done + fi + + echo "** Hardlinking staging directory (${TARGET})" perform_hardlink "${TARGET}" fi