revamp sync

This commit is contained in:
Louis Abel 2023-05-17 17:19:35 -07:00
parent 76ea9657cc
commit 0f6804ae01
Signed by untrusted user: label
GPG Key ID: B37E62D143879B36
2 changed files with 154 additions and 4 deletions

View File

@ -147,6 +147,54 @@ function perform_hardlink() {
hardlink -x '.*\.xml.*' "${TARGET}" hardlink -x '.*\.xml.*' "${TARGET}"
} }
function createrepo_only() {
local TARGET="${1}"
local TARGET_REV="${2}"
createrepo "${TARGET}" \
"--distro=cpe:/o:rocky:rocky:${TARGET_REV:0:1},Rocky Linux ${TARGET_REV:0:1}" \
--workers 12
}
function createrepo_comps() {
local TARGET="${1}"
local TARGET_REV="${2}"
local TARGET_COMPS="${3}"
createrepo --update "${TARGET}" \
--groupfile="${TARGET_COMPS}" \
--xz --revision="${TARGET_REV}" \
"--distro=cpe:/o:rocky:rocky:${TARGET_REV:0:1},Rocky Linux ${TARGET_REV:0:1}" \
--workers 12 --checksum=sha256
}
function modifyrepo_module() {
local TARGET="${1}"
local TARGET_YAML="${2}"
/bin/cp "${TARGET_YAML}" /tmp/modules.yaml
modifyrepo --mdtype=modules /tmp/modules.yaml \
"${TARGET}" --compress --compress-type=xz
/bin/rm /tmp/modules.yaml
}
function sign_data() {
local TARGET="${1}"
test -f /root/bin/sign-repo.sh && /root/bin/sign-repo.sh \
"${TARGET}"
}
function fix_metadata() {
local TARGET="${1}"
sed -i '/<open-size><\/open-size>/d' "${TARGET}"
}
function createrepo_updates() {
local TARGET="${1}"
local TARGET_REV="${2}"
createrepo "${TARGET}" \
"--distro=cpe:/o:rocky:rocky:${TARGET_REV:0:1},Rocky Linux ${TARGET_REV:0:1}" \
--workers 12 --update
}
export -f parallel_rsync_no_delete_staging export -f parallel_rsync_no_delete_staging
export -f parallel_rsync_no_delete_prod export -f parallel_rsync_no_delete_prod
export -f parallel_rsync_delete_staging export -f parallel_rsync_delete_staging
@ -157,3 +205,8 @@ export -f rsync_no_delete_prod
export -f rsync_delete_staging export -f rsync_delete_staging
export -f rsync_delete_prod export -f rsync_delete_prod
export -f perform_hardlink export -f perform_hardlink
export -f createrepo_only
export -f createrepo_comps
export -f modifyrepo_module
export -f sign_data
export -f createrepo_updates

View File

@ -17,7 +17,7 @@ if [[ "${RLVER}" -ne "8" ]]; then
fi fi
# Major Version (eg, 8) # Major Version (eg, 8)
MAJ=${RLVER} #MAJ=${RLVER}
#cd "${RELEASE_COMPOSE_ROOT}/compose" || { echo "Failed to change directory"; ret_val=1; exit 1; } #cd "${RELEASE_COMPOSE_ROOT}/compose" || { echo "Failed to change directory"; ret_val=1; exit 1; }
cd "${RELEASE_COMPOSE_ROOT}/" || { echo "Failed to change directory"; ret_val=1; exit 1; } cd "${RELEASE_COMPOSE_ROOT}/" || { echo "Failed to change directory"; ret_val=1; exit 1; }
@ -29,9 +29,13 @@ if [ $ret_val -eq "0" ]; then
mkdir -p "${TARGET}" mkdir -p "${TARGET}"
# Find all directories for this compose # Find all directories for this compose
repo_dirs=( $(find compose -name repodata -type d | sed 's/compose\///g') ) mapfile -t repo_dirs < <(find compose -name repodata -type d | sed 's/compose\///g')
mapfile -t src_dirs < <(find compose -name repodata -type d | sed 's/compose\///g ; s/\/repodata//g' | grep source)
mapfile -t arch_dirs < <(find compose -name repodata -type d | sed 's/compose\///g ; s/\/repodata//g' | grep -v source)
mapfile -t debug_dirs < <(find compose -name repodata -type d | sed 's/compose\///g ; s/\/repodata//g' | grep debug)
# Delete all repodata for this compose # Delete all repodata for this compose
echo "** Removing all current repo data"
for x in "${repo_dirs[@]}"; do for x in "${repo_dirs[@]}"; do
test -d "${TARGET}/${x}" test -d "${TARGET}/${x}"
ret_val=$? ret_val=$?
@ -42,8 +46,101 @@ if [ $ret_val -eq "0" ]; then
fi fi
done done
# We need to delete the old repodata # Now that we've deleted the repo data, we need to sync
echo "** Syncing all new content"
rsync_no_delete_staging_pungi "${TARGET}" rsync_no_delete_staging_pungi "${TARGET}"
echo "Hardlinking staging directory (${TARGET})"
# Now we need to createrepo
echo "** Running createrepo on source repos"
for src_repo in "${src_dirs[@]}"; do
echo "Trying ${src_repo}..."
test -d "${TARGET}/${src_repo}"
ret_val=$?
if [ $ret_val -eq "0" ]; then
createrepo_only "${TARGET}/${src_repo}" "${REVISION}"
fix_metadata "${TARGET}/${src_repo}/repodata/repomd.xml"
sign_data "${TARGET}/${src_repo}/repodata/repomd.xml"
else
echo "${src_repo} not found"
fi
done
# We need to be specific here. If the short name is "Rocky" we have extra
# work. Otherwise, normal createrepo is fine.
echo "** Running createrepo on arch repos"
if [[ "${SHORT}" == "Rocky" ]]; then
echo "** Updating all debug repos"
for debug_repo in "${debug_dirs[@]}"; do
echo "Trying ${debug_repo}..."
test -d "${TARGET}/${debug_repo}"
ret_val=$?
if [ $ret_val -eq "0" ]; then
createrepo_only "${TARGET}/${debug_repo}" "${REVISION}"
fix_metadata "${TARGET}/${debug_repo}/repodata/repomd.xml"
sign_data "${TARGET}/${debug_repo}/repodata/repomd.xml"
else
echo "${debug_repo} not found"
fi
done
echo "** Updating all repos with comps/groups"
for arch in "${ARCHES[@]}"; do
for comp_repo in "${MODS_REPOS[@]}"; do
echo "Trying ${arch} ${comp_repo}..."
REPO_PATH="${TARGET}/${comp_repo}/${arch}/os"
COMP_PATH="${RELEASE_COMPOSE_ROOT}/work/${arch}/comps/comps-${comp_repo}.${arch}.xml"
test -d "${REPO_PATH}"
ret_val=$?
if [ $ret_val -eq "0" ]; then
createrepo_comps "${REPO_PATH}" "${REVISION}" "${COMP_PATH}"
fix_metadata "${REPO_PATH}/repodata/repomd.xml"
sign_data "${REPO_PATH}/repodata/repomd.xml"
else
echo "${comp_repo} not found"
fi
done
done
echo "** Updating modules"
for arch in "${ARCHES[@]}"; do
for mod_repo in "${MODS_REPOS[@]}"; do
echo "Trying ${arch} ${mod_repo}..."
MOD_PATH="${TARGET}/${mod_repo}/${arch}/os/repodata"
MOD_YAML="/mnt/compose/${RLVER}_metadata/${arch}/${mod_repo}-modules.yaml"
test -f "${MOD_YAML}"
ret_val=$?
if [ $ret_val -ne 0 ]; then
echo "Module yaml not found"
continue
fi
test -d "${MOD_PATH}"
ret_val=$?
if [ $ret_val -eq 0 ]; then
modifyrepo_module "${MOD_PATH}" "${MOD_YAML}"
fix_metadata "${MOD_PATH}/repomd.xml"
sign_data "${MOD_PATH}/repomd.xml"
else
echo "${mod_repo} not found"
fi
done
done
else
for arch_repo in "${arch_dirs[@]}"; do
echo "Trying ${arch_repo}..."
test -d "${TARGET}/${arch_repo}"
ret_val=$?
if [ $ret_val -eq "0" ]; then
createrepo_only "${TARGET}/${arch_repo}" "${REVISION}"
fix_metadata "${TARGET}/${arch_repo}/repodata/repomd.xml"
sign_data "${TARGET}/${arch_repo}/repodata/repomd.xml"
else
echo "${arch_repo} not found"
fi
done
fi
echo "** Hardlinking staging directory (${TARGET})"
perform_hardlink "${TARGET}" perform_hardlink "${TARGET}"
fi fi