kemotaha/toolkit
1
0
Fork 0
forked from sig_core/toolkit
Code Pull requests Activity

make sync scripts friendly for next push

Browse source
This commit is contained in:
Louis Abel 2023-05-17 14:51:12 -07:00
parent 8392927586
commit 76ea9657cc
Signed by untrusted user: label
GPG key ID: B37E62D143879B36
7 changed files with 79 additions and 96 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

57
sync/beta-release-sync-to-staging.sh
View file

@ -14,7 +14,8 @@ MAJ=${RLVER}
# sync all pieces of a release, including extras, nfv, etc
for COMPOSE in "${NONSIG_COMPOSE[@]}"; do
echo "${COMPOSE}: Syncing"
pushd "${RELEASE_COMPOSE_ROOT}/compose" || { echo "${COMPOSE}: Failed to change directory"; break; }
SYNCSRC="/mnt/compose/${MAJ}-BETA/latest-${COMPOSE}-${MAJ}"
pushd "${SYNCSRC}/compose" || { echo "${COMPOSE}: Failed to change directory"; break; }
if [[ "${COMPOSE}" == "Rocky" ]]; then
# ISO Work before syncing
@ -40,30 +41,20 @@ for COMPOSE in "${NONSIG_COMPOSE[@]}"; do
done
popd || { echo "Could not change directory"; break; }
done
mkdir -p live/x86_64
ln -s live Live
fi
popd || { echo "${COMPOSE}: Failed to change directory"; break; }
TARGET="${STAGING_ROOT}/${CATEGORY_STUB}/${REV}"
mkdir -p "${TARGET}"
# disabling because none of our files should be starting with dashes. If they
# are something is *seriously* wrong here.
# shellcheck disable=SC2035
sudo -l && find **/* -maxdepth 0 -type d | parallel --will-cite -j 18 sudo rsync -av --chown=10004:10005 --progress --relative --human-readable \
{} "${TARGET}"
if [[ "${COMPOSE}" == "Rocky" ]]; then
cp COMPOSE_ID "${TARGET}"
chown 10004:10005 "${TARGET}/COMPOSE_ID"
rsync -av --chown=10004:10005 --progress --relative --human-readable metadata "${TARGET}"
pushd "${SYNCSRC}" || { echo "${COMPOSE}: Failed to change directory"; break; }
if [[ "${COMPOSE}" != "Rocky" ]]; then
rsync_no_delete_staging_with_excludes "${TARGET}" "metadata"
else
rsync_no_delete_staging "${TARGET}"
fi
# Return back to where we started
popd || { echo "${COMPOSE}: Failed to change back"; break; }
# Create extra stuff
# pushd "${TARGET}" || { echo "${COMPOSE}: Failed to change directory"; break; }
# mkdir -p Live/x86_64
# ln -s Live live
# popd || { echo "${COMPOSE}: Failed to change back"; break; }
popd || { echo "${COMPOSE}: Failed to change directory"; break; }
done
@ -73,36 +64,10 @@ for LINK in "${LINK_REPOS[@]}"; do
"${STAGING_ROOT}/${CATEGORY_STUB}/${REV}/${LINK_REPOS[$LINK]}"
done
# make a kickstart directory
#for ARCH in "${ARCHES[@]}"; do
# for REPO in "${MODS_REPOS[@]}"; do
# SOURCE="${STAGING_ROOT}/${CATEGORY_STUB}/${REV}/${REPO}/${ARCH}/os"
# TARGET="${STAGING_ROOT}/${CATEGORY_STUB}/${REV}/${REPO}/${ARCH}/kickstart"
# echo "Making golden kickstart directory"
# cp -na "${SOURCE}" "${TARGET}"
# done
#done
# fix treeinfo
for ARCH in "${ARCHES[@]}"; do
echo "Ensuring treeinfo is correct"
treeinfoModder "${ARCH}"
# treeinfoModderKickstart "${ARCH}"
treeinfoSaver "${ARCH}"
done
# sign all repos
#echo "Signing all repositories"
#test -f $(dirname "$0")/sign-repos-only.sh
#ret_val=$?
#if [ "$ret_val" -eq 0 ]; then
# $(dirname "$0")/sign-repos-only.sh
#fi
# Change Symlink if required
#echo "Setting symlink to ${REV}"
#pushd "${STAGING_ROOT}/${CATEGORY_STUB}" || exit
#/bin/rm "${STAGING_ROOT}/${CATEGORY_STUB}/latest-8"
#ln -sr "${STAGING_ROOT}/${CATEGORY_STUB}/${REV}" latest-8
#popd || exit

24
sync/common
View file

@ -97,10 +97,32 @@ function rsync_no_delete_staging() {
sudo -l && rsync -vrlptDSH --chown=10004:10005 --progress --human-readable compose/ "${TARGET}"
}
function rsync_no_delete_staging_pungi() {
local TARGET="${1}"
sudo -l && rsync -vrlptDSH --chown=10004:10005 --progress --human-readable \
--exclude ppc \
--exclude images \
--exclude boot \
--exclude .discinfo \
--exclude .treeinfo \
--exclude EFI \
--exclude isolinux \
--exclude media.repo \
--exclude EULA \
--exclude LICENSE \
--exclude COMMUNITY-CHARTER \
--exclude extra_files.json \
--exclude Minimal \
compose/ "${TARGET}"
}
function rsync_no_delete_staging_with_excludes() {
local TARGET="${1}"
local EXCLUDE="${2}"
sudo -l && rsync -vrlptDSH --chown=10004:10005 --progress --human-readable --exclude "${EXCLUDE}" compose/ "${TARGET}"
sudo -l && rsync -vrlptDSH --chown=10004:10005 --progress --human-readable \
--exclude "${EXCLUDE}" \
--exclude Minimal \
compose/ "${TARGET}"
}
function rsync_no_delete_prod() {

2
sync/common_8
View file

@ -36,7 +36,7 @@ RELEASE_COMPOSE_ROOT="${COMPOSE_ROOT}/${MAJOR}${COMPOSE_APPEND}/latest-${SHORT}-
# consistency.
NONMODS_REPOS=(
extras
Devel
devel
plus
)

57
sync/lh-release-sync-to-staging.sh
View file

@ -14,7 +14,8 @@ MAJ=${RLVER}
# sync all pieces of a release, including extras, nfv, etc
for COMPOSE in "${NONSIG_COMPOSE[@]}"; do
echo "${COMPOSE}: Syncing"
pushd "${RELEASE_COMPOSE_ROOT}/compose" || { echo "${COMPOSE}: Failed to change directory"; break; }
SYNCSRC="/mnt/compose/${MAJ}-LookAhead/latest-${COMPOSE}-${MAJ}"
pushd "${SYNCSRC}/compose" || { echo "${COMPOSE}: Failed to change directory"; break; }
if [[ "${COMPOSE}" == "Rocky" ]]; then
# ISO Work before syncing
@ -40,30 +41,20 @@ for COMPOSE in "${NONSIG_COMPOSE[@]}"; do
done
popd || { echo "Could not change directory"; break; }
done
mkdir -p live/x86_64
ln -s live Live
fi
popd || { echo "${COMPOSE}: Failed to change directory"; break; }
TARGET="${STAGING_ROOT}/${CATEGORY_STUB}/${REV}"
mkdir -p "${TARGET}"
# disabling because none of our files should be starting with dashes. If they
# are something is *seriously* wrong here.
# shellcheck disable=SC2035
sudo -l && find **/* -maxdepth 0 -type d | parallel --will-cite -j 18 sudo rsync -av --chown=10004:10005 --progress --relative --human-readable \
{} "${TARGET}"
if [[ "${COMPOSE}" == "Rocky" ]]; then
cp COMPOSE_ID "${TARGET}"
chown 10004:10005 "${TARGET}/COMPOSE_ID"
rsync -av --chown=10004:10005 --progress --relative --human-readable metadata "${TARGET}"
pushd "${SYNCSRC}" || { echo "${COMPOSE}: Failed to change directory"; break; }
if [[ "${COMPOSE}" != "Rocky" ]]; then
rsync_no_delete_staging_with_excludes "${TARGET}" "metadata"
else
rsync_no_delete_staging "${TARGET}"
fi
# Return back to where we started
popd || { echo "${COMPOSE}: Failed to change back"; break; }
# Create extra stuff
# pushd "${TARGET}" || { echo "${COMPOSE}: Failed to change directory"; break; }
# mkdir -p Live/x86_64
# ln -s Live live
# popd || { echo "${COMPOSE}: Failed to change back"; break; }
popd || { echo "${COMPOSE}: Failed to change directory"; break; }
done
@ -73,36 +64,10 @@ for LINK in "${LINK_REPOS[@]}"; do
"${STAGING_ROOT}/${CATEGORY_STUB}/${REV}/${LINK_REPOS[$LINK]}"
done
# make a kickstart directory
#for ARCH in "${ARCHES[@]}"; do
# for REPO in "${MODS_REPOS[@]}"; do
# SOURCE="${STAGING_ROOT}/${CATEGORY_STUB}/${REV}/${REPO}/${ARCH}/os"
# TARGET="${STAGING_ROOT}/${CATEGORY_STUB}/${REV}/${REPO}/${ARCH}/kickstart"
# echo "Making golden kickstart directory"
# cp -na "${SOURCE}" "${TARGET}"
# done
#done
# fix treeinfo
for ARCH in "${ARCHES[@]}"; do
echo "Ensuring treeinfo is correct"
treeinfoModder "${ARCH}"
# treeinfoModderKickstart "${ARCH}"
treeinfoSaver "${ARCH}"
done
# sign all repos
#echo "Signing all repositories"
#test -f $(dirname "$0")/sign-repos-only.sh
#ret_val=$?
#if [ "$ret_val" -eq 0 ]; then
# $(dirname "$0")/sign-repos-only.sh
#fi
# Change Symlink if required
#echo "Setting symlink to ${REV}"
#pushd "${STAGING_ROOT}/${CATEGORY_STUB}" || exit
#/bin/rm "${STAGING_ROOT}/${CATEGORY_STUB}/latest-8"
#ln -sr "${STAGING_ROOT}/${CATEGORY_STUB}/${REV}" latest-8
#popd || exit

10
sync/minor-release-sync-to-staging.sh
View file

@ -42,9 +42,17 @@ for COMPOSE in "${NONSIG_COMPOSE[@]}"; do
"$(stat -c %s ${file})" \
"$(sha256sum --tag ${file})" \
| sudo tee -a CHECKSUM;
printf "# %s: %s bytes\n%s\n" \
"${file}" \
"$(stat -c %s ${file})" \
"$(sha256sum --tag ${file})" \
| sudo tee -a "${file}.CHECKSUM"
done
popd || { echo "Could not change directory"; break; }
done
rm -rf Minimal
mkdir -p live/x86_64
ln -s live Live
fi
@ -100,4 +108,6 @@ echo "Setting symlink to ${REV}"
pushd "${STAGING_ROOT}/${CATEGORY_STUB}" || exit
/bin/rm "${STAGING_ROOT}/${CATEGORY_STUB}/latest-8"
ln -sr "${STAGING_ROOT}/${CATEGORY_STUB}/${REV}" latest-8
echo "Attempting hard link"
perform_hardlink "${STAGING_ROOT}/${CATEGORY_STUB}/${REV}"
popd || exit

2
sync/sync-to-prod.sh
View file

@ -17,7 +17,7 @@ ret_val=$?
if [ $ret_val -eq "0" ]; then
TARGET="${PRODUCTION_ROOT}/${CATEGORY_STUB}/${REV:0:3}"
mkdir -p "${TARGET}"
rsync_no_delete_prod "${REV}" "${TARGET}"
rsync_delete_prod "${REV}" "${TARGET}"
echo "Syncing to prod completed. Please run the file list script."
fi

23
sync/sync-to-staging.sh
View file

@ -12,6 +12,10 @@ if [[ $# -eq 0 ]]; then
exit 1
fi
if [[ "${RLVER}" -ne "8" ]]; then
echo "This is only used for Rocky Linux 8 releases."
fi
# Major Version (eg, 8)
MAJ=${RLVER}
@ -21,8 +25,25 @@ ret_val=$?
if [ $ret_val -eq "0" ]; then
TARGET="${STAGING_ROOT}/${CATEGORY_STUB}/${REV}"
# The target should already exist as this is used to do simple syncs.
mkdir -p "${TARGET}"
rsync_no_delete_staging "${TARGET}"
# Find all directories for this compose
repo_dirs=( $(find compose -name repodata -type d | sed 's/compose\///g') )
# Delete all repodata for this compose
for x in "${repo_dirs[@]}"; do
test -d "${TARGET}/${x}"
ret_val=$?
if [ $ret_val -eq "0" ]; then
/bin/rm "${TARGET:?}/${x}/"*
else
echo "${x} not found"
fi
done
# We need to delete the old repodata
rsync_no_delete_staging_pungi "${TARGET}"
echo "Hardlinking staging directory (${TARGET})"
perform_hardlink "${TARGET}"
fi