mirror of
https://github.com/rocky-linux/rocky-tools.git
synced 2024-12-04 18:36:27 +00:00
Initial import of getsrc command
This commit is contained in:
parent
84e8d81bef
commit
e1b1d47e91
15
getsrc/README.md
Normal file
15
getsrc/README.md
Normal file
@ -0,0 +1,15 @@
|
|||||||
|
# getsrc
|
||||||
|
|
||||||
|
An automatic lookaside grabber supporting all flavors of: Fedora, Rocky Linux, CentOS, CentOS Stream. Easy to extend to other lookaside-based distros as well.
|
||||||
|
|
||||||
|
## Usage
|
||||||
|
|
||||||
|
- Download getsrc.sh , save to somewhere in your $PATH
|
||||||
|
- Run getsrc.sh inside a checked-out repo. It will analyze the lookaside list file (".firefox.metadata", for example, or just "sources") and download the source tarballs for you
|
||||||
|
- Now you can do a local build or edit of the package
|
||||||
|
|
||||||
|
<br />
|
||||||
|
|
||||||
|
## Audience
|
||||||
|
|
||||||
|
I hope this will help anyone who needs to work with packages from different sources/repos in the RPM world.
|
||||||
234
getsrc/getsrc.sh
Executable file
234
getsrc/getsrc.sh
Executable file
@ -0,0 +1,234 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
#
|
||||||
|
# Skip's "universal" lookaside grabber
|
||||||
|
# Updated by Peter Ajamian
|
||||||
|
#
|
||||||
|
# Run this in a Fedora/Rocky/CentOS/CentOS Stream source directory, and it will retrieve the lookaside sources (tarballs) into the current directory
|
||||||
|
#
|
||||||
|
|
||||||
|
shopt -s nullglob extglob
|
||||||
|
|
||||||
|
# List of lookaside locations and their patterns
|
||||||
|
# This can be easily edited to add more distro locations, or change their order for less 404 misses:
|
||||||
|
lookasides=(
|
||||||
|
https://rocky-linux-sources-staging.a1.rockylinux.org/%HASH%
|
||||||
|
https://sources.build.resf.org/%HASH%
|
||||||
|
https://git.centos.org/sources/%PKG%/%BRANCH%/%HASH%
|
||||||
|
https://sources.stream.centos.org/sources/rpms/%PKG%/%FILENAME%/%SHATYPE%/%HASH%/%FILENAME%
|
||||||
|
https://src.fedoraproject.org/repo/pkgs/%PKG%/%FILENAME%/%SHATYPE%/%HASH%/%FILENAME%
|
||||||
|
)
|
||||||
|
|
||||||
|
# These are glob patterns. They should be in the same order as the lookasides
|
||||||
|
# above and need to be quoted to avoid early glob expansion.
|
||||||
|
remotes=(
|
||||||
|
'@(git@|http?(s)://)git.rockylinux.org*'
|
||||||
|
'@(git@|http?(s)://)git.rockylinux.org*'
|
||||||
|
'@(ssh://git@|http?(s)://)git.centos.org/*'
|
||||||
|
'@(ssh://git@|http?(s)://)gitlab.com[:/]redhat/centos-stream/*'
|
||||||
|
'@(ssh://git@|http?(s)://)src.fedoraproject.org/*'
|
||||||
|
)
|
||||||
|
|
||||||
|
# These are branch names that will be glob-matched to to the lookasides above.
|
||||||
|
# Missing entries or empty strings will default to matching any branch.
|
||||||
|
branches=(
|
||||||
|
r8
|
||||||
|
r9
|
||||||
|
''
|
||||||
|
''
|
||||||
|
''
|
||||||
|
)
|
||||||
|
|
||||||
|
declare -A macros
|
||||||
|
|
||||||
|
#
|
||||||
|
# Get the hash type for a given hash. Based on the length of the hash.
|
||||||
|
#
|
||||||
|
shasizes=(
|
||||||
|
[32]=md5
|
||||||
|
[40]=sha1
|
||||||
|
[64]=sha256
|
||||||
|
[96]=sha384
|
||||||
|
[128]=sha512
|
||||||
|
)
|
||||||
|
hashtype () {
|
||||||
|
printf '%s' "${shasizes[${#1}]}"
|
||||||
|
}
|
||||||
|
|
||||||
|
#
|
||||||
|
# Validate a file against the passed hash.
|
||||||
|
# Synopsis: check_file filename hash [hashtype]
|
||||||
|
#
|
||||||
|
check_file () {
|
||||||
|
[[ -r $1 ]] || return
|
||||||
|
local type
|
||||||
|
if (( $# >= 3 )); then
|
||||||
|
type=$3
|
||||||
|
else
|
||||||
|
type=$(hashtype "$2")
|
||||||
|
fi
|
||||||
|
|
||||||
|
# We use one of the "sum" commands, so the command name is the type followed
|
||||||
|
# by "sum".
|
||||||
|
"${type}sum" --status -c - <<<"$2 $1"
|
||||||
|
}
|
||||||
|
|
||||||
|
###
|
||||||
|
# Function that actually downloads a lookaside source
|
||||||
|
# Takes HASH / FILENAME / BRANCH / PKG / SHATYPE as arguments $1 / $2 / $3 / $4 / $5
|
||||||
|
function download {
|
||||||
|
# If the file already exists and matches the checksum then we don't need to
|
||||||
|
# download it again.
|
||||||
|
if check_file "${macros[FILENAME]}" "${macros[HASH]}" "${macros[SHATYPE]}"
|
||||||
|
then
|
||||||
|
printf 'File %s already exists and matches the passed hash ... %s\n' \
|
||||||
|
"${macros[FILENAME]}" 'skipping.'
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
|
||||||
|
# We need to re-order the lookasides according to the remote and branch
|
||||||
|
# macro entries.
|
||||||
|
local -a urls
|
||||||
|
local -A tried
|
||||||
|
|
||||||
|
# Start by looking for matching entries
|
||||||
|
if [[ ${macros[REMOTE]} ]]; then
|
||||||
|
shopt -s nocasematch
|
||||||
|
for ((i=0; i<${#lookasides[@]}; i++)); do
|
||||||
|
# shellcheck disable=SC2053
|
||||||
|
[[ ${macros[REMOTE]} == ${remotes[i]} ]] || continue
|
||||||
|
# shellcheck disable=SC2053
|
||||||
|
[[ ${macros[BRANCH]} == ${branches[i]:=\*} ]] || continue
|
||||||
|
|
||||||
|
urls+=("${lookasides[i]}")
|
||||||
|
tried[${lookasides[i]}]=1
|
||||||
|
done
|
||||||
|
shopt -u nocasematch
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Then pile the rest of the URLs onto the end.
|
||||||
|
for url in "${lookasides[@]}"; do
|
||||||
|
[[ ${tried[$url]} ]] && continue
|
||||||
|
urls+=("$url")
|
||||||
|
tried[$url]=1
|
||||||
|
done
|
||||||
|
|
||||||
|
for url in "${urls[@]}"; do
|
||||||
|
# Substitute each of our macros (%PKG%, %HASH%, etc.):
|
||||||
|
for k in "${!macros[@]}"; do
|
||||||
|
v=${macros[$k]}
|
||||||
|
url=${url//"%$k%"/$v}
|
||||||
|
done
|
||||||
|
|
||||||
|
# Download the file with curl, return if successful.
|
||||||
|
printf 'Trying: %s\n' "$url"
|
||||||
|
curl --create-dirs -sfLRo "${macros[FILENAME]}" "$url" || continue
|
||||||
|
check_file "${macros[FILENAME]}" "${macros[HASH]}" \
|
||||||
|
"${macros[SHATYPE]}" || {
|
||||||
|
printf 'Invalid or corrupted file downloaded. Trying next URL.\n'
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
printf 'Downloaded: %s -----> %s\n' "$url" "${macros[FILENAME]}"
|
||||||
|
return
|
||||||
|
done
|
||||||
|
|
||||||
|
echo "ERROR: Unable to find lookaside file with the following HASH / FILENAME / BRANCH / PKG / SHATYPE :"
|
||||||
|
echo "${macros[HASH]} / ${macros[FILENAME]} / ${macros[BRANCH]} / ${macros[PKG]} / ${macros[SHATYPE]}"
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
###
|
||||||
|
# discover our list of lookaside sources. They are either in a "sources" file (new), or the older ".packagename.metadata" format (old)
|
||||||
|
sourcesfiles=(.*.metadata sources)
|
||||||
|
mapfile -t sourcelines < <(cat "${sourcesfiles[@]}" 2>/dev/null)
|
||||||
|
|
||||||
|
if (( ${#sourcelines[@]} == 0 )); then
|
||||||
|
echo "ERROR: Cannot find .*.metadata or sources file listing sources. Are you in the right directory?"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
|
||||||
|
# Current git branch. We don't error out if this fails, as we may not necessarily need this info
|
||||||
|
macros[BRANCH]=$(git status | sed -n 's/.*On branch //p')
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
# Source package name should match the specfile - we'll use that in lieu of parsing "Name:" out of it
|
||||||
|
# There could def. be a better way to do this....
|
||||||
|
# UPDATE: The better way is to use rpmspec, but this may not be installed, so
|
||||||
|
# fall back to the old way if it isn't.
|
||||||
|
specfile=(*.spec SPECS/*.spec)
|
||||||
|
if (( ${#specfile[@]}!= 1 )); then
|
||||||
|
echo "ERROR: Exactly one spec file expected, ${#specfile[@]} found."
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
macros[PKG]=$(rpmspec -q --qf '%{NAME}\n' --srpm "${specfile[0]}" 2>/dev/null) || {
|
||||||
|
pkg=${specfile[0]##*/}
|
||||||
|
macros[PKG]=${pkg%.spec}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (( ${#macros[PKG]} < 2 )); then
|
||||||
|
echo "ERROR: Having trouble finding the name of the package based on the name of the .spec file."
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Get the remote origin from git if we can. This is not required but it will
|
||||||
|
# help us to determine which lookaside URL to try first.
|
||||||
|
# We look for a fetch remote tagged with origin, otherwise we get the first
|
||||||
|
# fetch remote that is returned.
|
||||||
|
macros[REMOTE]=""
|
||||||
|
while read -r name url direction; do
|
||||||
|
# Make sure the direction is fetch.
|
||||||
|
[[ $direction == '(fetch)' ]] || continue
|
||||||
|
|
||||||
|
# If the name is "origin" we need to use this url.
|
||||||
|
if [[ $name == 'origin' ]]; then
|
||||||
|
macros[REMOTE]=$url
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Otherwise we set the first url we encounter here.
|
||||||
|
[[ ${macros[REMOTE]} ]] || macros[REMOTE]=$url
|
||||||
|
done < <(git remote -v)
|
||||||
|
|
||||||
|
# Loop through each line of our looksaide, and download the file:
|
||||||
|
# Regexes to determine which type of line it is and match the fields.
|
||||||
|
new_re='^([a-z]+[0-9]+) \(([^\)]+)\) = ([0-9a-f]+)$'
|
||||||
|
old_re='^([0-9a-f]+) ([^ ]+)$'
|
||||||
|
# Regex used for skipping lines with only whitespace.
|
||||||
|
skip_re='^[[:space:]]*$'
|
||||||
|
for line in "${sourcelines[@]}"; do
|
||||||
|
macros[SHATYPE]=""
|
||||||
|
shopt -s nocasematch
|
||||||
|
if [[ $line =~ $new_re ]]; then
|
||||||
|
# This is a new-style line: "SHATYPE (NAME) = HASH"
|
||||||
|
macros[SHATYPE]=${BASH_REMATCH[1],,}
|
||||||
|
macros[FILENAME]=${BASH_REMATCH[2]}
|
||||||
|
macros[HASH]=${BASH_REMATCH[3]}
|
||||||
|
elif [[ $line =~ $old_re ]]; then
|
||||||
|
# This is an old-style line: "HASH NAME"
|
||||||
|
macros[HASH]=${BASH_REMATCH[1]}
|
||||||
|
macros[FILENAME]=${BASH_REMATCH[2]}
|
||||||
|
elif [[ $line =~ $skip_re ]]; then
|
||||||
|
# This line just has whitespace, skip it.
|
||||||
|
continue
|
||||||
|
else
|
||||||
|
echo "ERROR: This lookaside line does not appear to have 2 or 4 space-separated fields. I don't know how to parse this line:"
|
||||||
|
printf '%s\n' "$line"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
shopt -u nocasematch
|
||||||
|
|
||||||
|
# We have a hash and a filename, now we need to find the hash type (based on string length):
|
||||||
|
# UPDATE: We don't need to do this if we already have it from the line.
|
||||||
|
if [[ ! ${macros[SHATYPE]} ]]; then
|
||||||
|
macros[SHATYPE]=$(hashtype "${macros[HASH]}")
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Finally, we have all our information call the download function with the relevant variables:
|
||||||
|
download
|
||||||
|
done
|
||||||
Loading…
Reference in New Issue
Block a user