mirror of
https://github.com/rocky-linux/srpmproc.git
synced 2024-06-01 16:00:14 +00:00
Compare commits
21 Commits
Author | SHA1 | Date | |
---|---|---|---|
f2864a875a | |||
|
82271f5d9e | ||
963b5d25dd | |||
a5f9082c15 | |||
f2374845e3 | |||
|
162e7cea34 | ||
|
edeb0239df | ||
40180a342b | |||
|
7f14563f93 | ||
6e2ed2eb23 | |||
|
44725a727d | ||
|
4f975913b4 | ||
|
3371826851 | ||
|
f4f2a0a9e4 | ||
|
5e8ed2e521 | ||
|
2eb0768f33 | ||
2c6d6f0b0a | |||
|
0ea6123b6e | ||
|
2cc3bcc3a3 | ||
|
3237794071 | ||
06ffd1e507 |
|
@ -17,11 +17,13 @@ builds:
|
||||||
- s390x
|
- s390x
|
||||||
- ppc64le
|
- ppc64le
|
||||||
archives:
|
archives:
|
||||||
- replacements:
|
- name_template: >-
|
||||||
darwin: Darwin
|
{{- .ProjectName }}_
|
||||||
linux: Linux
|
{{- .Version }}
|
||||||
amd64: x86_64
|
{{- title .Os }}_
|
||||||
arm64: aarch64
|
{{- if eq .Arch "amd64" }}x86_64
|
||||||
|
{{- else if eq .Arch "arm64" }}aarch64
|
||||||
|
{{- else }}{{ .Arch }}{{ end }}{{ end -}}
|
||||||
checksum:
|
checksum:
|
||||||
name_template: 'checksums.txt'
|
name_template: 'checksums.txt'
|
||||||
snapshot:
|
snapshot:
|
||||||
|
|
42
README.md
42
README.md
|
@ -1,14 +1,14 @@
|
||||||
# srpmproc
|
# srpmproc
|
||||||
Upstream package importer with auto patching. Reference implementation for OpenPatch
|
Upstream package importer with auto patching. Reference implementation for OpenPatch
|
||||||
|
|
||||||
# Usage
|
## Usage
|
||||||
```
|
```
|
||||||
Usage:
|
Usage:
|
||||||
srpmproc [flags]
|
srpmproc [flags]
|
||||||
srpmproc [command]
|
srpmproc [command]
|
||||||
|
|
||||||
Available Commands:
|
Available Commands:
|
||||||
fetch
|
fetch
|
||||||
help Help about any command
|
help Help about any command
|
||||||
|
|
||||||
Flags:
|
Flags:
|
||||||
|
@ -16,7 +16,8 @@ Flags:
|
||||||
--basic-username string Basic auth username
|
--basic-username string Basic auth username
|
||||||
--branch-prefix string Branch prefix (replaces import-branch-prefix) (default "r")
|
--branch-prefix string Branch prefix (replaces import-branch-prefix) (default "r")
|
||||||
--branch-suffix string Branch suffix to use for imported branches
|
--branch-suffix string Branch suffix to use for imported branches
|
||||||
--cdn-url string CDN URL to download blobs from (default "https://git.centos.org/sources")
|
--cdn string CDN URL shortcuts for well-known distros, auto-assigns --cdn-url. Valid values: rocky8, rocky, fedora, centos, centos-stream. Setting this overrides --cdn-url
|
||||||
|
--cdn-url string CDN URL to download blobs from. Simple URL follows default rocky/centos patterns. Can be customized using macros (see docs) (default "https://git.centos.org/sources")
|
||||||
--git-committer-email string Email of committer (default "rockyautomation@rockylinux.org")
|
--git-committer-email string Email of committer (default "rockyautomation@rockylinux.org")
|
||||||
--git-committer-name string Name of committer (default "rockyautomation")
|
--git-committer-name string Name of committer (default "rockyautomation")
|
||||||
-h, --help help for srpmproc
|
-h, --help help for srpmproc
|
||||||
|
@ -28,6 +29,8 @@ Flags:
|
||||||
--no-dup-mode If enabled, skips already imported tags
|
--no-dup-mode If enabled, skips already imported tags
|
||||||
--no-storage-download If enabled, blobs are always downloaded from upstream
|
--no-storage-download If enabled, blobs are always downloaded from upstream
|
||||||
--no-storage-upload If enabled, blobs are not uploaded to blob storage
|
--no-storage-upload If enabled, blobs are not uploaded to blob storage
|
||||||
|
--package-release string Package release to fetch
|
||||||
|
--package-version string Package version to fetch
|
||||||
--rpm-prefix string Where to retrieve SRPM content. Only used when source-rpm is not a local file (default "https://git.centos.org/rpms")
|
--rpm-prefix string Where to retrieve SRPM content. Only used when source-rpm is not a local file (default "https://git.centos.org/rpms")
|
||||||
--single-tag string If set, only this tag is imported
|
--single-tag string If set, only this tag is imported
|
||||||
--source-rpm string Location of RPM to process
|
--source-rpm string Location of RPM to process
|
||||||
|
@ -35,9 +38,42 @@ Flags:
|
||||||
--ssh-user string SSH User (default "git")
|
--ssh-user string SSH User (default "git")
|
||||||
--storage-addr string Bucket to use as blob storage
|
--storage-addr string Bucket to use as blob storage
|
||||||
--strict-branch-mode If enabled, only branches with the calculated name are imported and not prefix only
|
--strict-branch-mode If enabled, only branches with the calculated name are imported and not prefix only
|
||||||
|
--taglessmode Tagless mode: If set, pull the latest commit from the branch and determine version numbers from spec file. This is auto-tried if tags aren't found.
|
||||||
--tmpfs-mode string If set, packages are imported to path and patched but not pushed
|
--tmpfs-mode string If set, packages are imported to path and patched but not pushed
|
||||||
--upstream-prefix string Upstream git repository prefix
|
--upstream-prefix string Upstream git repository prefix
|
||||||
--version int Upstream version
|
--version int Upstream version
|
||||||
|
|
||||||
Use "srpmproc [command] --help" for more information about a command.
|
Use "srpmproc [command] --help" for more information about a command.
|
||||||
```
|
```
|
||||||
|
|
||||||
|
<br />
|
||||||
|
|
||||||
|
## Examples:
|
||||||
|
|
||||||
|
1. Import the kernel package from git.centos.org/rpms/, to local folder /opt/gitroot/rpms/kernel.git/ . Download the lookaside source tarballs from the default CentOS file server location to local folder `/opt/fake_s3/` . We want to grab branch "c8" (import prefix plus RHEL version), and it will be committed as branch "r8" (branch prefix plus RHEL version). This assumes that `/opt/fake_s3` exists, and `/opt/gitroot/rpms/kernel.git` exists and is a git repository of some kind (even an empty one).
|
||||||
|
|
||||||
|
```
|
||||||
|
srpmproc --branch-prefix "r" --import-branch-prefix "c" --rpm-prefix "https://git.centos.org/rpms" --version 8 --storage-addr file:///opt/fake_s3 --upstream-prefix file:///opt/gitroot --cdn centos --strict-branch-mode --source-rpm kernel
|
||||||
|
```
|
||||||
|
|
||||||
|
<br />
|
||||||
|
|
||||||
|
## CDN and --cdn-url
|
||||||
|
The --cdn-url option allows for Go-style templates to craft complex URL patterns. These templates are: `{{.Name}}` (package name), `{{.Hash}}` (hash of lookaside file), `{{.Hashtype}}` (hash type of file, like "sha256" or "sha512"), `{{.Branch}}` (the branch we are importing), and `{{.Filename}}` (the lookaside file's name as it appears in SOURCES/). You can add these values as part of --cdn-url to craft your lookaside pattern.
|
||||||
|
|
||||||
|
|
||||||
|
For example, if I wanted my lookaside downloads to come from CentOS 9 Stream, I would use as part of my command:
|
||||||
|
```
|
||||||
|
--cdn-url "https://sources.stream.centos.org/sources/rpms/{{.Name}}/{{.Filename}}/{{.Hashtype}}/{{.Hash}}/{{.Filename}}"
|
||||||
|
```
|
||||||
|
|
||||||
|
|
||||||
|
**Default Behavior:** If these templates are not used, the default behavior of `--cdn-url` is to fall back on the traditional RHEL import pattern: `<CDN_URL>/<NAME>/<BRANCH>/<HASH>` . If that fails, a further fallback is attempted, the simple: `<CDN_URL>/<HASH>`. These cover the common Rocky Linux and RHEL/CentOS imports if the base lookaside URL is the only thing given. If no `--cdn-url` is specified, it defaults to "https://git.centos.org/sources" (for RHEL imports into Rocky Linux)
|
||||||
|
|
||||||
|
|
||||||
|
**CDN Shorthand:** For convenience, some lookaside patterns for popular distros are provided via the `--cdn` option. You can specify this without needing to use the longer `--cdn-url`. For example, when importing from CentOS 9 Stream, you could use `--cdn centos-stream`
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -58,7 +58,8 @@ var (
|
||||||
packageVersion string
|
packageVersion string
|
||||||
packageRelease string
|
packageRelease string
|
||||||
taglessMode bool
|
taglessMode bool
|
||||||
altLookAside bool
|
cdn string
|
||||||
|
moduleBranchNames bool
|
||||||
)
|
)
|
||||||
|
|
||||||
var root = &cobra.Command{
|
var root = &cobra.Command{
|
||||||
|
@ -96,7 +97,8 @@ func mn(_ *cobra.Command, _ []string) {
|
||||||
PackageVersion: packageVersion,
|
PackageVersion: packageVersion,
|
||||||
PackageRelease: packageRelease,
|
PackageRelease: packageRelease,
|
||||||
TaglessMode: taglessMode,
|
TaglessMode: taglessMode,
|
||||||
AltLookAside: altLookAside,
|
Cdn: cdn,
|
||||||
|
ModuleBranchNames: moduleBranchNames,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Fatal(err)
|
log.Fatal(err)
|
||||||
|
@ -131,7 +133,7 @@ func main() {
|
||||||
root.Flags().StringVar(&rpmPrefix, "rpm-prefix", "https://git.centos.org/rpms", "Where to retrieve SRPM content. Only used when source-rpm is not a local file")
|
root.Flags().StringVar(&rpmPrefix, "rpm-prefix", "https://git.centos.org/rpms", "Where to retrieve SRPM content. Only used when source-rpm is not a local file")
|
||||||
root.Flags().StringVar(&importBranchPrefix, "import-branch-prefix", "c", "Import branch prefix")
|
root.Flags().StringVar(&importBranchPrefix, "import-branch-prefix", "c", "Import branch prefix")
|
||||||
root.Flags().StringVar(&branchPrefix, "branch-prefix", "r", "Branch prefix (replaces import-branch-prefix)")
|
root.Flags().StringVar(&branchPrefix, "branch-prefix", "r", "Branch prefix (replaces import-branch-prefix)")
|
||||||
root.Flags().StringVar(&cdnUrl, "cdn-url", "https://git.centos.org/sources", "CDN URL to download blobs from")
|
root.Flags().StringVar(&cdnUrl, "cdn-url", "https://git.centos.org/sources", "CDN URL to download blobs from. Simple URL follows default rocky/centos patterns. Can be customized using macros (see docs)")
|
||||||
root.Flags().StringVar(&singleTag, "single-tag", "", "If set, only this tag is imported")
|
root.Flags().StringVar(&singleTag, "single-tag", "", "If set, only this tag is imported")
|
||||||
root.Flags().BoolVar(&noDupMode, "no-dup-mode", false, "If enabled, skips already imported tags")
|
root.Flags().BoolVar(&noDupMode, "no-dup-mode", false, "If enabled, skips already imported tags")
|
||||||
root.Flags().BoolVar(&moduleMode, "module-mode", false, "If enabled, imports a module instead of a package")
|
root.Flags().BoolVar(&moduleMode, "module-mode", false, "If enabled, imports a module instead of a package")
|
||||||
|
@ -146,8 +148,9 @@ func main() {
|
||||||
root.Flags().StringVar(&basicPassword, "basic-password", "", "Basic auth password")
|
root.Flags().StringVar(&basicPassword, "basic-password", "", "Basic auth password")
|
||||||
root.Flags().StringVar(&packageVersion, "package-version", "", "Package version to fetch")
|
root.Flags().StringVar(&packageVersion, "package-version", "", "Package version to fetch")
|
||||||
root.Flags().StringVar(&packageRelease, "package-release", "", "Package release to fetch")
|
root.Flags().StringVar(&packageRelease, "package-release", "", "Package release to fetch")
|
||||||
root.Flags().BoolVar(&taglessMode, "taglessmode", false, "Tagless mode: If set, pull the latest commit from a branch, and determine version info from spec file (aka upstream versions aren't tagged)")
|
root.Flags().BoolVar(&taglessMode, "taglessmode", false, "Tagless mode: If set, pull the latest commit from the branch and determine version numbers from spec file. This is auto-tried if tags aren't found.")
|
||||||
root.Flags().BoolVar(&altLookAside, "altlookaside", false, "If set, uses the new CentOS Stream lookaside pattern (https://<SITE_PREFIX>/<RPM_NAME>/<FILE_NAME>/<SHA_VERSION>/<SHA_SUM>/<FILE_NAME>)")
|
root.Flags().StringVar(&cdn, "cdn", "", "CDN URL shortcuts for well-known distros, auto-assigns --cdn-url. Valid values: rocky8, rocky, fedora, centos, centos-stream. Setting this overrides --cdn-url")
|
||||||
|
root.Flags().BoolVar(&moduleBranchNames, "module-branch-names-only", false, "If enabled, module imports will use the branch name that is being imported, rather than use the commit hash.")
|
||||||
|
|
||||||
if err := root.Execute(); err != nil {
|
if err := root.Execute(); err != nil {
|
||||||
log.Fatal(err)
|
log.Fatal(err)
|
||||||
|
|
|
@ -59,5 +59,6 @@ type ProcessData struct {
|
||||||
PackageVersion string
|
PackageVersion string
|
||||||
PackageRelease string
|
PackageRelease string
|
||||||
TaglessMode bool
|
TaglessMode bool
|
||||||
AltLookAside bool
|
Cdn string
|
||||||
|
ModuleBranchNames bool
|
||||||
}
|
}
|
||||||
|
|
|
@ -38,8 +38,8 @@ func GetTagImportRegex(pd *data.ProcessData) *regexp.Regexp {
|
||||||
// if we are ok with importing that reference. We are looking for the traditional <prefix><version><suffix> pattern, like "c9s", and also the
|
// if we are ok with importing that reference. We are looking for the traditional <prefix><version><suffix> pattern, like "c9s", and also the
|
||||||
// modular "stream-<NAME>-<VERSION>-rhel-<VERSION> branch pattern as well
|
// modular "stream-<NAME>-<VERSION>-rhel-<VERSION> branch pattern as well
|
||||||
func TaglessRefOk(tag string, pd *data.ProcessData) bool {
|
func TaglessRefOk(tag string, pd *data.ProcessData) bool {
|
||||||
// First case is very easy: if we are "refs/heads/<prefix><version><suffix>" , then this is def. a branch we should import
|
// First case is very easy: if we are exactly "refs/heads/<prefix><version><suffix>" , then this is def. a branch we should import
|
||||||
if strings.HasPrefix(tag, fmt.Sprintf("refs/heads/%s%d%s", pd.ImportBranchPrefix, pd.Version, pd.BranchSuffix)) {
|
if tag == fmt.Sprintf("refs/heads/%s%d%s", pd.ImportBranchPrefix, pd.Version, pd.BranchSuffix) {
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
122
pkg/modes/git.go
122
pkg/modes/git.go
|
@ -21,12 +21,15 @@
|
||||||
package modes
|
package modes
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bytes"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io/ioutil"
|
"io/ioutil"
|
||||||
|
"log"
|
||||||
"net/http"
|
"net/http"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
|
"text/template"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/go-git/go-git/v5/plumbing/transport"
|
"github.com/go-git/go-git/v5/plumbing/transport"
|
||||||
|
@ -46,6 +49,15 @@ type remoteTarget struct {
|
||||||
when time.Time
|
when time.Time
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Struct to define the possible template values ( {{.Value}} in CDN URL strings:
|
||||||
|
type Lookaside struct {
|
||||||
|
Name string
|
||||||
|
Branch string
|
||||||
|
Hash string
|
||||||
|
Hashtype string
|
||||||
|
Filename string
|
||||||
|
}
|
||||||
|
|
||||||
type remoteTargetSlice []remoteTarget
|
type remoteTargetSlice []remoteTarget
|
||||||
|
|
||||||
func (p remoteTargetSlice) Len() int {
|
func (p remoteTargetSlice) Len() int {
|
||||||
|
@ -341,44 +353,66 @@ func (g *GitMode) WriteSource(pd *data.ProcessData, md *data.ModeData) error {
|
||||||
} else {
|
} else {
|
||||||
|
|
||||||
url := ""
|
url := ""
|
||||||
// Alternate lookaside logic: if enabled, we pull from a new URL pattern
|
|
||||||
if !pd.AltLookAside {
|
// We need to figure out the hashtype for templating purposes:
|
||||||
url = fmt.Sprintf("%s/%s/%s/%s", pd.CdnUrl, md.Name, branchName, hash)
|
hashType := "sha512"
|
||||||
} else {
|
switch len(hash) {
|
||||||
// We first need the hash algorithm based on length of hash:
|
case 128:
|
||||||
hashType := "sha512"
|
hashType = "sha512"
|
||||||
switch len(hash) {
|
case 64:
|
||||||
case 128:
|
hashType = "sha256"
|
||||||
hashType = "sha512"
|
case 40:
|
||||||
case 64:
|
hashType = "sha1"
|
||||||
hashType = "sha256"
|
case 32:
|
||||||
case 40:
|
hashType = "md5"
|
||||||
hashType = "sha1"
|
}
|
||||||
case 32:
|
|
||||||
hashType = "md5"
|
// need the name of the file without "SOURCES/":
|
||||||
|
fileName := strings.Split(path, "/")[1]
|
||||||
|
|
||||||
|
// Feed our template info to ProcessUrl and transform to the real values: ( {{.Name}}, {{.Branch}}, {{.Hash}}, {{.Hashtype}}, {{.Filename}} )
|
||||||
|
url, hasTemplate := ProcessUrl(pd.CdnUrl, md.Name, branchName, hash, hashType, fileName)
|
||||||
|
|
||||||
|
var req *http.Request
|
||||||
|
var resp *http.Response
|
||||||
|
|
||||||
|
// Download the --cdn-url given, but *only* if it contains template strings ( {{.Name}} , {{.Hash}} , etc. )
|
||||||
|
// Otherwise we need to fall back to the traditional cdn-url patterns
|
||||||
|
if hasTemplate {
|
||||||
|
pd.Log.Printf("downloading %s", url)
|
||||||
|
|
||||||
|
req, err := http.NewRequest("GET", url, nil)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("could not create new http request: %v", err)
|
||||||
}
|
}
|
||||||
|
req.Header.Set("Accept-Encoding", "*")
|
||||||
|
|
||||||
// need the name of the file without "SOURCES/":
|
resp, err = client.Do(req)
|
||||||
fileName := strings.Split(path, "/")[1]
|
if err != nil {
|
||||||
|
return fmt.Errorf("could not download dist-git file: %v", err)
|
||||||
// Alt. lookaside url is of the form: <cdn> / <name> / <filename> / <hashtype> / <hash> / <filename>
|
}
|
||||||
url = fmt.Sprintf("%s/%s/%s/%s/%s/%s", pd.CdnUrl, md.Name, fileName, hashType, hash, fileName)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pd.Log.Printf("downloading %s", url)
|
// Default cdn-url: If we don't have a templated download string, try the default <SITE>/<PKG>/<BRANCH>/<HASH> pattern:
|
||||||
|
if resp == nil || resp.StatusCode != http.StatusOK {
|
||||||
req, err := http.NewRequest("GET", url, nil)
|
url = fmt.Sprintf("%s/%s/%s/%s", pd.CdnUrl, md.Name, branchName, hash)
|
||||||
if err != nil {
|
pd.Log.Printf("Attempting default URL: %s", url)
|
||||||
return fmt.Errorf("could not create new http request: %v", err)
|
req, err = http.NewRequest("GET", url, nil)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("could not create new http request: %v", err)
|
||||||
|
}
|
||||||
|
req.Header.Set("Accept-Encoding", "*")
|
||||||
|
resp, err = client.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("could not download dist-git file: %v", err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
req.Header.Set("Accept-Encoding", "*")
|
|
||||||
|
|
||||||
resp, err := client.Do(req)
|
// If the default URL fails, we have one more pattern to try. The simple <SITE>/<HASH> pattern
|
||||||
if err != nil {
|
// If this one fails, we are truly lost, and have to bail out w/ an error:
|
||||||
return fmt.Errorf("could not download dist-git file: %v", err)
|
if resp == nil || resp.StatusCode != http.StatusOK {
|
||||||
}
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
|
||||||
url = fmt.Sprintf("%s/%s", pd.CdnUrl, hash)
|
url = fmt.Sprintf("%s/%s", pd.CdnUrl, hash)
|
||||||
|
pd.Log.Printf("Attempting 2nd fallback URL: %s", url)
|
||||||
req, err = http.NewRequest("GET", url, nil)
|
req, err = http.NewRequest("GET", url, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("could not create new http request: %v", err)
|
return fmt.Errorf("could not create new http request: %v", err)
|
||||||
|
@ -458,3 +492,29 @@ func (g *GitMode) ImportName(pd *data.ProcessData, md *data.ModeData) string {
|
||||||
|
|
||||||
return strings.Replace(strings.TrimPrefix(md.TagBranch, "refs/heads/"), "%", "_", -1)
|
return strings.Replace(strings.TrimPrefix(md.TagBranch, "refs/heads/"), "%", "_", -1)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Given a cdnUrl string as input, return same string, but with substituted
|
||||||
|
// template values ( {{.Name}} , {{.Hash}}, {{.Filename}}, etc. )
|
||||||
|
func ProcessUrl(cdnUrl string, name string, branch string, hash string, hashtype string, filename string) (string, bool) {
|
||||||
|
tmpUrl := Lookaside{name, branch, hash, hashtype, filename}
|
||||||
|
|
||||||
|
// Return cdnUrl as-is if we don't have any templates ("{{ .Variable }}") to process:
|
||||||
|
if !(strings.Contains(cdnUrl, "{{") && strings.Contains(cdnUrl, "}}")) {
|
||||||
|
return cdnUrl, false
|
||||||
|
}
|
||||||
|
|
||||||
|
// If we run into trouble with our template parsing, we'll just return the cdnUrl, exactly as we found it
|
||||||
|
tmpl, err := template.New("").Parse(cdnUrl)
|
||||||
|
if err != nil {
|
||||||
|
return cdnUrl, false
|
||||||
|
}
|
||||||
|
|
||||||
|
var result bytes.Buffer
|
||||||
|
err = tmpl.Execute(&result, tmpUrl)
|
||||||
|
if err != nil {
|
||||||
|
log.Fatalf("ERROR: Could not process CDN URL template(s) from URL string: %s\n", cdnUrl)
|
||||||
|
}
|
||||||
|
|
||||||
|
return result.String(), true
|
||||||
|
|
||||||
|
}
|
||||||
|
|
|
@ -3,4 +3,4 @@ package rpmutils
|
||||||
import "regexp"
|
import "regexp"
|
||||||
|
|
||||||
// Nvr is a regular expression that matches a NVR.
|
// Nvr is a regular expression that matches a NVR.
|
||||||
var Nvr = regexp.MustCompile("^(\\S+)-([\\w~%.+]+)-(\\w+(?:\\.[\\w+]+)+?)(?:\\.rpm)?$")
|
var Nvr = regexp.MustCompile("^(\\S+)-([\\w~%.+]+)-(\\w+(?:\\.[\\w~%+]+)+?)(?:\\.rpm)?$")
|
||||||
|
|
|
@ -23,7 +23,7 @@ package srpmproc
|
||||||
import (
|
import (
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io/ioutil"
|
"io"
|
||||||
"log"
|
"log"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
@ -48,10 +48,16 @@ import (
|
||||||
|
|
||||||
func cfgPatches(pd *data.ProcessData, md *data.ModeData, patchTree *git.Worktree, pushTree *git.Worktree) error {
|
func cfgPatches(pd *data.ProcessData, md *data.ModeData, patchTree *git.Worktree, pushTree *git.Worktree) error {
|
||||||
// check CFG patches
|
// check CFG patches
|
||||||
_, err := patchTree.Filesystem.Stat("ROCKY/CFG")
|
// use PATCHES directory if it exists otherwise ROCKY/CFG
|
||||||
|
cfgdir := "PATCHES"
|
||||||
|
_, err := patchTree.Filesystem.Stat(cfgdir)
|
||||||
|
if err != nil {
|
||||||
|
cfgdir = "ROCKY/CFG"
|
||||||
|
}
|
||||||
|
_, err = patchTree.Filesystem.Stat(cfgdir)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
// iterate through patches
|
// iterate through patches
|
||||||
infos, err := patchTree.Filesystem.ReadDir("ROCKY/CFG")
|
infos, err := patchTree.Filesystem.ReadDir(cfgdir)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("could not walk patches: %v", err)
|
return fmt.Errorf("could not walk patches: %v", err)
|
||||||
}
|
}
|
||||||
|
@ -63,12 +69,12 @@ func cfgPatches(pd *data.ProcessData, md *data.ModeData, patchTree *git.Worktree
|
||||||
}
|
}
|
||||||
|
|
||||||
pd.Log.Printf("applying directive %s", info.Name())
|
pd.Log.Printf("applying directive %s", info.Name())
|
||||||
filePath := filepath.Join("ROCKY/CFG", info.Name())
|
filePath := filepath.Join(cfgdir, info.Name())
|
||||||
directive, err := patchTree.Filesystem.Open(filePath)
|
directive, err := patchTree.Filesystem.Open(filePath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("could not open directive file %s: %v", info.Name(), err)
|
return fmt.Errorf("could not open directive file %s: %v", info.Name(), err)
|
||||||
}
|
}
|
||||||
directiveBytes, err := ioutil.ReadAll(directive)
|
directiveBytes, err := io.ReadAll(directive)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("could not read directive file: %v", err)
|
return fmt.Errorf("could not read directive file: %v", err)
|
||||||
}
|
}
|
||||||
|
@ -96,7 +102,12 @@ func cfgPatches(pd *data.ProcessData, md *data.ModeData, patchTree *git.Worktree
|
||||||
|
|
||||||
func applyPatches(pd *data.ProcessData, md *data.ModeData, patchTree *git.Worktree, pushTree *git.Worktree) error {
|
func applyPatches(pd *data.ProcessData, md *data.ModeData, patchTree *git.Worktree, pushTree *git.Worktree) error {
|
||||||
// check if patches exist
|
// check if patches exist
|
||||||
_, err := patchTree.Filesystem.Stat("ROCKY")
|
cfgdir := "PATCHES"
|
||||||
|
_, err := patchTree.Filesystem.Stat(cfgdir)
|
||||||
|
if err != nil {
|
||||||
|
cfgdir = "ROCKY"
|
||||||
|
}
|
||||||
|
_, err = patchTree.Filesystem.Stat(cfgdir)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
err := cfgPatches(pd, md, patchTree, pushTree)
|
err := cfgPatches(pd, md, patchTree, pushTree)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -315,7 +326,7 @@ func patchModuleYaml(pd *data.ProcessData, md *data.ModeData) error {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
content, err := ioutil.ReadAll(f)
|
content, err := io.ReadAll(f)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("could not read modulemd file: %v", err)
|
return fmt.Errorf("could not read modulemd file: %v", err)
|
||||||
}
|
}
|
||||||
|
@ -397,6 +408,12 @@ func patchModuleYaml(pd *data.ProcessData, md *data.ModeData) error {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// If we got this far, we're good. This means the repos, branches, and commits exist.
|
||||||
|
// If this option is on, just use the branch name.
|
||||||
|
if pd.ModuleBranchNames {
|
||||||
|
tipHash = md.PushBranch
|
||||||
|
}
|
||||||
|
|
||||||
rpm.Ref = tipHash
|
rpm.Ref = tipHash
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -101,8 +101,15 @@ type ProcessDataRequest struct {
|
||||||
PackageVersion string
|
PackageVersion string
|
||||||
PackageRelease string
|
PackageRelease string
|
||||||
|
|
||||||
TaglessMode bool
|
TaglessMode bool
|
||||||
AltLookAside bool
|
Cdn string
|
||||||
|
|
||||||
|
ModuleBranchNames bool
|
||||||
|
}
|
||||||
|
|
||||||
|
type LookasidePath struct {
|
||||||
|
Distro string
|
||||||
|
Url string
|
||||||
}
|
}
|
||||||
|
|
||||||
func gitlabify(str string) string {
|
func gitlabify(str string) string {
|
||||||
|
@ -113,7 +120,57 @@ func gitlabify(str string) string {
|
||||||
return strings.Replace(str, "+", "plus", -1)
|
return strings.Replace(str, "+", "plus", -1)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// List of distros and their lookaside patterns
|
||||||
|
// If we find one of these passed as --cdn (ex: "--cdn fedora"), then we override, and assign this URL to be our --cdn-url
|
||||||
|
func StaticLookasides() []LookasidePath {
|
||||||
|
centos := LookasidePath{
|
||||||
|
Distro: "centos",
|
||||||
|
Url: "https://git.centos.org/sources/{{.Name}}/{{.Branch}}/{{.Hash}}",
|
||||||
|
}
|
||||||
|
centosStream := LookasidePath{
|
||||||
|
Distro: "centos-stream",
|
||||||
|
Url: "https://sources.stream.centos.org/sources/rpms/{{.Name}}/{{.Filename}}/{{.Hashtype}}/{{.Hash}}/{{.Filename}}",
|
||||||
|
}
|
||||||
|
rocky8 := LookasidePath{
|
||||||
|
Distro: "rocky8",
|
||||||
|
Url: "https://rocky-linux-sources-staging.a1.rockylinux.org/{{.Hash}}",
|
||||||
|
}
|
||||||
|
rocky := LookasidePath{
|
||||||
|
Distro: "rocky",
|
||||||
|
Url: "https://sources.build.resf.org/{{.Hash}}",
|
||||||
|
}
|
||||||
|
fedora := LookasidePath{
|
||||||
|
Distro: "fedora",
|
||||||
|
Url: "https://src.fedoraproject.org/repo/pkgs/{{.Name}}/{{.Filename}}/{{.Hashtype}}/{{.Hash}}/{{.Filename}}",
|
||||||
|
}
|
||||||
|
|
||||||
|
return []LookasidePath{centos, centosStream, rocky8, rocky, fedora}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
// Given a "--cdn" entry like "centos", we can search through our struct list of distros, and return the proper lookaside URL
|
||||||
|
// If we can't find it, we return false and the calling function will error out
|
||||||
|
func FindDistro(cdn string) (string, bool) {
|
||||||
|
var cdnUrl = ""
|
||||||
|
|
||||||
|
// Loop through each distro in the static list defined, try to find a match with "--cdn":
|
||||||
|
for _, distro := range StaticLookasides() {
|
||||||
|
if distro.Distro == strings.ToLower(cdn) {
|
||||||
|
cdnUrl = distro.Url
|
||||||
|
return cdnUrl, true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
|
||||||
func NewProcessData(req *ProcessDataRequest) (*data.ProcessData, error) {
|
func NewProcessData(req *ProcessDataRequest) (*data.ProcessData, error) {
|
||||||
|
// Build the logger to use for the data import
|
||||||
|
var writer io.Writer = os.Stdout
|
||||||
|
if req.LogWriter != nil {
|
||||||
|
writer = req.LogWriter
|
||||||
|
}
|
||||||
|
logger := log.New(writer, "", log.LstdFlags)
|
||||||
|
|
||||||
// Set defaults
|
// Set defaults
|
||||||
if req.ModulePrefix == "" {
|
if req.ModulePrefix == "" {
|
||||||
req.ModulePrefix = ModulePrefixCentOS
|
req.ModulePrefix = ModulePrefixCentOS
|
||||||
|
@ -139,13 +196,22 @@ func NewProcessData(req *ProcessDataRequest) (*data.ProcessData, error) {
|
||||||
if req.BranchPrefix == "" {
|
if req.BranchPrefix == "" {
|
||||||
req.BranchPrefix = "r"
|
req.BranchPrefix = "r"
|
||||||
}
|
}
|
||||||
if req.CdnUrl == "" && !req.AltLookAside {
|
if req.CdnUrl == "" {
|
||||||
req.CdnUrl = "https://git.centos.org/sources"
|
req.CdnUrl = "https://git.centos.org/sources"
|
||||||
}
|
}
|
||||||
// If altlookaside is enabled, and the CdnUrl hasn't been changed, then automatically set it to the default
|
|
||||||
// CentOS Stream (the new pattern very much won't work with the old git.centos.org/sources site)
|
// If a Cdn distro is defined, we try to find a match from StaticLookasides() array of structs
|
||||||
if (req.CdnUrl == "https://git.centos.org/sources" || req.CdnUrl == "") && req.AltLookAside {
|
// see if we have a match to --cdn (matching values are things like fedora, centos, rocky8, etc.)
|
||||||
req.CdnUrl = "https://sources.stream.centos.org/sources/rpms"
|
// If we match, then we want to short-circuit the CdnUrl to the assigned distro's one
|
||||||
|
if req.Cdn != "" {
|
||||||
|
newCdn, foundDistro := FindDistro(req.Cdn)
|
||||||
|
|
||||||
|
if !foundDistro {
|
||||||
|
return nil, fmt.Errorf("Error, distro name given as --cdn argument is not valid.")
|
||||||
|
}
|
||||||
|
|
||||||
|
req.CdnUrl = newCdn
|
||||||
|
logger.Printf("Discovered --cdn distro: %s . Using override CDN URL Pattern: %s", req.Cdn, req.CdnUrl)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Validate required
|
// Validate required
|
||||||
|
@ -214,12 +280,6 @@ func NewProcessData(req *ProcessDataRequest) (*data.ProcessData, error) {
|
||||||
reqFsCreator = req.FsCreator
|
reqFsCreator = req.FsCreator
|
||||||
}
|
}
|
||||||
|
|
||||||
var writer io.Writer = os.Stdout
|
|
||||||
if req.LogWriter != nil {
|
|
||||||
writer = req.LogWriter
|
|
||||||
}
|
|
||||||
logger := log.New(writer, "", log.LstdFlags)
|
|
||||||
|
|
||||||
if req.TmpFsMode != "" {
|
if req.TmpFsMode != "" {
|
||||||
logger.Printf("using tmpfs dir: %s", req.TmpFsMode)
|
logger.Printf("using tmpfs dir: %s", req.TmpFsMode)
|
||||||
fsCreator = func(branch string) (billy.Filesystem, error) {
|
fsCreator = func(branch string) (billy.Filesystem, error) {
|
||||||
|
@ -276,7 +336,8 @@ func NewProcessData(req *ProcessDataRequest) (*data.ProcessData, error) {
|
||||||
PackageVersion: req.PackageVersion,
|
PackageVersion: req.PackageVersion,
|
||||||
PackageRelease: req.PackageRelease,
|
PackageRelease: req.PackageRelease,
|
||||||
TaglessMode: req.TaglessMode,
|
TaglessMode: req.TaglessMode,
|
||||||
AltLookAside: req.AltLookAside,
|
Cdn: req.Cdn,
|
||||||
|
ModuleBranchNames: req.ModuleBranchNames,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -334,6 +395,7 @@ func ProcessRPM(pd *data.ProcessData) (*srpmprocpb.ProcessResponse, error) {
|
||||||
list, err := remote.List(&git.ListOptions{
|
list, err := remote.List(&git.ListOptions{
|
||||||
Auth: pd.Authenticator,
|
Auth: pd.Authenticator,
|
||||||
})
|
})
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Println("ignoring no-dup-mode")
|
log.Println("ignoring no-dup-mode")
|
||||||
} else {
|
} else {
|
||||||
|
@ -354,17 +416,17 @@ func ProcessRPM(pd *data.ProcessData) (*srpmprocpb.ProcessResponse, error) {
|
||||||
if pd.SingleTag != "" {
|
if pd.SingleTag != "" {
|
||||||
md.Branches = []string{fmt.Sprintf("refs/tags/%s", pd.SingleTag)}
|
md.Branches = []string{fmt.Sprintf("refs/tags/%s", pd.SingleTag)}
|
||||||
} else if len(pd.ManualCommits) > 0 {
|
} else if len(pd.ManualCommits) > 0 {
|
||||||
md.Branches = []string{}
|
log.Println("Manual commits were listed for import. Switching to perform a tagless import of these commit(s).")
|
||||||
for _, commit := range pd.ManualCommits {
|
pd.TaglessMode = true
|
||||||
branchCommit := strings.Split(commit, ":")
|
return processRPMTagless(pd)
|
||||||
if len(branchCommit) != 2 {
|
}
|
||||||
return nil, fmt.Errorf("invalid manual commit list")
|
|
||||||
}
|
|
||||||
|
|
||||||
head := fmt.Sprintf("refs/tags/imports/%s/%s-%s", branchCommit[0], md.Name, branchCommit[1])
|
// If we have no valid branches to consider, then we'll automatically switch to attempt a tagless import:
|
||||||
md.Branches = append(md.Branches, head)
|
if len(md.Branches) == 0 {
|
||||||
commitPin[head] = branchCommit[1]
|
log.Println("No valid tags (refs/tags/imports/*) found in repository! Switching to perform a tagless import.")
|
||||||
}
|
pd.TaglessMode = true
|
||||||
|
result, err := processRPMTagless(pd)
|
||||||
|
return result, err
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, branch := range md.Branches {
|
for _, branch := range md.Branches {
|
||||||
|
@ -527,7 +589,7 @@ func ProcessRPM(pd *data.ProcessData) (*srpmprocpb.ProcessResponse, error) {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("could not open ignored source file %s: %v", sourcePath, err)
|
return nil, fmt.Errorf("could not open ignored source file %s: %v", sourcePath, err)
|
||||||
}
|
}
|
||||||
sourceFileBts, err := ioutil.ReadAll(sourceFile)
|
sourceFileBts, err := io.ReadAll(sourceFile)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("could not read the whole of ignored source file: %v", err)
|
return nil, fmt.Errorf("could not read the whole of ignored source file: %v", err)
|
||||||
}
|
}
|
||||||
|
@ -596,6 +658,12 @@ func ProcessRPM(pd *data.ProcessData) (*srpmprocpb.ProcessResponse, error) {
|
||||||
|
|
||||||
// show status
|
// show status
|
||||||
status, _ := w.Status()
|
status, _ := w.Status()
|
||||||
|
if !pd.ModuleMode {
|
||||||
|
if status.IsClean() {
|
||||||
|
pd.Log.Printf("No changes detected. Our downstream is up to date.")
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
pd.Log.Printf("successfully processed:\n%s", status)
|
pd.Log.Printf("successfully processed:\n%s", status)
|
||||||
|
|
||||||
statusLines := strings.Split(status.String(), "\n")
|
statusLines := strings.Split(status.String(), "\n")
|
||||||
|
@ -681,7 +749,6 @@ func ProcessRPM(pd *data.ProcessData) (*srpmprocpb.ProcessResponse, error) {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Process for when we want to import a tagless repo (like from CentOS Stream)
|
// Process for when we want to import a tagless repo (like from CentOS Stream)
|
||||||
//
|
|
||||||
func processRPMTagless(pd *data.ProcessData) (*srpmprocpb.ProcessResponse, error) {
|
func processRPMTagless(pd *data.ProcessData) (*srpmprocpb.ProcessResponse, error) {
|
||||||
pd.Log.Println("Tagless mode detected, attempting import of latest commit")
|
pd.Log.Println("Tagless mode detected, attempting import of latest commit")
|
||||||
|
|
||||||
|
@ -746,6 +813,20 @@ func processRPMTagless(pd *data.ProcessData) (*srpmprocpb.ProcessResponse, error
|
||||||
sourceWorktree := *md.Worktree
|
sourceWorktree := *md.Worktree
|
||||||
localPath := ""
|
localPath := ""
|
||||||
|
|
||||||
|
// if a manual commit list is provided, we want to create our md.Branches[] array in a special format:
|
||||||
|
if len(pd.ManualCommits) > 0 {
|
||||||
|
md.Branches = []string{}
|
||||||
|
for _, commit := range pd.ManualCommits {
|
||||||
|
branchCommit := strings.Split(commit, ":")
|
||||||
|
if len(branchCommit) != 2 {
|
||||||
|
return nil, fmt.Errorf("invalid manual commit list")
|
||||||
|
}
|
||||||
|
|
||||||
|
head := fmt.Sprintf("COMMIT:%s:%s", branchCommit[0], branchCommit[1])
|
||||||
|
md.Branches = append(md.Branches, head)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
for _, branch := range md.Branches {
|
for _, branch := range md.Branches {
|
||||||
md.Repo = &sourceRepo
|
md.Repo = &sourceRepo
|
||||||
md.Worktree = &sourceWorktree
|
md.Worktree = &sourceWorktree
|
||||||
|
@ -765,13 +846,35 @@ func processRPMTagless(pd *data.ProcessData) (*srpmprocpb.ProcessResponse, error
|
||||||
return nil, fmt.Errorf("Could not create temporary directory: %s", localPath)
|
return nil, fmt.Errorf("Could not create temporary directory: %s", localPath)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// we'll make our branch we're processing more presentable if it's in the COMMIT:<branch>:<hash> format:
|
||||||
|
if strings.HasPrefix(branch, "COMMIT:") {
|
||||||
|
branch = fmt.Sprintf("refs/heads/%s", strings.Split(branch, ":")[1])
|
||||||
|
}
|
||||||
|
|
||||||
// Clone repo into the temporary path, but only the tag we're interested in:
|
// Clone repo into the temporary path, but only the tag we're interested in:
|
||||||
// (TODO: will probably need to assign this a variable or use the md struct gitrepo object to perform a successful tag+push later)
|
// (TODO: will probably need to assign this a variable or use the md struct gitrepo object to perform a successful tag+push later)
|
||||||
_, _ = git.PlainClone(localPath, false, &git.CloneOptions{
|
rTmp, err := git.PlainClone(localPath, false, &git.CloneOptions{
|
||||||
URL: pd.RpmLocation,
|
URL: pd.RpmLocation,
|
||||||
SingleBranch: true,
|
SingleBranch: true,
|
||||||
ReferenceName: plumbing.ReferenceName(branch),
|
ReferenceName: plumbing.ReferenceName(branch),
|
||||||
})
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// If we're dealing with a special manual commit to import ("COMMIT:<branch>:<githash>"), then we need to check out that
|
||||||
|
// specific hash from the repo we are importing from:
|
||||||
|
if strings.HasPrefix(md.TagBranch, "COMMIT:") {
|
||||||
|
commitList := strings.Split(md.TagBranch, ":")
|
||||||
|
|
||||||
|
wTmp, _ := rTmp.Worktree()
|
||||||
|
err = wTmp.Checkout(&git.CheckoutOptions{
|
||||||
|
Hash: plumbing.NewHash(commitList[2]),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("Could not find manual commit %s in the repository. Must be a valid commit hash.", commitList[2])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Now that we're cloned into localPath, we need to "covert" the import into the old format
|
// Now that we're cloned into localPath, we need to "covert" the import into the old format
|
||||||
// We want sources to become .PKGNAME.metadata, we want SOURCES and SPECS folders, etc.
|
// We want sources to become .PKGNAME.metadata, we want SOURCES and SPECS folders, etc.
|
||||||
|
@ -788,9 +891,9 @@ func processRPMTagless(pd *data.ProcessData) (*srpmprocpb.ProcessResponse, error
|
||||||
|
|
||||||
// get name-version-release of tagless repo, only if we're not a module repo:
|
// get name-version-release of tagless repo, only if we're not a module repo:
|
||||||
if !pd.ModuleMode {
|
if !pd.ModuleMode {
|
||||||
nvrString := getVersionFromSpec(md.Name, localPath, pd.Version)
|
nvrString, err := getVersionFromSpec(localPath, pd.Version)
|
||||||
if nvrString == "" {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("Error using rpm or rpmbuild to build SRPM and determine version info! (tagless mode)")
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// Set version and release fields we extracted (name|version|release are separated by pipes)
|
// Set version and release fields we extracted (name|version|release are separated by pipes)
|
||||||
|
@ -852,6 +955,12 @@ func processRPMTagless(pd *data.ProcessData) (*srpmprocpb.ProcessResponse, error
|
||||||
Force: true,
|
Force: true,
|
||||||
})
|
})
|
||||||
|
|
||||||
|
// These os commands actually move the data from our cloned import repo to our cloned "push" repo (upstream -> downstream)
|
||||||
|
// First we clobber the target push repo's data, and then move the new data into it from the upstream repo we cloned earlier
|
||||||
|
os.RemoveAll(fmt.Sprintf("%s_gitpush/SPECS", localPath))
|
||||||
|
os.RemoveAll(fmt.Sprintf("%s_gitpush/SOURCES", localPath))
|
||||||
|
os.RemoveAll(fmt.Sprintf("%s_gitpush/.gitignore", localPath))
|
||||||
|
os.RemoveAll(fmt.Sprintf("%s_gitpush/%s.metadata", localPath, md.Name))
|
||||||
os.Rename(fmt.Sprintf("%s/SPECS", localPath), fmt.Sprintf("%s_gitpush/SPECS", localPath))
|
os.Rename(fmt.Sprintf("%s/SPECS", localPath), fmt.Sprintf("%s_gitpush/SPECS", localPath))
|
||||||
os.Rename(fmt.Sprintf("%s/SOURCES", localPath), fmt.Sprintf("%s_gitpush/SOURCES", localPath))
|
os.Rename(fmt.Sprintf("%s/SOURCES", localPath), fmt.Sprintf("%s_gitpush/SOURCES", localPath))
|
||||||
os.Rename(fmt.Sprintf("%s/.gitignore", localPath), fmt.Sprintf("%s_gitpush/.gitignore", localPath))
|
os.Rename(fmt.Sprintf("%s/.gitignore", localPath), fmt.Sprintf("%s_gitpush/.gitignore", localPath))
|
||||||
|
@ -892,6 +1001,12 @@ func processRPMTagless(pd *data.ProcessData) (*srpmprocpb.ProcessResponse, error
|
||||||
}
|
}
|
||||||
|
|
||||||
status, err := w.Status()
|
status, err := w.Status()
|
||||||
|
if !pd.ModuleMode {
|
||||||
|
if status.IsClean() {
|
||||||
|
pd.Log.Printf("No changes detected. Our downstream is up to date.")
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
pd.Log.Printf("successfully processed:\n%s", status)
|
pd.Log.Printf("successfully processed:\n%s", status)
|
||||||
|
|
||||||
// assign tag for our new remote we're about to push (derived from the SRPM version)
|
// assign tag for our new remote we're about to push (derived from the SRPM version)
|
||||||
|
@ -985,7 +1100,6 @@ func processRPMTagless(pd *data.ProcessData) (*srpmprocpb.ProcessResponse, error
|
||||||
Version: pd.PackageVersion,
|
Version: pd.PackageVersion,
|
||||||
Release: pd.PackageRelease,
|
Release: pd.PackageRelease,
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// return struct with all our branch:commit and branch:version+release mappings
|
// return struct with all our branch:commit and branch:version+release mappings
|
||||||
|
@ -1108,113 +1222,50 @@ func convertMetaData(pkgName string, localRepo string) bool {
|
||||||
// Given a local checked out folder and package name, including SPECS/ , SOURCES/ , and .package.metadata, this will:
|
// Given a local checked out folder and package name, including SPECS/ , SOURCES/ , and .package.metadata, this will:
|
||||||
// - create a "dummy" SRPM (using dummy sources files we use to populate tarballs from lookaside)
|
// - create a "dummy" SRPM (using dummy sources files we use to populate tarballs from lookaside)
|
||||||
// - extract RPM version info from that SRPM, and return it
|
// - extract RPM version info from that SRPM, and return it
|
||||||
|
//
|
||||||
// If we are in tagless mode, we need to get a package version somehow!
|
// If we are in tagless mode, we need to get a package version somehow!
|
||||||
func getVersionFromSpec(pkgName string, localRepo string, majorVersion int) string {
|
func getVersionFromSpec(localRepo string, majorVersion int) (string, error) {
|
||||||
// Make sure we have "rpm" and "rpmbuild" and "cp" available in our PATH. Otherwise, this won't work:
|
// Make sure we have "rpm" and "rpmbuild" and "cp" available in our PATH. Otherwise, this won't work:
|
||||||
_, err := exec.LookPath("rpm")
|
_, err := exec.LookPath("rpmspec")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return ""
|
return "", fmt.Errorf("Could not find rpmspec program in PATH")
|
||||||
}
|
}
|
||||||
|
|
||||||
_, err = exec.LookPath("rpmbuild")
|
// Read the first file from SPECS/ to get our spec file
|
||||||
|
// (there should only be one file - we check that it ends in ".spec" just to be sure!)
|
||||||
|
lsTmp, err := ioutil.ReadDir(fmt.Sprintf("%s/SPECS/", localRepo))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return ""
|
return "", err
|
||||||
|
}
|
||||||
|
specFile := lsTmp[0].Name()
|
||||||
|
|
||||||
|
if !strings.HasSuffix(specFile, ".spec") {
|
||||||
|
return "", fmt.Errorf("First file found in SPECS/ is not a .spec file! Check the SPECS/ directory in the repo?")
|
||||||
}
|
}
|
||||||
|
|
||||||
_, err = exec.LookPath("cp")
|
// Call the rpmspec binary to extract the version-release info out of it, and tack on ".el<VERSION>" at the end:
|
||||||
|
cmdArgs := []string{
|
||||||
|
"--srpm",
|
||||||
|
fmt.Sprintf(`--define=dist .el%d`, majorVersion),
|
||||||
|
fmt.Sprintf(`--define=_topdir %s`, localRepo),
|
||||||
|
"-q",
|
||||||
|
"--queryformat",
|
||||||
|
`%{NAME}|%{VERSION}|%{RELEASE}\n`,
|
||||||
|
fmt.Sprintf("%s/SPECS/%s", localRepo, specFile),
|
||||||
|
}
|
||||||
|
cmd := exec.Command("rpmspec", cmdArgs...)
|
||||||
|
nvrTmp, err := cmd.Output()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return ""
|
return "", fmt.Errorf("Error running rpmspec command to determine RPM name-version-release identifier. \nCommand attempted: %s \nCommand output: %s", cmd.String(), string(nvrTmp))
|
||||||
}
|
}
|
||||||
|
|
||||||
// create separate temp folder space to do our RPM work - we don't want to accidentally contaminate the main Git area:
|
// Pull first line of the version output to get the name-version-release number (there should only be 1 line)
|
||||||
rpmBuildPath := fmt.Sprintf("%s_rpm", localRepo)
|
|
||||||
os.Mkdir(rpmBuildPath, 0o755)
|
|
||||||
|
|
||||||
// Copy SOURCES/ and SPECS/ into the temp rpmbuild directory recursively
|
|
||||||
// Yes, we could create or import an elaborate Go-native way to do this, but damnit this is easier:
|
|
||||||
cmdArgs := strings.Fields(fmt.Sprintf("cp -rp %s/SOURCES %s/SPECS %s/", localRepo, localRepo, rpmBuildPath))
|
|
||||||
if err := exec.Command(cmdArgs[0], cmdArgs[1:]...).Run(); err != nil {
|
|
||||||
log.Println(err)
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
|
|
||||||
// Loop through .<package>.metadata and get the file names we need to make our SRPM:
|
|
||||||
lookAside, err := os.Open(fmt.Sprintf("%s/.%s.metadata", localRepo, pkgName))
|
|
||||||
if err != nil {
|
|
||||||
log.Println(err)
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
|
|
||||||
// Split file into lines and start processing:
|
|
||||||
scanner := bufio.NewScanner(lookAside)
|
|
||||||
scanner.Split(bufio.ScanLines)
|
|
||||||
|
|
||||||
// loop through each line, and:
|
|
||||||
// - isolate the SOURCES/filename entry
|
|
||||||
// - write out a dummy file of the same name to rpmBuildPath/SOURCES
|
|
||||||
for scanner.Scan() {
|
|
||||||
|
|
||||||
// lookaside source is always the 2nd part of the line (after the long SHA sum)
|
|
||||||
srcFile := strings.Fields(scanner.Text())[1]
|
|
||||||
|
|
||||||
// write a dummy file of the same name into the rpmbuild SOURCES/ directory:
|
|
||||||
dummyFile, err := os.OpenFile(fmt.Sprintf("%s/%s", rpmBuildPath, srcFile), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644)
|
|
||||||
if err != nil {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
writer := bufio.NewWriter(dummyFile)
|
|
||||||
_, _ = writer.WriteString("This is a dummy lookaside file generated by srpmproc. It is only needed to get a working SRPM and extract version information. Please disregard\n")
|
|
||||||
writer.Flush()
|
|
||||||
dummyFile.Close()
|
|
||||||
}
|
|
||||||
|
|
||||||
lookAside.Close()
|
|
||||||
|
|
||||||
// Now, call rpmbuild to produce the dummy src file:
|
|
||||||
// Example: rpmbuild --define "_topdir /tmp/srpmproctmp_httpd1988142783_rpm" -bs /tmp/srpmproctmp_httpd1988142783_rpm/SPECS/*.spec
|
|
||||||
cmd := exec.Command("rpmbuild", fmt.Sprintf(`--define=_topdir %s`, rpmBuildPath), fmt.Sprintf(`--define=dist .el%d`, majorVersion), "-bs", fmt.Sprintf("%s/SPECS/%s.spec", rpmBuildPath, pkgName))
|
|
||||||
if err := cmd.Run(); err != nil {
|
|
||||||
log.Println(err)
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
|
|
||||||
// Read the first file from the SRPMS/ folder in rpmBuildPath. It should be the SRPM that rpmbuild produced above
|
|
||||||
// (there should only be one file - we check that it ends in ".rpm" just to be sure!)
|
|
||||||
lsTmp, err := ioutil.ReadDir(fmt.Sprintf("%s/SRPMS/", rpmBuildPath))
|
|
||||||
if err != nil {
|
|
||||||
log.Println(err)
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
|
|
||||||
srpmFile := lsTmp[0].Name()
|
|
||||||
|
|
||||||
if !strings.HasSuffix(srpmFile, ".rpm") {
|
|
||||||
log.Println("Error, file found in dummy SRPMS directory did not have an .rpm extension! Perhaps rpmbuild didn't produce a proper source RPM?")
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
|
|
||||||
// Call the rpm binary to extract the version-release info out of it, and tack on ".el<VERSION>" at the end:
|
|
||||||
cmd = exec.Command("rpm", "-qp", "--qf", `%{NAME}|%{VERSION}|%{RELEASE}\n`, fmt.Sprintf("%s/SRPMS/%s", rpmBuildPath, srpmFile))
|
|
||||||
nvrTmp, err := cmd.CombinedOutput()
|
|
||||||
if err != nil {
|
|
||||||
log.Println("Error running rpm command to extract temporary SRPM name-version-release identifiers.")
|
|
||||||
log.Println("rpmbuild output: ", string(nvrTmp))
|
|
||||||
log.Println("rpmbuild command: ", cmd.String())
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
|
|
||||||
// Pull first line of the rpm command's output to get the name-version-release number (there should only be 1 line)
|
|
||||||
nvr := string(nvrTmp)
|
nvr := string(nvrTmp)
|
||||||
nvr = strings.Fields(nvr)[0]
|
nvr = strings.Fields(nvr)[0]
|
||||||
|
|
||||||
// Clean up: delete the temporary directory
|
|
||||||
if err := os.RemoveAll(rpmBuildPath); err != nil {
|
|
||||||
log.Printf("Error cleaning up temporary RPM directory %s . Non-fatal, continuing anyway...\n", rpmBuildPath)
|
|
||||||
}
|
|
||||||
|
|
||||||
// return name-version-release string we derived:
|
// return name-version-release string we derived:
|
||||||
log.Printf("Derived NVR %s from tagless repo via temporary SRPM build\n", nvr)
|
log.Printf("Derived NVR %s from tagless repo via rpmspec command\n", nvr)
|
||||||
return nvr
|
return nvr, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// We need to loop through the lookaside blob files ("SourcesToIgnore"),
|
// We need to loop through the lookaside blob files ("SourcesToIgnore"),
|
||||||
|
@ -1248,7 +1299,7 @@ func processLookasideSources(pd *data.ProcessData, md *data.ModeData, localDir s
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("could not open ignored source file %s: %v", sourcePath, err)
|
return fmt.Errorf("could not open ignored source file %s: %v", sourcePath, err)
|
||||||
}
|
}
|
||||||
sourceFileBts, err := ioutil.ReadAll(sourceFile)
|
sourceFileBts, err := io.ReadAll(sourceFile)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("could not read the whole of ignored source file: %v", err)
|
return fmt.Errorf("could not read the whole of ignored source file: %v", err)
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue
Block a user