Compare commits
28 Commits
Author | SHA1 | Date |
---|---|---|
Mustafa Gezen | f2864a875a | |
Dirk Müller | 82271f5d9e | |
Mustafa Gezen | 963b5d25dd | |
Mustafa Gezen | a5f9082c15 | |
Mustafa Gezen | f2374845e3 | |
Louis Abel | 162e7cea34 | |
Louis Abel | edeb0239df | |
Mustafa Gezen | 40180a342b | |
skip77 | 7f14563f93 | |
Mustafa Gezen | 6e2ed2eb23 | |
Skip Grube | 44725a727d | |
Skip Grube | 4f975913b4 | |
Skip Grube | 3371826851 | |
skip77 | f4f2a0a9e4 | |
skip77 | 5e8ed2e521 | |
Skip Grube | 2eb0768f33 | |
Mustafa Gezen | 2c6d6f0b0a | |
Skip Grube | 0ea6123b6e | |
Skip Grube | 2cc3bcc3a3 | |
Skip Grube | 3237794071 | |
Mustafa Gezen | 06ffd1e507 | |
Mustafa Gezen | 29778c7ea2 | |
Mustafa Gezen | 2a2b10ba10 | |
Mustafa Gezen | e859f54e5e | |
Mustafa Gezen | f3ac46447d | |
skip77 | cff0cc0fbf | |
Mustafa Gezen | 9b44bc6e50 | |
jarod.w | d7ed1c7a7f |
|
@ -16,7 +16,7 @@ jobs:
|
|||
- name: Set up Go
|
||||
uses: actions/setup-go@v2
|
||||
with:
|
||||
go-version: 1.16
|
||||
go-version: 1.18
|
||||
- name: Run GoReleaser
|
||||
uses: goreleaser/goreleaser-action@v2
|
||||
with:
|
||||
|
|
|
@ -17,11 +17,13 @@ builds:
|
|||
- s390x
|
||||
- ppc64le
|
||||
archives:
|
||||
- replacements:
|
||||
darwin: Darwin
|
||||
linux: Linux
|
||||
amd64: x86_64
|
||||
arm64: aarch64
|
||||
- name_template: >-
|
||||
{{- .ProjectName }}_
|
||||
{{- .Version }}
|
||||
{{- title .Os }}_
|
||||
{{- if eq .Arch "amd64" }}x86_64
|
||||
{{- else if eq .Arch "arm64" }}aarch64
|
||||
{{- else }}{{ .Arch }}{{ end }}{{ end -}}
|
||||
checksum:
|
||||
name_template: 'checksums.txt'
|
||||
snapshot:
|
||||
|
|
42
README.md
42
README.md
|
@ -1,14 +1,14 @@
|
|||
# srpmproc
|
||||
Upstream package importer with auto patching. Reference implementation for OpenPatch
|
||||
|
||||
# Usage
|
||||
## Usage
|
||||
```
|
||||
Usage:
|
||||
srpmproc [flags]
|
||||
srpmproc [command]
|
||||
|
||||
Available Commands:
|
||||
fetch
|
||||
fetch
|
||||
help Help about any command
|
||||
|
||||
Flags:
|
||||
|
@ -16,7 +16,8 @@ Flags:
|
|||
--basic-username string Basic auth username
|
||||
--branch-prefix string Branch prefix (replaces import-branch-prefix) (default "r")
|
||||
--branch-suffix string Branch suffix to use for imported branches
|
||||
--cdn-url string CDN URL to download blobs from (default "https://git.centos.org/sources")
|
||||
--cdn string CDN URL shortcuts for well-known distros, auto-assigns --cdn-url. Valid values: rocky8, rocky, fedora, centos, centos-stream. Setting this overrides --cdn-url
|
||||
--cdn-url string CDN URL to download blobs from. Simple URL follows default rocky/centos patterns. Can be customized using macros (see docs) (default "https://git.centos.org/sources")
|
||||
--git-committer-email string Email of committer (default "rockyautomation@rockylinux.org")
|
||||
--git-committer-name string Name of committer (default "rockyautomation")
|
||||
-h, --help help for srpmproc
|
||||
|
@ -28,6 +29,8 @@ Flags:
|
|||
--no-dup-mode If enabled, skips already imported tags
|
||||
--no-storage-download If enabled, blobs are always downloaded from upstream
|
||||
--no-storage-upload If enabled, blobs are not uploaded to blob storage
|
||||
--package-release string Package release to fetch
|
||||
--package-version string Package version to fetch
|
||||
--rpm-prefix string Where to retrieve SRPM content. Only used when source-rpm is not a local file (default "https://git.centos.org/rpms")
|
||||
--single-tag string If set, only this tag is imported
|
||||
--source-rpm string Location of RPM to process
|
||||
|
@ -35,9 +38,42 @@ Flags:
|
|||
--ssh-user string SSH User (default "git")
|
||||
--storage-addr string Bucket to use as blob storage
|
||||
--strict-branch-mode If enabled, only branches with the calculated name are imported and not prefix only
|
||||
--taglessmode Tagless mode: If set, pull the latest commit from the branch and determine version numbers from spec file. This is auto-tried if tags aren't found.
|
||||
--tmpfs-mode string If set, packages are imported to path and patched but not pushed
|
||||
--upstream-prefix string Upstream git repository prefix
|
||||
--version int Upstream version
|
||||
|
||||
Use "srpmproc [command] --help" for more information about a command.
|
||||
```
|
||||
|
||||
<br />
|
||||
|
||||
## Examples:
|
||||
|
||||
1. Import the kernel package from git.centos.org/rpms/, to local folder /opt/gitroot/rpms/kernel.git/ . Download the lookaside source tarballs from the default CentOS file server location to local folder `/opt/fake_s3/` . We want to grab branch "c8" (import prefix plus RHEL version), and it will be committed as branch "r8" (branch prefix plus RHEL version). This assumes that `/opt/fake_s3` exists, and `/opt/gitroot/rpms/kernel.git` exists and is a git repository of some kind (even an empty one).
|
||||
|
||||
```
|
||||
srpmproc --branch-prefix "r" --import-branch-prefix "c" --rpm-prefix "https://git.centos.org/rpms" --version 8 --storage-addr file:///opt/fake_s3 --upstream-prefix file:///opt/gitroot --cdn centos --strict-branch-mode --source-rpm kernel
|
||||
```
|
||||
|
||||
<br />
|
||||
|
||||
## CDN and --cdn-url
|
||||
The --cdn-url option allows for Go-style templates to craft complex URL patterns. These templates are: `{{.Name}}` (package name), `{{.Hash}}` (hash of lookaside file), `{{.Hashtype}}` (hash type of file, like "sha256" or "sha512"), `{{.Branch}}` (the branch we are importing), and `{{.Filename}}` (the lookaside file's name as it appears in SOURCES/). You can add these values as part of --cdn-url to craft your lookaside pattern.
|
||||
|
||||
|
||||
For example, if I wanted my lookaside downloads to come from CentOS 9 Stream, I would use as part of my command:
|
||||
```
|
||||
--cdn-url "https://sources.stream.centos.org/sources/rpms/{{.Name}}/{{.Filename}}/{{.Hashtype}}/{{.Hash}}/{{.Filename}}"
|
||||
```
|
||||
|
||||
|
||||
**Default Behavior:** If these templates are not used, the default behavior of `--cdn-url` is to fall back on the traditional RHEL import pattern: `<CDN_URL>/<NAME>/<BRANCH>/<HASH>` . If that fails, a further fallback is attempted, the simple: `<CDN_URL>/<HASH>`. These cover the common Rocky Linux and RHEL/CentOS imports if the base lookaside URL is the only thing given. If no `--cdn-url` is specified, it defaults to "https://git.centos.org/sources" (for RHEL imports into Rocky Linux)
|
||||
|
||||
|
||||
**CDN Shorthand:** For convenience, some lookaside patterns for popular distros are provided via the `--cdn` option. You can specify this without needing to use the longer `--cdn-url`. For example, when importing from CentOS 9 Stream, you could use `--cdn centos-stream`
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
|
|
@ -21,11 +21,12 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"log"
|
||||
"os"
|
||||
|
||||
"github.com/go-git/go-billy/v5/osfs"
|
||||
"github.com/rocky-linux/srpmproc/pkg/srpmproc"
|
||||
"github.com/spf13/cobra"
|
||||
"log"
|
||||
"os"
|
||||
)
|
||||
|
||||
var fetch = &cobra.Command{
|
||||
|
|
|
@ -22,10 +22,11 @@ package main
|
|||
|
||||
import (
|
||||
"encoding/json"
|
||||
"github.com/rocky-linux/srpmproc/pkg/srpmproc"
|
||||
"log"
|
||||
"os"
|
||||
|
||||
"github.com/rocky-linux/srpmproc/pkg/srpmproc"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
|
@ -56,6 +57,9 @@ var (
|
|||
basicPassword string
|
||||
packageVersion string
|
||||
packageRelease string
|
||||
taglessMode bool
|
||||
cdn string
|
||||
moduleBranchNames bool
|
||||
)
|
||||
|
||||
var root = &cobra.Command{
|
||||
|
@ -92,6 +96,9 @@ func mn(_ *cobra.Command, _ []string) {
|
|||
HttpPassword: basicPassword,
|
||||
PackageVersion: packageVersion,
|
||||
PackageRelease: packageRelease,
|
||||
TaglessMode: taglessMode,
|
||||
Cdn: cdn,
|
||||
ModuleBranchNames: moduleBranchNames,
|
||||
})
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
|
@ -106,7 +113,6 @@ func mn(_ *cobra.Command, _ []string) {
|
|||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func main() {
|
||||
|
@ -127,7 +133,7 @@ func main() {
|
|||
root.Flags().StringVar(&rpmPrefix, "rpm-prefix", "https://git.centos.org/rpms", "Where to retrieve SRPM content. Only used when source-rpm is not a local file")
|
||||
root.Flags().StringVar(&importBranchPrefix, "import-branch-prefix", "c", "Import branch prefix")
|
||||
root.Flags().StringVar(&branchPrefix, "branch-prefix", "r", "Branch prefix (replaces import-branch-prefix)")
|
||||
root.Flags().StringVar(&cdnUrl, "cdn-url", "https://git.centos.org/sources", "CDN URL to download blobs from")
|
||||
root.Flags().StringVar(&cdnUrl, "cdn-url", "https://git.centos.org/sources", "CDN URL to download blobs from. Simple URL follows default rocky/centos patterns. Can be customized using macros (see docs)")
|
||||
root.Flags().StringVar(&singleTag, "single-tag", "", "If set, only this tag is imported")
|
||||
root.Flags().BoolVar(&noDupMode, "no-dup-mode", false, "If enabled, skips already imported tags")
|
||||
root.Flags().BoolVar(&moduleMode, "module-mode", false, "If enabled, imports a module instead of a package")
|
||||
|
@ -142,6 +148,9 @@ func main() {
|
|||
root.Flags().StringVar(&basicPassword, "basic-password", "", "Basic auth password")
|
||||
root.Flags().StringVar(&packageVersion, "package-version", "", "Package version to fetch")
|
||||
root.Flags().StringVar(&packageRelease, "package-release", "", "Package release to fetch")
|
||||
root.Flags().BoolVar(&taglessMode, "taglessmode", false, "Tagless mode: If set, pull the latest commit from the branch and determine version numbers from spec file. This is auto-tried if tags aren't found.")
|
||||
root.Flags().StringVar(&cdn, "cdn", "", "CDN URL shortcuts for well-known distros, auto-assigns --cdn-url. Valid values: rocky8, rocky, fedora, centos, centos-stream. Setting this overrides --cdn-url")
|
||||
root.Flags().BoolVar(&moduleBranchNames, "module-branch-names-only", false, "If enabled, module imports will use the branch name that is being imported, rather than use the commit hash.")
|
||||
|
||||
if err := root.Execute(); err != nil {
|
||||
log.Fatal(err)
|
||||
|
|
47
go.mod
47
go.mod
|
@ -1,6 +1,6 @@
|
|||
module github.com/rocky-linux/srpmproc
|
||||
|
||||
go 1.15
|
||||
go 1.18
|
||||
|
||||
require (
|
||||
cloud.google.com/go/storage v1.12.0
|
||||
|
@ -13,3 +13,48 @@ require (
|
|||
google.golang.org/protobuf v1.25.0
|
||||
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
|
||||
)
|
||||
|
||||
require (
|
||||
cloud.google.com/go v0.66.0 // indirect
|
||||
github.com/emirpasic/gods v1.12.0 // indirect
|
||||
github.com/fsnotify/fsnotify v1.4.7 // indirect
|
||||
github.com/go-git/gcfg v1.5.0 // indirect
|
||||
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
|
||||
github.com/golang/protobuf v1.4.2 // indirect
|
||||
github.com/googleapis/gax-go/v2 v2.0.5 // indirect
|
||||
github.com/hashicorp/hcl v1.0.0 // indirect
|
||||
github.com/imdario/mergo v0.3.9 // indirect
|
||||
github.com/inconshreveable/mousetrap v1.0.0 // indirect
|
||||
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
|
||||
github.com/jmespath/go-jmespath v0.4.0 // indirect
|
||||
github.com/jstemmer/go-junit-report v0.9.1 // indirect
|
||||
github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd // indirect
|
||||
github.com/magiconair/properties v1.8.1 // indirect
|
||||
github.com/mitchellh/go-homedir v1.1.0 // indirect
|
||||
github.com/mitchellh/mapstructure v1.1.2 // indirect
|
||||
github.com/pelletier/go-toml v1.2.0 // indirect
|
||||
github.com/sergi/go-diff v1.1.0 // indirect
|
||||
github.com/spf13/afero v1.1.2 // indirect
|
||||
github.com/spf13/cast v1.3.0 // indirect
|
||||
github.com/spf13/jwalterweatherman v1.0.0 // indirect
|
||||
github.com/spf13/pflag v1.0.5 // indirect
|
||||
github.com/subosito/gotenv v1.2.0 // indirect
|
||||
github.com/xanzy/ssh-agent v0.2.1 // indirect
|
||||
go.opencensus.io v0.22.4 // indirect
|
||||
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 // indirect
|
||||
golang.org/x/lint v0.0.0-20200302205851-738671d3881b // indirect
|
||||
golang.org/x/mod v0.3.0 // indirect
|
||||
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b // indirect
|
||||
golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43 // indirect
|
||||
golang.org/x/sys v0.1.0 // indirect
|
||||
golang.org/x/text v0.3.3 // indirect
|
||||
golang.org/x/tools v0.0.0-20200918232735-d647fc253266 // indirect
|
||||
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
|
||||
google.golang.org/api v0.32.0 // indirect
|
||||
google.golang.org/appengine v1.6.6 // indirect
|
||||
google.golang.org/genproto v0.0.0-20200921151605-7abf4a1a14d5 // indirect
|
||||
google.golang.org/grpc v1.32.0 // indirect
|
||||
gopkg.in/ini.v1 v1.51.0 // indirect
|
||||
gopkg.in/warnings.v0 v0.1.2 // indirect
|
||||
gopkg.in/yaml.v2 v2.2.8 // indirect
|
||||
)
|
||||
|
|
4
go.sum
4
go.sum
|
@ -85,7 +85,6 @@ github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.m
|
|||
github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
|
||||
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
|
||||
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
|
||||
github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BHsljHzVlRcyQhjrss6TZTdY2VfCqZPbv5k3iBFa2ZQ=
|
||||
github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
|
||||
github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
|
||||
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
|
||||
|
@ -440,8 +439,9 @@ golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7w
|
|||
golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200828194041-157a740278f4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f h1:+Nyd8tzPX9R7BWHguqsrbFdRx3WQ/1ib8I44HXV5yTA=
|
||||
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U=
|
||||
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
|
|
|
@ -22,6 +22,7 @@ package modulemd
|
|||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/go-git/go-billy/v5"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
@ -160,58 +161,49 @@ type Defaults struct {
|
|||
Data *DefaultsData `yaml:"data,omitempty"`
|
||||
}
|
||||
|
||||
func Parse(input []byte) (*ModuleMd, error) {
|
||||
type NotBackwardsCompatibleModuleMd struct {
|
||||
V2 *ModuleMd
|
||||
V3 *V3
|
||||
}
|
||||
|
||||
func Parse(input []byte) (*NotBackwardsCompatibleModuleMd, error) {
|
||||
var detect DetectVersionDocument
|
||||
err := yaml.Unmarshal(input, &detect)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error detecting document version: %s", err)
|
||||
}
|
||||
|
||||
var ret ModuleMd
|
||||
var ret NotBackwardsCompatibleModuleMd
|
||||
|
||||
if detect.Version == 2 {
|
||||
err = yaml.Unmarshal(input, &ret)
|
||||
var v2 ModuleMd
|
||||
err = yaml.Unmarshal(input, &v2)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error parsing modulemd: %s", err)
|
||||
}
|
||||
ret.V2 = &v2
|
||||
} else if detect.Version == 3 {
|
||||
var v3 V3
|
||||
err = yaml.Unmarshal(input, &v3)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error parsing modulemd: %s", err)
|
||||
}
|
||||
|
||||
ret = ModuleMd{
|
||||
Document: v3.Document,
|
||||
Version: v3.Version,
|
||||
Data: &Data{
|
||||
Name: v3.Data.Name,
|
||||
Stream: v3.Data.Stream,
|
||||
Summary: v3.Data.Summary,
|
||||
Description: v3.Data.Description,
|
||||
License: &License{
|
||||
Module: v3.Data.License,
|
||||
},
|
||||
Xmd: v3.Data.Xmd,
|
||||
References: v3.Data.References,
|
||||
Profiles: v3.Data.Profiles,
|
||||
Profile: v3.Data.Profile,
|
||||
API: v3.Data.API,
|
||||
Filter: v3.Data.Filter,
|
||||
BuildOpts: &BuildOpts{
|
||||
Rpms: v3.Data.Configurations[0].BuildOpts.Rpms,
|
||||
Arches: v3.Data.Configurations[0].BuildOpts.Arches,
|
||||
},
|
||||
Components: v3.Data.Components,
|
||||
},
|
||||
}
|
||||
ret.V3 = &v3
|
||||
}
|
||||
|
||||
return &ret, nil
|
||||
}
|
||||
|
||||
func (m *ModuleMd) Marshal(fs billy.Filesystem, path string) error {
|
||||
bts, err := yaml.Marshal(m)
|
||||
func (m *NotBackwardsCompatibleModuleMd) Marshal(fs billy.Filesystem, path string) error {
|
||||
var bts []byte
|
||||
|
||||
var err error
|
||||
if m.V2 != nil {
|
||||
bts, err = yaml.Marshal(m.V2)
|
||||
}
|
||||
if m.V3 != nil {
|
||||
bts, err = yaml.Marshal(m.V3)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
|
@ -38,7 +38,7 @@ func New(path string) *File {
|
|||
}
|
||||
|
||||
func (f *File) Write(path string, content []byte) error {
|
||||
w, err := os.OpenFile(filepath.Join(f.path, path), os.O_CREATE|os.O_TRUNC|os.O_RDWR, 0644)
|
||||
w, err := os.OpenFile(filepath.Join(f.path, path), os.O_CREATE|os.O_TRUNC|os.O_RDWR, 0o644)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not open file: %v", err)
|
||||
}
|
||||
|
@ -57,7 +57,7 @@ func (f *File) Write(path string, content []byte) error {
|
|||
}
|
||||
|
||||
func (f *File) Read(path string) ([]byte, error) {
|
||||
r, err := os.OpenFile(filepath.Join(f.path, path), os.O_RDONLY, 0644)
|
||||
r, err := os.OpenFile(filepath.Join(f.path, path), os.O_RDONLY, 0o644)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil, nil
|
||||
|
|
|
@ -21,10 +21,11 @@
|
|||
package gcs
|
||||
|
||||
import (
|
||||
"cloud.google.com/go/storage"
|
||||
"context"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
|
||||
"cloud.google.com/go/storage"
|
||||
)
|
||||
|
||||
type GCS struct {
|
||||
|
|
|
@ -22,6 +22,8 @@ package s3
|
|||
|
||||
import (
|
||||
"bytes"
|
||||
"io/ioutil"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/aws/awserr"
|
||||
"github.com/aws/aws-sdk-go/aws/credentials"
|
||||
|
@ -29,7 +31,6 @@ import (
|
|||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
"github.com/aws/aws-sdk-go/service/s3/s3manager"
|
||||
"github.com/spf13/viper"
|
||||
"io/ioutil"
|
||||
)
|
||||
|
||||
type S3 struct {
|
||||
|
|
|
@ -21,8 +21,9 @@
|
|||
package data
|
||||
|
||||
import (
|
||||
"github.com/go-git/go-git/v5"
|
||||
"hash"
|
||||
|
||||
"github.com/go-git/go-git/v5"
|
||||
)
|
||||
|
||||
type ImportMode interface {
|
||||
|
|
|
@ -21,10 +21,11 @@
|
|||
package data
|
||||
|
||||
import (
|
||||
"log"
|
||||
|
||||
"github.com/go-git/go-billy/v5"
|
||||
"github.com/go-git/go-git/v5/plumbing/transport"
|
||||
"github.com/rocky-linux/srpmproc/pkg/blob"
|
||||
"log"
|
||||
)
|
||||
|
||||
type FsCreatorFunc func(branch string) (billy.Filesystem, error)
|
||||
|
@ -57,4 +58,7 @@ type ProcessData struct {
|
|||
Log *log.Logger
|
||||
PackageVersion string
|
||||
PackageRelease string
|
||||
TaglessMode bool
|
||||
Cdn string
|
||||
ModuleBranchNames bool
|
||||
}
|
||||
|
|
|
@ -27,11 +27,12 @@ import (
|
|||
"crypto/sha512"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"github.com/go-git/go-billy/v5"
|
||||
"hash"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/go-git/go-billy/v5"
|
||||
)
|
||||
|
||||
func CopyFromFs(from billy.Filesystem, to billy.Filesystem, path string) error {
|
||||
|
@ -44,7 +45,7 @@ func CopyFromFs(from billy.Filesystem, to billy.Filesystem, path string) error {
|
|||
fullPath := filepath.Join(path, fi.Name())
|
||||
|
||||
if fi.IsDir() {
|
||||
_ = to.MkdirAll(fullPath, 0755)
|
||||
_ = to.MkdirAll(fullPath, 0o755)
|
||||
err := CopyFromFs(from, to, fullPath)
|
||||
if err != nil {
|
||||
return err
|
||||
|
|
|
@ -50,7 +50,7 @@ func add(cfg *srpmprocpb.Cfg, pd *data.ProcessData, md *data.ModeData, patchTree
|
|||
case *srpmprocpb.Add_File:
|
||||
filePath = checkAddPrefix(eitherString(filepath.Base(addType.File), add.Name))
|
||||
|
||||
fPatch, err := patchTree.Filesystem.OpenFile(addType.File, os.O_RDONLY, 0644)
|
||||
fPatch, err := patchTree.Filesystem.OpenFile(addType.File, os.O_RDONLY, 0o644)
|
||||
if err != nil {
|
||||
return errors.New(fmt.Sprintf("COULD_NOT_OPEN_FROM:%s", addType.File))
|
||||
}
|
||||
|
@ -80,7 +80,7 @@ func add(cfg *srpmprocpb.Cfg, pd *data.ProcessData, md *data.ModeData, patchTree
|
|||
break
|
||||
}
|
||||
|
||||
f, err := pushTree.Filesystem.OpenFile(filePath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
|
||||
f, err := pushTree.Filesystem.OpenFile(filePath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0o644)
|
||||
if err != nil {
|
||||
return errors.New(fmt.Sprintf("COULD_NOT_OPEN_DESTINATION:%s", filePath))
|
||||
}
|
||||
|
|
|
@ -128,7 +128,7 @@ func lookaside(cfg *srpmprocpb.Cfg, _ *data.ProcessData, md *data.ModeData, patc
|
|||
}
|
||||
|
||||
path := filepath.Join("SOURCES", fmt.Sprintf("%s.tar.gz", directive.ArchiveName))
|
||||
pushF, err := pushTree.Filesystem.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_RDWR, 0644)
|
||||
pushF, err := pushTree.Filesystem.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_RDWR, 0o644)
|
||||
if err != nil {
|
||||
return errors.New(fmt.Sprintf("COULD_NOT_CREATE_TAR_FILE:%s", path))
|
||||
}
|
||||
|
|
|
@ -24,6 +24,7 @@ import (
|
|||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/bluekeyes/go-gitdiff/gitdiff"
|
||||
"github.com/go-git/go-git/v5"
|
||||
srpmprocpb "github.com/rocky-linux/srpmproc/pb"
|
||||
|
|
|
@ -51,7 +51,7 @@ func replace(cfg *srpmprocpb.Cfg, pd *data.ProcessData, _ *data.ModeData, patchT
|
|||
|
||||
switch replacing := replace.Replacing.(type) {
|
||||
case *srpmprocpb.Replace_WithFile:
|
||||
fPatch, err := patchTree.Filesystem.OpenFile(replacing.WithFile, os.O_RDONLY, 0644)
|
||||
fPatch, err := patchTree.Filesystem.OpenFile(replacing.WithFile, os.O_RDONLY, 0o644)
|
||||
if err != nil {
|
||||
return errors.New(fmt.Sprintf("COULD_NOT_OPEN_REPLACING:%s", replacing.WithFile))
|
||||
}
|
||||
|
|
|
@ -41,9 +41,7 @@ const (
|
|||
sectionChangelog = "%changelog"
|
||||
)
|
||||
|
||||
var (
|
||||
sections = []string{"%description", "%prep", "%build", "%install", "%files", "%changelog"}
|
||||
)
|
||||
var sections = []string{"%description", "%prep", "%build", "%install", "%files", "%changelog"}
|
||||
|
||||
type sourcePatchOperationInLoopRequest struct {
|
||||
cfg *srpmprocpb.Cfg
|
||||
|
@ -212,7 +210,7 @@ func specChange(cfg *srpmprocpb.Cfg, pd *data.ProcessData, md *data.ModeData, _
|
|||
return errors.New("COULD_NOT_STAT_SPEC_FILE")
|
||||
}
|
||||
|
||||
specFile, err := pushTree.Filesystem.OpenFile(filePath, os.O_RDONLY, 0644)
|
||||
specFile, err := pushTree.Filesystem.OpenFile(filePath, os.O_RDONLY, 0o644)
|
||||
if err != nil {
|
||||
return errors.New("COULD_NOT_READ_SPEC_FILE")
|
||||
}
|
||||
|
|
|
@ -2,9 +2,11 @@ package misc
|
|||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/rocky-linux/srpmproc/pkg/data"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/rocky-linux/srpmproc/pkg/data"
|
||||
)
|
||||
|
||||
func GetTagImportRegex(pd *data.ProcessData) *regexp.Regexp {
|
||||
|
@ -31,3 +33,21 @@ func GetTagImportRegex(pd *data.ProcessData) *regexp.Regexp {
|
|||
|
||||
return regexp.MustCompile(regex)
|
||||
}
|
||||
|
||||
// Given a git reference in tagless mode (like "refs/heads/c9s", or "refs/heads/stream-httpd-2.4-rhel-9.1.0"), determine
|
||||
// if we are ok with importing that reference. We are looking for the traditional <prefix><version><suffix> pattern, like "c9s", and also the
|
||||
// modular "stream-<NAME>-<VERSION>-rhel-<VERSION> branch pattern as well
|
||||
func TaglessRefOk(tag string, pd *data.ProcessData) bool {
|
||||
// First case is very easy: if we are exactly "refs/heads/<prefix><version><suffix>" , then this is def. a branch we should import
|
||||
if tag == fmt.Sprintf("refs/heads/%s%d%s", pd.ImportBranchPrefix, pd.Version, pd.BranchSuffix) {
|
||||
return true
|
||||
}
|
||||
|
||||
// Less easy: if a modular branch is present (starts w/ "stream-"), we need to check if it's part of our major version, and return true if it is
|
||||
// (major version means we look for the text "rhel-X." in the branch name, like "rhel-9.1.0")
|
||||
if strings.HasPrefix(tag, "refs/heads/stream-") && strings.Contains(tag, fmt.Sprintf("rhel-%d.", pd.Version)) {
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
|
244
pkg/modes/git.go
244
pkg/modes/git.go
|
@ -21,16 +21,20 @@
|
|||
package modes
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"github.com/go-git/go-git/v5/plumbing/transport"
|
||||
"github.com/rocky-linux/srpmproc/pkg/misc"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"net/http"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"text/template"
|
||||
"time"
|
||||
|
||||
"github.com/go-git/go-git/v5/plumbing/transport"
|
||||
"github.com/rocky-linux/srpmproc/pkg/misc"
|
||||
|
||||
"github.com/go-git/go-billy/v5/memfs"
|
||||
"github.com/go-git/go-git/v5"
|
||||
"github.com/go-git/go-git/v5/config"
|
||||
|
@ -45,6 +49,15 @@ type remoteTarget struct {
|
|||
when time.Time
|
||||
}
|
||||
|
||||
// Struct to define the possible template values ( {{.Value}} in CDN URL strings:
|
||||
type Lookaside struct {
|
||||
Name string
|
||||
Branch string
|
||||
Hash string
|
||||
Hashtype string
|
||||
Filename string
|
||||
}
|
||||
|
||||
type remoteTargetSlice []remoteTarget
|
||||
|
||||
func (p remoteTargetSlice) Len() int {
|
||||
|
@ -116,7 +129,6 @@ func (g *GitMode) RetrieveSource(pd *data.ProcessData) (*data.ModeData, error) {
|
|||
if exists != nil && exists.when.After(tag.Tagger.When) {
|
||||
return nil
|
||||
}
|
||||
|
||||
latestTags[match[2]] = &remoteTarget{
|
||||
remote: refSpec,
|
||||
when: tag.Tagger.When,
|
||||
|
@ -126,11 +138,38 @@ func (g *GitMode) RetrieveSource(pd *data.ProcessData) (*data.ModeData, error) {
|
|||
return nil
|
||||
}
|
||||
|
||||
// In case of "tagless mode", we need to get the head ref of the branch instead
|
||||
// This is a kind of alternative implementation of the above tagAdd assignment
|
||||
refAdd := func(tag *object.Tag) error {
|
||||
if misc.TaglessRefOk(tag.Name, pd) {
|
||||
pd.Log.Printf("Tagless mode: Identified tagless commit for import: %s\n", tag.Name)
|
||||
refSpec := fmt.Sprintf(tag.Name)
|
||||
|
||||
// We split the string by "/", the branch name we're looking for to pass to latestTags is always last
|
||||
// (ex: "refs/heads/c9s" ---> we want latestTags[c9s]
|
||||
tmpRef := strings.Split(refSpec, "/")
|
||||
tmpBranchName := tmpRef[(len(tmpRef) - 1)]
|
||||
|
||||
latestTags[tmpBranchName] = &remoteTarget{
|
||||
remote: refSpec,
|
||||
when: tag.Tagger.When,
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
tagIter, err := repo.TagObjects()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("could not get tag objects: %v", err)
|
||||
}
|
||||
_ = tagIter.ForEach(tagAdd)
|
||||
|
||||
// tagless mode means we use "refAdd" (add commit by reference)
|
||||
// normal mode means we can rely on "tagAdd" (the tag should be present for us in the source repo)
|
||||
if pd.TaglessMode {
|
||||
_ = tagIter.ForEach(refAdd)
|
||||
} else {
|
||||
_ = tagIter.ForEach(tagAdd)
|
||||
}
|
||||
|
||||
listOpts := &git.ListOptions{
|
||||
Auth: pd.Authenticator,
|
||||
|
@ -157,17 +196,26 @@ func (g *GitMode) RetrieveSource(pd *data.ProcessData) (*data.ModeData, error) {
|
|||
if err != nil {
|
||||
continue
|
||||
}
|
||||
_ = tagAdd(&object.Tag{
|
||||
Name: strings.TrimPrefix(string(ref.Name()), "refs/tags/"),
|
||||
Tagger: commit.Committer,
|
||||
})
|
||||
|
||||
// Call refAdd instead of tagAdd in the case of TaglessMode enabled
|
||||
if pd.TaglessMode {
|
||||
_ = refAdd(&object.Tag{
|
||||
Name: string(ref.Name()),
|
||||
Tagger: commit.Committer,
|
||||
})
|
||||
} else {
|
||||
_ = tagAdd(&object.Tag{
|
||||
Name: strings.TrimPrefix(string(ref.Name()), "refs/tags/"),
|
||||
Tagger: commit.Committer,
|
||||
})
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
for _, branch := range latestTags {
|
||||
pd.Log.Printf("tag: %s", strings.TrimPrefix(branch.remote, "refs/tags/"))
|
||||
branches = append(branches, *branch)
|
||||
}
|
||||
|
||||
sort.Sort(branches)
|
||||
|
||||
var sortedBranches []string
|
||||
|
@ -186,53 +234,64 @@ func (g *GitMode) RetrieveSource(pd *data.ProcessData) (*data.ModeData, error) {
|
|||
|
||||
func (g *GitMode) WriteSource(pd *data.ProcessData, md *data.ModeData) error {
|
||||
remote, err := md.Repo.Remote("upstream")
|
||||
if err != nil {
|
||||
|
||||
if err != nil && !pd.TaglessMode {
|
||||
return fmt.Errorf("could not get upstream remote: %v", err)
|
||||
}
|
||||
|
||||
var refspec config.RefSpec
|
||||
var branchName string
|
||||
|
||||
if strings.HasPrefix(md.TagBranch, "refs/heads") {
|
||||
refspec = config.RefSpec(fmt.Sprintf("+%s:%s", md.TagBranch, md.TagBranch))
|
||||
branchName = strings.TrimPrefix(md.TagBranch, "refs/heads/")
|
||||
} else {
|
||||
match := misc.GetTagImportRegex(pd).FindStringSubmatch(md.TagBranch)
|
||||
branchName = match[2]
|
||||
refspec = config.RefSpec(fmt.Sprintf("+refs/heads/%s:%s", branchName, md.TagBranch))
|
||||
}
|
||||
pd.Log.Printf("checking out upstream refspec %s", refspec)
|
||||
fetchOpts := &git.FetchOptions{
|
||||
Auth: pd.Authenticator,
|
||||
RemoteName: "upstream",
|
||||
RefSpecs: []config.RefSpec{refspec},
|
||||
Tags: git.AllTags,
|
||||
Force: true,
|
||||
}
|
||||
err = remote.Fetch(fetchOpts)
|
||||
if err != nil && err != git.NoErrAlreadyUpToDate {
|
||||
if err == transport.ErrInvalidAuthMethod || err == transport.ErrAuthenticationRequired {
|
||||
fetchOpts.Auth = nil
|
||||
err = remote.Fetch(fetchOpts)
|
||||
if err != nil && err != git.NoErrAlreadyUpToDate {
|
||||
// In the case of tagless mode, we already have the transformed repo sitting in the worktree,
|
||||
// and don't need to perform any checkout or fetch operations
|
||||
if !pd.TaglessMode {
|
||||
if strings.HasPrefix(md.TagBranch, "refs/heads") {
|
||||
refspec = config.RefSpec(fmt.Sprintf("+%s:%s", md.TagBranch, md.TagBranch))
|
||||
branchName = strings.TrimPrefix(md.TagBranch, "refs/heads/")
|
||||
} else {
|
||||
match := misc.GetTagImportRegex(pd).FindStringSubmatch(md.TagBranch)
|
||||
branchName = match[2]
|
||||
refspec = config.RefSpec(fmt.Sprintf("+refs/heads/%s:%s", branchName, md.TagBranch))
|
||||
fmt.Println("Found branchname that does not start w/ refs/heads :: ", branchName)
|
||||
}
|
||||
pd.Log.Printf("checking out upstream refspec %s", refspec)
|
||||
|
||||
fetchOpts := &git.FetchOptions{
|
||||
Auth: pd.Authenticator,
|
||||
RemoteName: "upstream",
|
||||
RefSpecs: []config.RefSpec{refspec},
|
||||
Tags: git.AllTags,
|
||||
Force: true,
|
||||
}
|
||||
err = remote.Fetch(fetchOpts)
|
||||
if err != nil && err != git.NoErrAlreadyUpToDate {
|
||||
if err == transport.ErrInvalidAuthMethod || err == transport.ErrAuthenticationRequired {
|
||||
fetchOpts.Auth = nil
|
||||
err = remote.Fetch(fetchOpts)
|
||||
if err != nil && err != git.NoErrAlreadyUpToDate {
|
||||
return fmt.Errorf("could not fetch upstream: %v", err)
|
||||
}
|
||||
} else {
|
||||
return fmt.Errorf("could not fetch upstream: %v", err)
|
||||
}
|
||||
} else {
|
||||
return fmt.Errorf("could not fetch upstream: %v", err)
|
||||
}
|
||||
|
||||
err = md.Worktree.Checkout(&git.CheckoutOptions{
|
||||
Branch: plumbing.ReferenceName(md.TagBranch),
|
||||
Force: true,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not checkout source from git: %v", err)
|
||||
}
|
||||
|
||||
_, err = md.Worktree.Add(".")
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not add Worktree: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
err = md.Worktree.Checkout(&git.CheckoutOptions{
|
||||
Branch: plumbing.ReferenceName(md.TagBranch),
|
||||
Force: true,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not checkout source from git: %v", err)
|
||||
}
|
||||
|
||||
_, err = md.Worktree.Add(".")
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not add Worktree: %v", err)
|
||||
if pd.TaglessMode {
|
||||
branchName = fmt.Sprintf("%s%d%s", pd.ImportBranchPrefix, pd.Version, pd.BranchSuffix)
|
||||
}
|
||||
|
||||
metadataPath := ""
|
||||
|
@ -292,21 +351,68 @@ func (g *GitMode) WriteSource(pd *data.ProcessData, md *data.ModeData) error {
|
|||
body = fromBlobStorage
|
||||
pd.Log.Printf("downloading %s from blob storage", hash)
|
||||
} else {
|
||||
url := fmt.Sprintf("%s/%s/%s/%s", pd.CdnUrl, md.Name, branchName, hash)
|
||||
pd.Log.Printf("downloading %s", url)
|
||||
|
||||
req, err := http.NewRequest("GET", url, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not create new http request: %v", err)
|
||||
}
|
||||
req.Header.Set("Accept-Encoding", "*")
|
||||
url := ""
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not download dist-git file: %v", err)
|
||||
// We need to figure out the hashtype for templating purposes:
|
||||
hashType := "sha512"
|
||||
switch len(hash) {
|
||||
case 128:
|
||||
hashType = "sha512"
|
||||
case 64:
|
||||
hashType = "sha256"
|
||||
case 40:
|
||||
hashType = "sha1"
|
||||
case 32:
|
||||
hashType = "md5"
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
|
||||
// need the name of the file without "SOURCES/":
|
||||
fileName := strings.Split(path, "/")[1]
|
||||
|
||||
// Feed our template info to ProcessUrl and transform to the real values: ( {{.Name}}, {{.Branch}}, {{.Hash}}, {{.Hashtype}}, {{.Filename}} )
|
||||
url, hasTemplate := ProcessUrl(pd.CdnUrl, md.Name, branchName, hash, hashType, fileName)
|
||||
|
||||
var req *http.Request
|
||||
var resp *http.Response
|
||||
|
||||
// Download the --cdn-url given, but *only* if it contains template strings ( {{.Name}} , {{.Hash}} , etc. )
|
||||
// Otherwise we need to fall back to the traditional cdn-url patterns
|
||||
if hasTemplate {
|
||||
pd.Log.Printf("downloading %s", url)
|
||||
|
||||
req, err := http.NewRequest("GET", url, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not create new http request: %v", err)
|
||||
}
|
||||
req.Header.Set("Accept-Encoding", "*")
|
||||
|
||||
resp, err = client.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not download dist-git file: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Default cdn-url: If we don't have a templated download string, try the default <SITE>/<PKG>/<BRANCH>/<HASH> pattern:
|
||||
if resp == nil || resp.StatusCode != http.StatusOK {
|
||||
url = fmt.Sprintf("%s/%s/%s/%s", pd.CdnUrl, md.Name, branchName, hash)
|
||||
pd.Log.Printf("Attempting default URL: %s", url)
|
||||
req, err = http.NewRequest("GET", url, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not create new http request: %v", err)
|
||||
}
|
||||
req.Header.Set("Accept-Encoding", "*")
|
||||
resp, err = client.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not download dist-git file: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// If the default URL fails, we have one more pattern to try. The simple <SITE>/<HASH> pattern
|
||||
// If this one fails, we are truly lost, and have to bail out w/ an error:
|
||||
if resp == nil || resp.StatusCode != http.StatusOK {
|
||||
url = fmt.Sprintf("%s/%s", pd.CdnUrl, hash)
|
||||
pd.Log.Printf("Attempting 2nd fallback URL: %s", url)
|
||||
req, err = http.NewRequest("GET", url, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not create new http request: %v", err)
|
||||
|
@ -386,3 +492,29 @@ func (g *GitMode) ImportName(pd *data.ProcessData, md *data.ModeData) string {
|
|||
|
||||
return strings.Replace(strings.TrimPrefix(md.TagBranch, "refs/heads/"), "%", "_", -1)
|
||||
}
|
||||
|
||||
// Given a cdnUrl string as input, return same string, but with substituted
|
||||
// template values ( {{.Name}} , {{.Hash}}, {{.Filename}}, etc. )
|
||||
func ProcessUrl(cdnUrl string, name string, branch string, hash string, hashtype string, filename string) (string, bool) {
|
||||
tmpUrl := Lookaside{name, branch, hash, hashtype, filename}
|
||||
|
||||
// Return cdnUrl as-is if we don't have any templates ("{{ .Variable }}") to process:
|
||||
if !(strings.Contains(cdnUrl, "{{") && strings.Contains(cdnUrl, "}}")) {
|
||||
return cdnUrl, false
|
||||
}
|
||||
|
||||
// If we run into trouble with our template parsing, we'll just return the cdnUrl, exactly as we found it
|
||||
tmpl, err := template.New("").Parse(cdnUrl)
|
||||
if err != nil {
|
||||
return cdnUrl, false
|
||||
}
|
||||
|
||||
var result bytes.Buffer
|
||||
err = tmpl.Execute(&result, tmpUrl)
|
||||
if err != nil {
|
||||
log.Fatalf("ERROR: Could not process CDN URL template(s) from URL string: %s\n", cdnUrl)
|
||||
}
|
||||
|
||||
return result.String(), true
|
||||
|
||||
}
|
||||
|
|
|
@ -2,10 +2,5 @@ package rpmutils
|
|||
|
||||
import "regexp"
|
||||
|
||||
var (
|
||||
Nvr = regexp.MustCompile("^(\\S+)-([\\w~%.+]+)-(\\w+(?:\\.[\\w+]+)+?)(?:\\.(\\w+))?(?:\\.rpm)?$")
|
||||
epoch = regexp.MustCompile("(\\d+):")
|
||||
module = regexp.MustCompile("^(.+)-(.+)-([0-9]{19})\\.((?:.+){8})$")
|
||||
dist = regexp.MustCompile("(\\.el\\d(?:_\\d|))")
|
||||
moduleDist = regexp.MustCompile("\\.module.+$")
|
||||
)
|
||||
// Nvr is a regular expression that matches a NVR.
|
||||
var Nvr = regexp.MustCompile("^(\\S+)-([\\w~%.+]+)-(\\w+(?:\\.[\\w~%+]+)+?)(?:\\.rpm)?$")
|
||||
|
|
|
@ -3,15 +3,16 @@ package srpmproc
|
|||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"github.com/go-git/go-billy/v5"
|
||||
"github.com/rocky-linux/srpmproc/pkg/blob"
|
||||
"github.com/rocky-linux/srpmproc/pkg/data"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"net/http"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/go-git/go-billy/v5"
|
||||
"github.com/rocky-linux/srpmproc/pkg/blob"
|
||||
"github.com/rocky-linux/srpmproc/pkg/data"
|
||||
)
|
||||
|
||||
func Fetch(logger io.Writer, cdnUrl string, dir string, fs billy.Filesystem, storage blob.Storage) error {
|
||||
|
@ -101,7 +102,7 @@ func Fetch(logger io.Writer, cdnUrl string, dir string, fs billy.Filesystem, sto
|
|||
return fmt.Errorf("checksum in metadata does not match dist-git file")
|
||||
}
|
||||
|
||||
err = fs.MkdirAll(filepath.Join(dir, filepath.Dir(path)), 0755)
|
||||
err = fs.MkdirAll(filepath.Join(dir, filepath.Dir(path)), 0o755)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not create all directories")
|
||||
}
|
||||
|
|
|
@ -23,15 +23,17 @@ package srpmproc
|
|||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"github.com/go-git/go-git/v5/plumbing/transport"
|
||||
"github.com/rocky-linux/srpmproc/pkg/misc"
|
||||
"io/ioutil"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/go-git/go-git/v5/plumbing/transport"
|
||||
"github.com/rocky-linux/srpmproc/pkg/misc"
|
||||
|
||||
"github.com/go-git/go-billy/v5"
|
||||
"github.com/go-git/go-billy/v5/memfs"
|
||||
"github.com/go-git/go-git/v5"
|
||||
"github.com/go-git/go-git/v5/config"
|
||||
|
@ -46,10 +48,16 @@ import (
|
|||
|
||||
func cfgPatches(pd *data.ProcessData, md *data.ModeData, patchTree *git.Worktree, pushTree *git.Worktree) error {
|
||||
// check CFG patches
|
||||
_, err := patchTree.Filesystem.Stat("ROCKY/CFG")
|
||||
// use PATCHES directory if it exists otherwise ROCKY/CFG
|
||||
cfgdir := "PATCHES"
|
||||
_, err := patchTree.Filesystem.Stat(cfgdir)
|
||||
if err != nil {
|
||||
cfgdir = "ROCKY/CFG"
|
||||
}
|
||||
_, err = patchTree.Filesystem.Stat(cfgdir)
|
||||
if err == nil {
|
||||
// iterate through patches
|
||||
infos, err := patchTree.Filesystem.ReadDir("ROCKY/CFG")
|
||||
infos, err := patchTree.Filesystem.ReadDir(cfgdir)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not walk patches: %v", err)
|
||||
}
|
||||
|
@ -61,12 +69,12 @@ func cfgPatches(pd *data.ProcessData, md *data.ModeData, patchTree *git.Worktree
|
|||
}
|
||||
|
||||
pd.Log.Printf("applying directive %s", info.Name())
|
||||
filePath := filepath.Join("ROCKY/CFG", info.Name())
|
||||
filePath := filepath.Join(cfgdir, info.Name())
|
||||
directive, err := patchTree.Filesystem.Open(filePath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not open directive file %s: %v", info.Name(), err)
|
||||
}
|
||||
directiveBytes, err := ioutil.ReadAll(directive)
|
||||
directiveBytes, err := io.ReadAll(directive)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not read directive file: %v", err)
|
||||
}
|
||||
|
@ -94,7 +102,12 @@ func cfgPatches(pd *data.ProcessData, md *data.ModeData, patchTree *git.Worktree
|
|||
|
||||
func applyPatches(pd *data.ProcessData, md *data.ModeData, patchTree *git.Worktree, pushTree *git.Worktree) error {
|
||||
// check if patches exist
|
||||
_, err := patchTree.Filesystem.Stat("ROCKY")
|
||||
cfgdir := "PATCHES"
|
||||
_, err := patchTree.Filesystem.Stat(cfgdir)
|
||||
if err != nil {
|
||||
cfgdir = "ROCKY"
|
||||
}
|
||||
_, err = patchTree.Filesystem.Stat(cfgdir)
|
||||
if err == nil {
|
||||
err := cfgPatches(pd, md, patchTree, pushTree)
|
||||
if err != nil {
|
||||
|
@ -286,17 +299,34 @@ func patchModuleYaml(pd *data.ProcessData, md *data.ModeData) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
mdTxtPath := "SOURCES/modulemd.src.txt"
|
||||
f, err := md.Worktree.Filesystem.Open(mdTxtPath)
|
||||
if err != nil {
|
||||
mdTxtPath = "SOURCES/modulemd.txt"
|
||||
mdTxtPath := ""
|
||||
var f billy.File
|
||||
|
||||
// tagless mode implies we're looking for CentOS Stream modules, which are generally "SOURCES/NAME.yaml" (copied to SOURCES/ from import)
|
||||
// if not tagless mode, proceed as usual with SOURCES/modulemd.*.txt
|
||||
if pd.TaglessMode {
|
||||
mdTxtPath = fmt.Sprintf("SOURCES/%s.yaml", md.Name)
|
||||
f, err = md.Worktree.Filesystem.Open(mdTxtPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not open modulemd file: %v", err)
|
||||
mdTxtPath = fmt.Sprintf("SOURCES/%s.yml", md.Name)
|
||||
f, err = md.Worktree.Filesystem.Open(mdTxtPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not open modulemd file: %v", err)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
mdTxtPath = "SOURCES/modulemd.src.txt"
|
||||
f, err = md.Worktree.Filesystem.Open(mdTxtPath)
|
||||
if err != nil {
|
||||
mdTxtPath = "SOURCES/modulemd.txt"
|
||||
f, err = md.Worktree.Filesystem.Open(mdTxtPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not open modulemd file: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
content, err := ioutil.ReadAll(f)
|
||||
content, err := io.ReadAll(f)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not read modulemd file: %v", err)
|
||||
}
|
||||
|
@ -307,13 +337,30 @@ func patchModuleYaml(pd *data.ProcessData, md *data.ModeData) error {
|
|||
}
|
||||
|
||||
// Get stream branch from tag
|
||||
match := misc.GetTagImportRegex(pd).FindStringSubmatch(md.TagBranch)
|
||||
streamBranch := strings.Split(match[2], "-")
|
||||
// Force stream to be the same as stream name in branch
|
||||
module.Data.Stream = streamBranch[len(streamBranch)-1]
|
||||
// (in tagless mode we are trusting the "Stream: <VERSION>" text in the source YAML to be accurate)
|
||||
if !pd.TaglessMode {
|
||||
match := misc.GetTagImportRegex(pd).FindStringSubmatch(md.TagBranch)
|
||||
streamBranch := strings.Split(match[2], "-")
|
||||
|
||||
// Force stream to be the same as stream name in branch
|
||||
if module.V2 != nil {
|
||||
module.V2.Data.Stream = streamBranch[len(streamBranch)-1]
|
||||
}
|
||||
if module.V3 != nil {
|
||||
module.V3.Data.Stream = streamBranch[len(streamBranch)-1]
|
||||
}
|
||||
}
|
||||
|
||||
var components *modulemd.Components
|
||||
if module.V2 != nil {
|
||||
components = module.V2.Data.Components
|
||||
}
|
||||
if module.V3 != nil {
|
||||
components = module.V3.Data.Components
|
||||
}
|
||||
|
||||
log.Println("This module contains the following rpms:")
|
||||
for name := range module.Data.Components.Rpms {
|
||||
for name := range components.Rpms {
|
||||
pd.Log.Printf("\t- %s", name)
|
||||
}
|
||||
|
||||
|
@ -322,7 +369,7 @@ func patchModuleYaml(pd *data.ProcessData, md *data.ModeData) error {
|
|||
defaultBranch = fmt.Sprintf("%s%d-stream-%s", pd.BranchPrefix, pd.Version, pd.ModuleFallbackStream)
|
||||
}
|
||||
|
||||
for name, rpm := range module.Data.Components.Rpms {
|
||||
for name, rpm := range components.Rpms {
|
||||
var tipHash string
|
||||
var pushBranch string
|
||||
|
||||
|
@ -361,6 +408,12 @@ func patchModuleYaml(pd *data.ProcessData, md *data.ModeData) error {
|
|||
}
|
||||
}
|
||||
|
||||
// If we got this far, we're good. This means the repos, branches, and commits exist.
|
||||
// If this option is on, just use the branch name.
|
||||
if pd.ModuleBranchNames {
|
||||
tipHash = md.PushBranch
|
||||
}
|
||||
|
||||
rpm.Ref = tipHash
|
||||
}
|
||||
|
||||
|
|
|
@ -21,10 +21,22 @@
|
|||
package srpmproc
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"os"
|
||||
"os/exec"
|
||||
"os/user"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/go-git/go-billy/v5"
|
||||
"github.com/go-git/go-billy/v5/osfs"
|
||||
"github.com/go-git/go-git/v5/plumbing/format/gitignore"
|
||||
"github.com/go-git/go-git/v5/plumbing/transport"
|
||||
"github.com/go-git/go-git/v5/plumbing/transport/http"
|
||||
"github.com/go-git/go-git/v5/plumbing/transport/ssh"
|
||||
|
@ -36,14 +48,6 @@ import (
|
|||
"github.com/rocky-linux/srpmproc/pkg/misc"
|
||||
"github.com/rocky-linux/srpmproc/pkg/modes"
|
||||
"github.com/rocky-linux/srpmproc/pkg/rpmutils"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"os"
|
||||
"os/user"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/go-git/go-billy/v5/memfs"
|
||||
"github.com/go-git/go-git/v5"
|
||||
|
@ -96,6 +100,16 @@ type ProcessDataRequest struct {
|
|||
|
||||
PackageVersion string
|
||||
PackageRelease string
|
||||
|
||||
TaglessMode bool
|
||||
Cdn string
|
||||
|
||||
ModuleBranchNames bool
|
||||
}
|
||||
|
||||
type LookasidePath struct {
|
||||
Distro string
|
||||
Url string
|
||||
}
|
||||
|
||||
func gitlabify(str string) string {
|
||||
|
@ -106,7 +120,57 @@ func gitlabify(str string) string {
|
|||
return strings.Replace(str, "+", "plus", -1)
|
||||
}
|
||||
|
||||
// List of distros and their lookaside patterns
|
||||
// If we find one of these passed as --cdn (ex: "--cdn fedora"), then we override, and assign this URL to be our --cdn-url
|
||||
func StaticLookasides() []LookasidePath {
|
||||
centos := LookasidePath{
|
||||
Distro: "centos",
|
||||
Url: "https://git.centos.org/sources/{{.Name}}/{{.Branch}}/{{.Hash}}",
|
||||
}
|
||||
centosStream := LookasidePath{
|
||||
Distro: "centos-stream",
|
||||
Url: "https://sources.stream.centos.org/sources/rpms/{{.Name}}/{{.Filename}}/{{.Hashtype}}/{{.Hash}}/{{.Filename}}",
|
||||
}
|
||||
rocky8 := LookasidePath{
|
||||
Distro: "rocky8",
|
||||
Url: "https://rocky-linux-sources-staging.a1.rockylinux.org/{{.Hash}}",
|
||||
}
|
||||
rocky := LookasidePath{
|
||||
Distro: "rocky",
|
||||
Url: "https://sources.build.resf.org/{{.Hash}}",
|
||||
}
|
||||
fedora := LookasidePath{
|
||||
Distro: "fedora",
|
||||
Url: "https://src.fedoraproject.org/repo/pkgs/{{.Name}}/{{.Filename}}/{{.Hashtype}}/{{.Hash}}/{{.Filename}}",
|
||||
}
|
||||
|
||||
return []LookasidePath{centos, centosStream, rocky8, rocky, fedora}
|
||||
|
||||
}
|
||||
|
||||
// Given a "--cdn" entry like "centos", we can search through our struct list of distros, and return the proper lookaside URL
|
||||
// If we can't find it, we return false and the calling function will error out
|
||||
func FindDistro(cdn string) (string, bool) {
|
||||
var cdnUrl = ""
|
||||
|
||||
// Loop through each distro in the static list defined, try to find a match with "--cdn":
|
||||
for _, distro := range StaticLookasides() {
|
||||
if distro.Distro == strings.ToLower(cdn) {
|
||||
cdnUrl = distro.Url
|
||||
return cdnUrl, true
|
||||
}
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
func NewProcessData(req *ProcessDataRequest) (*data.ProcessData, error) {
|
||||
// Build the logger to use for the data import
|
||||
var writer io.Writer = os.Stdout
|
||||
if req.LogWriter != nil {
|
||||
writer = req.LogWriter
|
||||
}
|
||||
logger := log.New(writer, "", log.LstdFlags)
|
||||
|
||||
// Set defaults
|
||||
if req.ModulePrefix == "" {
|
||||
req.ModulePrefix = ModulePrefixCentOS
|
||||
|
@ -136,6 +200,20 @@ func NewProcessData(req *ProcessDataRequest) (*data.ProcessData, error) {
|
|||
req.CdnUrl = "https://git.centos.org/sources"
|
||||
}
|
||||
|
||||
// If a Cdn distro is defined, we try to find a match from StaticLookasides() array of structs
|
||||
// see if we have a match to --cdn (matching values are things like fedora, centos, rocky8, etc.)
|
||||
// If we match, then we want to short-circuit the CdnUrl to the assigned distro's one
|
||||
if req.Cdn != "" {
|
||||
newCdn, foundDistro := FindDistro(req.Cdn)
|
||||
|
||||
if !foundDistro {
|
||||
return nil, fmt.Errorf("Error, distro name given as --cdn argument is not valid.")
|
||||
}
|
||||
|
||||
req.CdnUrl = newCdn
|
||||
logger.Printf("Discovered --cdn distro: %s . Using override CDN URL Pattern: %s", req.Cdn, req.CdnUrl)
|
||||
}
|
||||
|
||||
// Validate required
|
||||
if req.Package == "" {
|
||||
return nil, fmt.Errorf("package cannot be empty")
|
||||
|
@ -193,7 +271,7 @@ func NewProcessData(req *ProcessDataRequest) (*data.ProcessData, error) {
|
|||
|
||||
fsCreator := func(branch string) (billy.Filesystem, error) {
|
||||
if req.TmpFsMode != "" {
|
||||
return osfs.New("."), nil
|
||||
return osfs.New(""), nil
|
||||
}
|
||||
return memfs.New(), nil
|
||||
}
|
||||
|
@ -202,12 +280,6 @@ func NewProcessData(req *ProcessDataRequest) (*data.ProcessData, error) {
|
|||
reqFsCreator = req.FsCreator
|
||||
}
|
||||
|
||||
var writer io.Writer = os.Stdout
|
||||
if req.LogWriter != nil {
|
||||
writer = req.LogWriter
|
||||
}
|
||||
logger := log.New(writer, "", log.LstdFlags)
|
||||
|
||||
if req.TmpFsMode != "" {
|
||||
logger.Printf("using tmpfs dir: %s", req.TmpFsMode)
|
||||
fsCreator = func(branch string) (billy.Filesystem, error) {
|
||||
|
@ -216,7 +288,7 @@ func NewProcessData(req *ProcessDataRequest) (*data.ProcessData, error) {
|
|||
return nil, err
|
||||
}
|
||||
tmpDir := filepath.Join(req.TmpFsMode, branch)
|
||||
err = fs.MkdirAll(tmpDir, 0755)
|
||||
err = fs.MkdirAll(tmpDir, 0o755)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("could not create tmpfs dir: %v", err)
|
||||
}
|
||||
|
@ -263,6 +335,9 @@ func NewProcessData(req *ProcessDataRequest) (*data.ProcessData, error) {
|
|||
Log: logger,
|
||||
PackageVersion: req.PackageVersion,
|
||||
PackageRelease: req.PackageRelease,
|
||||
TaglessMode: req.TaglessMode,
|
||||
Cdn: req.Cdn,
|
||||
ModuleBranchNames: req.ModuleBranchNames,
|
||||
}, nil
|
||||
}
|
||||
|
||||
|
@ -274,6 +349,13 @@ func NewProcessData(req *ProcessDataRequest) (*data.ProcessData, error) {
|
|||
// all files that are remote goes into .gitignore
|
||||
// all ignored files' hash goes into .{Name}.metadata
|
||||
func ProcessRPM(pd *data.ProcessData) (*srpmprocpb.ProcessResponse, error) {
|
||||
// if we are using "tagless mode", then we need to jump to a completely different import process:
|
||||
// Version info needs to be derived from rpmbuild + spec file, not tags
|
||||
if pd.TaglessMode {
|
||||
result, err := processRPMTagless(pd)
|
||||
return result, err
|
||||
}
|
||||
|
||||
md, err := pd.Importer.RetrieveSource(pd)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
@ -313,6 +395,7 @@ func ProcessRPM(pd *data.ProcessData) (*srpmprocpb.ProcessResponse, error) {
|
|||
list, err := remote.List(&git.ListOptions{
|
||||
Auth: pd.Authenticator,
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
log.Println("ignoring no-dup-mode")
|
||||
} else {
|
||||
|
@ -333,17 +416,17 @@ func ProcessRPM(pd *data.ProcessData) (*srpmprocpb.ProcessResponse, error) {
|
|||
if pd.SingleTag != "" {
|
||||
md.Branches = []string{fmt.Sprintf("refs/tags/%s", pd.SingleTag)}
|
||||
} else if len(pd.ManualCommits) > 0 {
|
||||
md.Branches = []string{}
|
||||
for _, commit := range pd.ManualCommits {
|
||||
branchCommit := strings.Split(commit, ":")
|
||||
if len(branchCommit) != 2 {
|
||||
return nil, fmt.Errorf("invalid manual commit list")
|
||||
}
|
||||
log.Println("Manual commits were listed for import. Switching to perform a tagless import of these commit(s).")
|
||||
pd.TaglessMode = true
|
||||
return processRPMTagless(pd)
|
||||
}
|
||||
|
||||
head := fmt.Sprintf("refs/tags/imports/%s/%s-%s", branchCommit[0], md.Name, branchCommit[1])
|
||||
md.Branches = append(md.Branches, head)
|
||||
commitPin[head] = branchCommit[1]
|
||||
}
|
||||
// If we have no valid branches to consider, then we'll automatically switch to attempt a tagless import:
|
||||
if len(md.Branches) == 0 {
|
||||
log.Println("No valid tags (refs/tags/imports/*) found in repository! Switching to perform a tagless import.")
|
||||
pd.TaglessMode = true
|
||||
result, err := processRPMTagless(pd)
|
||||
return result, err
|
||||
}
|
||||
|
||||
for _, branch := range md.Branches {
|
||||
|
@ -372,7 +455,9 @@ func ProcessRPM(pd *data.ProcessData) (*srpmprocpb.ProcessResponse, error) {
|
|||
}
|
||||
|
||||
match := misc.GetTagImportRegex(pd).FindStringSubmatch(matchString)
|
||||
|
||||
md.PushBranch = pd.BranchPrefix + strings.TrimPrefix(match[2], pd.ImportBranchPrefix)
|
||||
|
||||
newTag := "imports/" + pd.BranchPrefix + strings.TrimPrefix(match[1], "imports/"+pd.ImportBranchPrefix)
|
||||
newTag = strings.Replace(newTag, "%", "_", -1)
|
||||
|
||||
|
@ -504,7 +589,7 @@ func ProcessRPM(pd *data.ProcessData) (*srpmprocpb.ProcessResponse, error) {
|
|||
if err != nil {
|
||||
return nil, fmt.Errorf("could not open ignored source file %s: %v", sourcePath, err)
|
||||
}
|
||||
sourceFileBts, err := ioutil.ReadAll(sourceFile)
|
||||
sourceFileBts, err := io.ReadAll(sourceFile)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("could not read the whole of ignored source file: %v", err)
|
||||
}
|
||||
|
@ -573,6 +658,12 @@ func ProcessRPM(pd *data.ProcessData) (*srpmprocpb.ProcessResponse, error) {
|
|||
|
||||
// show status
|
||||
status, _ := w.Status()
|
||||
if !pd.ModuleMode {
|
||||
if status.IsClean() {
|
||||
pd.Log.Printf("No changes detected. Our downstream is up to date.")
|
||||
continue
|
||||
}
|
||||
}
|
||||
pd.Log.Printf("successfully processed:\n%s", status)
|
||||
|
||||
statusLines := strings.Split(status.String(), "\n")
|
||||
|
@ -656,3 +747,631 @@ func ProcessRPM(pd *data.ProcessData) (*srpmprocpb.ProcessResponse, error) {
|
|||
BranchVersions: versionForBranch,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Process for when we want to import a tagless repo (like from CentOS Stream)
|
||||
func processRPMTagless(pd *data.ProcessData) (*srpmprocpb.ProcessResponse, error) {
|
||||
pd.Log.Println("Tagless mode detected, attempting import of latest commit")
|
||||
|
||||
// In tagless mode, we *automatically* set StrictBranchMode to true
|
||||
// Only the exact <PREFIX><VERSION><SUFFIX> branch should be pulled from the source repo
|
||||
pd.StrictBranchMode = true
|
||||
|
||||
// our return values: a mapping of branches -> commits (1:1) that we're bringing in,
|
||||
// and a mapping of branches to: version = X, release = Y
|
||||
latestHashForBranch := map[string]string{}
|
||||
versionForBranch := map[string]*srpmprocpb.VersionRelease{}
|
||||
|
||||
md, err := pd.Importer.RetrieveSource(pd)
|
||||
if err != nil {
|
||||
pd.Log.Println("Error detected in RetrieveSource!")
|
||||
return nil, err
|
||||
}
|
||||
|
||||
md.BlobCache = map[string][]byte{}
|
||||
|
||||
// TODO: add tagless module support
|
||||
remotePrefix := "rpms"
|
||||
if pd.ModuleMode {
|
||||
remotePrefix = "modules"
|
||||
}
|
||||
|
||||
// Set up our remote URL for pushing our repo to
|
||||
var tagIgnoreList []string
|
||||
if pd.NoDupMode {
|
||||
repo, err := git.Init(memory.NewStorage(), memfs.New())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("could not init git repo: %v", err)
|
||||
}
|
||||
remoteUrl := fmt.Sprintf("%s/%s/%s.git", pd.UpstreamPrefix, remotePrefix, gitlabify(md.Name))
|
||||
refspec := config.RefSpec("+refs/heads/*:refs/remotes/origin/*")
|
||||
|
||||
remote, err := repo.CreateRemote(&config.RemoteConfig{
|
||||
Name: "origin",
|
||||
URLs: []string{remoteUrl},
|
||||
Fetch: []config.RefSpec{refspec},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("could not create remote: %v", err)
|
||||
}
|
||||
|
||||
list, err := remote.List(&git.ListOptions{
|
||||
Auth: pd.Authenticator,
|
||||
})
|
||||
if err != nil {
|
||||
log.Println("ignoring no-dup-mode")
|
||||
} else {
|
||||
for _, ref := range list {
|
||||
if !strings.HasPrefix(string(ref.Name()), "refs/tags/imports") {
|
||||
continue
|
||||
}
|
||||
tagIgnoreList = append(tagIgnoreList, string(ref.Name()))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
sourceRepo := *md.Repo
|
||||
sourceWorktree := *md.Worktree
|
||||
localPath := ""
|
||||
|
||||
// if a manual commit list is provided, we want to create our md.Branches[] array in a special format:
|
||||
if len(pd.ManualCommits) > 0 {
|
||||
md.Branches = []string{}
|
||||
for _, commit := range pd.ManualCommits {
|
||||
branchCommit := strings.Split(commit, ":")
|
||||
if len(branchCommit) != 2 {
|
||||
return nil, fmt.Errorf("invalid manual commit list")
|
||||
}
|
||||
|
||||
head := fmt.Sprintf("COMMIT:%s:%s", branchCommit[0], branchCommit[1])
|
||||
md.Branches = append(md.Branches, head)
|
||||
}
|
||||
}
|
||||
|
||||
for _, branch := range md.Branches {
|
||||
md.Repo = &sourceRepo
|
||||
md.Worktree = &sourceWorktree
|
||||
md.TagBranch = branch
|
||||
|
||||
for _, source := range md.SourcesToIgnore {
|
||||
source.Expired = true
|
||||
}
|
||||
|
||||
// Create a temporary place to check out our tag/branch : /tmp/srpmproctmp_<PKG_NAME><RANDOMSTRING>/
|
||||
localPath, _ = os.MkdirTemp("/tmp", fmt.Sprintf("srpmproctmp_%s", md.Name))
|
||||
|
||||
if err := os.RemoveAll(localPath); err != nil {
|
||||
return nil, fmt.Errorf("Could not remove previous temporary directory: %s", localPath)
|
||||
}
|
||||
if err := os.Mkdir(localPath, 0o755); err != nil {
|
||||
return nil, fmt.Errorf("Could not create temporary directory: %s", localPath)
|
||||
}
|
||||
|
||||
// we'll make our branch we're processing more presentable if it's in the COMMIT:<branch>:<hash> format:
|
||||
if strings.HasPrefix(branch, "COMMIT:") {
|
||||
branch = fmt.Sprintf("refs/heads/%s", strings.Split(branch, ":")[1])
|
||||
}
|
||||
|
||||
// Clone repo into the temporary path, but only the tag we're interested in:
|
||||
// (TODO: will probably need to assign this a variable or use the md struct gitrepo object to perform a successful tag+push later)
|
||||
rTmp, err := git.PlainClone(localPath, false, &git.CloneOptions{
|
||||
URL: pd.RpmLocation,
|
||||
SingleBranch: true,
|
||||
ReferenceName: plumbing.ReferenceName(branch),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// If we're dealing with a special manual commit to import ("COMMIT:<branch>:<githash>"), then we need to check out that
|
||||
// specific hash from the repo we are importing from:
|
||||
if strings.HasPrefix(md.TagBranch, "COMMIT:") {
|
||||
commitList := strings.Split(md.TagBranch, ":")
|
||||
|
||||
wTmp, _ := rTmp.Worktree()
|
||||
err = wTmp.Checkout(&git.CheckoutOptions{
|
||||
Hash: plumbing.NewHash(commitList[2]),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Could not find manual commit %s in the repository. Must be a valid commit hash.", commitList[2])
|
||||
}
|
||||
}
|
||||
|
||||
// Now that we're cloned into localPath, we need to "covert" the import into the old format
|
||||
// We want sources to become .PKGNAME.metadata, we want SOURCES and SPECS folders, etc.
|
||||
repoFixed, _ := convertLocalRepo(md.Name, localPath)
|
||||
if !repoFixed {
|
||||
return nil, fmt.Errorf("Error converting repository into SOURCES + SPECS + .package.metadata format")
|
||||
}
|
||||
|
||||
// call extra function to determine the proper way to convert the tagless branch name.
|
||||
// c9s becomes r9s (in the usual case), or in the modular case, stream-httpd-2.4-rhel-9.1.0 becomes r9s-stream-httpd-2.4_r9.1.0
|
||||
md.PushBranch = taglessBranchName(branch, pd)
|
||||
|
||||
rpmVersion := ""
|
||||
|
||||
// get name-version-release of tagless repo, only if we're not a module repo:
|
||||
if !pd.ModuleMode {
|
||||
nvrString, err := getVersionFromSpec(localPath, pd.Version)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Set version and release fields we extracted (name|version|release are separated by pipes)
|
||||
pd.PackageVersion = strings.Split(nvrString, "|")[1]
|
||||
pd.PackageRelease = strings.Split(nvrString, "|")[2]
|
||||
|
||||
// Set full rpm version: name-version-release (for tagging properly)
|
||||
rpmVersion = fmt.Sprintf("%s-%s-%s", md.Name, pd.PackageVersion, pd.PackageRelease)
|
||||
|
||||
pd.Log.Println("Successfully determined version of tagless checkout: ", rpmVersion)
|
||||
} else {
|
||||
// In case of module mode, we just set rpmVersion to the current date - that's what our tag will end up being
|
||||
rpmVersion = time.Now().Format("2006-01-02")
|
||||
}
|
||||
|
||||
// Make an initial repo we will use to push to our target
|
||||
pushRepo, err := git.PlainInit(localPath+"_gitpush", false)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("could not create new dist Repo: %v", err)
|
||||
}
|
||||
|
||||
w, err := pushRepo.Worktree()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("could not get dist Worktree: %v", err)
|
||||
}
|
||||
|
||||
// Create a remote "origin" in our empty git, make the upstream equal to the branch we want to modify
|
||||
pushUrl := fmt.Sprintf("%s/%s/%s.git", pd.UpstreamPrefix, remotePrefix, gitlabify(md.Name))
|
||||
refspec := config.RefSpec(fmt.Sprintf("+refs/heads/%s:refs/remotes/origin/%s", md.PushBranch, md.PushBranch))
|
||||
|
||||
// Make our remote repo the target one - the one we want to push our update to
|
||||
pushRepoRemote, err := pushRepo.CreateRemote(&config.RemoteConfig{
|
||||
Name: "origin",
|
||||
URLs: []string{pushUrl},
|
||||
Fetch: []config.RefSpec{refspec},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("could not create remote: %v", err)
|
||||
}
|
||||
|
||||
// fetch our branch data (md.PushBranch) into this new repo
|
||||
err = pushRepo.Fetch(&git.FetchOptions{
|
||||
RemoteName: "origin",
|
||||
RefSpecs: []config.RefSpec{refspec},
|
||||
Auth: pd.Authenticator,
|
||||
})
|
||||
|
||||
refName := plumbing.NewBranchReferenceName(md.PushBranch)
|
||||
|
||||
var hash plumbing.Hash
|
||||
h := plumbing.NewSymbolicReference(plumbing.HEAD, refName)
|
||||
if err := pushRepo.Storer.CheckAndSetReference(h, nil); err != nil {
|
||||
return nil, fmt.Errorf("Could not set symbolic reference: %v", err)
|
||||
}
|
||||
|
||||
err = w.Checkout(&git.CheckoutOptions{
|
||||
Branch: plumbing.NewRemoteReferenceName("origin", md.PushBranch),
|
||||
Hash: hash,
|
||||
Force: true,
|
||||
})
|
||||
|
||||
// These os commands actually move the data from our cloned import repo to our cloned "push" repo (upstream -> downstream)
|
||||
// First we clobber the target push repo's data, and then move the new data into it from the upstream repo we cloned earlier
|
||||
os.RemoveAll(fmt.Sprintf("%s_gitpush/SPECS", localPath))
|
||||
os.RemoveAll(fmt.Sprintf("%s_gitpush/SOURCES", localPath))
|
||||
os.RemoveAll(fmt.Sprintf("%s_gitpush/.gitignore", localPath))
|
||||
os.RemoveAll(fmt.Sprintf("%s_gitpush/%s.metadata", localPath, md.Name))
|
||||
os.Rename(fmt.Sprintf("%s/SPECS", localPath), fmt.Sprintf("%s_gitpush/SPECS", localPath))
|
||||
os.Rename(fmt.Sprintf("%s/SOURCES", localPath), fmt.Sprintf("%s_gitpush/SOURCES", localPath))
|
||||
os.Rename(fmt.Sprintf("%s/.gitignore", localPath), fmt.Sprintf("%s_gitpush/.gitignore", localPath))
|
||||
os.Rename(fmt.Sprintf("%s/.%s.metadata", localPath, md.Name), fmt.Sprintf("%s_gitpush/.%s.metadata", localPath, md.Name))
|
||||
|
||||
md.Repo = pushRepo
|
||||
md.Worktree = w
|
||||
|
||||
// Download lookaside sources (tarballs) into the push git repo:
|
||||
err = pd.Importer.WriteSource(pd, md)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Call function to upload source to target lookaside and
|
||||
// ensure the sources are added to .gitignore
|
||||
err = processLookasideSources(pd, md, localPath+"_gitpush")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Apply patch(es) if needed:
|
||||
if pd.ModuleMode {
|
||||
err := patchModuleYaml(pd, md)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
} else {
|
||||
err := executePatchesRpm(pd, md)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
err = w.AddWithOptions(&git.AddOptions{All: true})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Error adding SOURCES/ , SPECS/ or .metadata file to commit list.")
|
||||
}
|
||||
|
||||
status, err := w.Status()
|
||||
if !pd.ModuleMode {
|
||||
if status.IsClean() {
|
||||
pd.Log.Printf("No changes detected. Our downstream is up to date.")
|
||||
continue
|
||||
}
|
||||
}
|
||||
pd.Log.Printf("successfully processed:\n%s", status)
|
||||
|
||||
// assign tag for our new remote we're about to push (derived from the SRPM version)
|
||||
newTag := "refs/tags/imports/" + md.PushBranch + "/" + rpmVersion
|
||||
newTag = strings.Replace(newTag, "%", "_", -1)
|
||||
|
||||
// pushRefspecs is a list of all the references we want to push (tags + heads)
|
||||
// It's an array of colon-separated strings which map local references to their remote counterparts
|
||||
var pushRefspecs []config.RefSpec
|
||||
|
||||
// We need to find out if the remote repo already has this branch
|
||||
// If it doesn't, we want to add *:* to our references for commit. This will allow us to push the new branch
|
||||
// If it does, we can simply push HEAD:refs/heads/<BRANCH>
|
||||
newRepo := true
|
||||
refList, _ := pushRepoRemote.List(&git.ListOptions{Auth: pd.Authenticator})
|
||||
for _, ref := range refList {
|
||||
if strings.HasSuffix(ref.Name().String(), fmt.Sprintf("heads/%s", md.PushBranch)) {
|
||||
newRepo = false
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if newRepo {
|
||||
pushRefspecs = append(pushRefspecs, config.RefSpec("*:*"))
|
||||
pd.Log.Printf("New remote repo detected, creating new remote branch")
|
||||
}
|
||||
|
||||
// Identify specific references we want to push
|
||||
// Should be refs/heads/<target_branch>, and a tag called imports/<target_branch>/<rpm_nvr>
|
||||
pushRefspecs = append(pushRefspecs, config.RefSpec(fmt.Sprintf("HEAD:refs/heads/%s", md.PushBranch)))
|
||||
pushRefspecs = append(pushRefspecs, config.RefSpec(fmt.Sprintf("HEAD:%s", newTag)))
|
||||
|
||||
// Actually do the commit (locally)
|
||||
commit, err := w.Commit("import from tagless source "+pd.Importer.ImportName(pd, md), &git.CommitOptions{
|
||||
Author: &object.Signature{
|
||||
Name: pd.GitCommitterName,
|
||||
Email: pd.GitCommitterEmail,
|
||||
When: time.Now(),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("could not commit object: %v", err)
|
||||
}
|
||||
|
||||
obj, err := pushRepo.CommitObject(commit)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("could not get commit object: %v", err)
|
||||
}
|
||||
|
||||
pd.Log.Printf("Committed local repo tagless mode transform:\n%s", obj.String())
|
||||
|
||||
// After commit, we will now tag our local repo on disk:
|
||||
_, err = pushRepo.CreateTag(newTag, commit, &git.CreateTagOptions{
|
||||
Tagger: &object.Signature{
|
||||
Name: pd.GitCommitterName,
|
||||
Email: pd.GitCommitterEmail,
|
||||
When: time.Now(),
|
||||
},
|
||||
Message: "import " + md.TagBranch + " from " + pd.RpmLocation + "(import from tagless source)",
|
||||
SignKey: nil,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("could not create tag: %v", err)
|
||||
}
|
||||
|
||||
pd.Log.Printf("Pushing these references to the remote: %+v \n", pushRefspecs)
|
||||
|
||||
// Do the actual push to the remote target repository
|
||||
err = pushRepo.Push(&git.PushOptions{
|
||||
RemoteName: "origin",
|
||||
Auth: pd.Authenticator,
|
||||
RefSpecs: pushRefspecs,
|
||||
Force: true,
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("could not push to remote: %v", err)
|
||||
}
|
||||
|
||||
if err := os.RemoveAll(localPath); err != nil {
|
||||
log.Printf("Error cleaning up temporary git checkout directory %s . Non-fatal, continuing anyway...\n", localPath)
|
||||
}
|
||||
if err := os.RemoveAll(fmt.Sprintf("%s_gitpush", localPath)); err != nil {
|
||||
log.Printf("Error cleaning up temporary git checkout directory %s . Non-fatal, continuing anyway...\n", fmt.Sprintf("%s_gitpush", localPath))
|
||||
}
|
||||
|
||||
// append our processed branch to the return structures:
|
||||
latestHashForBranch[md.PushBranch] = obj.Hash.String()
|
||||
|
||||
versionForBranch[md.PushBranch] = &srpmprocpb.VersionRelease{
|
||||
Version: pd.PackageVersion,
|
||||
Release: pd.PackageRelease,
|
||||
}
|
||||
}
|
||||
|
||||
// return struct with all our branch:commit and branch:version+release mappings
|
||||
return &srpmprocpb.ProcessResponse{
|
||||
BranchCommits: latestHashForBranch,
|
||||
BranchVersions: versionForBranch,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Given a local repo on disk, ensure it's in the "traditional" format. This means:
|
||||
// - metadata file is named .pkgname.metadata
|
||||
// - metadata file has the old "<SHASUM> SOURCES/<filename>" format
|
||||
// - SPECS/ and SOURCES/ exist and are populated correctly
|
||||
func convertLocalRepo(pkgName string, localRepo string) (bool, error) {
|
||||
// Make sure we have a SPECS and SOURCES folder made:
|
||||
if err := os.MkdirAll(fmt.Sprintf("%s/SOURCES", localRepo), 0o755); err != nil {
|
||||
return false, fmt.Errorf("Could not create SOURCES directory in: %s", localRepo)
|
||||
}
|
||||
|
||||
if err := os.MkdirAll(fmt.Sprintf("%s/SPECS", localRepo), 0o755); err != nil {
|
||||
return false, fmt.Errorf("Could not create SPECS directory in: %s", localRepo)
|
||||
}
|
||||
|
||||
// Loop through each file/folder and operate accordingly:
|
||||
files, err := ioutil.ReadDir(localRepo)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
for _, f := range files {
|
||||
// We don't want to process SOURCES, SPECS, or any of our .git folders
|
||||
if f.Name() == "SOURCES" || f.Name() == "SPECS" || strings.HasPrefix(f.Name(), ".git") || f.Name() == "."+pkgName+".metadata" {
|
||||
continue
|
||||
}
|
||||
|
||||
// If we have a metadata "sources" file, we need to read it and convert to the old .<pkgname>.metadata format
|
||||
if f.Name() == "sources" {
|
||||
convertStatus := convertMetaData(pkgName, localRepo)
|
||||
|
||||
if convertStatus != true {
|
||||
return false, fmt.Errorf("Error converting sources metadata file to .metadata format")
|
||||
}
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
// Any file that ends in a ".spec" should be put into SPECS/
|
||||
if strings.HasSuffix(f.Name(), ".spec") {
|
||||
err := os.Rename(fmt.Sprintf("%s/%s", localRepo, f.Name()), fmt.Sprintf("%s/SPECS/%s", localRepo, f.Name()))
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("Error moving .spec file to SPECS/")
|
||||
}
|
||||
}
|
||||
|
||||
// if a file isn't skipped in one of the above checks, then it must be a file that belongs in SOURCES/
|
||||
os.Rename(fmt.Sprintf("%s/%s", localRepo, f.Name()), fmt.Sprintf("%s/SOURCES/%s", localRepo, f.Name()))
|
||||
}
|
||||
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// Given a local "sources" metadata file (new CentOS Stream format), convert it into the older
|
||||
// classic CentOS style: "<HASH> SOURCES/<FILENAME>"
|
||||
func convertMetaData(pkgName string, localRepo string) bool {
|
||||
lookAside, err := os.Open(fmt.Sprintf("%s/sources", localRepo))
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
// Split file into lines and start processing:
|
||||
scanner := bufio.NewScanner(lookAside)
|
||||
scanner.Split(bufio.ScanLines)
|
||||
|
||||
// convertedLA is our array of new "converted" lookaside lines
|
||||
var convertedLA []string
|
||||
|
||||
// loop through each line, and:
|
||||
// - split by whitespace
|
||||
// - check each line begins with "SHA" or "MD" - validate
|
||||
// - take the
|
||||
// Then check
|
||||
for scanner.Scan() {
|
||||
tmpLine := strings.Fields(scanner.Text())
|
||||
// make sure line starts with a "SHA" or "MD" before processing - otherwise it might not be a valid format lookaside line!
|
||||
if !(strings.HasPrefix(tmpLine[0], "SHA") || strings.HasPrefix(tmpLine[0], "MD")) {
|
||||
continue
|
||||
}
|
||||
|
||||
// Strip out "( )" characters from file name and prepend SOURCES/ to it
|
||||
tmpLine[1] = strings.ReplaceAll(tmpLine[1], "(", "")
|
||||
tmpLine[1] = strings.ReplaceAll(tmpLine[1], ")", "")
|
||||
tmpLine[1] = fmt.Sprintf("SOURCES/%s", tmpLine[1])
|
||||
|
||||
convertedLA = append(convertedLA, fmt.Sprintf("%s %s", tmpLine[3], tmpLine[1]))
|
||||
}
|
||||
lookAside.Close()
|
||||
|
||||
// open .<NAME>.metadata file for writing our old-format lines
|
||||
lookAside, err = os.OpenFile(fmt.Sprintf("%s/.%s.metadata", localRepo, pkgName), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644)
|
||||
if err != nil {
|
||||
fmt.Errorf("Error opening new .metadata file for writing.")
|
||||
return false
|
||||
}
|
||||
|
||||
writer := bufio.NewWriter(lookAside)
|
||||
|
||||
for _, convertedLine := range convertedLA {
|
||||
_, _ = writer.WriteString(convertedLine + "\n")
|
||||
}
|
||||
|
||||
writer.Flush()
|
||||
lookAside.Close()
|
||||
|
||||
// Remove old "sources" metadata file - we don't need it now that conversion is complete
|
||||
os.Remove(fmt.Sprintf("%s/sources", localRepo))
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
// Given a local checked out folder and package name, including SPECS/ , SOURCES/ , and .package.metadata, this will:
|
||||
// - create a "dummy" SRPM (using dummy sources files we use to populate tarballs from lookaside)
|
||||
// - extract RPM version info from that SRPM, and return it
|
||||
//
|
||||
// If we are in tagless mode, we need to get a package version somehow!
|
||||
func getVersionFromSpec(localRepo string, majorVersion int) (string, error) {
|
||||
// Make sure we have "rpm" and "rpmbuild" and "cp" available in our PATH. Otherwise, this won't work:
|
||||
_, err := exec.LookPath("rpmspec")
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("Could not find rpmspec program in PATH")
|
||||
}
|
||||
|
||||
// Read the first file from SPECS/ to get our spec file
|
||||
// (there should only be one file - we check that it ends in ".spec" just to be sure!)
|
||||
lsTmp, err := ioutil.ReadDir(fmt.Sprintf("%s/SPECS/", localRepo))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
specFile := lsTmp[0].Name()
|
||||
|
||||
if !strings.HasSuffix(specFile, ".spec") {
|
||||
return "", fmt.Errorf("First file found in SPECS/ is not a .spec file! Check the SPECS/ directory in the repo?")
|
||||
}
|
||||
|
||||
// Call the rpmspec binary to extract the version-release info out of it, and tack on ".el<VERSION>" at the end:
|
||||
cmdArgs := []string{
|
||||
"--srpm",
|
||||
fmt.Sprintf(`--define=dist .el%d`, majorVersion),
|
||||
fmt.Sprintf(`--define=_topdir %s`, localRepo),
|
||||
"-q",
|
||||
"--queryformat",
|
||||
`%{NAME}|%{VERSION}|%{RELEASE}\n`,
|
||||
fmt.Sprintf("%s/SPECS/%s", localRepo, specFile),
|
||||
}
|
||||
cmd := exec.Command("rpmspec", cmdArgs...)
|
||||
nvrTmp, err := cmd.Output()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("Error running rpmspec command to determine RPM name-version-release identifier. \nCommand attempted: %s \nCommand output: %s", cmd.String(), string(nvrTmp))
|
||||
}
|
||||
|
||||
// Pull first line of the version output to get the name-version-release number (there should only be 1 line)
|
||||
nvr := string(nvrTmp)
|
||||
nvr = strings.Fields(nvr)[0]
|
||||
|
||||
// return name-version-release string we derived:
|
||||
log.Printf("Derived NVR %s from tagless repo via rpmspec command\n", nvr)
|
||||
return nvr, nil
|
||||
}
|
||||
|
||||
// We need to loop through the lookaside blob files ("SourcesToIgnore"),
|
||||
// and upload them to our target storage (usually an S3 bucket, but could be a local folder)
|
||||
//
|
||||
// We also need to add the source paths to .gitignore in the git repo, so we don't accidentally commit + push them
|
||||
func processLookasideSources(pd *data.ProcessData, md *data.ModeData, localDir string) error {
|
||||
w := md.Worktree
|
||||
metadata, err := w.Filesystem.Create(fmt.Sprintf(".%s.metadata", md.Name))
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not create metadata file: %v", err)
|
||||
}
|
||||
|
||||
// Keep track of files we've already uploaded - don't want duplicates!
|
||||
var alreadyUploadedBlobs []string
|
||||
|
||||
gitIgnore, err := os.OpenFile(fmt.Sprintf("%s/.gitignore", localDir), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, source := range md.SourcesToIgnore {
|
||||
|
||||
sourcePath := source.Name
|
||||
_, err := w.Filesystem.Stat(sourcePath)
|
||||
if source.Expired || err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
sourceFile, err := w.Filesystem.Open(sourcePath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not open ignored source file %s: %v", sourcePath, err)
|
||||
}
|
||||
sourceFileBts, err := io.ReadAll(sourceFile)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not read the whole of ignored source file: %v", err)
|
||||
}
|
||||
|
||||
source.HashFunction.Reset()
|
||||
_, err = source.HashFunction.Write(sourceFileBts)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not write bytes to hash function: %v", err)
|
||||
}
|
||||
checksum := hex.EncodeToString(source.HashFunction.Sum(nil))
|
||||
checksumLine := fmt.Sprintf("%s %s\n", checksum, sourcePath)
|
||||
_, err = metadata.Write([]byte(checksumLine))
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not write to metadata file: %v", err)
|
||||
}
|
||||
|
||||
if data.StrContains(alreadyUploadedBlobs, checksum) {
|
||||
continue
|
||||
}
|
||||
exists, err := pd.BlobStorage.Exists(checksum)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !exists && !pd.NoStorageUpload {
|
||||
err := pd.BlobStorage.Write(checksum, sourceFileBts)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
pd.Log.Printf("wrote %s to blob storage", checksum)
|
||||
}
|
||||
alreadyUploadedBlobs = append(alreadyUploadedBlobs, checksum)
|
||||
|
||||
// Add this SOURCES/ lookaside file to be excluded
|
||||
w.Excludes = append(w.Excludes, gitignore.ParsePattern(sourcePath, nil))
|
||||
|
||||
// Append the SOURCES/<file> path to .gitignore:
|
||||
_, err = gitIgnore.Write([]byte(fmt.Sprintf("%s\n", sourcePath)))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
err = gitIgnore.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Given an input branch name to import from, like "refs/heads/c9s", produce the tagless branch name we want to commit to, like "r9s"
|
||||
// Modular translation of CentOS stream branches i is also done - branch stream-maven-3.8-rhel-9.1.0 ----> r9s-stream-maven-3.8_9.1.0
|
||||
func taglessBranchName(fullBranch string, pd *data.ProcessData) string {
|
||||
// Split the full branch name "refs/heads/blah" to only get the short name - last entry
|
||||
tmpBranch := strings.Split(fullBranch, "/")
|
||||
branch := tmpBranch[len(tmpBranch)-1]
|
||||
|
||||
// Simple case: if our branch is not a modular stream branch, just return the normal <prefix><version><suffix> pattern
|
||||
if !strings.HasPrefix(branch, "stream-") {
|
||||
return fmt.Sprintf("%s%d%s", pd.BranchPrefix, pd.Version, pd.BranchSuffix)
|
||||
}
|
||||
|
||||
// index where the "-rhel-" starts near the end of the string
|
||||
rhelSpot := strings.LastIndex(branch, "-rhel-")
|
||||
|
||||
// module name will be everything from the start until that "-rhel-" string (like "stream-httpd-2.4")
|
||||
moduleString := branch[0:rhelSpot]
|
||||
|
||||
// major minor version is everything after the "-rhel-" string
|
||||
majorMinor := branch[rhelSpot+6:]
|
||||
|
||||
// return translated modular branch:
|
||||
return fmt.Sprintf("%s%d%s-%s_%s", pd.BranchPrefix, pd.Version, pd.BranchSuffix, moduleString, majorMinor)
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue