From 42c043f8d691ee44d689858f164684ded2bc8ff7 Mon Sep 17 00:00:00 2001 From: Mustafa Gezen Date: Wed, 8 Sep 2021 17:00:56 +0200 Subject: [PATCH] enhancement: add more info for each processed package as well as print more info on cli --- cmd/srpmproc/main.go | 6 +- gen.go | 2 +- go.mod | 1 + go.sum | 11 ++ pb/cfg.pb.go | 9 +- pb/response.pb.go | 249 ++++++++++++++++++++++++++++++++++++++++ pkg/rpmutils/regex.go | 11 ++ pkg/srpmproc/patch.go | 3 +- pkg/srpmproc/process.go | 36 ++++-- proto/response.proto | 15 +++ 10 files changed, 322 insertions(+), 21 deletions(-) create mode 100644 pb/response.pb.go create mode 100644 pkg/rpmutils/regex.go create mode 100644 proto/response.proto diff --git a/cmd/srpmproc/main.go b/cmd/srpmproc/main.go index 3810fe6..a45c6c7 100644 --- a/cmd/srpmproc/main.go +++ b/cmd/srpmproc/main.go @@ -81,17 +81,18 @@ func mn(_ *cobra.Command, _ []string) { NoStorageUpload: noStorageUpload, NoStorageDownload: noStorageDownload, SingleTag: singleTag, + CdnUrl: cdnUrl, }) if err != nil { log.Fatal(err) } - latestHashForBranch, err := srpmproc.ProcessRPM(pd) + res, err := srpmproc.ProcessRPM(pd) if err != nil { log.Fatal(err) } - err = json.NewEncoder(os.Stdout).Encode(latestHashForBranch) + err = json.NewEncoder(os.Stdout).Encode(res) if err != nil { log.Fatal(err) } @@ -116,6 +117,7 @@ func main() { root.Flags().StringVar(&rpmPrefix, "rpm-prefix", "https://git.centos.org/rpms", "Where to retrieve SRPM content. Only used when source-rpm is not a local file") root.Flags().StringVar(&importBranchPrefix, "import-branch-prefix", "c", "Import branch prefix") root.Flags().StringVar(&branchPrefix, "branch-prefix", "r", "Branch prefix (replaces import-branch-prefix)") + root.Flags().StringVar(&cdnUrl, "cdn-url", "https://git.centos.org/sources", "CDN URL to download blobs from") root.Flags().StringVar(&singleTag, "single-tag", "", "If set, only this tag is imported") root.Flags().BoolVar(&noDupMode, "no-dup-mode", false, "If enabled, skips already imported tags") root.Flags().BoolVar(&moduleMode, "module-mode", false, "If enabled, imports a module instead of a package") diff --git a/gen.go b/gen.go index 4ac6eae..51b6cb7 100644 --- a/gen.go +++ b/gen.go @@ -18,5 +18,5 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. -//go:generate protoc -Iproto --go_opt=paths=source_relative --go_out=pb proto/cfg.proto +//go:generate protoc -Iproto --go_opt=paths=source_relative --go_out=pb proto/cfg.proto proto/response.proto package srpmproc diff --git a/go.mod b/go.mod index 878295f..9df80b1 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ require ( github.com/go-git/go-git/v5 v5.2.0 github.com/golang/protobuf v1.4.2 github.com/spf13/cobra v1.1.1 + github.com/spf13/viper v1.7.0 google.golang.org/protobuf v1.25.0 gopkg.in/yaml.v2 v2.2.8 ) diff --git a/go.sum b/go.sum index 497d839..7f746f6 100644 --- a/go.sum +++ b/go.sum @@ -87,6 +87,7 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= +github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= @@ -179,6 +180,7 @@ github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/b github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= @@ -212,6 +214,7 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= @@ -225,6 +228,7 @@ github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eI github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= @@ -232,6 +236,7 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -265,20 +270,25 @@ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v1.1.1 h1:KfztREH0tPxJJ+geloSLaAkaPkr4ki2Er5quFV1TDo4= github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI= +github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.7.0 h1:xVKxvI7ouOI5I+U9s2eeiUfMaWBVoXA3AWskkrqK0VM= github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/xanzy/ssh-agent v0.2.1 h1:TCbipTQL2JiiCprBWx9frJ2eJlCYT00NmctrHxVAr70= @@ -587,6 +597,7 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= diff --git a/pb/cfg.pb.go b/pb/cfg.pb.go index b3a7dd4..1886bd1 100644 --- a/pb/cfg.pb.go +++ b/pb/cfg.pb.go @@ -1,13 +1,12 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.24.0-devel -// protoc v3.14.0 +// protoc-gen-go v1.27.1 +// protoc v3.17.3 // source: cfg.proto package srpmprocpb import ( - proto "github.com/golang/protobuf/proto" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" @@ -21,10 +20,6 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) -// This is a compile-time assertion that a sufficiently up-to-date version -// of the legacy proto package is being used. -const _ = proto.ProtoPackageIsVersion4 - type SpecChange_FileOperation_Type int32 const ( diff --git a/pb/response.pb.go b/pb/response.pb.go new file mode 100644 index 0000000..0181fea --- /dev/null +++ b/pb/response.pb.go @@ -0,0 +1,249 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.27.1 +// protoc v3.17.3 +// source: response.proto + +package srpmprocpb + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type VersionRelease struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Version string `protobuf:"bytes,1,opt,name=version,proto3" json:"version,omitempty"` + Release string `protobuf:"bytes,2,opt,name=release,proto3" json:"release,omitempty"` +} + +func (x *VersionRelease) Reset() { + *x = VersionRelease{} + if protoimpl.UnsafeEnabled { + mi := &file_response_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *VersionRelease) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*VersionRelease) ProtoMessage() {} + +func (x *VersionRelease) ProtoReflect() protoreflect.Message { + mi := &file_response_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use VersionRelease.ProtoReflect.Descriptor instead. +func (*VersionRelease) Descriptor() ([]byte, []int) { + return file_response_proto_rawDescGZIP(), []int{0} +} + +func (x *VersionRelease) GetVersion() string { + if x != nil { + return x.Version + } + return "" +} + +func (x *VersionRelease) GetRelease() string { + if x != nil { + return x.Release + } + return "" +} + +type ProcessResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + BranchCommits map[string]string `protobuf:"bytes,1,rep,name=branch_commits,json=branchCommits,proto3" json:"branch_commits,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + BranchVersions map[string]*VersionRelease `protobuf:"bytes,2,rep,name=branch_versions,json=branchVersions,proto3" json:"branch_versions,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` +} + +func (x *ProcessResponse) Reset() { + *x = ProcessResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_response_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ProcessResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ProcessResponse) ProtoMessage() {} + +func (x *ProcessResponse) ProtoReflect() protoreflect.Message { + mi := &file_response_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ProcessResponse.ProtoReflect.Descriptor instead. +func (*ProcessResponse) Descriptor() ([]byte, []int) { + return file_response_proto_rawDescGZIP(), []int{1} +} + +func (x *ProcessResponse) GetBranchCommits() map[string]string { + if x != nil { + return x.BranchCommits + } + return nil +} + +func (x *ProcessResponse) GetBranchVersions() map[string]*VersionRelease { + if x != nil { + return x.BranchVersions + } + return nil +} + +var File_response_proto protoreflect.FileDescriptor + +var file_response_proto_rawDesc = []byte{ + 0x0a, 0x0e, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x12, 0x08, 0x73, 0x72, 0x70, 0x6d, 0x70, 0x72, 0x6f, 0x63, 0x22, 0x44, 0x0a, 0x0e, 0x56, 0x65, + 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x12, 0x18, 0x0a, 0x07, + 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, + 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x18, 0x0a, 0x07, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, + 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, + 0x22, 0xdd, 0x02, 0x0a, 0x0f, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x53, 0x0a, 0x0e, 0x62, 0x72, 0x61, 0x6e, 0x63, 0x68, 0x5f, 0x63, + 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2c, 0x2e, 0x73, + 0x72, 0x70, 0x6d, 0x70, 0x72, 0x6f, 0x63, 0x2e, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x72, 0x61, 0x6e, 0x63, 0x68, 0x43, 0x6f, + 0x6d, 0x6d, 0x69, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0d, 0x62, 0x72, 0x61, 0x6e, + 0x63, 0x68, 0x43, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x73, 0x12, 0x56, 0x0a, 0x0f, 0x62, 0x72, 0x61, + 0x6e, 0x63, 0x68, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x73, 0x72, 0x70, 0x6d, 0x70, 0x72, 0x6f, 0x63, 0x2e, 0x50, 0x72, + 0x6f, 0x63, 0x65, 0x73, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x72, + 0x61, 0x6e, 0x63, 0x68, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72, + 0x79, 0x52, 0x0e, 0x62, 0x72, 0x61, 0x6e, 0x63, 0x68, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, + 0x73, 0x1a, 0x40, 0x0a, 0x12, 0x42, 0x72, 0x61, 0x6e, 0x63, 0x68, 0x43, 0x6f, 0x6d, 0x6d, 0x69, + 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, + 0x02, 0x38, 0x01, 0x1a, 0x5b, 0x0a, 0x13, 0x42, 0x72, 0x61, 0x6e, 0x63, 0x68, 0x56, 0x65, 0x72, + 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, + 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x2e, 0x0a, 0x05, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x73, 0x72, + 0x70, 0x6d, 0x70, 0x72, 0x6f, 0x63, 0x2e, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, + 0x6c, 0x65, 0x61, 0x73, 0x65, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, + 0x42, 0x2f, 0x5a, 0x2d, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x72, + 0x6f, 0x63, 0x6b, 0x79, 0x2d, 0x6c, 0x69, 0x6e, 0x75, 0x78, 0x2f, 0x73, 0x72, 0x70, 0x6d, 0x70, + 0x72, 0x6f, 0x63, 0x2f, 0x70, 0x62, 0x3b, 0x73, 0x72, 0x70, 0x6d, 0x70, 0x72, 0x6f, 0x63, 0x70, + 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_response_proto_rawDescOnce sync.Once + file_response_proto_rawDescData = file_response_proto_rawDesc +) + +func file_response_proto_rawDescGZIP() []byte { + file_response_proto_rawDescOnce.Do(func() { + file_response_proto_rawDescData = protoimpl.X.CompressGZIP(file_response_proto_rawDescData) + }) + return file_response_proto_rawDescData +} + +var file_response_proto_msgTypes = make([]protoimpl.MessageInfo, 4) +var file_response_proto_goTypes = []interface{}{ + (*VersionRelease)(nil), // 0: srpmproc.VersionRelease + (*ProcessResponse)(nil), // 1: srpmproc.ProcessResponse + nil, // 2: srpmproc.ProcessResponse.BranchCommitsEntry + nil, // 3: srpmproc.ProcessResponse.BranchVersionsEntry +} +var file_response_proto_depIdxs = []int32{ + 2, // 0: srpmproc.ProcessResponse.branch_commits:type_name -> srpmproc.ProcessResponse.BranchCommitsEntry + 3, // 1: srpmproc.ProcessResponse.branch_versions:type_name -> srpmproc.ProcessResponse.BranchVersionsEntry + 0, // 2: srpmproc.ProcessResponse.BranchVersionsEntry.value:type_name -> srpmproc.VersionRelease + 3, // [3:3] is the sub-list for method output_type + 3, // [3:3] is the sub-list for method input_type + 3, // [3:3] is the sub-list for extension type_name + 3, // [3:3] is the sub-list for extension extendee + 0, // [0:3] is the sub-list for field type_name +} + +func init() { file_response_proto_init() } +func file_response_proto_init() { + if File_response_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_response_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*VersionRelease); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_response_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ProcessResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_response_proto_rawDesc, + NumEnums: 0, + NumMessages: 4, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_response_proto_goTypes, + DependencyIndexes: file_response_proto_depIdxs, + MessageInfos: file_response_proto_msgTypes, + }.Build() + File_response_proto = out.File + file_response_proto_rawDesc = nil + file_response_proto_goTypes = nil + file_response_proto_depIdxs = nil +} diff --git a/pkg/rpmutils/regex.go b/pkg/rpmutils/regex.go new file mode 100644 index 0000000..b3094fe --- /dev/null +++ b/pkg/rpmutils/regex.go @@ -0,0 +1,11 @@ +package rpmutils + +import "regexp" + +var ( + Nvr = regexp.MustCompile("^(\\S+)-([\\w.]+)-(\\w+(?:\\.[\\w+]+)+?)(?:\\.(\\w+))?(?:\\.rpm)?$") + epoch = regexp.MustCompile("(\\d+):") + module = regexp.MustCompile("^(.+)-(.+)-([0-9]{19})\\.((?:.+){8})$") + dist = regexp.MustCompile("(\\.el8(?:(?:_\\d)|))") + moduleDist = regexp.MustCompile("\\.module.+$") +) diff --git a/pkg/srpmproc/patch.go b/pkg/srpmproc/patch.go index 9d3edc4..d96f4e0 100644 --- a/pkg/srpmproc/patch.go +++ b/pkg/srpmproc/patch.go @@ -22,6 +22,7 @@ package srpmproc import ( "fmt" + "github.com/rocky-linux/srpmproc/pkg/misc" "io/ioutil" "log" "path/filepath" @@ -284,7 +285,7 @@ func patchModuleYaml(pd *data.ProcessData, md *data.ModeData) error { } // Get stream branch from tag - match := tagImportRegex.FindStringSubmatch(md.TagBranch) + match := misc.GetTagImportRegex(pd.ImportBranchPrefix, pd.AllowStreamBranches).FindStringSubmatch(md.TagBranch) streamBranch := strings.Split(match[2], "-") // Force stream to be the same as stream name in branch module.Data.Stream = streamBranch[len(streamBranch)-1] diff --git a/pkg/srpmproc/process.go b/pkg/srpmproc/process.go index abf4cb5..3f71c41 100644 --- a/pkg/srpmproc/process.go +++ b/pkg/srpmproc/process.go @@ -25,16 +25,18 @@ import ( "fmt" "github.com/go-git/go-billy/v5" "github.com/go-git/go-git/v5/plumbing/transport/ssh" + srpmprocpb "github.com/rocky-linux/srpmproc/pb" "github.com/rocky-linux/srpmproc/pkg/blob" "github.com/rocky-linux/srpmproc/pkg/blob/file" "github.com/rocky-linux/srpmproc/pkg/blob/gcs" "github.com/rocky-linux/srpmproc/pkg/blob/s3" + "github.com/rocky-linux/srpmproc/pkg/misc" "github.com/rocky-linux/srpmproc/pkg/modes" + "github.com/rocky-linux/srpmproc/pkg/rpmutils" "io/ioutil" "log" "os/user" "path/filepath" - "regexp" "strings" "time" @@ -55,8 +57,6 @@ const ( UpstreamPrefixRocky = "https://git.rockylinux.org/staging" ) -var tagImportRegex *regexp.Regexp - type ProcessDataRequest struct { // Required Version int @@ -178,14 +178,15 @@ func NewProcessData(req *ProcessDataRequest) (*data.ProcessData, error) { fsCreator := func(branch string) (billy.Filesystem, error) { return memfs.New(), nil } + reqFsCreator := fsCreator if req.FsCreator != nil { - fsCreator = req.FsCreator + reqFsCreator = req.FsCreator } if req.TmpFsMode != "" { log.Printf("using tmpfs dir: %s", req.TmpFsMode) fsCreator = func(branch string) (billy.Filesystem, error) { - fs, err := fsCreator(branch) + fs, err := reqFsCreator(branch) if err != nil { return nil, err } @@ -201,6 +202,8 @@ func NewProcessData(req *ProcessDataRequest) (*data.ProcessData, error) { return nFs, nil } + } else { + fsCreator = reqFsCreator } var manualCs []string @@ -232,6 +235,7 @@ func NewProcessData(req *ProcessDataRequest) (*data.ProcessData, error) { ModuleFallbackStream: req.ModuleFallbackStream, AllowStreamBranches: req.AllowStreamBranches, FsCreator: fsCreator, + CdnUrl: req.CdnUrl, }, nil } @@ -242,7 +246,7 @@ func NewProcessData(req *ProcessDataRequest) (*data.ProcessData, error) { // source files goes into -> SOURCES // all files that are remote goes into .gitignore // all ignored files' hash goes into .{Name}.metadata -func ProcessRPM(pd *data.ProcessData) (map[string]string, error) { +func ProcessRPM(pd *data.ProcessData) (*srpmprocpb.ProcessResponse, error) { md, err := pd.Importer.RetrieveSource(pd) if err != nil { return nil, err @@ -255,6 +259,7 @@ func ProcessRPM(pd *data.ProcessData) (map[string]string, error) { } latestHashForBranch := map[string]string{} + versionForBranch := map[string]*srpmprocpb.VersionRelease{} // already uploaded blobs are skipped var alreadyUploadedBlobs []string @@ -327,7 +332,7 @@ func ProcessRPM(pd *data.ProcessData) (map[string]string, error) { } var matchString string - if !tagImportRegex.MatchString(md.TagBranch) { + if !misc.GetTagImportRegex(pd.ImportBranchPrefix, pd.AllowStreamBranches).MatchString(md.TagBranch) { if pd.ModuleMode { prefix := fmt.Sprintf("refs/heads/%s%d", pd.ImportBranchPrefix, pd.Version) if strings.HasPrefix(md.TagBranch, prefix) { @@ -336,14 +341,14 @@ func ProcessRPM(pd *data.ProcessData) (map[string]string, error) { log.Printf("using match string: %s", matchString) } } - if !tagImportRegex.MatchString(matchString) { + if !misc.GetTagImportRegex(pd.ImportBranchPrefix, pd.AllowStreamBranches).MatchString(matchString) { continue } } else { matchString = md.TagBranch } - match := tagImportRegex.FindStringSubmatch(matchString) + match := misc.GetTagImportRegex(pd.ImportBranchPrefix, pd.AllowStreamBranches).FindStringSubmatch(matchString) md.PushBranch = pd.BranchPrefix + strings.TrimPrefix(match[2], pd.ImportBranchPrefix) newTag := "imports/" + pd.BranchPrefix + strings.TrimPrefix(match[1], "imports/"+pd.ImportBranchPrefix) newTag = strings.Replace(newTag, "%", "_", -1) @@ -504,6 +509,14 @@ func ProcessRPM(pd *data.ProcessData) (map[string]string, error) { } } + nvrMatch := rpmutils.Nvr.FindStringSubmatch(match[3]) + if len(nvrMatch) >= 4 { + versionForBranch[md.PushBranch] = &srpmprocpb.VersionRelease{ + Version: nvrMatch[2], + Release: nvrMatch[3], + } + } + if pd.TmpFsMode != "" { continue } @@ -593,5 +606,8 @@ func ProcessRPM(pd *data.ProcessData) (map[string]string, error) { latestHashForBranch[md.PushBranch] = hashString } - return latestHashForBranch, nil + return &srpmprocpb.ProcessResponse{ + BranchCommits: latestHashForBranch, + BranchVersions: versionForBranch, + }, nil } diff --git a/proto/response.proto b/proto/response.proto new file mode 100644 index 0000000..8a0def2 --- /dev/null +++ b/proto/response.proto @@ -0,0 +1,15 @@ +syntax = "proto3"; + +option go_package = "github.com/rocky-linux/srpmproc/pb;srpmprocpb"; + +package srpmproc; + +message VersionRelease { + string version = 1; + string release = 2; +} + +message ProcessResponse { + map branch_commits = 1; + map branch_versions = 2; +}