diff --git a/cmd/srpmproc/fetch.go b/cmd/srpmproc/fetch.go index ca05a3f..65a0107 100644 --- a/cmd/srpmproc/fetch.go +++ b/cmd/srpmproc/fetch.go @@ -9,7 +9,6 @@ import ( "log" "net/http" "os" - "os/exec" "path/filepath" "strings" ) @@ -58,13 +57,6 @@ func runFetch(_ *cobra.Command, _ []string) { log.Fatalf("could not read metadata file: %v", err) } - cmd := exec.Command("git", "branch", "--show-current") - branchByte, err := cmd.Output() - if err != nil { - log.Fatalf("could not get branch: %v", err) - } - branch := strings.TrimSuffix(string(branchByte), "\n") - client := &http.Client{ Transport: &http.Transport{ DisableCompression: false, @@ -80,7 +72,7 @@ func runFetch(_ *cobra.Command, _ []string) { hash := lineInfo[0] path := lineInfo[1] - url := fmt.Sprintf("%s/%s/%s/%s", cdnUrl, name, branch, hash) + url := fmt.Sprintf("%s/%s/%s", cdnUrl, name, hash) log.Printf("downloading %s", url) req, err := http.NewRequest("GET", url, nil) diff --git a/cmd/srpmproc/main.go b/cmd/srpmproc/main.go index ef1f603..be940cf 100644 --- a/cmd/srpmproc/main.go +++ b/cmd/srpmproc/main.go @@ -27,6 +27,7 @@ var ( modulePrefix string rpmPrefix string noDupMode bool + moduleMode bool ) var root = &cobra.Command{ @@ -58,7 +59,11 @@ func mn(_ *cobra.Command, _ []string) { sourceRpmLocation = strings.TrimPrefix(sourceRpm, "file://") importer = &internal.SrpmMode{} } else { - sourceRpmLocation = fmt.Sprintf("%s/%s", rpmPrefix, sourceRpm) + if moduleMode { + sourceRpmLocation = fmt.Sprintf("%s/%s", modulePrefix, sourceRpm) + } else { + sourceRpmLocation = fmt.Sprintf("%s/%s", rpmPrefix, sourceRpm) + } importer = &internal.GitMode{} } @@ -89,6 +94,7 @@ func mn(_ *cobra.Command, _ []string) { ModulePrefix: modulePrefix, Authenticator: authenticator, NoDupMode: noDupMode, + ModuleMode: moduleMode, }) } @@ -109,6 +115,7 @@ func main() { root.Flags().StringVar(&modulePrefix, "module-prefix", "https://git.centos.org/modules", "Where to retrieve modules if exists. Only used when source-rpm is a git repo") root.Flags().StringVar(&rpmPrefix, "rpm-prefix", "https://git.centos.org/rpms", "Where to retrieve SRPM content. Only used when source-rpm is not a local file") root.Flags().BoolVar(&noDupMode, "no-dup-mode", false, "If enabled, skips already imported tags") + root.Flags().BoolVar(&moduleMode, "module-mode", false, "If enabled, imports a module instead of a package") if err := root.Execute(); err != nil { log.Fatal(err) diff --git a/go.mod b/go.mod index b94af20..599b557 100644 --- a/go.mod +++ b/go.mod @@ -13,4 +13,5 @@ require ( github.com/golang/protobuf v1.4.2 github.com/spf13/cobra v1.1.1 google.golang.org/protobuf v1.25.0 + gopkg.in/yaml.v2 v2.2.8 ) diff --git a/go.sum b/go.sum index e8a64fa..a4024fa 100644 --- a/go.sum +++ b/go.sum @@ -591,6 +591,7 @@ gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bl gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/internal/git.go b/internal/git.go index e1c4c7e..b54ec33 100644 --- a/internal/git.go +++ b/internal/git.go @@ -52,6 +52,17 @@ func (g *GitMode) RetrieveSource(pd *ProcessData) *modeData { branches = append(branches, name) } } + + // no tags? fetch branches + if len(branches) == 0 { + for _, ref := range list { + name := string(ref.Name()) + prefix := fmt.Sprintf("refs/heads/c%d", pd.Version) + if strings.HasPrefix(name, prefix) { + branches = append(branches, name) + } + } + } sort.Strings(branches) return &modeData{ @@ -69,8 +80,17 @@ func (g *GitMode) WriteSource(md *modeData) { log.Fatalf("could not get upstream remote: %v", err) } - match := tagImportRegex.FindStringSubmatch(md.tagBranch) - refspec := config.RefSpec(fmt.Sprintf("+refs/heads/%s:%s", match[2], md.tagBranch)) + var refspec config.RefSpec + var branchName string + + if strings.HasPrefix(md.tagBranch, "refs/heads") { + refspec = config.RefSpec(fmt.Sprintf("+%s:%s", md.tagBranch, md.tagBranch)) + branchName = strings.TrimPrefix(md.tagBranch, "refs/heads/") + } else { + match := tagImportRegex.FindStringSubmatch(md.tagBranch) + branchName = match[2] + refspec = config.RefSpec(fmt.Sprintf("+refs/heads/%s:%s", branchName, md.tagBranch)) + } log.Printf("checking out upstream refspec %s", refspec) err = remote.Fetch(&git.FetchOptions{ RemoteName: "upstream", @@ -97,7 +117,8 @@ func (g *GitMode) WriteSource(md *modeData) { metadataFile, err := md.worktree.Filesystem.Open(fmt.Sprintf(".%s.metadata", md.rpmFile.Name())) if err != nil { - log.Fatalf("could not open metadata file: %v", err) + log.Printf("warn: could not open metadata file, so skipping: %v", err) + return } fileBytes, err := ioutil.ReadAll(metadataFile) @@ -120,7 +141,7 @@ func (g *GitMode) WriteSource(md *modeData) { hash := strings.TrimSpace(lineInfo[0]) path := strings.TrimSpace(lineInfo[1]) - url := fmt.Sprintf("https://git.centos.org/sources/%s/%s/%s", md.rpmFile.Name(), match[2], hash) + url := fmt.Sprintf("https://git.centos.org/sources/%s/%s/%s", md.rpmFile.Name(), branchName, hash) log.Printf("downloading %s", url) req, err := http.NewRequest("GET", url, nil) @@ -181,6 +202,10 @@ func (g *GitMode) PostProcess(md *modeData) { } func (g *GitMode) ImportName(_ *ProcessData, md *modeData) string { - match := tagImportRegex.FindStringSubmatch(md.tagBranch) - return match[3] + if tagImportRegex.MatchString(md.tagBranch) { + match := tagImportRegex.FindStringSubmatch(md.tagBranch) + return match[3] + } + + return strings.TrimPrefix(md.tagBranch, "refs/heads/") } diff --git a/internal/patch.go b/internal/patch.go index 674ac6c..ee6307c 100644 --- a/internal/patch.go +++ b/internal/patch.go @@ -10,6 +10,7 @@ import ( "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/storage/memory" "github.com/mstg/srpmproc/internal/directives" + "github.com/mstg/srpmproc/modulemd" srpmprocpb "github.com/mstg/srpmproc/pb" "google.golang.org/protobuf/encoding/prototext" "io/ioutil" @@ -165,7 +166,7 @@ func applyPatches(patchTree *git.Worktree, pushTree *git.Worktree) { } } -func executePatches(pd *ProcessData, md *modeData) { +func executePatchesRpm(pd *ProcessData, md *modeData) { // fetch patch repository repo, err := git.Init(memory.NewStorage(), memfs.New()) if err != nil { @@ -225,3 +226,95 @@ func executePatches(pd *ProcessData, md *modeData) { } } } + +func getTipStream(pd *ProcessData, module string, pushBranch string) string { + repo, err := git.Init(memory.NewStorage(), memfs.New()) + if err != nil { + log.Fatalf("could not init git repo: %v", err) + } + + remoteUrl := fmt.Sprintf("%s/dist/%s.git", pd.UpstreamPrefix, module) + refspec := config.RefSpec("+refs/heads/*:refs/remotes/origin/*") + remote, err := repo.CreateRemote(&config.RemoteConfig{ + Name: "origin", + URLs: []string{remoteUrl}, + Fetch: []config.RefSpec{refspec}, + }) + if err != nil { + log.Fatalf("could not create remote: %v", err) + } + + list, err := remote.List(&git.ListOptions{ + Auth: pd.Authenticator, + }) + if err != nil { + log.Fatalf("could not get rpm refs. import the rpm before the module: %v", err) + } + + var tipHash string + + for _, ref := range list { + prefix := fmt.Sprintf("refs/heads/%s", pushBranch) + if strings.HasPrefix(ref.Name().String(), prefix) { + tipHash = ref.Hash().String() + } + } + + if tipHash == "" { + log.Fatal("could not find tip hash") + } + + return tipHash +} + +func patchModuleYaml(pd *ProcessData, md *modeData) { + // special case for platform.yaml + _, err := md.worktree.Filesystem.Open("platform.yaml") + if err == nil { + return + } + + mdTxtPath := "SOURCES/modulemd.src.txt" + f, err := md.worktree.Filesystem.Open(mdTxtPath) + if err != nil { + log.Fatalf("could not open modulemd file: %v", err) + } + + content, err := ioutil.ReadAll(f) + if err != nil { + log.Fatalf("could not read modulemd file: %v", err) + } + + module, err := modulemd.Parse(content) + if err != nil { + log.Fatalf("could not parse modulemd file: %v", err) + } + + var tipHash string + name := md.rpmFile.Name() + module.Data.Name = name + ref := module.Data.Components.Rpms[name].Ref + if strings.HasPrefix(ref, "stream-") { + module.Data.Components.Rpms[name].Ref = md.pushBranch + tipHash = getTipStream(pd, name, md.pushBranch) + } else { + log.Fatal("could not recognize modulemd file, no stream- ref?") + } + + err = module.Marshal(md.worktree.Filesystem, mdTxtPath) + if err != nil { + log.Fatalf("could not marshal modulemd: %v", err) + } + + module.Data.Components.Rpms[name].Ref = tipHash + rootModule := fmt.Sprintf("%s.yaml", name) + err = module.Marshal(md.worktree.Filesystem, rootModule) + if err != nil { + log.Fatalf("could not marshal root modulemd: %v", err) + } + + _, err = md.worktree.Add(rootModule) + if err != nil { + log.Fatalf("could not add root modulemd: %v", err) + } +} diff --git a/internal/process.go b/internal/process.go index 1622629..b1bebf3 100644 --- a/internal/process.go +++ b/internal/process.go @@ -15,6 +15,7 @@ import ( "hash" "io/ioutil" "log" + "path/filepath" "regexp" "strings" "time" @@ -36,6 +37,7 @@ type ProcessData struct { Importer ImportMode BlobStorage blob.Storage NoDupMode bool + ModuleMode bool } type ignoredSource struct { @@ -64,6 +66,11 @@ type modeData struct { func ProcessRPM(pd *ProcessData) { md := pd.Importer.RetrieveSource(pd) + remotePrefix := "dist" + if pd.ModuleMode { + remotePrefix = "modules" + } + // if no-dup-mode is enabled then skip already imported versions var tagIgnoreList []string if pd.NoDupMode { @@ -71,7 +78,7 @@ func ProcessRPM(pd *ProcessData) { if err != nil { log.Fatalf("could not init git repo: %v", err) } - remoteUrl := fmt.Sprintf("%s/dist/%s.git", pd.UpstreamPrefix, md.rpmFile.Name()) + remoteUrl := fmt.Sprintf("%s/%s/%s.git", pd.UpstreamPrefix, remotePrefix, md.rpmFile.Name()) refspec := config.RefSpec("+refs/heads/*:refs/remotes/origin/*") remote, err := repo.CreateRemote(&config.RemoteConfig{ @@ -87,7 +94,7 @@ func ProcessRPM(pd *ProcessData) { Auth: pd.Authenticator, }) if err != nil { - log.Println("ignoring no-dup-mode") + log.Println("ignoring no-dup-mode") } else { for _, ref := range list { if !strings.HasPrefix(string(ref.Name()), "refs/tags/imports") { @@ -118,11 +125,24 @@ func ProcessRPM(pd *ProcessData) { log.Fatalf("could not get dist worktree: %v", err) } + var matchString string if !tagImportRegex.MatchString(md.tagBranch) { - log.Fatal("import tag invalid") + if pd.ModuleMode { + prefix := fmt.Sprintf("refs/heads/c%d", pd.Version) + if strings.HasPrefix(md.tagBranch, prefix) { + replace := strings.Replace(md.tagBranch, "refs/heads/", "", 1) + matchString = fmt.Sprintf("refs/tags/imports/%s/%s", replace, filepath.Base(pd.RpmLocation)) + log.Printf("using match string: %s", matchString) + } + } + if !tagImportRegex.MatchString(matchString) { + log.Fatal("import tag invalid") + } + } else { + matchString = md.tagBranch } - match := tagImportRegex.FindStringSubmatch(md.tagBranch) + match := tagImportRegex.FindStringSubmatch(matchString) md.pushBranch = "rocky" + strings.TrimPrefix(match[2], "c") newTag := "imports/rocky" + strings.TrimPrefix(match[1], "imports/c") @@ -138,7 +158,7 @@ func ProcessRPM(pd *ProcessData) { } // create a new remote - remoteUrl := fmt.Sprintf("%s/dist/%s.git", pd.UpstreamPrefix, rpmFile.Name()) + remoteUrl := fmt.Sprintf("%s/%s/%s.git", pd.UpstreamPrefix, remotePrefix, rpmFile.Name()) log.Printf("using remote: %s", remoteUrl) refspec := config.RefSpec(fmt.Sprintf("+refs/heads/%s:refs/remotes/origin/%s", md.pushBranch, md.pushBranch)) log.Printf("using refspec: %s", refspec) @@ -182,7 +202,11 @@ func ProcessRPM(pd *ProcessData) { md.repo = repo md.worktree = w - executePatches(pd, md) + if pd.ModuleMode { + patchModuleYaml(pd, md) + } else { + executePatchesRpm(pd, md) + } // get ignored files hash and add to .{name}.metadata metadataFile := fmt.Sprintf(".%s.metadata", rpmFile.Name()) @@ -213,7 +237,7 @@ func ProcessRPM(pd *ProcessData) { log.Fatalf("could not write to metadata file: %v", err) } - path := fmt.Sprintf("%s/%s/%s", rpmFile.Name(), md.pushBranch, checksum) + path := fmt.Sprintf("%s/%s", rpmFile.Name(), checksum) pd.BlobStorage.Write(path, sourceFileBts) log.Printf("wrote %s to blob storage", path) } @@ -225,9 +249,12 @@ func ProcessRPM(pd *ProcessData) { lastFilesToAdd := []string{".gitignore", "SPECS"} for _, f := range lastFilesToAdd { - _, err := w.Add(f) - if err != nil { - log.Fatalf("could not add metadata file: %v", err) + _, err := w.Filesystem.Stat(f) + if err == nil { + _, err := w.Add(f) + if err != nil { + log.Fatalf("could not add %s: %v", f, err) + } } } diff --git a/modulemd/modulemd.go b/modulemd/modulemd.go new file mode 100644 index 0000000..cbe9901 --- /dev/null +++ b/modulemd/modulemd.go @@ -0,0 +1,76 @@ +package modulemd + +import ( + "github.com/go-git/go-billy/v5" + "gopkg.in/yaml.v2" +) + +type ModuleMd struct { + Document string `yaml:"document"` + Version int `yaml:"version"` + Data struct { + Name string `yaml:"name"` + Stream string `yaml:"stream"` + Summary string `yaml:"summary"` + Description string `yaml:"description"` + License struct { + Module []string `yaml:"module"` + } `yaml:"license"` + Dependencies []struct { + BuildRequires struct { + Platform []string `yaml:"platform"` + } `yaml:"buildrequires"` + Requires struct { + Platform []string `yaml:"platform"` + } `yaml:"requires"` + } `yaml:"dependencies"` + References struct { + Documentation string `yaml:"documentation"` + Tracker string `yaml:"tracker"` + } `yaml:"references"` + Profiles struct { + Common struct { + Rpms []string `yaml:"rpms"` + } `yaml:"common"` + } `yaml:"profiles"` + API struct { + Rpms []string `yaml:"rpms"` + } `yaml:"api"` + Components struct { + Rpms map[string]*struct { + Rationale string `yaml:"rationale"` + Ref string `yaml:"ref"` + } `yaml:"rpms"` + } `yaml:"components"` + } `yaml:"data"` +} + +func Parse(input []byte) (*ModuleMd, error) { + var ret ModuleMd + err := yaml.Unmarshal(input, &ret) + if err != nil { + return nil, err + } + + return &ret, nil +} + +func (m *ModuleMd) Marshal(fs billy.Filesystem, path string) error { + bts, err := yaml.Marshal(m) + if err != nil { + return err + } + + _ = fs.Remove(path) + f, err := fs.Create(path) + if err != nil { + return err + } + _, err = f.Write(bts) + if err != nil { + return err + } + _ = f.Close() + + return nil +}