sig_core/srpmproc
8
0
Fork 0
mirror of https://github.com/rocky-linux/srpmproc.git synced 2024-06-18 07:30:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

add module mode

Browse Source
This commit is contained in:
Mustafa Gezen 2020-12-22 17:28:04 +01:00
parent 49a22b9061
commit 600ba21558
8 changed files with 249 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
cmd/srpmproc/fetch.go
View File

@ -9,7 +9,6 @@ import (
"log"
"net/http"
"os"
"os/exec"
"path/filepath"
"strings"
)
@ -58,13 +57,6 @@ func runFetch(_ *cobra.Command, _ []string) {
log.Fatalf("could not read metadata file: %v", err)
}
cmd := exec.Command("git", "branch", "--show-current")
branchByte, err := cmd.Output()
if err != nil {
log.Fatalf("could not get branch: %v", err)
}
branch := strings.TrimSuffix(string(branchByte), "\n")
client := &http.Client{
Transport: &http.Transport{
DisableCompression: false,
@ -80,7 +72,7 @@ func runFetch(_ *cobra.Command, _ []string) {
hash := lineInfo[0]
path := lineInfo[1]
url := fmt.Sprintf("%s/%s/%s/%s", cdnUrl, name, branch, hash)
url := fmt.Sprintf("%s/%s/%s", cdnUrl, name, hash)
log.Printf("downloading %s", url)
req, err := http.NewRequest("GET", url, nil)

9
cmd/srpmproc/main.go
View File

@ -27,6 +27,7 @@ var (
modulePrefix string
rpmPrefix string
noDupMode bool
moduleMode bool
)
var root = &cobra.Command{
@ -58,7 +59,11 @@ func mn(_ *cobra.Command, _ []string) {
sourceRpmLocation = strings.TrimPrefix(sourceRpm, "file://")
importer = &internal.SrpmMode{}
} else {
sourceRpmLocation = fmt.Sprintf("%s/%s", rpmPrefix, sourceRpm)
if moduleMode {
sourceRpmLocation = fmt.Sprintf("%s/%s", modulePrefix, sourceRpm)
} else {
sourceRpmLocation = fmt.Sprintf("%s/%s", rpmPrefix, sourceRpm)
}
importer = &internal.GitMode{}
}
@ -89,6 +94,7 @@ func mn(_ *cobra.Command, _ []string) {
ModulePrefix: modulePrefix,
Authenticator: authenticator,
NoDupMode: noDupMode,
ModuleMode: moduleMode,
})
}
@ -109,6 +115,7 @@ func main() {
root.Flags().StringVar(&modulePrefix, "module-prefix", "https://git.centos.org/modules", "Where to retrieve modules if exists. Only used when source-rpm is a git repo")
root.Flags().StringVar(&rpmPrefix, "rpm-prefix", "https://git.centos.org/rpms", "Where to retrieve SRPM content. Only used when source-rpm is not a local file")
root.Flags().BoolVar(&noDupMode, "no-dup-mode", false, "If enabled, skips already imported tags")
root.Flags().BoolVar(&moduleMode, "module-mode", false, "If enabled, imports a module instead of a package")
if err := root.Execute(); err != nil {
log.Fatal(err)

1
go.mod
View File

@ -13,4 +13,5 @@ require (
github.com/golang/protobuf v1.4.2
github.com/spf13/cobra v1.1.1
google.golang.org/protobuf v1.25.0
gopkg.in/yaml.v2 v2.2.8
)

1
go.sum
View File

@ -591,6 +591,7 @@ gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bl
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

37
internal/git.go
View File

@ -52,6 +52,17 @@ func (g *GitMode) RetrieveSource(pd *ProcessData) *modeData {
branches = append(branches, name)
}
}
// no tags? fetch branches
if len(branches) == 0 {
for _, ref := range list {
name := string(ref.Name())
prefix := fmt.Sprintf("refs/heads/c%d", pd.Version)
if strings.HasPrefix(name, prefix) {
branches = append(branches, name)
}
}
}
sort.Strings(branches)
return &modeData{
@ -69,8 +80,17 @@ func (g *GitMode) WriteSource(md *modeData) {
log.Fatalf("could not get upstream remote: %v", err)
}
match := tagImportRegex.FindStringSubmatch(md.tagBranch)
refspec := config.RefSpec(fmt.Sprintf("+refs/heads/%s:%s", match[2], md.tagBranch))
var refspec config.RefSpec
var branchName string
if strings.HasPrefix(md.tagBranch, "refs/heads") {
refspec = config.RefSpec(fmt.Sprintf("+%s:%s", md.tagBranch, md.tagBranch))
branchName = strings.TrimPrefix(md.tagBranch, "refs/heads/")
} else {
match := tagImportRegex.FindStringSubmatch(md.tagBranch)
branchName = match[2]
refspec = config.RefSpec(fmt.Sprintf("+refs/heads/%s:%s", branchName, md.tagBranch))
}
log.Printf("checking out upstream refspec %s", refspec)
err = remote.Fetch(&git.FetchOptions{
RemoteName: "upstream",
@ -97,7 +117,8 @@ func (g *GitMode) WriteSource(md *modeData) {
metadataFile, err := md.worktree.Filesystem.Open(fmt.Sprintf(".%s.metadata", md.rpmFile.Name()))
if err != nil {
log.Fatalf("could not open metadata file: %v", err)
log.Printf("warn: could not open metadata file, so skipping: %v", err)
return
}
fileBytes, err := ioutil.ReadAll(metadataFile)
@ -120,7 +141,7 @@ func (g *GitMode) WriteSource(md *modeData) {
hash := strings.TrimSpace(lineInfo[0])
path := strings.TrimSpace(lineInfo[1])
url := fmt.Sprintf("https://git.centos.org/sources/%s/%s/%s", md.rpmFile.Name(), match[2], hash)
url := fmt.Sprintf("https://git.centos.org/sources/%s/%s/%s", md.rpmFile.Name(), branchName, hash)
log.Printf("downloading %s", url)
req, err := http.NewRequest("GET", url, nil)
@ -181,6 +202,10 @@ func (g *GitMode) PostProcess(md *modeData) {
}
func (g *GitMode) ImportName(_ *ProcessData, md *modeData) string {
match := tagImportRegex.FindStringSubmatch(md.tagBranch)
return match[3]
if tagImportRegex.MatchString(md.tagBranch) {
match := tagImportRegex.FindStringSubmatch(md.tagBranch)
return match[3]
}
return strings.TrimPrefix(md.tagBranch, "refs/heads/")
}

95
internal/patch.go
View File

@ -10,6 +10,7 @@ import (
"github.com/go-git/go-git/v5/plumbing"
"github.com/go-git/go-git/v5/storage/memory"
"github.com/mstg/srpmproc/internal/directives"
"github.com/mstg/srpmproc/modulemd"
srpmprocpb "github.com/mstg/srpmproc/pb"
"google.golang.org/protobuf/encoding/prototext"
"io/ioutil"
@ -165,7 +166,7 @@ func applyPatches(patchTree *git.Worktree, pushTree *git.Worktree) {
}
}
func executePatches(pd *ProcessData, md *modeData) {
func executePatchesRpm(pd *ProcessData, md *modeData) {
// fetch patch repository
repo, err := git.Init(memory.NewStorage(), memfs.New())
if err != nil {
@ -225,3 +226,95 @@ func executePatches(pd *ProcessData, md *modeData) {
}
}
}
func getTipStream(pd *ProcessData, module string, pushBranch string) string {
repo, err := git.Init(memory.NewStorage(), memfs.New())
if err != nil {
log.Fatalf("could not init git repo: %v", err)
}
remoteUrl := fmt.Sprintf("%s/dist/%s.git", pd.UpstreamPrefix, module)
refspec := config.RefSpec("+refs/heads/*:refs/remotes/origin/*")
remote, err := repo.CreateRemote(&config.RemoteConfig{
Name: "origin",
URLs: []string{remoteUrl},
Fetch: []config.RefSpec{refspec},
})
if err != nil {
log.Fatalf("could not create remote: %v", err)
}
list, err := remote.List(&git.ListOptions{
Auth: pd.Authenticator,
})
if err != nil {
log.Fatalf("could not get rpm refs. import the rpm before the module: %v", err)
}
var tipHash string
for _, ref := range list {
prefix := fmt.Sprintf("refs/heads/%s", pushBranch)
if strings.HasPrefix(ref.Name().String(), prefix) {
tipHash = ref.Hash().String()
}
}
if tipHash == "" {
log.Fatal("could not find tip hash")
}
return tipHash
}
func patchModuleYaml(pd *ProcessData, md *modeData) {
// special case for platform.yaml
_, err := md.worktree.Filesystem.Open("platform.yaml")
if err == nil {
return
}
mdTxtPath := "SOURCES/modulemd.src.txt"
f, err := md.worktree.Filesystem.Open(mdTxtPath)
if err != nil {
log.Fatalf("could not open modulemd file: %v", err)
}
content, err := ioutil.ReadAll(f)
if err != nil {
log.Fatalf("could not read modulemd file: %v", err)
}
module, err := modulemd.Parse(content)
if err != nil {
log.Fatalf("could not parse modulemd file: %v", err)
}
var tipHash string
name := md.rpmFile.Name()
module.Data.Name = name
ref := module.Data.Components.Rpms[name].Ref
if strings.HasPrefix(ref, "stream-") {
module.Data.Components.Rpms[name].Ref = md.pushBranch
tipHash = getTipStream(pd, name, md.pushBranch)
} else {
log.Fatal("could not recognize modulemd file, no stream- ref?")
}
err = module.Marshal(md.worktree.Filesystem, mdTxtPath)
if err != nil {
log.Fatalf("could not marshal modulemd: %v", err)
}
module.Data.Components.Rpms[name].Ref = tipHash
rootModule := fmt.Sprintf("%s.yaml", name)
err = module.Marshal(md.worktree.Filesystem, rootModule)
if err != nil {
log.Fatalf("could not marshal root modulemd: %v", err)
}
_, err = md.worktree.Add(rootModule)
if err != nil {
log.Fatalf("could not add root modulemd: %v", err)
}
}

47
internal/process.go
View File

@ -15,6 +15,7 @@ import (
"hash"
"io/ioutil"
"log"
"path/filepath"
"regexp"
"strings"
"time"
@ -36,6 +37,7 @@ type ProcessData struct {
Importer ImportMode
BlobStorage blob.Storage
NoDupMode bool
ModuleMode bool
}
type ignoredSource struct {
@ -64,6 +66,11 @@ type modeData struct {
func ProcessRPM(pd *ProcessData) {
md := pd.Importer.RetrieveSource(pd)
remotePrefix := "dist"
if pd.ModuleMode {
remotePrefix = "modules"
}
// if no-dup-mode is enabled then skip already imported versions
var tagIgnoreList []string
if pd.NoDupMode {
@ -71,7 +78,7 @@ func ProcessRPM(pd *ProcessData) {
if err != nil {
log.Fatalf("could not init git repo: %v", err)
}
remoteUrl := fmt.Sprintf("%s/dist/%s.git", pd.UpstreamPrefix, md.rpmFile.Name())
remoteUrl := fmt.Sprintf("%s/%s/%s.git", pd.UpstreamPrefix, remotePrefix, md.rpmFile.Name())
refspec := config.RefSpec("+refs/heads/*:refs/remotes/origin/*")
remote, err := repo.CreateRemote(&config.RemoteConfig{
@ -87,7 +94,7 @@ func ProcessRPM(pd *ProcessData) {
Auth: pd.Authenticator,
})
if err != nil {
log.Println("ignoring no-dup-mode")
log.Println("ignoring no-dup-mode")
} else {
for _, ref := range list {
if !strings.HasPrefix(string(ref.Name()), "refs/tags/imports") {
@ -118,11 +125,24 @@ func ProcessRPM(pd *ProcessData) {
log.Fatalf("could not get dist worktree: %v", err)
}
var matchString string
if !tagImportRegex.MatchString(md.tagBranch) {
log.Fatal("import tag invalid")
if pd.ModuleMode {
prefix := fmt.Sprintf("refs/heads/c%d", pd.Version)
if strings.HasPrefix(md.tagBranch, prefix) {
replace := strings.Replace(md.tagBranch, "refs/heads/", "", 1)
matchString = fmt.Sprintf("refs/tags/imports/%s/%s", replace, filepath.Base(pd.RpmLocation))
log.Printf("using match string: %s", matchString)
}
}
if !tagImportRegex.MatchString(matchString) {
log.Fatal("import tag invalid")
}
} else {
matchString = md.tagBranch
}
match := tagImportRegex.FindStringSubmatch(md.tagBranch)
match := tagImportRegex.FindStringSubmatch(matchString)
md.pushBranch = "rocky" + strings.TrimPrefix(match[2], "c")
newTag := "imports/rocky" + strings.TrimPrefix(match[1], "imports/c")
@ -138,7 +158,7 @@ func ProcessRPM(pd *ProcessData) {
}
// create a new remote
remoteUrl := fmt.Sprintf("%s/dist/%s.git", pd.UpstreamPrefix, rpmFile.Name())
remoteUrl := fmt.Sprintf("%s/%s/%s.git", pd.UpstreamPrefix, remotePrefix, rpmFile.Name())
log.Printf("using remote: %s", remoteUrl)
refspec := config.RefSpec(fmt.Sprintf("+refs/heads/%s:refs/remotes/origin/%s", md.pushBranch, md.pushBranch))
log.Printf("using refspec: %s", refspec)
@ -182,7 +202,11 @@ func ProcessRPM(pd *ProcessData) {
md.repo = repo
md.worktree = w
executePatches(pd, md)
if pd.ModuleMode {
patchModuleYaml(pd, md)
} else {
executePatchesRpm(pd, md)
}
// get ignored files hash and add to .{name}.metadata
metadataFile := fmt.Sprintf(".%s.metadata", rpmFile.Name())
@ -213,7 +237,7 @@ func ProcessRPM(pd *ProcessData) {
log.Fatalf("could not write to metadata file: %v", err)
}
path := fmt.Sprintf("%s/%s/%s", rpmFile.Name(), md.pushBranch, checksum)
path := fmt.Sprintf("%s/%s", rpmFile.Name(), checksum)
pd.BlobStorage.Write(path, sourceFileBts)
log.Printf("wrote %s to blob storage", path)
}
@ -225,9 +249,12 @@ func ProcessRPM(pd *ProcessData) {
lastFilesToAdd := []string{".gitignore", "SPECS"}
for _, f := range lastFilesToAdd {
_, err := w.Add(f)
if err != nil {
log.Fatalf("could not add metadata file: %v", err)
_, err := w.Filesystem.Stat(f)
if err == nil {
_, err := w.Add(f)
if err != nil {
log.Fatalf("could not add %s: %v", f, err)
}
}
}

76
modulemd/modulemd.go Normal file
View File

@ -0,0 +1,76 @@
package modulemd
import (
"github.com/go-git/go-billy/v5"
"gopkg.in/yaml.v2"
)
type ModuleMd struct {
Document string `yaml:"document"`
Version int `yaml:"version"`
Data struct {
Name string `yaml:"name"`
Stream string `yaml:"stream"`
Summary string `yaml:"summary"`
Description string `yaml:"description"`
License struct {
Module []string `yaml:"module"`
} `yaml:"license"`
Dependencies []struct {
BuildRequires struct {
Platform []string `yaml:"platform"`
} `yaml:"buildrequires"`
Requires struct {
Platform []string `yaml:"platform"`
} `yaml:"requires"`
} `yaml:"dependencies"`
References struct {
Documentation string `yaml:"documentation"`
Tracker string `yaml:"tracker"`
} `yaml:"references"`
Profiles struct {
Common struct {
Rpms []string `yaml:"rpms"`
} `yaml:"common"`
} `yaml:"profiles"`
API struct {
Rpms []string `yaml:"rpms"`
} `yaml:"api"`
Components struct {
Rpms map[string]*struct {
Rationale string `yaml:"rationale"`
Ref string `yaml:"ref"`
} `yaml:"rpms"`
} `yaml:"components"`
} `yaml:"data"`
}
func Parse(input []byte) (*ModuleMd, error) {
var ret ModuleMd
err := yaml.Unmarshal(input, &ret)
if err != nil {
return nil, err
}
return &ret, nil
}
func (m *ModuleMd) Marshal(fs billy.Filesystem, path string) error {
bts, err := yaml.Marshal(m)
if err != nil {
return err
}
_ = fs.Remove(path)
f, err := fs.Create(path)
if err != nil {
return err
}
_, err = f.Write(bts)
if err != nil {
return err
}
_ = f.Close()
return nil
}