sig_core
/
srpmproc
mirror of https://github.com/rocky-linux/srpmproc.git
8
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

abstract modes and blob storage. support import from git.c.o

This commit is contained in:
Mustafa Gezen 2020-12-20 10:36:13 +01:00
parent 777015bb1d
commit 1dcceac63a
13 changed files with 928 additions and 374 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

74
cmd/srpmproc/main.go
View File

@ -1,9 +1,14 @@
package main
import (
"cloud.google.com/go/storage"
"context"
"fmt"
"github.com/go-git/go-git/v5/plumbing/transport/ssh"
"github.com/mstg/srpmproc/internal/blob"
"github.com/mstg/srpmproc/internal/blob/gcs"
"github.com/mstg/srpmproc/internal/blob/s3"
"log"
"os/user"
"path/filepath"
"strings"
"github.com/mstg/srpmproc/internal"
@ -15,10 +20,12 @@ var (
sshKeyLocation string
sshUser string
upstreamPrefix string
branch string
gcsBucket string
version int
storageAddr string
gitCommitterName string
gitCommitterEmail string
modulePrefix string
rpmPrefix string
)
var root = &cobra.Command{
@ -27,28 +34,59 @@ var root = &cobra.Command{
}
func mn(_ *cobra.Command, _ []string) {
ctx := context.Background()
client, err := storage.NewClient(ctx)
if err != nil {
log.Fatalf("could not create gcloud client: %v", err)
switch version {
case 8:
break
default:
log.Fatalf("unsupported upstream version %d", version)
}
var importer internal.ImportMode
var blobStorage blob.Storage
if strings.HasPrefix(storageAddr, "gs://") {
blobStorage = gcs.New(strings.Replace(storageAddr, "gs://", "", 1))
} else if strings.HasPrefix(storageAddr, "s3://") {
blobStorage = s3.New(strings.Replace(storageAddr, "s3://", "", 1))
} else {
log.Fatalf("invalid blob storage")
}
sourceRpmLocation := ""
if strings.HasPrefix(sourceRpm, "file://") {
sourceRpmLocation = strings.TrimPrefix(sourceRpm, "file://")
importer = &internal.SrpmMode{}
} else {
log.Fatal("non-local SRPMs are currently not supported")
sourceRpmLocation = fmt.Sprintf("%s/%s", rpmPrefix, sourceRpm)
importer = &internal.GitMode{}
}
lastKeyLocation := sshKeyLocation
if lastKeyLocation == "" {
usr, err := user.Current()
if err != nil {
log.Fatalf("could not get user: %v", err)
}
lastKeyLocation = filepath.Join(usr.HomeDir, ".ssh/id_rsa")
}
// create ssh key authenticator
authenticator, err := ssh.NewPublicKeysFromFile(sshUser, lastKeyLocation, "")
if err != nil {
log.Fatalf("could not get git authenticator: %v", err)
}
internal.ProcessRPM(&internal.ProcessData{
Importer: importer,
RpmLocation: sourceRpmLocation,
UpstreamPrefix: upstreamPrefix,
SshKeyLocation: sshKeyLocation,
SshUser: sshUser,
Branch: branch,
Bucket: client.Bucket(gcsBucket),
Version: version,
BlobStorage: blobStorage,
GitCommitterName: gitCommitterName,
GitCommitterEmail: gitCommitterEmail,
ModulePrefix: modulePrefix,
Authenticator: authenticator,
})
}
@ -57,15 +95,17 @@ func main() {
_ = root.MarkFlagRequired("source-rpm")
root.Flags().StringVar(&upstreamPrefix, "upstream-prefix", "", "Upstream git repository prefix")
_ = root.MarkFlagRequired("upstream-prefix")
root.Flags().StringVar(&branch, "branch", "", "Upstream branch")
_ = root.MarkFlagRequired("branch")
root.Flags().StringVar(&gcsBucket, "gcs-bucket", "", "Bucket to use as blob storage")
_ = root.MarkFlagRequired("gcs-bucket")
root.Flags().StringVar(&gitCommitterName, "git-committer-name", "distrobuild-bot", "Name of committer")
root.Flags().StringVar(&gitCommitterEmail, "git-committer-email", "mustafa+distrobuild@bycrates.com", "Email of committer")
root.Flags().IntVar(&version, "version", 0, "Upstream version")
_ = root.MarkFlagRequired("version")
root.Flags().StringVar(&storageAddr, "storage-addr", "", "Bucket to use as blob storage")
_ = root.MarkFlagRequired("storage-addr")
root.Flags().StringVar(&sshKeyLocation, "ssh-key-location", "", "Location of the SSH key to use to authenticate against upstream")
root.Flags().StringVar(&sshUser, "ssh-user", "git", "SSH User")
root.Flags().StringVar(&gitCommitterName, "git-committer-name", "distrobuild-bot", "Name of committer")
root.Flags().StringVar(&gitCommitterEmail, "git-committer-email", "mustafa+distrobuild@bycrates.com", "Email of committer")
root.Flags().StringVar(&modulePrefix, "module-prefix", "https://git.centos.org/modules", "Where to retrieve modules if exists. Only used when source-rpm is a git repo")
root.Flags().StringVar(&rpmPrefix, "rpm-prefix", "https://git.centos.org/rpms", "Where to retrieve SRPM content. Only used when source-rpm is not a local file")
if err := root.Execute(); err != nil {
log.Fatal(err)

3
go.mod
View File

@ -4,12 +4,11 @@ go 1.15
require (
cloud.google.com/go/storage v1.12.0
github.com/aws/aws-sdk-go v1.36.12
github.com/bluekeyes/go-gitdiff v0.5.0
github.com/cavaliercoder/go-cpio v0.0.0-20180626203310-925f9528c45e
github.com/cavaliercoder/go-rpm v0.0.0-20200122174316-8cb9fd9c31a8
github.com/go-git/go-billy/v5 v5.0.0
github.com/go-git/go-git/v5 v5.2.0
github.com/sirupsen/logrus v1.2.0
github.com/spf13/cobra v1.1.1
google.golang.org/api v0.32.0
)

10
go.sum
View File

@ -47,6 +47,8 @@ github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hC
github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
github.com/aws/aws-sdk-go v1.36.12 h1:YJpKFEMbqEoo+incs5qMe61n1JH3o4O1IMkMexLzJG8=
github.com/aws/aws-sdk-go v1.36.12/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
@ -188,6 +190,9 @@ github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANyt
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A=
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo=
github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
@ -228,6 +233,7 @@ github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FI
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
@ -369,6 +375,8 @@ golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81R
golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20200904194848-62affa334b73 h1:MXfv8rhZWmFeqX3GNZRsd6vOLoaCHjYEX3qkRo3YBUA=
golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b h1:uwuIcX0g4Yl1NC5XAz37xsr2lTtcqevgzYNVt49waME=
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@ -419,6 +427,8 @@ golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20200828194041-157a740278f4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f h1:Fqb3ao1hUmOR3GkUOg/Y+BadLwykBIzs5q8Ez2SbHyc=
golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f h1:+Nyd8tzPX9R7BWHguqsrbFdRx3WQ/1ib8I44HXV5yTA=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

5
internal/blob/blob.go Normal file
View File

@ -0,0 +1,5 @@
package blob
type Storage interface {
Write(path string, content []byte)
}

39
internal/blob/gcs/gcs.go Normal file
View File

@ -0,0 +1,39 @@
package gcs
import (
"cloud.google.com/go/storage"
"context"
"log"
)
type GCS struct {
bucket *storage.BucketHandle
}
func New(name string) *GCS {
ctx := context.Background()
client, err := storage.NewClient(ctx)
if err != nil {
log.Fatalf("could not create gcloud client: %v", err)
}
return &GCS{
bucket: client.Bucket(name),
}
}
func (g *GCS) Write(path string, content []byte) {
ctx := context.Background()
obj := g.bucket.Object(path)
w := obj.NewWriter(ctx)
_, err := w.Write(content)
if err != nil {
log.Fatalf("could not write file to gcs: %v", err)
}
// Close, just like writing a file.
if err := w.Close(); err != nil {
log.Fatalf("could not close gcs writer to source: %v", err)
}
}

37
internal/blob/s3/s3.go Normal file
View File

@ -0,0 +1,37 @@
package s3
import (
"bytes"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
"log"
)
type S3 struct {
bucket string
uploader *s3manager.Uploader
}
func New(name string) *S3 {
sess := session.Must(session.NewSession())
uploader := s3manager.NewUploader(sess)
return &S3{
bucket: name,
uploader: uploader,
}
}
func (s *S3) Write(path string, content []byte) {
buf := bytes.NewBuffer(content)
_, err := s.uploader.Upload(&s3manager.UploadInput{
Bucket: aws.String(s.bucket),
Key: aws.String(path),
Body: buf,
})
if err != nil {
log.Fatalf("failed to upload file to s3, %v", err)
}
}

185
internal/git.go Normal file
View File

@ -0,0 +1,185 @@
package internal
import (
"fmt"
"github.com/go-git/go-billy/v5/memfs"
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/config"
"github.com/go-git/go-git/v5/plumbing"
"github.com/go-git/go-git/v5/storage/memory"
"io/ioutil"
"log"
"net/http"
"path/filepath"
"sort"
"strings"
)
type GitMode struct{}
func (g *GitMode) RetrieveSource(pd *ProcessData) *modeData {
repo, err := git.Init(memory.NewStorage(), memfs.New())
if err != nil {
log.Fatalf("could not init git repo: %v", err)
}
w, err := repo.Worktree()
if err != nil {
log.Fatalf("could not get worktree: %v", err)
}
refspec := config.RefSpec("+refs/heads/*:refs/remotes/*")
remote, err := repo.CreateRemote(&config.RemoteConfig{
Name: "upstream",
URLs: []string{pd.RpmLocation},
Fetch: []config.RefSpec{refspec},
})
if err != nil {
log.Fatalf("could not create remote: %v", err)
}
list, err := remote.List(&git.ListOptions{})
if err != nil {
log.Fatalf("could not list remote: %v", err)
}
var branches []string
for _, ref := range list {
log.Println(ref.String())
name := string(ref.Name())
if strings.HasPrefix(name, fmt.Sprintf("refs/tags/imports/c%d", pd.Version)) {
branches = append(branches, name)
}
}
sort.Strings(branches)
return &modeData{
repo: repo,
worktree: w,
rpmFile: createPackageFile(filepath.Base(pd.RpmLocation)),
fileWrites: nil,
branches: branches,
}
}
func (g *GitMode) WriteSource(md *modeData) {
remote, err := md.repo.Remote("upstream")
if err != nil {
log.Fatalf("could not get upstream remote: %v", err)
}
match := tagImportRegex.FindStringSubmatch(md.tagBranch)
refspec := config.RefSpec(fmt.Sprintf("+refs/heads/%s:%s", match[2], md.tagBranch))
err = remote.Fetch(&git.FetchOptions{
RemoteName: "upstream",
RefSpecs: []config.RefSpec{refspec},
Tags: git.AllTags,
Force: true,
})
if err != nil {
log.Fatalf("could not fetch upstream: %v", err)
}
err = md.worktree.Checkout(&git.CheckoutOptions{
Branch: plumbing.ReferenceName(md.tagBranch),
Force: true,
})
if err != nil {
log.Fatalf("could not checkout source from git: %v", err)
}
_, err = md.worktree.Add(".")
if err != nil {
log.Fatalf("could not add worktree: %v", err)
}
metadataFile, err := md.worktree.Filesystem.Open(fmt.Sprintf(".%s.metadata", md.rpmFile.Name()))
if err != nil {
log.Fatalf("could not open metadata file: %v", err)
}
fileBytes, err := ioutil.ReadAll(metadataFile)
if err != nil {
log.Fatalf("could not read metadata file: %v", err)
}
client := &http.Client{
Transport: &http.Transport{
DisableCompression: false,
},
}
fileContent := strings.Split(string(fileBytes), "\n")
for _, line := range fileContent {
if strings.TrimSpace(line) == "" {
continue
}
lineInfo := strings.Split(line, " ")
hash := lineInfo[0]
path := lineInfo[1]
url := fmt.Sprintf("https://git.centos.org/sources/%s/%s/%s", md.rpmFile.Name(), match[2], hash)
log.Printf("downloading %s", url)
req, err := http.NewRequest("GET", url, nil)
if err != nil {
log.Fatalf("could not create new http request: %v", err)
}
req.Header.Set("Accept-Encoding", "*")
resp, err := client.Do(req)
if err != nil {
log.Fatalf("could not download dist-git file: %v", err)
}
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Fatalf("could not read the whole dist-git file: %v", err)
}
err = resp.Body.Close()
if err != nil {
log.Fatalf("could not close body handle: %v", err)
}
f, err := md.worktree.Filesystem.Create(path)
if err != nil {
log.Fatalf("could not open file pointer: %v", err)
}
hasher := compareHash(body, hash)
if hasher == nil {
log.Fatal("checksum in metadata does not match dist-git file")
}
md.sourcesToIgnore = append(md.sourcesToIgnore, &ignoredSource{
name: filepath.Base(path),
hashFunction: hasher,
})
_, err = f.Write(body)
if err != nil {
log.Fatalf("could not copy dist-git file to in-tree: %v", err)
}
_ = f.Close()
}
}
func (g *GitMode) PostProcess(md *modeData) {
for _, source := range md.sourcesToIgnore {
err := md.worktree.Filesystem.Remove(filepath.Join("SOURCES", source.name))
if err != nil {
log.Fatalf("could not remove dist-git file: %v", err)
}
}
_, err := md.worktree.Add(".")
if err != nil {
log.Fatalf("could not add git sources: %v", err)
}
}
func (g *GitMode) ImportName(_ *ProcessData, md *modeData) string {
match := tagImportRegex.FindStringSubmatch(md.tagBranch)
return match[3]
}

31
internal/gitrpm.go Normal file
View File

@ -0,0 +1,31 @@
package internal
import (
"github.com/cavaliercoder/go-rpm"
)
// TODO: ugly hack, should create an interface
// since GitMode does not parse an RPM file, we just mimick
// the headers of an actual file to reuse rpmFile.Name()
func createPackageFile(name string) *rpm.PackageFile {
return &rpm.PackageFile{
Lead: rpm.Lead{},
Headers: []rpm.Header{
{},
{
Version: 0,
IndexCount: 1,
Length: 1,
Indexes: []rpm.IndexEntry{
{
Tag: 1000,
Type: rpm.IndexDataTypeStringArray,
Offset: 0,
ItemCount: 1,
Value: []string{name},
},
},
},
},
}
}

8
internal/mode.go Normal file
View File

@ -0,0 +1,8 @@
package internal
type ImportMode interface {
RetrieveSource(pd *ProcessData) *modeData
WriteSource(md *modeData)
PostProcess(md *modeData)
ImportName(pd *ProcessData, md *modeData) string
}

118
internal/patch.go Normal file
View File

@ -0,0 +1,118 @@
package internal
import (
"bytes"
"github.com/bluekeyes/go-gitdiff/gitdiff"
"github.com/go-git/go-git/v5"
"log"
"path/filepath"
"strings"
)
func srpmPatches(w *git.Worktree) {
// check SRPM patches
_, err := w.Filesystem.Stat("ROCKY/SRPM")
if err == nil {
// iterate through patches
infos, err := w.Filesystem.ReadDir("ROCKY/SRPM")
if err != nil {
log.Fatalf("could not walk patches: %v", err)
}
for _, info := range infos {
// can only process .patch files
if !strings.HasSuffix(info.Name(), ".patch") {
continue
}
log.Printf("applying %s", info.Name())
filePath := filepath.Join("ROCKY/SRPM", info.Name())
patch, err := w.Filesystem.Open(filePath)
if err != nil {
log.Fatalf("could not open patch file %s: %v", info.Name(), err)
}
files, _, err := gitdiff.Parse(patch)
if err != nil {
log.Fatalf("could not parse patch file: %v", err)
}
for _, patchedFile := range files {
srcPath := patchedFile.NewName
if !strings.HasPrefix(srcPath, "SPECS") {
srcPath = filepath.Join("SOURCES", patchedFile.NewName)
}
var output bytes.Buffer
if !patchedFile.IsDelete && !patchedFile.IsNew {
patchSubjectFile, err := w.Filesystem.Open(srcPath)
if err != nil {
log.Fatalf("could not open patch subject: %v", err)
}
err = gitdiff.NewApplier(patchSubjectFile).ApplyFile(&output, patchedFile)
if err != nil {
log.Fatalf("could not apply patch: %v", err)
}
}
oldName := filepath.Join("SOURCES", patchedFile.OldName)
_ = w.Filesystem.Remove(oldName)
_ = w.Filesystem.Remove(srcPath)
if patchedFile.IsNew {
newFile, err := w.Filesystem.Create(srcPath)
if err != nil {
log.Fatalf("could not create new file: %v", err)
}
err = gitdiff.NewApplier(newFile).ApplyFile(&output, patchedFile)
if err != nil {
log.Fatalf("could not apply patch: %v", err)
}
_, err = newFile.Write(output.Bytes())
if err != nil {
log.Fatalf("could not write post-patch file: %v", err)
}
_, err = w.Add(srcPath)
if err != nil {
log.Fatalf("could not add file %s to git: %v", srcPath, err)
}
log.Printf("git add %s", srcPath)
} else if !patchedFile.IsDelete {
newFile, err := w.Filesystem.Create(srcPath)
if err != nil {
log.Fatalf("could not create post-patch file: %v", err)
}
_, err = newFile.Write(output.Bytes())
if err != nil {
log.Fatalf("could not write post-patch file: %v", err)
}
_, err = w.Add(srcPath)
if err != nil {
log.Fatalf("could not add file %s to git: %v", srcPath, err)
}
log.Printf("git add %s", srcPath)
} else {
_, err = w.Remove(oldName)
if err != nil {
log.Fatalf("could not remove file %s to git: %v", oldName, err)
}
log.Printf("git rm %s", oldName)
}
}
_, err = w.Add(filePath)
if err != nil {
log.Fatalf("could not add file %s to git: %v", filePath, err)
}
log.Printf("git add %s", filePath)
}
}
}
func executePatches(w *git.Worktree) {
// check if patches exist
_, err := w.Filesystem.Stat("ROCKY")
if err == nil {
srpmPatches(w)
}
}

533
internal/process.go
View File

@ -1,14 +1,8 @@
package internal
import (
"bytes"
"cloud.google.com/go/storage"
"context"
"crypto/sha1"
"encoding/hex"
"fmt"
"github.com/bluekeyes/go-gitdiff/gitdiff"
"github.com/cavaliercoder/go-cpio"
"github.com/cavaliercoder/go-rpm"
"github.com/go-git/go-billy/v5/memfs"
"github.com/go-git/go-git/v5"
@ -17,36 +11,46 @@ import (
"github.com/go-git/go-git/v5/plumbing/object"
"github.com/go-git/go-git/v5/plumbing/transport/ssh"
"github.com/go-git/go-git/v5/storage/memory"
"io"
"github.com/mstg/srpmproc/internal/blob"
"hash"
"io/ioutil"
"log"
"os"
"os/exec"
"os/user"
"path/filepath"
"regexp"
"strings"
"time"
)
var tagImportRegex = regexp.MustCompile("refs/tags/(imports/(.*)/(.*))")
type ProcessData struct {
RpmLocation string
UpstreamPrefix string
SshKeyLocation string
SshUser string
Branch string
Bucket *storage.BucketHandle
Version int
GitCommitterName string
GitCommitterEmail string
Mode int
ModulePrefix string
Authenticator *ssh.PublicKeys
Importer ImportMode
BlobStorage blob.Storage
}
func strContains(a []string, b string) bool {
for _, val := range a {
if val == b {
return true
}
}
type ignoredSource struct {
name string
hashFunction hash.Hash
}
return false
type modeData struct {
repo *git.Repository
worktree *git.Worktree
rpmFile *rpm.PackageFile
fileWrites map[string][]byte
tagBranch string
pushBranch string
branches []string
sourcesToIgnore []*ignoredSource
}
// ProcessRPM checks the RPM specs and discards any remote files
@ -57,375 +61,194 @@ func strContains(a []string, b string) bool {
// all files that are remote goes into .gitignore
// all ignored files' hash goes into .{name}.metadata
func ProcessRPM(pd *ProcessData) {
cmd := exec.Command("rpm2cpio", pd.RpmLocation)
cpioBytes, err := cmd.Output()
if err != nil {
log.Fatalf("could not convert to cpio (maybe rpm2cpio is missing): %v", err)
}
md := pd.Importer.RetrieveSource(pd)
// create in memory git repository
repo, err := git.Init(memory.NewStorage(), memfs.New())
if err != nil {
log.Fatalf("could not init git repo: %v", err)
}
sourceRepo := *md.repo
sourceWorktree := *md.worktree
// read the rpm in cpio format
buf := bytes.NewReader(cpioBytes)
r := cpio.NewReader(buf)
fileWrites := map[string][]byte{}
for {
hdr, err := r.Next()
if err == io.EOF {
// end of cpio archive
break
}
for _, branch := range md.branches {
md.repo = &sourceRepo
md.worktree = &sourceWorktree
md.tagBranch = branch
md.sourcesToIgnore = []*ignoredSource{}
rpmFile := md.rpmFile
// create new repo for final dist
repo, err := git.Init(memory.NewStorage(), memfs.New())
if err != nil {
log.Fatalln(err)
log.Fatalf("could not create new dist repo: %v", err)
}
bts, err := ioutil.ReadAll(r)
w, err := repo.Worktree()
if err != nil {
log.Fatalf("could not copy file to virtual filesystem: %v", err)
log.Fatalf("could not get dist worktree: %v", err)
}
fileWrites[hdr.Name] = bts
}
w, err := repo.Worktree()
if err != nil {
log.Fatalf("could not get worktree: %v", err)
}
// create structure
err = w.Filesystem.MkdirAll("SPECS", 0755)
if err != nil {
log.Fatalf("could not create SPECS dir in vfs: %v", err)
}
err = w.Filesystem.MkdirAll("SOURCES", 0755)
if err != nil {
log.Fatalf("could not create SOURCES dir in vfs: %v", err)
}
f, err := os.Open(pd.RpmLocation)
if err != nil {
log.Fatalf("could not open the file again: %v", err)
}
rpmFile, err := rpm.ReadPackageFile(f)
if err != nil {
log.Fatalf("could not read package, invalid?: %v", err)
}
var sourcesToIgnore []string
for _, source := range rpmFile.Source() {
if strings.Contains(source, ".tar") {
sourcesToIgnore = append(sourcesToIgnore, source)
if !tagImportRegex.MatchString(md.tagBranch) {
log.Fatal("import tag invalid")
}
}
// create a new remote
remoteUrl := fmt.Sprintf("%s/dist/%s.git", pd.UpstreamPrefix, rpmFile.Name())
log.Printf("using remote: %s", remoteUrl)
refspec := config.RefSpec(fmt.Sprintf("+refs/heads/%s:refs/remotes/origin/%s", pd.Branch, pd.Branch))
log.Printf("using refspec: %s", refspec)
match := tagImportRegex.FindStringSubmatch(md.tagBranch)
md.pushBranch = "rocky" + strings.TrimPrefix(match[2], "c")
_, err = repo.CreateRemote(&config.RemoteConfig{
Name: "origin",
URLs: []string{remoteUrl},
Fetch: []config.RefSpec{refspec},
})
if err != nil {
log.Fatalf("could not create remote: %v", err)
}
// create a new remote
remoteUrl := fmt.Sprintf("%s/dist/%s.git", pd.UpstreamPrefix, rpmFile.Name())
log.Printf("using remote: %s", remoteUrl)
refspec := config.RefSpec(fmt.Sprintf("+refs/heads/%s:refs/remotes/origin/%s", md.pushBranch, md.pushBranch))
log.Printf("using refspec: %s", refspec)
lastKeyLocation := pd.SshKeyLocation
if lastKeyLocation == "" {
usr, err := user.Current()
if err != nil {
log.Fatalf("could not get user: %v", err)
}
lastKeyLocation = filepath.Join(usr.HomeDir, ".ssh/id_rsa")
}
// create ssh key authenticator
authenticator, err := ssh.NewPublicKeysFromFile(pd.SshUser, lastKeyLocation, "")
if err != nil {
log.Fatalf("could not get git authenticator: %v", err)
}
err = repo.Fetch(&git.FetchOptions{
RemoteName: "origin",
RefSpecs: []config.RefSpec{refspec},
Auth: authenticator,
})
refName := plumbing.NewBranchReferenceName(pd.Branch)
log.Printf("set reference to ref: %s", refName)
if err != nil {
h := plumbing.NewSymbolicReference(plumbing.HEAD, refName)
if err := repo.Storer.CheckAndSetReference(h, nil); err != nil {
log.Fatalf("could not set reference: %v", err)
}
} else {
err = w.Checkout(&git.CheckoutOptions{
Branch: plumbing.NewRemoteReferenceName("origin", pd.Branch),
Force: true,
_, err = repo.CreateRemote(&config.RemoteConfig{
Name: "origin",
URLs: []string{remoteUrl},
Fetch: []config.RefSpec{refspec},
})
if err != nil {
log.Fatalf("could not checkout: %v", err)
log.Fatalf("could not create remote: %v", err)
}
}
// TODO: Remove dangling files
err = repo.Fetch(&git.FetchOptions{
RemoteName: "origin",
RefSpecs: []config.RefSpec{refspec},
Auth: pd.Authenticator,
})
for fileName, contents := range fileWrites {
var newPath string
if filepath.Ext(fileName) == ".spec" {
newPath = filepath.Join("SPECS", fileName)
refName := plumbing.NewBranchReferenceName(md.pushBranch)
log.Printf("set reference to ref: %s", refName)
if err != nil {
h := plumbing.NewSymbolicReference(plumbing.HEAD, refName)
if err := repo.Storer.CheckAndSetReference(h, nil); err != nil {
log.Fatalf("could not set reference: %v", err)
}
} else {
newPath = filepath.Join("SOURCES", fileName)
}
mode := os.FileMode(0666)
for _, file := range rpmFile.Files() {
if file.Name() == fileName {
mode = file.Mode()
}
}
// add the file to the virtual filesystem
// we will move it to correct destination later
f, err := w.Filesystem.OpenFile(newPath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, mode)
if err != nil {
log.Fatalf("could not create file %s: %v", fileName, err)
}
_, err = f.Write(contents)
if err != nil {
log.Fatalf("could not write to file %s: %v", fileName, err)
}
_ = f.Close()
// don't add ignored file to git
if strContains(sourcesToIgnore, fileName) {
continue
}
_, err = w.Add(newPath)
if err != nil {
log.Fatalf("could not add source file: %v", err)
}
}
// add sources to ignore (remote sources)
gitIgnore, err := w.Filesystem.Create(".gitignore")
if err != nil {
log.Fatalf("could not create .gitignore: %v", err)
}
for _, ignore := range sourcesToIgnore {
line := fmt.Sprintf("SOURCES/%s\n", ignore)
_, err := gitIgnore.Write([]byte(line))
if err != nil {
log.Fatalf("could not write line to .gitignore: %v", err)
}
}
err = gitIgnore.Close()
if err != nil {
log.Fatalf("could not close .gitignore: %v", err)
}
// check if patches exist
_, err = w.Filesystem.Stat("ROCKY")
if err == nil {
// check SRPM patches
_, err = w.Filesystem.Stat("ROCKY/SRPM")
if err == nil {
// iterate through patches
infos, err := w.Filesystem.ReadDir("ROCKY/SRPM")
err = w.Checkout(&git.CheckoutOptions{
Branch: plumbing.NewRemoteReferenceName("origin", md.pushBranch),
Force: true,
})
if err != nil {
log.Fatalf("could not walk patches: %v", err)
}
for _, info := range infos {
// can only process .patch files
if !strings.HasSuffix(info.Name(), ".patch") {
continue
}
log.Printf("applying %s", info.Name())
filePath := filepath.Join("ROCKY/SRPM", info.Name())
patch, err := w.Filesystem.Open(filePath)
if err != nil {
log.Fatalf("could not open patch file %s: %v", info.Name(), err)
}
files, _, err := gitdiff.Parse(patch)
if err != nil {
log.Fatalf("could not parse patch file: %v", err)
}
for _, patchedFile := range files {
srcPath := patchedFile.NewName
if !strings.HasPrefix(srcPath, "SPECS") {
srcPath = filepath.Join("SOURCES", patchedFile.NewName)
}
var output bytes.Buffer
if !patchedFile.IsDelete && !patchedFile.IsNew {
patchSubjectFile, err := w.Filesystem.Open(srcPath)
if err != nil {
log.Fatalf("could not open patch subject: %v", err)
}
err = gitdiff.NewApplier(patchSubjectFile).ApplyFile(&output, patchedFile)
if err != nil {
log.Fatalf("could not apply patch: %v", err)
}
}
oldName := filepath.Join("SOURCES", patchedFile.OldName)
_ = w.Filesystem.Remove(oldName)
_ = w.Filesystem.Remove(srcPath)
if patchedFile.IsNew {
newFile, err := w.Filesystem.Create(srcPath)
if err != nil {
log.Fatalf("could not create new file: %v", err)
}
err = gitdiff.NewApplier(newFile).ApplyFile(&output, patchedFile)
if err != nil {
log.Fatalf("could not apply patch: %v", err)
}
_, err = newFile.Write(output.Bytes())
if err != nil {
log.Fatalf("could not write post-patch file: %v", err)
}
_, err = w.Add(srcPath)
if err != nil {
log.Fatalf("could not add file %s to git: %v", srcPath, err)
}
log.Printf("git add %s", srcPath)
} else if !patchedFile.IsDelete {
newFile, err := w.Filesystem.Create(srcPath)
if err != nil {
log.Fatalf("could not create post-patch file: %v", err)
}
_, err = newFile.Write(output.Bytes())
if err != nil {
log.Fatalf("could not write post-patch file: %v", err)
}
_, err = w.Add(srcPath)
if err != nil {
log.Fatalf("could not add file %s to git: %v", srcPath, err)
}
log.Printf("git add %s", srcPath)
} else {
_, err = w.Remove(oldName)
if err != nil {
log.Fatalf("could not remove file %s to git: %v", oldName, err)
}
log.Printf("git rm %s", oldName)
}
}
_, err = w.Add(filePath)
if err != nil {
log.Fatalf("could not add file %s to git: %v", filePath, err)
}
log.Printf("git add %s", filePath)
log.Fatalf("could not checkout: %v", err)
}
}
}
// get ignored files sha1 hash and add to .{name}.metadata
metadataFile := fmt.Sprintf(".%s.metadata", rpmFile.Name())
metadata, err := w.Filesystem.Create(metadataFile)
if err != nil {
log.Fatalf("could not create metadata file: %v", err)
}
for _, source := range sourcesToIgnore {
sourcePath := "SOURCES/" + source
sourceFile, err := w.Filesystem.Open(sourcePath)
pd.Importer.WriteSource(md)
copyFromFs(md.worktree.Filesystem, w.Filesystem, ".")
md.repo = repo
md.worktree = w
executePatches(w)
// get ignored files hash and add to .{name}.metadata
metadataFile := fmt.Sprintf(".%s.metadata", rpmFile.Name())
metadata, err := w.Filesystem.Create(metadataFile)
if err != nil {
log.Fatalf("could not open ignored source file %s: %v", sourcePath, err)
log.Fatalf("could not create metadata file: %v", err)
}
sourceFileBts, err := ioutil.ReadAll(sourceFile)
if err != nil {
log.Fatalf("could not read the whole of ignored source file: %v", err)
for _, source := range md.sourcesToIgnore {
sourcePath := "SOURCES/" + source.name
sourceFile, err := w.Filesystem.Open(sourcePath)
if err != nil {
log.Fatalf("could not open ignored source file %s: %v", sourcePath, err)
}
sourceFileBts, err := ioutil.ReadAll(sourceFile)
if err != nil {
log.Fatalf("could not read the whole of ignored source file: %v", err)
}
path := fmt.Sprintf("%s-%s/%s", rpmFile.Name(), md.pushBranch, source.name)
pd.BlobStorage.Write(path, sourceFileBts)
log.Printf("wrote %s to blob storage", path)
source.hashFunction.Reset()
_, err = source.hashFunction.Write(sourceFileBts)
if err != nil {
log.Fatalf("could not write bytes to hash function: %v", err)
}
checksum := source.hashFunction.Sum(nil)
checksumLine := fmt.Sprintf("%s %s\n", hex.EncodeToString(checksum), sourcePath)
_, err = metadata.Write([]byte(checksumLine))
if err != nil {
log.Fatalf("could not write to metadata file: %v", err)
}
}
ctx := context.Background()
gcsPath := fmt.Sprintf("%s-%s/%s", rpmFile.Name(), pd.Branch, source)
obj := pd.Bucket.Object(gcsPath)
w := obj.NewWriter(ctx)
_, err = w.Write(sourceFileBts)
if err != nil {
log.Fatalf("could not write tarball to gcs: %v", err)
}
// Close, just like writing a file.
if err := w.Close(); err != nil {
log.Fatalf("could not close gcs writer to source %s: %v", source, err)
}
log.Printf("wrote %s to gcs", gcsPath)
checksum := sha1.Sum(sourceFileBts)
checksumLine := fmt.Sprintf("%s %s\n", hex.EncodeToString(checksum[:]), sourcePath)
_, err = metadata.Write([]byte(checksumLine))
if err != nil {
log.Fatalf("could not write to metadata file: %v", err)
}
}
lastFilesToAdd := []string{metadataFile, ".gitignore", "SPECS"}
for _, f := range lastFilesToAdd {
_, err := w.Add(f)
_, err = w.Add(metadataFile)
if err != nil {
log.Fatalf("could not add metadata file: %v", err)
}
}
// show status
status, _ := w.Status()
log.Printf("successfully processed:\n%s", status)
lastFilesToAdd := []string{".gitignore", "SPECS"}
for _, f := range lastFilesToAdd {
_, err := w.Add(f)
if err != nil {
log.Fatalf("could not add metadata file: %v", err)
}
}
var hashes []plumbing.Hash
var pushRefspecs []config.RefSpec
pd.Importer.PostProcess(md)
head, err := repo.Head()
if err != nil {
hashes = nil
pushRefspecs = nil
} else {
log.Printf("tip %s", head.String())
hashes = append(hashes, head.Hash())
refOrigin := "refs/heads/" + pd.Branch
pushRefspecs = append(pushRefspecs, config.RefSpec(fmt.Sprintf("HEAD:%s", refOrigin)))
}
// show status
status, _ := w.Status()
log.Printf("successfully processed:\n%s", status)
// we are now finished with the tree and are going to push it to the src repo
// create import commit
commit, err := w.Commit("import "+filepath.Base(pd.RpmLocation), &git.CommitOptions{
Author: &object.Signature{
Name: pd.GitCommitterName,
Email: pd.GitCommitterEmail,
When: time.Now(),
},
Parents: hashes,
})
if err != nil {
log.Fatalf("could not commit object: %v", err)
}
var hashes []plumbing.Hash
var pushRefspecs []config.RefSpec
obj, err := repo.CommitObject(commit)
if err != nil {
log.Fatalf("could not get commit object: %v", err)
}
head, err := repo.Head()
if err != nil {
hashes = nil
pushRefspecs = append(pushRefspecs, "*:*")
} else {
log.Printf("tip %s", head.String())
hashes = append(hashes, head.Hash())
refOrigin := "refs/heads/" + md.pushBranch
pushRefspecs = append(pushRefspecs, config.RefSpec(fmt.Sprintf("HEAD:%s", refOrigin)))
}
log.Printf("committed:\n%s", obj.String())
// we are now finished with the tree and are going to push it to the src repo
// create import commit
commit, err := w.Commit("import "+pd.Importer.ImportName(pd, md), &git.CommitOptions{
Author: &object.Signature{
Name: pd.GitCommitterName,
Email: pd.GitCommitterEmail,
When: time.Now(),
},
Parents: hashes,
})
if err != nil {
log.Fatalf("could not commit object: %v", err)
}
err = repo.Push(&git.PushOptions{
RemoteName: "origin",
Auth: authenticator,
RefSpecs: pushRefspecs,
})
if err != nil {
log.Fatalf("could not push to remote: %v", err)
obj, err := repo.CommitObject(commit)
if err != nil {
log.Fatalf("could not get commit object: %v", err)
}
log.Printf("committed:\n%s", obj.String())
newTag := "imports/rocky" + strings.TrimPrefix(match[1], "imports/c")
_, err = repo.CreateTag(newTag, commit, &git.CreateTagOptions{
Tagger: &object.Signature{
Name: pd.GitCommitterName,
Email: pd.GitCommitterEmail,
When: time.Now(),
},
Message: "import " + md.tagBranch + " from " + pd.RpmLocation,
SignKey: nil,
})
if err != nil {
log.Fatalf("could not create tag: %v", err)
}
pushRefspecs = append(pushRefspecs, config.RefSpec(fmt.Sprintf("HEAD:%s", plumbing.NewTagReferenceName(newTag))))
err = repo.Push(&git.PushOptions{
RemoteName: "origin",
Auth: pd.Authenticator,
RefSpecs: pushRefspecs,
Force: true,
})
if err != nil {
log.Fatalf("could not push to remote: %v", err)
}
}
}

165
internal/srpm.go Normal file
View File

@ -0,0 +1,165 @@
package internal
import (
"bytes"
"crypto/sha256"
"fmt"
"github.com/cavaliercoder/go-cpio"
"github.com/cavaliercoder/go-rpm"
"github.com/go-git/go-billy/v5/memfs"
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/storage/memory"
"io"
"io/ioutil"
"log"
"os"
"os/exec"
"path/filepath"
"strings"
)
type SrpmMode struct{}
func (s *SrpmMode) RetrieveSource(pd *ProcessData) *modeData {
cmd := exec.Command("rpm2cpio", pd.RpmLocation)
cpioBytes, err := cmd.Output()
if err != nil {
log.Fatalf("could not convert to cpio (maybe rpm2cpio is missing): %v", err)
}
// create in memory git repository
repo, err := git.Init(memory.NewStorage(), memfs.New())
if err != nil {
log.Fatalf("could not init git repo: %v", err)
}
// read the rpm in cpio format
buf := bytes.NewReader(cpioBytes)
r := cpio.NewReader(buf)
fileWrites := map[string][]byte{}
for {
hdr, err := r.Next()
if err == io.EOF {
// end of cpio archive
break
}
if err != nil {
log.Fatalln(err)
}
bts, err := ioutil.ReadAll(r)
if err != nil {
log.Fatalf("could not copy file to virtual filesystem: %v", err)
}
fileWrites[hdr.Name] = bts
}
w, err := repo.Worktree()
if err != nil {
log.Fatalf("could not get worktree: %v", err)
}
// create structure
err = w.Filesystem.MkdirAll("SPECS", 0755)
if err != nil {
log.Fatalf("could not create SPECS dir in vfs: %v", err)
}
err = w.Filesystem.MkdirAll("SOURCES", 0755)
if err != nil {
log.Fatalf("could not create SOURCES dir in vfs: %v", err)
}
f, err := os.Open(pd.RpmLocation)
if err != nil {
log.Fatalf("could not open the file again: %v", err)
}
rpmFile, err := rpm.ReadPackageFile(f)
if err != nil {
log.Fatalf("could not read package, invalid?: %v", err)
}
var sourcesToIgnore []*ignoredSource
for _, source := range rpmFile.Source() {
if strings.Contains(source, ".tar") {
sourcesToIgnore = append(sourcesToIgnore, &ignoredSource{
name: source,
hashFunction: sha256.New(),
})
}
}
branch := fmt.Sprintf("rocky%d", pd.Version)
return &modeData{
repo: repo,
worktree: w,
rpmFile: rpmFile,
fileWrites: fileWrites,
branches: []string{branch},
sourcesToIgnore: sourcesToIgnore,
}
}
func (s *SrpmMode) WriteSource(md *modeData) {
for fileName, contents := range md.fileWrites {
var newPath string
if filepath.Ext(fileName) == ".spec" {
newPath = filepath.Join("SPECS", fileName)
} else {
newPath = filepath.Join("SOURCES", fileName)
}
mode := os.FileMode(0666)
for _, file := range md.rpmFile.Files() {
if file.Name() == fileName {
mode = file.Mode()
}
}
// add the file to the virtual filesystem
// we will move it to correct destination later
f, err := md.worktree.Filesystem.OpenFile(newPath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, mode)
if err != nil {
log.Fatalf("could not create file %s: %v", fileName, err)
}
_, err = f.Write(contents)
if err != nil {
log.Fatalf("could not write to file %s: %v", fileName, err)
}
_ = f.Close()
// don't add ignored file to git
if ignoredContains(md.sourcesToIgnore, fileName) {
continue
}
_, err = md.worktree.Add(newPath)
if err != nil {
log.Fatalf("could not add source file: %v", err)
}
}
// add sources to ignore (remote sources)
gitIgnore, err := md.worktree.Filesystem.Create(".gitignore")
if err != nil {
log.Fatalf("could not create .gitignore: %v", err)
}
for _, ignore := range md.sourcesToIgnore {
line := fmt.Sprintf("SOURCES/%s\n", ignore)
_, err := gitIgnore.Write([]byte(line))
if err != nil {
log.Fatalf("could not write line to .gitignore: %v", err)
}
}
err = gitIgnore.Close()
if err != nil {
log.Fatalf("could not close .gitignore: %v", err)
}
}
func (s *SrpmMode) PostProcess(_ *modeData) {}
func (s *SrpmMode) ImportName(pd *ProcessData, _ *modeData) string {
return filepath.Base(pd.RpmLocation)
}

94
internal/utils.go Normal file
View File

@ -0,0 +1,94 @@
package internal
import (
"crypto/md5"
"crypto/sha1"
"crypto/sha256"
"crypto/sha512"
"encoding/hex"
"github.com/go-git/go-billy/v5"
"hash"
"io"
"log"
"os"
"path/filepath"
)
func copyFromFs(from billy.Filesystem, to billy.Filesystem, path string) {
read, err := from.ReadDir(path)
if err != nil {
log.Fatalf("could not read dir: %v", err)
}
for _, fi := range read {
fullPath := filepath.Join(path, fi.Name())
if fi.IsDir() {
_ = to.MkdirAll(fullPath, 0755)
copyFromFs(from, to, fullPath)
} else {
_ = to.Remove(fullPath)
f, err := to.OpenFile(fullPath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, fi.Mode())
if err != nil {
log.Fatalf("could not open file: %v", err)
}
oldFile, err := from.Open(fullPath)
if err != nil {
log.Fatalf("could not open from file: %v", err)
}
_, err = io.Copy(f, oldFile)
if err != nil {
log.Fatalf("could not copy from oldFile to new: %v", err)
}
}
}
}
func ignoredContains(a []*ignoredSource, b string) bool {
for _, val := range a {
if val.name == b {
return true
}
}
return false
}
// check if content and checksum matches
// returns the hash type if success else nil
func compareHash(content []byte, checksum string) hash.Hash {
var hashType hash.Hash
switch len(checksum) {
case 128:
hashType = sha512.New()
break
case 64:
hashType = sha256.New()
break
case 40:
hashType = sha1.New()
break
case 32:
hashType = md5.New()
break
default:
return nil
}
hashType.Reset()
_, err := hashType.Write(content)
if err != nil {
return nil
}
calculated := hex.EncodeToString(hashType.Sum(nil))
if calculated != checksum {
return nil
}
return hashType
}