srpmproc/pkg/modes/git.go

// Copyright (c) 2021 The Srpmproc Authors
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

package modes

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"net/http"
	"path/filepath"
	"sort"
	"strings"
	"text/template"
	"time"

	"github.com/go-git/go-git/v5/plumbing/transport"
	"github.com/rocky-linux/srpmproc/pkg/misc"

	"github.com/go-git/go-billy/v5/memfs"
	"github.com/go-git/go-git/v5"
	"github.com/go-git/go-git/v5/config"
	"github.com/go-git/go-git/v5/plumbing"
	"github.com/go-git/go-git/v5/plumbing/object"
	"github.com/go-git/go-git/v5/storage/memory"
	"github.com/rocky-linux/srpmproc/pkg/data"
)

type remoteTarget struct {
	remote string
	when   time.Time
}

type remoteTargetSlice []remoteTarget

func (p remoteTargetSlice) Len() int {
	return len(p)
}

func (p remoteTargetSlice) Less(i, j int) bool {
	return p[i].when.Before(p[j].when)
}

func (p remoteTargetSlice) Swap(i, j int) {
	p[i], p[j] = p[j], p[i]
}

type GitMode struct{}

func (g *GitMode) RetrieveSource(pd *data.ProcessData) (*data.ModeData, error) {
	repo, err := git.Init(memory.NewStorage(), memfs.New())
	if err != nil {
		return nil, fmt.Errorf("could not init git Repo: %v", err)
	}

	w, err := repo.Worktree()
	if err != nil {
		return nil, fmt.Errorf("could not get Worktree: %v", err)
	}

	refspec := config.RefSpec("+refs/heads/*:refs/remotes/*")
	remote, err := repo.CreateRemote(&config.RemoteConfig{
		Name:  "upstream",
		URLs:  []string{fmt.Sprintf("%s.git", pd.RpmLocation)},
		Fetch: []config.RefSpec{refspec},
	})
	if err != nil {
		return nil, fmt.Errorf("could not create remote: %v", err)
	}

	fetchOpts := &git.FetchOptions{
		Auth:     pd.Authenticator,
		RefSpecs: []config.RefSpec{refspec},
		Tags:     git.AllTags,
		Force:    true,
	}

	err = remote.Fetch(fetchOpts)
	if err != nil {
		if err == transport.ErrInvalidAuthMethod || err == transport.ErrAuthenticationRequired {
			fetchOpts.Auth = nil
			err = remote.Fetch(fetchOpts)
			if err != nil {
				return nil, fmt.Errorf("could not fetch upstream: %v", err)
			}
		} else {
			return nil, fmt.Errorf("could not fetch upstream: %v", err)
		}
	}

	var branches remoteTargetSlice

	latestTags := map[string]*remoteTarget{}

	tagAdd := func(tag *object.Tag) error {
		if strings.HasPrefix(tag.Name, fmt.Sprintf("imports/%s%d", pd.ImportBranchPrefix, pd.Version)) {
			refSpec := fmt.Sprintf("refs/tags/%s", tag.Name)
			if misc.GetTagImportRegex(pd).MatchString(refSpec) {
				match := misc.GetTagImportRegex(pd).FindStringSubmatch(refSpec)

				exists := latestTags[match[2]]
				if exists != nil && exists.when.After(tag.Tagger.When) {
					return nil
				}
				latestTags[match[2]] = &remoteTarget{
					remote: refSpec,
					when:   tag.Tagger.When,
				}
			}
		}
		return nil
	}

	// In case of "tagless mode", we need to get the head ref of the branch instead
	// This is a kind of alternative implementation of the above tagAdd assignment
	refAdd := func(tag *object.Tag) error {
		if misc.TaglessRefOk(tag.Name, pd) {
			pd.Log.Printf("Tagless mode:  Identified tagless commit for import: %s\n", tag.Name)
			refSpec := fmt.Sprintf(tag.Name)

			// We split the string by "/", the branch name we're looking for to pass to latestTags is always last
			// (ex: "refs/heads/c9s" ---> we want latestTags[c9s]
			tmpRef := strings.Split(refSpec, "/")
			tmpBranchName := tmpRef[(len(tmpRef) - 1)]

			latestTags[tmpBranchName] = &remoteTarget{
				remote: refSpec,
				when:   tag.Tagger.When,
			}
		}
		return nil
	}

	tagIter, err := repo.TagObjects()
	if err != nil {
		return nil, fmt.Errorf("could not get tag objects: %v", err)
	}

	// tagless mode means we use "refAdd" (add commit by reference)
	// normal mode means we can rely on "tagAdd" (the tag should be present for us in the source repo)
	if pd.TaglessMode {
		_ = tagIter.ForEach(refAdd)
	} else {
		_ = tagIter.ForEach(tagAdd)
	}

	listOpts := &git.ListOptions{
		Auth: pd.Authenticator,
	}
	list, err := remote.List(listOpts)
	if err != nil {
		if err == transport.ErrInvalidAuthMethod || err == transport.ErrAuthenticationRequired {
			listOpts.Auth = nil
			list, err = remote.List(listOpts)
			if err != nil {
				return nil, fmt.Errorf("could not list upstream: %v", err)
			}
		} else {
			return nil, fmt.Errorf("could not list upstream: %v", err)
		}
	}

	for _, ref := range list {
		if ref.Hash().IsZero() {
			continue
		}

		commit, err := repo.CommitObject(ref.Hash())
		if err != nil {
			continue
		}

		// Call refAdd instead of tagAdd in the case of TaglessMode enabled
		if pd.TaglessMode {
			_ = refAdd(&object.Tag{
				Name:   string(ref.Name()),
				Tagger: commit.Committer,
			})
		} else {
			_ = tagAdd(&object.Tag{
				Name:   strings.TrimPrefix(string(ref.Name()), "refs/tags/"),
				Tagger: commit.Committer,
			})
		}

	}

	for _, branch := range latestTags {
		pd.Log.Printf("tag: %s", strings.TrimPrefix(branch.remote, "refs/tags/"))
		branches = append(branches, *branch)
	}
	sort.Sort(branches)

	var sortedBranches []string
	for _, branch := range branches {
		sortedBranches = append(sortedBranches, branch.remote)
	}

	return &data.ModeData{
		Name:       filepath.Base(pd.RpmLocation),
		Repo:       repo,
		Worktree:   w,
		FileWrites: nil,
		Branches:   sortedBranches,
	}, nil
}

func (g *GitMode) WriteSource(pd *data.ProcessData, md *data.ModeData) error {
	remote, err := md.Repo.Remote("upstream")

	if err != nil && !pd.TaglessMode {
		return fmt.Errorf("could not get upstream remote: %v", err)
	}

	var refspec config.RefSpec
	var branchName string

	// In the case of tagless mode, we already have the transformed repo sitting in the worktree,
	// and don't need to perform any checkout or fetch operations
	if !pd.TaglessMode {
		if strings.HasPrefix(md.TagBranch, "refs/heads") {
			refspec = config.RefSpec(fmt.Sprintf("+%s:%s", md.TagBranch, md.TagBranch))
			branchName = strings.TrimPrefix(md.TagBranch, "refs/heads/")
		} else {
			match := misc.GetTagImportRegex(pd).FindStringSubmatch(md.TagBranch)
			branchName = match[2]
			refspec = config.RefSpec(fmt.Sprintf("+refs/heads/%s:%s", branchName, md.TagBranch))
			fmt.Println("Found branchname that does not start w/ refs/heads :: ", branchName)
		}
		pd.Log.Printf("checking out upstream refspec %s", refspec)

		fetchOpts := &git.FetchOptions{
			Auth:       pd.Authenticator,
			RemoteName: "upstream",
			RefSpecs:   []config.RefSpec{refspec},
			Tags:       git.AllTags,
			Force:      true,
		}
		err = remote.Fetch(fetchOpts)
		if err != nil && err != git.NoErrAlreadyUpToDate {
			if err == transport.ErrInvalidAuthMethod || err == transport.ErrAuthenticationRequired {
				fetchOpts.Auth = nil
				err = remote.Fetch(fetchOpts)
				if err != nil && err != git.NoErrAlreadyUpToDate {
					return fmt.Errorf("could not fetch upstream: %v", err)
				}
			} else {
				return fmt.Errorf("could not fetch upstream: %v", err)
			}
		}

		err = md.Worktree.Checkout(&git.CheckoutOptions{
			Branch: plumbing.ReferenceName(md.TagBranch),
			Force:  true,
		})
		if err != nil {
			return fmt.Errorf("could not checkout source from git: %v", err)
		}

		_, err = md.Worktree.Add(".")
		if err != nil {
			return fmt.Errorf("could not add Worktree: %v", err)
		}
	}

	if pd.TaglessMode {
		branchName = fmt.Sprintf("%s%d%s", pd.ImportBranchPrefix, pd.Version, pd.BranchSuffix)
	}

	metadataPath := ""
	ls, err := md.Worktree.Filesystem.ReadDir(".")
	if err != nil {
		return fmt.Errorf("could not read directory: %v", err)
	}
	for _, f := range ls {
		if strings.HasSuffix(f.Name(), ".metadata") {
			if metadataPath != "" {
				return fmt.Errorf("multiple metadata files found")
			}
			metadataPath = f.Name()
		}
	}
	if metadataPath == "" {
		metadataPath = fmt.Sprintf(".%s.metadata", md.Name)
	}

	metadataFile, err := md.Worktree.Filesystem.Open(metadataPath)
	if err != nil {
		pd.Log.Printf("warn: could not open metadata file, so skipping: %v", err)
		return nil
	}

	fileBytes, err := ioutil.ReadAll(metadataFile)
	if err != nil {
		return fmt.Errorf("could not read metadata file: %v", err)
	}

	client := &http.Client{
		Transport: &http.Transport{
			DisableCompression: false,
		},
	}
	fileContent := strings.Split(string(fileBytes), "\n")
	for _, line := range fileContent {
		if strings.TrimSpace(line) == "" {
			continue
		}

		lineInfo := strings.SplitN(line, " ", 2)
		hash := strings.TrimSpace(lineInfo[0])
		path := strings.TrimSpace(lineInfo[1])

		var body []byte

		if md.BlobCache[hash] != nil {
			body = md.BlobCache[hash]
			pd.Log.Printf("retrieving %s from cache", hash)
		} else {
			fromBlobStorage, err := pd.BlobStorage.Read(hash)
			if err != nil {
				return err
			}
			if fromBlobStorage != nil && !pd.NoStorageDownload {
				body = fromBlobStorage
				pd.Log.Printf("downloading %s from blob storage", hash)
			} else {

				url := ""

				// We need to figure out the hashtype for templating purposes:
				hashType := "sha512"
				switch len(hash) {
				case 128:
					hashType = "sha512"
				case 64:
					hashType = "sha256"
				case 40:
					hashType = "sha1"
				case 32:
					hashType = "md5"
				}

				// need the name of the file without "SOURCES/":
				fileName := strings.Split(path, "/")[1]

				// Feed our template info to ProcessUrl and transform to the real values: ( {{.Name}}, {{.Branch}}, {{.Hash}}, {{.Hashtype}}, {{.Filename}} )
				url, err = ProcessUrl(pd.CdnUrl, md.Name, branchName, hash, hashType, fileName)
				if err != nil {
					return fmt.Errorf("Could not process CDN URL template(s) in string ( {{ .Variable }} )")
				}

				var req *http.Request
				var resp *http.Response

				// Download the --cdn-url given, but *only* if it contains template strings ( {{.Name}} , {{.Hash}} , etc. )
				// Otherwise we need to fall back to the traditional cdn-url patterns
				if strings.Contains(pd.CdnUrl, "{{") && strings.Contains(pd.CdnUrl, "}}") {
					pd.Log.Printf("downloading %s", url)

					req, err := http.NewRequest("GET", url, nil)
					if err != nil {
						return fmt.Errorf("could not create new http request: %v", err)
					}
					req.Header.Set("Accept-Encoding", "*")

					resp, err = client.Do(req)
					if err != nil {
						return fmt.Errorf("could not download dist-git file: %v", err)
					}
				}

				// Default cdn-url:  If we don't have a templated download string, try the default <SITE>/<PKG>/<BRANCH>/<HASH> pattern:
				if resp == nil || resp.StatusCode != http.StatusOK {
					url = fmt.Sprintf("%s/%s/%s/%s", pd.CdnUrl, md.Name, branchName, hash)
					pd.Log.Printf("Attempting default URL: %s", url)
					req, err = http.NewRequest("GET", url, nil)
					if err != nil {
						return fmt.Errorf("could not create new http request: %v", err)
					}
					req.Header.Set("Accept-Encoding", "*")
					resp, err = client.Do(req)
					if err != nil {
						return fmt.Errorf("could not download dist-git file: %v", err)
					}
				}

				// If the default URL fails, we have one more pattern to try.  The simple <SITE>/<HASH> pattern
				// If this one fails, we are truly lost, and have to bail out w/ an error:
				if resp == nil || resp.StatusCode != http.StatusOK {
					url = fmt.Sprintf("%s/%s", pd.CdnUrl, hash)
					pd.Log.Printf("Attempting 2nd fallback URL: %s", url)
					req, err = http.NewRequest("GET", url, nil)
					if err != nil {
						return fmt.Errorf("could not create new http request: %v", err)
					}
					req.Header.Set("Accept-Encoding", "*")
					resp, err = client.Do(req)
					if err != nil {
						return fmt.Errorf("could not download dist-git file: %v", err)
					}
					if resp.StatusCode != http.StatusOK {
						return fmt.Errorf("could not download dist-git file (status code %d): %v", resp.StatusCode, err)
					}
				}

				body, err = ioutil.ReadAll(resp.Body)
				if err != nil {
					return fmt.Errorf("could not read the whole dist-git file: %v", err)
				}
				err = resp.Body.Close()
				if err != nil {
					return fmt.Errorf("could not close body handle: %v", err)
				}
			}

			md.BlobCache[hash] = body
		}

		f, err := md.Worktree.Filesystem.Create(path)
		if err != nil {
			return fmt.Errorf("could not open file pointer: %v", err)
		}

		hasher := pd.CompareHash(body, hash)
		if hasher == nil {
			return fmt.Errorf("checksum in metadata does not match dist-git file")
		}

		md.SourcesToIgnore = append(md.SourcesToIgnore, &data.IgnoredSource{
			Name:         path,
			HashFunction: hasher,
		})

		_, err = f.Write(body)
		if err != nil {
			return fmt.Errorf("could not copy dist-git file to in-tree: %v", err)
		}
		_ = f.Close()
	}

	return nil
}

func (g *GitMode) PostProcess(md *data.ModeData) error {
	for _, source := range md.SourcesToIgnore {
		_, err := md.Worktree.Filesystem.Stat(source.Name)
		if err == nil {
			err := md.Worktree.Filesystem.Remove(source.Name)
			if err != nil {
				return fmt.Errorf("could not remove dist-git file: %v", err)
			}
		}
	}

	_, err := md.Worktree.Add(".")
	if err != nil {
		return fmt.Errorf("could not add git sources: %v", err)
	}

	return nil
}

func (g *GitMode) ImportName(pd *data.ProcessData, md *data.ModeData) string {
	if misc.GetTagImportRegex(pd).MatchString(md.TagBranch) {
		match := misc.GetTagImportRegex(pd).FindStringSubmatch(md.TagBranch)
		return match[3]
	}

	return strings.Replace(strings.TrimPrefix(md.TagBranch, "refs/heads/"), "%", "_", -1)
}

// Given a cdnUrl string as input, return same string, but with substituted
// template values ( {{.Name}} , {{.Hash}}, {{.Filename}}, etc. )
func ProcessUrl(cdnUrl string, name string, branch string, hash string, hashtype string, filename string) (string, error) {

	// These 5 {{ .Value }} items are possible in our templated string:
	type Lookaside struct {
		Name     string
		Branch   string
		Hash     string
		Hashtype string
		Filename string
	}

	tmpUrl := Lookaside{name, branch, hash, hashtype, filename}

	tmpl, err := template.New("").Parse(cdnUrl)
	if err != nil {
		panic(err)
	}

	var result bytes.Buffer
	err = tmpl.Execute(&result, tmpUrl)

	if err != nil {
		panic(err)
	}

	return result.String(), nil

}