521 lines
15 KiB
Go
521 lines
15 KiB
Go
// Copyright (c) All respective contributors to the Peridot Project. All rights reserved.
|
|
// Copyright (c) 2021-2022 Rocky Enterprise Software Foundation, Inc. All rights reserved.
|
|
// Copyright (c) 2021-2022 Ctrl IQ, Inc. All rights reserved.
|
|
//
|
|
// Redistribution and use in source and binary forms, with or without
|
|
// modification, are permitted provided that the following conditions are met:
|
|
//
|
|
// 1. Redistributions of source code must retain the above copyright notice,
|
|
// this list of conditions and the following disclaimer.
|
|
//
|
|
// 2. Redistributions in binary form must reproduce the above copyright notice,
|
|
// this list of conditions and the following disclaimer in the documentation
|
|
// and/or other materials provided with the distribution.
|
|
//
|
|
// 3. Neither the name of the copyright holder nor the names of its contributors
|
|
// may be used to endorse or promote products derived from this software without
|
|
// specific prior written permission.
|
|
//
|
|
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
|
// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
|
// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
|
// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
|
// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
|
// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
|
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
|
// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
|
// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
|
// POSSIBILITY OF SUCH DAMAGE.
|
|
|
|
package workflow
|
|
|
|
import (
|
|
"bytes"
|
|
"compress/gzip"
|
|
"context"
|
|
"crypto/sha256"
|
|
"database/sql"
|
|
"encoding/hex"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"io/fs"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/cavaliergopher/rpm"
|
|
"github.com/go-git/go-billy/v5/osfs"
|
|
"github.com/go-git/go-git/v5"
|
|
"github.com/go-git/go-git/v5/config"
|
|
"github.com/go-git/go-git/v5/plumbing"
|
|
"github.com/rocky-linux/srpmproc/pkg/srpmproc"
|
|
"google.golang.org/protobuf/types/known/anypb"
|
|
"google.golang.org/protobuf/types/known/wrapperspb"
|
|
"peridot.resf.org/peridot/db/models"
|
|
peridotpb "peridot.resf.org/peridot/pb"
|
|
"peridot.resf.org/peridot/rpmbuild"
|
|
)
|
|
|
|
func gitlabify(str string) string {
|
|
if str == "tree" {
|
|
return "treepkg"
|
|
}
|
|
|
|
return strings.Replace(str, "+", "plus", -1)
|
|
}
|
|
|
|
func findSpec() (string, error) {
|
|
var specFilePath string
|
|
err := filepath.Walk(filepath.Join(rpmbuild.GetCloneDirectory(), "SPECS"), func(path string, info fs.FileInfo, err error) error {
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if info.IsDir() {
|
|
return nil
|
|
}
|
|
|
|
if strings.HasSuffix(filepath.Base(path), ".spec") {
|
|
specFilePath = path
|
|
return nil
|
|
}
|
|
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
if specFilePath == "" {
|
|
return "", fmt.Errorf("could not find a valid spec file")
|
|
}
|
|
|
|
return specFilePath, nil
|
|
}
|
|
|
|
func findSrpm() (string, error) {
|
|
var srpmFilePath string
|
|
err := filepath.Walk(rpmbuild.GetCloneDirectory()+"/SRPMS", func(path string, info fs.FileInfo, err error) error {
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if info.IsDir() {
|
|
return nil
|
|
}
|
|
|
|
if strings.HasSuffix(filepath.Base(path), ".src.rpm") {
|
|
srpmFilePath = path
|
|
return nil
|
|
}
|
|
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
if srpmFilePath == "" {
|
|
return "", fmt.Errorf("could not find a valid srpm file")
|
|
}
|
|
|
|
return srpmFilePath, nil
|
|
}
|
|
|
|
func decompressGz(path string) ([]byte, error) {
|
|
if filepath.Ext(path) != ".gz" {
|
|
return nil, errors.New("gz file must end with .gz")
|
|
}
|
|
|
|
f, err := os.Open(path)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
r, err := gzip.NewReader(f)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
var buf bytes.Buffer
|
|
_, err = io.Copy(&buf, r)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return buf.Bytes(), nil
|
|
}
|
|
|
|
func (c *Controller) uploadArtifact(projectId string, parentTaskId string, filePath string, arch string, taskType peridotpb.TaskType) (*UploadActivityResult, error) {
|
|
task, err := c.db.CreateTask(nil, "noarch", taskType, &projectId, &parentTaskId)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
err = c.db.SetTaskStatus(task.ID.String(), peridotpb.TaskStatus_TASK_STATUS_RUNNING)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
defer func() {
|
|
err := c.db.SetTaskStatus(task.ID.String(), task.Status)
|
|
if err != nil {
|
|
c.log.Errorf("could not set task status in uploadArtifact: %v", err)
|
|
}
|
|
}()
|
|
|
|
// should fall back to FAILED in case it actually fails before we
|
|
// can set it to SUCCEEDED
|
|
task.Status = peridotpb.TaskStatus_TASK_STATUS_FAILED
|
|
|
|
// if the artifact is a rpm, generate repo and save metadata
|
|
var metadata *anypb.Any
|
|
|
|
objectName := filepath.Join(parentTaskId, filepath.Base(filePath))
|
|
f, err := os.Open(filePath)
|
|
if err != nil {
|
|
_ = c.logToMon([]string{fmt.Sprintf("could not open file %s: %v", filePath, err)}, task.ID.String(), parentTaskId)
|
|
return nil, fmt.Errorf("could not open file: %v", err)
|
|
}
|
|
|
|
hasher := sha256.New()
|
|
buf := make([]byte, 1024*1024)
|
|
for {
|
|
bytesRead, err := f.Read(buf)
|
|
if err != nil {
|
|
if err == io.EOF {
|
|
break
|
|
}
|
|
return nil, fmt.Errorf("could not read file: %v", err)
|
|
}
|
|
|
|
hasher.Write(buf[:bytesRead])
|
|
}
|
|
hash := hex.EncodeToString(hasher.Sum(nil))
|
|
exists, err := c.storage.Exists(objectName)
|
|
if exists || err == nil {
|
|
_ = c.logToMon([]string{fmt.Sprintf("skipping upload of %s, already exists", objectName)}, task.ID.String(), parentTaskId)
|
|
task.Status = peridotpb.TaskStatus_TASK_STATUS_SUCCEEDED
|
|
|
|
return &UploadActivityResult{
|
|
ObjectName: objectName,
|
|
Subtask: task,
|
|
HashSha256: hash,
|
|
Arch: arch,
|
|
Skip: true,
|
|
}, nil
|
|
}
|
|
|
|
if filepath.Ext(filePath) == ".rpm" {
|
|
tmp, err := os.MkdirTemp("", "")
|
|
if err != nil {
|
|
return nil, fmt.Errorf("could not create temp dir: %v", err)
|
|
}
|
|
err = os.Link(filePath, filepath.Join(tmp, filepath.Base(filePath)))
|
|
if err != nil {
|
|
_ = c.logToMon([]string{fmt.Sprintf("could not link file: %v", err)}, task.ID.String(), parentTaskId)
|
|
return nil, fmt.Errorf("could not link %s to %s: %v", filePath, filepath.Join(tmp, filepath.Base(filePath)), err)
|
|
}
|
|
err = runCmd("createrepo_c", "--basedir="+tmp, tmp)
|
|
if err != nil {
|
|
_ = c.logToMon([]string{fmt.Sprintf("could not create repo: %v", err)}, task.ID.String(), parentTaskId)
|
|
return nil, fmt.Errorf("could not create repo: %v", err)
|
|
}
|
|
|
|
var primaryGzPath string
|
|
var filelistsGzPath string
|
|
var otherGzPath string
|
|
err = filepath.Walk(filepath.Join(tmp, "repodata"), func(path string, info fs.FileInfo, err error) error {
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if strings.HasSuffix(path, "primary.xml.gz") {
|
|
primaryGzPath = path
|
|
} else if strings.HasSuffix(path, "filelists.xml.gz") {
|
|
filelistsGzPath = path
|
|
} else if strings.HasSuffix(path, "other.xml.gz") {
|
|
otherGzPath = path
|
|
}
|
|
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
primaryXmlBytes, err := decompressGz(primaryGzPath)
|
|
if err != nil {
|
|
_ = c.logToMon([]string{fmt.Sprintf("could not decompress primary.xml.gz: %v", err)}, task.ID.String(), parentTaskId)
|
|
return nil, fmt.Errorf("could not decompress primary.xml.gz: %v", err)
|
|
}
|
|
filelistsXmlBytes, err := decompressGz(filelistsGzPath)
|
|
if err != nil {
|
|
_ = c.logToMon([]string{fmt.Sprintf("could not decompress filelists.xml.gz: %v", err)}, task.ID.String(), parentTaskId)
|
|
return nil, fmt.Errorf("could not decompress filelists.xml.gz: %v", err)
|
|
}
|
|
otherXmlBytes, err := decompressGz(otherGzPath)
|
|
if err != nil {
|
|
_ = c.logToMon([]string{fmt.Sprintf("could not decompress other.xml.gz: %v", err)}, task.ID.String(), parentTaskId)
|
|
return nil, fmt.Errorf("could not decompress other.xml.gz: %v", err)
|
|
}
|
|
|
|
f, err := os.Open(filePath)
|
|
if err != nil {
|
|
_ = c.logToMon([]string{fmt.Sprintf("could not open file: %v", err)}, task.ID.String(), parentTaskId)
|
|
return nil, fmt.Errorf("could not open file %s: %v", filePath, err)
|
|
}
|
|
rpmPkg, err := rpm.Read(f)
|
|
if err != nil {
|
|
_ = c.logToMon([]string{fmt.Sprintf("could not read rpm: %v", err)}, task.ID.String(), parentTaskId)
|
|
return nil, fmt.Errorf("could not read rpm: %v", err)
|
|
}
|
|
// Use header tags in RPM headers directly
|
|
// todo(mustafa): Abstract away
|
|
// Source: https://github.com/rpm-software-management/rpm/blob/82dafa39a2dfd3e24858681ca75f467c1e1b3635/lib/rpmtag.h
|
|
buildArch := rpmPkg.Header.GetTag(1089).StringSlice()
|
|
excludeArch := rpmPkg.Header.GetTag(1059).StringSlice()
|
|
exclusiveArch := rpmPkg.Header.GetTag(1061).StringSlice()
|
|
|
|
rpmMetadata := &peridotpb.RpmArtifactMetadata{
|
|
Primary: primaryXmlBytes,
|
|
Filelists: filelistsXmlBytes,
|
|
Other: otherXmlBytes,
|
|
ExcludeArch: excludeArch,
|
|
ExclusiveArch: exclusiveArch,
|
|
BuildArch: buildArch,
|
|
}
|
|
metadata, err = anypb.New(rpmMetadata)
|
|
if err != nil {
|
|
_ = c.logToMon([]string{fmt.Sprintf("could not create metadata: %v", err)}, task.ID.String(), parentTaskId)
|
|
return nil, fmt.Errorf("could not create metadata: %v", err)
|
|
}
|
|
}
|
|
|
|
_, err = c.storage.PutObject(objectName, filePath)
|
|
if err != nil {
|
|
_ = c.logToMon([]string{fmt.Sprintf("could not upload file %s: %v", filePath, err)}, task.ID.String(), parentTaskId)
|
|
return nil, fmt.Errorf("could not upload artifact: %v", err)
|
|
}
|
|
_ = c.logToMon(
|
|
[]string{fmt.Sprintf("uploaded %s to blob storage", filePath)},
|
|
task.ID.String(),
|
|
parentTaskId,
|
|
)
|
|
|
|
err = c.db.AttachArtifactToTask(objectName, hash, arch, metadata, task.ID.String())
|
|
if err != nil {
|
|
_ = c.logToMon([]string{fmt.Sprintf("could not attach artifact to task: %v", err)}, task.ID.String(), parentTaskId)
|
|
return nil, fmt.Errorf("could not attach artifact to task: %v", err)
|
|
}
|
|
err = c.db.SetTaskMetadata(task.ID.String(), metadata)
|
|
if err != nil {
|
|
_ = c.logToMon([]string{fmt.Sprintf("could not set task metadata: %v", err)}, task.ID.String(), parentTaskId)
|
|
return nil, fmt.Errorf("could not set task metadata: %v", err)
|
|
}
|
|
|
|
task.Status = peridotpb.TaskStatus_TASK_STATUS_SUCCEEDED
|
|
|
|
return &UploadActivityResult{
|
|
ObjectName: objectName,
|
|
Subtask: task,
|
|
HashSha256: hash,
|
|
Arch: arch,
|
|
}, nil
|
|
}
|
|
|
|
func (c *Controller) BuildSRPMActivity(ctx context.Context, upstreamPrefix string, scmHash string, projectId string, packageName string, packageVersion *models.PackageVersion, task *models.Task, extraOptions *peridotpb.ExtraBuildOptions) error {
|
|
stopChan := makeHeartbeat(ctx, 30*time.Second)
|
|
defer func() { stopChan <- true }()
|
|
|
|
err := c.db.SetTaskStatus(task.ID.String(), peridotpb.TaskStatus_TASK_STATUS_RUNNING)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
defer func() {
|
|
err := c.db.SetTaskStatus(task.ID.String(), task.Status)
|
|
if err != nil {
|
|
c.log.Errorf("could not set task status in BuildSRPMActivity: %v", err)
|
|
}
|
|
}()
|
|
|
|
// should fall back to FAILED in case it actually fails before we
|
|
// can set it to SUCCEEDED
|
|
task.Status = peridotpb.TaskStatus_TASK_STATUS_FAILED
|
|
|
|
projects, err := c.db.ListProjects(&peridotpb.ProjectFilters{Id: wrapperspb.String(projectId)})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
project := projects[0]
|
|
|
|
pkgEo, err := c.db.GetExtraOptionsForPackage(project.ID.String(), packageName)
|
|
if err != nil && err != sql.ErrNoRows {
|
|
return err
|
|
}
|
|
|
|
authenticator, _ := c.getAuthenticator(projectId)
|
|
repoUrl := fmt.Sprintf("%s/rpms/%s.git", upstreamPrefix, gitlabify(packageName))
|
|
r, err := git.PlainClone(rpmbuild.GetCloneDirectory(), false, &git.CloneOptions{
|
|
Auth: authenticator,
|
|
URL: repoUrl,
|
|
Tags: git.AllTags,
|
|
})
|
|
if err != nil {
|
|
return fmt.Errorf("could not clone rpmbuild repo %s: %v", repoUrl, err)
|
|
}
|
|
|
|
err = r.Fetch(&git.FetchOptions{
|
|
RefSpecs: []config.RefSpec{"+refs/heads/*:refs/remotes/*"},
|
|
Auth: authenticator,
|
|
Tags: git.AllTags,
|
|
Force: true,
|
|
})
|
|
if err != nil {
|
|
return fmt.Errorf("could not fetch rpmbuild repo: %v", err)
|
|
}
|
|
|
|
w, err := r.Worktree()
|
|
if err != nil {
|
|
return fmt.Errorf("could not get worktree: %v", err)
|
|
}
|
|
|
|
err = w.Checkout(&git.CheckoutOptions{
|
|
Hash: plumbing.NewHash(scmHash),
|
|
Force: true,
|
|
Keep: false,
|
|
})
|
|
if err != nil {
|
|
return fmt.Errorf("could not checkout %s: %v", scmHash, err)
|
|
}
|
|
|
|
cloneDir := rpmbuild.GetCloneDirectory()
|
|
err = os.MkdirAll(filepath.Join(cloneDir, "SRPMS"), 0755)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
err = srpmproc.Fetch(os.Stdout, "", cloneDir, osfs.New("/"), c.storage)
|
|
if err != nil {
|
|
return fmt.Errorf("could not import using srpmproc: %v", err)
|
|
}
|
|
|
|
err = runCmd("chown", "-R", "peridotbuilder:mock", cloneDir)
|
|
if err != nil {
|
|
return fmt.Errorf("could not chown clone dir: %v", err)
|
|
}
|
|
|
|
specFilePath, err := findSpec()
|
|
if err != nil {
|
|
return fmt.Errorf("could not find spec file: %v", err)
|
|
}
|
|
|
|
var pkgGroup = DefaultSrpmBuildPkgGroup
|
|
|
|
if len(project.SrpmStagePackages) != 0 {
|
|
pkgGroup = project.SrpmStagePackages
|
|
}
|
|
|
|
var enableModules []string
|
|
var disableModules []string
|
|
err = ParsePackageExtraOptions(pkgEo, &pkgGroup, &enableModules, &disableModules)
|
|
|
|
if err != nil {
|
|
c.log.Infof("no extra options to process for package")
|
|
}
|
|
|
|
extraOptions.DisabledModules = disableModules
|
|
extraOptions.Modules = enableModules
|
|
|
|
hostArch := os.Getenv("REAL_BUILD_ARCH")
|
|
extraOptions.EnableNetworking = true
|
|
err = c.writeMockConfig(&project, packageVersion, extraOptions, "noarch", hostArch, pkgGroup)
|
|
if err != nil {
|
|
return fmt.Errorf("could not write mock config: %v", err)
|
|
}
|
|
// The SOURCES dir should always be available. Some packages don't have that
|
|
// and Mock complains. Loudly. About that
|
|
_ = os.MkdirAll(filepath.Join(cloneDir, "SOURCES"), 0644)
|
|
|
|
args := []string{
|
|
"mock",
|
|
"--isolation=simple",
|
|
"-r",
|
|
"/var/peridot/mock.cfg",
|
|
"--target",
|
|
"noarch",
|
|
"--resultdir",
|
|
filepath.Join(cloneDir, "SRPMS"),
|
|
"--sources",
|
|
filepath.Join(cloneDir, "SOURCES"),
|
|
}
|
|
if pkgEo != nil {
|
|
for _, with := range pkgEo.WithFlags {
|
|
args = append(args, "--with="+with)
|
|
}
|
|
for _, without := range pkgEo.WithoutFlags {
|
|
args = append(args, "--without="+without)
|
|
}
|
|
}
|
|
args = append(args, []string{"--buildsrpm", "--spec", specFilePath}...)
|
|
|
|
cmd := exec.Command("/bundle/fork-exec.py", args...)
|
|
cmd.Stdout = os.Stdout
|
|
cmd.Stderr = os.Stderr
|
|
err = cmd.Run()
|
|
if err != nil {
|
|
return fmt.Errorf("could not mock build: %v", err)
|
|
}
|
|
|
|
task.Status = peridotpb.TaskStatus_TASK_STATUS_SUCCEEDED
|
|
|
|
return nil
|
|
}
|
|
|
|
func ParsePackageExtraOptions(pkgEo *models.ExtraOptions, pkgGroup *[]string, enableModules *[]string, disableModules *[]string) error {
|
|
|
|
if pkgEo == nil {
|
|
return fmt.Errorf("no extra options to parse for package")
|
|
}
|
|
|
|
if len(pkgEo.DependsOn) != 0 {
|
|
for _, pkg := range pkgEo.DependsOn {
|
|
*pkgGroup = append(*pkgGroup, pkg)
|
|
}
|
|
}
|
|
|
|
if len(pkgEo.EnableModule) != 0 {
|
|
for _, pkg := range pkgEo.EnableModule {
|
|
*enableModules = append(*enableModules, pkg)
|
|
}
|
|
}
|
|
|
|
if len(pkgEo.DisableModule) != 0 {
|
|
for _, pkg := range pkgEo.DisableModule {
|
|
*disableModules = append(*disableModules, pkg)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
type UploadActivityResult struct {
|
|
ObjectName string `json:"objectName"`
|
|
Subtask *models.Task `json:"subtask"`
|
|
HashSha256 string `json:"hashSha256"`
|
|
Arch string `json:"arch"`
|
|
Skip bool `json:"skip"`
|
|
}
|
|
|
|
func (c *Controller) UploadSRPMActivity(ctx context.Context, projectId string, parentTaskId string) (*UploadActivityResult, error) {
|
|
stopChan := makeHeartbeat(ctx, 4*time.Second)
|
|
defer func() { stopChan <- true }()
|
|
|
|
srpmFilePath, err := findSrpm()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return c.uploadArtifact(projectId, parentTaskId, srpmFilePath, "src", peridotpb.TaskType_TASK_TYPE_BUILD_SRPM_UPLOAD)
|
|
}
|