368 lines
13 KiB
Go
368 lines
13 KiB
Go
// Copyright (c) All respective contributors to the Peridot Project. All rights reserved.
|
|
// Copyright (c) 2021-2022 Rocky Enterprise Software Foundation, Inc. All rights reserved.
|
|
// Copyright (c) 2021-2022 Ctrl IQ, Inc. All rights reserved.
|
|
//
|
|
// Redistribution and use in source and binary forms, with or without
|
|
// modification, are permitted provided that the following conditions are met:
|
|
//
|
|
// 1. Redistributions of source code must retain the above copyright notice,
|
|
// this list of conditions and the following disclaimer.
|
|
//
|
|
// 2. Redistributions in binary form must reproduce the above copyright notice,
|
|
// this list of conditions and the following disclaimer in the documentation
|
|
// and/or other materials provided with the distribution.
|
|
//
|
|
// 3. Neither the name of the copyright holder nor the names of its contributors
|
|
// may be used to endorse or promote products derived from this software without
|
|
// specific prior written permission.
|
|
//
|
|
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
|
// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
|
// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
|
// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
|
// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
|
// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
|
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
|
// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
|
// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
|
// POSSIBILITY OF SUCH DAMAGE.
|
|
|
|
package workflow
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"encoding/base64"
|
|
"encoding/json"
|
|
"encoding/xml"
|
|
"fmt"
|
|
"github.com/google/uuid"
|
|
"go.temporal.io/sdk/temporal"
|
|
"go.temporal.io/sdk/workflow"
|
|
"io/ioutil"
|
|
"os"
|
|
"path/filepath"
|
|
"peridot.resf.org/peridot/db/models"
|
|
keykeeperpb "peridot.resf.org/peridot/keykeeper/pb"
|
|
peridotpb "peridot.resf.org/peridot/pb"
|
|
"peridot.resf.org/peridot/yummeta"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
func (c *Controller) CreateHashedRepositoriesWorkflow(ctx workflow.Context, req *peridotpb.CreateHashedRepositoriesRequest, task *models.Task) (*peridotpb.CreateHashedRepositoriesTask, error) {
|
|
ret := peridotpb.CreateHashedRepositoriesTask{}
|
|
|
|
deferTask, errorDetails, err := c.commonCreateTask(task, &ret)
|
|
defer deferTask()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
task.Status = peridotpb.TaskStatus_TASK_STATUS_FAILED
|
|
|
|
// Let's provision an ephemeral worker
|
|
taskQueue, cleanupWorker, err := c.provisionWorker(ctx, &ProvisionWorkerRequest{
|
|
TaskId: task.ID.String(),
|
|
ParentTaskId: task.ParentTaskId,
|
|
Purpose: "sync",
|
|
Arch: "noarch",
|
|
ProjectId: req.ProjectId.Value,
|
|
})
|
|
if err != nil {
|
|
setInternalError(errorDetails, err)
|
|
return nil, err
|
|
}
|
|
defer cleanupWorker()
|
|
|
|
createRepositoriesCtx := workflow.WithActivityOptions(ctx, workflow.ActivityOptions{
|
|
ScheduleToStartTimeout: 25 * time.Minute,
|
|
StartToCloseTimeout: 15 * time.Minute,
|
|
TaskQueue: taskQueue,
|
|
RetryPolicy: &temporal.RetryPolicy{
|
|
MaximumAttempts: 1,
|
|
},
|
|
})
|
|
err = workflow.ExecuteActivity(createRepositoriesCtx, c.CreateHashedRepositoriesActivity, req).Get(ctx, &ret)
|
|
if err != nil {
|
|
setActivityError(errorDetails, err)
|
|
return nil, err
|
|
}
|
|
|
|
task.Status = peridotpb.TaskStatus_TASK_STATUS_SUCCEEDED
|
|
|
|
return &ret, nil
|
|
}
|
|
|
|
func (c *Controller) CreateHashedRepositoriesActivity(req *peridotpb.CreateHashedRepositoriesRequest) (*peridotpb.CreateHashedRepositoriesTask, error) {
|
|
ret := peridotpb.CreateHashedRepositoriesTask{}
|
|
|
|
projects, err := c.db.ListProjects(&peridotpb.ProjectFilters{
|
|
Id: req.ProjectId,
|
|
})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if len(projects) != 1 {
|
|
return nil, fmt.Errorf("project not found")
|
|
}
|
|
project := projects[0]
|
|
|
|
beginTx, err := c.db.Begin()
|
|
if err != nil {
|
|
return nil, fmt.Errorf("begin transaction: %w", err)
|
|
}
|
|
tx := c.db.UseTransaction(beginTx)
|
|
|
|
key, err := tx.GetDefaultKeyForProject(project.ID.String())
|
|
if err != nil && err != sql.ErrNoRows {
|
|
return nil, fmt.Errorf("could not get default key for project %s: %v", project.ID.String(), err)
|
|
}
|
|
|
|
for _, repoName := range req.Repositories {
|
|
arches := append(project.Archs, "src")
|
|
for _, arch := range project.Archs {
|
|
arches = append(arches, arch+"-debug")
|
|
}
|
|
for _, arch := range arches {
|
|
tmpDir, err := os.MkdirTemp("", "")
|
|
if err != nil {
|
|
return nil, fmt.Errorf("could not create temporary directory: %v", err)
|
|
}
|
|
|
|
//goland:noinspection GoDeferInLoop
|
|
defer func(path string) {
|
|
err := os.RemoveAll(path)
|
|
if err != nil {
|
|
c.log.Errorf("could not remove temporary directory: %v", err)
|
|
}
|
|
}(tmpDir)
|
|
|
|
repodataDir := filepath.Join(tmpDir, "repodata")
|
|
err = os.Mkdir(repodataDir, 0755)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("could not create temporary repodata directory: %v", err)
|
|
}
|
|
|
|
_, err = c.db.GetRepository(nil, &repoName, &req.ProjectId.Value)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("get repository: %w", err)
|
|
}
|
|
|
|
revision, err := c.db.GetLatestActiveRepositoryRevisionByProjectIdAndNameAndArch(project.ID.String(), repoName, arch)
|
|
if err != nil {
|
|
if err != sql.ErrNoRows {
|
|
return nil, fmt.Errorf("could not get latest active revision: %v", err)
|
|
}
|
|
continue
|
|
}
|
|
|
|
filelistsGz, err := base64.StdEncoding.DecodeString(revision.FilelistsXml)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("decode filelists xml: %w", err)
|
|
}
|
|
otherGz, err := base64.StdEncoding.DecodeString(revision.OtherXml)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("decode other xml: %w", err)
|
|
}
|
|
groupsGz, err := base64.StdEncoding.DecodeString(revision.GroupsXml)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("decode groups xml: %w", err)
|
|
}
|
|
var groupsXml []byte
|
|
err = decompressWithGz(groupsGz, &groupsXml)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("decompress groups xml: %w", err)
|
|
}
|
|
|
|
// Decode primary and re-write urls
|
|
primaryXmlGz, err := base64.StdEncoding.DecodeString(revision.PrimaryXml)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("decode primary xml: %w", err)
|
|
}
|
|
var primaryXml []byte
|
|
err = decompressWithGz(primaryXmlGz, &primaryXml)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("decompress primary xml: %w", err)
|
|
}
|
|
primaryXml = []byte(strings.ReplaceAll(string(primaryXml), "rpm:", "rpm_"))
|
|
if err != nil {
|
|
return nil, fmt.Errorf("could not decompress primary.xml: %v", err)
|
|
}
|
|
var primaryRoot yummeta.PrimaryRoot
|
|
err = xml.Unmarshal(primaryXml, &primaryRoot)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("could not unmarshal primary.xml: %v", err)
|
|
}
|
|
urlMappings := map[string]string{}
|
|
for _, pkg := range primaryRoot.Packages {
|
|
rpmFileName := filepath.Base(pkg.Location.Href)
|
|
firstLetterLower := strings.ToLower(string(rpmFileName[0]))
|
|
newHref := fmt.Sprintf("%s/%s", firstLetterLower, rpmFileName)
|
|
urlMappings[newHref] = strings.TrimPrefix(pkg.Location.Href, "Packages/")
|
|
pkg.Location.Href = fmt.Sprintf("Packages/%s", newHref)
|
|
}
|
|
|
|
// Re-set namespaces because of stupid Go quirks with XML namespaces
|
|
primaryRoot.Rpm = "http://linux.duke.edu/metadata/rpm"
|
|
primaryRoot.XmlnsRpm = "http://linux.duke.edu/metadata/rpm"
|
|
|
|
// Remarshal primary and recalculate checksum
|
|
primaryXml, err = xml.Marshal(primaryRoot)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("could not marshal primary.xml: %v", err)
|
|
}
|
|
primaryXml = []byte(strings.ReplaceAll(string(primaryXml), "rpm_", "rpm:"))
|
|
var newPrimaryXmlGz []byte
|
|
err = compressWithGz(primaryXml, &newPrimaryXmlGz)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("could not compress primary.xml: %v", err)
|
|
}
|
|
newChecksums, err := getChecksums(primaryXml, newPrimaryXmlGz)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("could not calculate checksums: %v", err)
|
|
}
|
|
|
|
// Fetch repomd and set primary checksum
|
|
newRevisionId := uuid.New()
|
|
repomdXml, err := base64.StdEncoding.DecodeString(revision.RepomdXml)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("decode repomd xml: %w", err)
|
|
}
|
|
var repomdRoot yummeta.RepoMdRoot
|
|
err = xml.Unmarshal(repomdXml, &repomdRoot)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("could not unmarshal repomd.xml: %v", err)
|
|
}
|
|
|
|
primaryXmlName := fmt.Sprintf("%s-PRIMARY.xml.gz", newRevisionId)
|
|
var filelistsXmlName string
|
|
var otherXmlName string
|
|
var groupsXmlName string
|
|
for _, data := range repomdRoot.Data {
|
|
if data.Type == "primary" {
|
|
data.OpenChecksum.Value = newChecksums[0]
|
|
data.Checksum.Value = newChecksums[1]
|
|
data.OpenSize = len(primaryXml)
|
|
data.Size = len(newPrimaryXmlGz)
|
|
data.Location.Href = fmt.Sprintf("repodata/%s", primaryXmlName)
|
|
} else if data.Type == "filelists" {
|
|
filelistsXmlName = filepath.Base(data.Location.Href)
|
|
} else if data.Type == "other" {
|
|
otherXmlName = filepath.Base(data.Location.Href)
|
|
} else if data.Type == "group" {
|
|
groupsXmlName = filepath.Base(data.Location.Href)
|
|
}
|
|
}
|
|
repomdRoot.Revision = newRevisionId.String()
|
|
repomdXml, err = xml.Marshal(repomdRoot)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("could not marshal repomd.xml: %v", err)
|
|
}
|
|
|
|
// Marshal url mappings to json
|
|
urlMappingsJson, err := json.Marshal(urlMappings)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("could not marshal url mappings: %v", err)
|
|
}
|
|
urlMappingsJsonStr := string(urlMappingsJson)
|
|
|
|
// Create new hashed repo and revision
|
|
primaryXmlB64 := base64.StdEncoding.EncodeToString(newPrimaryXmlGz)
|
|
|
|
// First check if hashed repo already exists
|
|
hashedRepoName := fmt.Sprintf("hashed-%s", repoName)
|
|
hashedRepo, err := c.db.GetRepository(nil, &hashedRepoName, &req.ProjectId.Value)
|
|
if err != nil {
|
|
if err != sql.ErrNoRows {
|
|
return nil, fmt.Errorf("get repository: %w", err)
|
|
}
|
|
hashedRepo, err = c.db.CreateRepositoryWithPackages(hashedRepoName, req.ProjectId.Value, true, []string{})
|
|
if err != nil {
|
|
return nil, fmt.Errorf("create repository: %w", err)
|
|
}
|
|
}
|
|
|
|
// Write repomd.xml, primary.xml.gz, filelists.xml.gz, other.xml.gz, groups.xml and groups.xml.gz to tmpDir to run sqliterepo_c
|
|
err = multiErrorCheck(
|
|
ioutil.WriteFile(filepath.Join(repodataDir, "repomd.xml"), repomdXml, 0644),
|
|
ioutil.WriteFile(filepath.Join(repodataDir, primaryXmlName), newPrimaryXmlGz, 0644),
|
|
ioutil.WriteFile(filepath.Join(repodataDir, filelistsXmlName), filelistsGz, 0644),
|
|
ioutil.WriteFile(filepath.Join(repodataDir, otherXmlName), otherGz, 0644),
|
|
)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("write repodata files: %w", err)
|
|
}
|
|
if groupsXmlName != "" {
|
|
err = multiErrorCheck(
|
|
ioutil.WriteFile(filepath.Join(repodataDir, groupsXmlName), groupsXml, 0644),
|
|
ioutil.WriteFile(filepath.Join(repodataDir, groupsXmlName+".gz"), groupsGz, 0644),
|
|
)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("write repodata (groups) files: %w", err)
|
|
}
|
|
}
|
|
err = runCmd("sqliterepo_c", "-f", "--compress-type=gzip", tmpDir)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("run sqliterepo_c: %w", err)
|
|
}
|
|
|
|
// Walk repodataDir and upload files ending with .sqlite.gz to S3
|
|
err = filepath.Walk(repodataDir, func(path string, info os.FileInfo, err error) error {
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if info.IsDir() {
|
|
return nil
|
|
}
|
|
if !strings.HasSuffix(path, ".sqlite.gz") {
|
|
return nil
|
|
}
|
|
_, err = c.storage.PutObject(filepath.Join("sqlite-files", filepath.Base(path)), path)
|
|
if err != nil {
|
|
return fmt.Errorf("put object: %w", err)
|
|
}
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
return nil, fmt.Errorf("walk repodata dir: %w", err)
|
|
}
|
|
|
|
repomdXml, err = ioutil.ReadFile(filepath.Join(repodataDir, "repomd.xml"))
|
|
if err != nil {
|
|
return nil, fmt.Errorf("read repomd.xml: %w", err)
|
|
}
|
|
repomdXmlB64 := base64.StdEncoding.EncodeToString(repomdXml)
|
|
|
|
if key != nil {
|
|
ctx := context.TODO()
|
|
signRes, err := c.keykeeper.SignText(ctx, &keykeeperpb.SignTextRequest{
|
|
Text: string(repomdXml),
|
|
KeyName: key.Name,
|
|
})
|
|
if err != nil {
|
|
return nil, fmt.Errorf("sign text: %w", err)
|
|
}
|
|
_, err = c.storage.PutObjectBytes(filepath.Join("repo-signatures", fmt.Sprintf("%s.xml.asc", newRevisionId.String())), []byte(signRes.Signature))
|
|
if err != nil {
|
|
return nil, fmt.Errorf("put object (repo signature): %w", err)
|
|
}
|
|
}
|
|
|
|
newRevision, err := tx.CreateRevisionForRepository(newRevisionId.String(), hashedRepo.ID.String(), revision.Arch, repomdXmlB64, primaryXmlB64, revision.FilelistsXml, revision.OtherXml, revision.UpdateinfoXml, revision.ModuleDefaultsYaml, revision.ModulesYaml, revision.GroupsXml, urlMappingsJsonStr)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("error creating new revision: %v", err)
|
|
}
|
|
ret.RepoRevisions = append(ret.RepoRevisions, newRevision.ID.String())
|
|
}
|
|
}
|
|
|
|
err = beginTx.Commit()
|
|
if err != nil {
|
|
return nil, fmt.Errorf("commit transaction: %w", err)
|
|
}
|
|
|
|
return &ret, nil
|
|
}
|