diff --git a/ci/resfdeploy.jsonnet b/ci/resfdeploy.jsonnet index d2a98ec..e799c3d 100644 --- a/ci/resfdeploy.jsonnet +++ b/ci/resfdeploy.jsonnet @@ -274,7 +274,7 @@ local manifestYamlStream = function (value, indent_array_in_object=false, c_docu ]), [svcVsDr]: manifestYamlStream( - [kubernetes.define_service( + ([kubernetes.define_service( metadata { name: srv.name, annotations: { @@ -286,8 +286,8 @@ local manifestYamlStream = function (value, indent_array_in_object=false, c_docu portName=srv.portName, selector=metadata.name, env=mappings.get_env_from_svc(srv.name) - ) for srv in services] + - if istio_mode then [] else [if std.objectHas(srv, 'expose') && srv.expose then kubernetes.define_ingress( + ) for srv in services]) + + (if istio_mode then [] else [if std.objectHas(srv, 'expose') && srv.expose then kubernetes.define_ingress( metadata { name: srv.name, annotations: ingress_annotations + { @@ -301,8 +301,8 @@ local manifestYamlStream = function (value, indent_array_in_object=false, c_docu host=if helm_mode then '{{ .Values.%s.ingressHost }}' % srv.portName else mappings.get(srv.name, user), port=srv.port, srvName=srv.name + '-service', - ) else null for srv in services] + - if !istio_mode then [] else [kubernetes.define_virtual_service(metadata { name: srv.name + '-internal' }, { + ) else null for srv in services]) + + (if !istio_mode then [] else [kubernetes.define_virtual_service(metadata { name: srv.name + '-internal' }, { hosts: [vshost(srv)], gateways: [], http: [ @@ -318,8 +318,8 @@ local manifestYamlStream = function (value, indent_array_in_object=false, c_docu } + (if std.objectHas(info, 'internal_route_options') then info.internal_route_options else {})], }, ], - },) for srv in services] + - if !istio_mode then [] else [if std.objectHas(srv, 'expose') && srv.expose then kubernetes.define_virtual_service( + },) for srv in services]) + + (if !istio_mode then [] else [if std.objectHas(srv, 'expose') && srv.expose then kubernetes.define_virtual_service( metadata { name: srv.name, annotations: { @@ -343,8 +343,8 @@ local manifestYamlStream = function (value, indent_array_in_object=false, c_docu }, ], } - ) else null for srv in services] + - if !istio_mode then [] else [{ + ) else null for srv in services]) + + (if !istio_mode then [] else [{ apiVersion: 'security.istio.io/v1beta1', kind: 'RequestAuthentication', metadata: metadata { @@ -364,8 +364,8 @@ local manifestYamlStream = function (value, indent_array_in_object=false, c_docu fromHeaders: [{ name: 'x-goog-iap-jwt-assertion' }], }] else [], }, - } for srv in services] + - if !istio_mode then [] else [{ + } for srv in services]) + + (if !istio_mode then [] else [{ apiVersion: 'security.istio.io/v1beta1', kind: 'AuthorizationPolicy', metadata: metadata { @@ -389,8 +389,8 @@ local manifestYamlStream = function (value, indent_array_in_object=false, c_docu }] }], }, - } for srv in services] + - if !istio_mode then [] else [kubernetes.define_destination_rule(metadata { name: srv.name }, { + } for srv in services]) + + (if !istio_mode then [] else [kubernetes.define_destination_rule(metadata { name: srv.name }, { host: vshost(srv), trafficPolicy: { tls: { @@ -406,7 +406,7 @@ local manifestYamlStream = function (value, indent_array_in_object=false, c_docu }, }, ], - },) for srv in services] + },) for srv in services]) ), [if std.objectHas(info, 'custom_job_items') then custom else null]: manifestYamlStream(if std.objectHas(info, 'custom_job_items') then info.custom_job_items(metadata, extra_info) else [{}]), diff --git a/hack/push_peridot b/hack/push_peridot index e661f46..b372898 100755 --- a/hack/push_peridot +++ b/hack/push_peridot @@ -46,3 +46,7 @@ targets=( for target in "${targets[@]}"; do br "$target" amd64 done + +for target in "${targets[@]}"; do + GIT_COMMIT="$(git rev-parse HEAD)-arm64" br "$target" arm64 +done diff --git a/package.json b/package.json index 23c6614..5dde71c 100644 --- a/package.json +++ b/package.json @@ -34,7 +34,7 @@ "@mui/styles": "^5.2.3", "@mui/system": "^5.10.6", "@mui/x-data-grid": "^5.2.0", - "@ory/hydra-client": "^1.10.6", + "@ory/hydra-client": "^2.0.2", "@pmmmwh/react-refresh-webpack-plugin": "^0.5.1", "@tailwindcss/forms": "^0.5.3", "@types/classnames": "^2.2.11", diff --git a/peridot/builder/v1/workflow/BUILD.bazel b/peridot/builder/v1/workflow/BUILD.bazel index 7af7bb6..e60c869 100644 --- a/peridot/builder/v1/workflow/BUILD.bazel +++ b/peridot/builder/v1/workflow/BUILD.bazel @@ -13,12 +13,14 @@ go_library( "rpmimport.go", "srpm.go", "sync.go", + "updateinfo.go", "workflow.go", "yumrepofs.go", ], importpath = "peridot.resf.org/peridot/builder/v1/workflow", visibility = ["//visibility:public"], deps = [ + "//apollo/proto/v1:pb", "//apollo/rpmutils", "//peridot/composetools", "//peridot/db", @@ -26,10 +28,12 @@ go_library( "//peridot/lookaside", "//peridot/plugin", "//peridot/proto/v1:pb", + "//peridot/proto/v1/admin:pb", "//peridot/proto/v1/keykeeper:pb", "//peridot/proto/v1/yumrepofs:pb", "//peridot/rpmbuild", "//peridot/yummeta", + "//publisher/updateinfo", "//servicecatalog", "//utils", "//vendor/cirello.io/dynamolock", diff --git a/peridot/builder/v1/workflow/arch.go b/peridot/builder/v1/workflow/arch.go index 1f0e15d..78ecd9b 100644 --- a/peridot/builder/v1/workflow/arch.go +++ b/peridot/builder/v1/workflow/arch.go @@ -50,7 +50,6 @@ import ( "time" "github.com/google/uuid" - "go.temporal.io/sdk/activity" "google.golang.org/protobuf/types/known/wrapperspb" "peridot.resf.org/apollo/rpmutils" "peridot.resf.org/peridot/db/models" @@ -368,7 +367,7 @@ func (c *Controller) buildMacros(project *models.Project, packageVersion *models } func (c *Controller) setBuildMacros(project *models.Project, packageVersion *models.PackageVersion) error { - err := os.Remove("/etc/rpm/macros.dist") + err := os.RemoveAll("/etc/rpm/macros.dist") if err != nil && !os.IsNotExist(err) { return err } @@ -474,6 +473,7 @@ config_opts['releasever'] = '{majorVersion}' config_opts['package_manager'] = 'dnf' config_opts['extra_chroot_dirs'] = [ '/run/lock' ] config_opts['rpmbuild_command'] = '{rpmbuildCommand}' +{additionalVendorConfig} config_opts['plugin_conf']['ccache_enable'] = False config_opts['plugin_conf']['root_cache_enable'] = False @@ -557,6 +557,14 @@ config_opts['dnf.conf'] = """ yumConfig := c.yumConfig(project) + additionalVendorConfig := "" + if project.TargetVendor == "suse" { + additionalVendorConfig = `config_opts['useradd'] = '/usr/sbin/useradd -o -m -u {{chrootuid}} -g {{chrootgid}} -d {{chroothome}} {{chrootuser}}' +config_opts['ssl_ca_bundle_path'] = '/var/lib/ca-certificates/ca-bundle.pem' +config_opts['package_manager_max_attempts'] = 4 +config_opts['package_manager_attempt_delay'] = 20` + } + rpmbuildNetworking := "False" if extra.EnableNetworking { rpmbuildNetworking = "True" @@ -574,6 +582,7 @@ config_opts['dnf.conf'] = """ "{targetVendor}", project.TargetVendor, "{moduleSetupCommands}", strings.Join(moduleSetupCommands, ","), "{rpmbuildNetworking}", rpmbuildNetworking, + "{additionalVendorConfig}", additionalVendorConfig, ).Replace(mockConfig) return rendered, nil @@ -597,12 +606,8 @@ func (c *Controller) writeMockConfig(project *models.Project, packageVersion *mo // Current implementation is broken for modules // todo(mustafa): Evaluate if we can skip chroot again func (c *Controller) BuildArchActivity(ctx context.Context, projectId string, packageName string, disableChecks bool, packageVersion *models.PackageVersion, uploadSRPMResult *UploadActivityResult, task *models.Task, arch string, extraOptions *peridotpb.ExtraBuildOptions) error { - go func() { - for { - activity.RecordHeartbeat(ctx) - time.Sleep(10 * time.Second) - } - }() + stopChan := makeHeartbeat(ctx, 10*time.Second) + defer func() { stopChan <- true }() err := c.db.SetTaskStatus(task.ID.String(), peridotpb.TaskStatus_TASK_STATUS_RUNNING) if err != nil { @@ -732,12 +737,8 @@ func (c *Controller) BuildArchActivity(ctx context.Context, projectId string, pa } func (c *Controller) UploadArchActivity(ctx context.Context, projectId string, parentTaskId string) ([]*UploadActivityResult, error) { - go func() { - for { - activity.RecordHeartbeat(ctx) - time.Sleep(4 * time.Second) - } - }() + stopChan := makeHeartbeat(ctx, 4*time.Second) + defer func() { stopChan <- true }() rpms, err := findRpms() if err != nil { diff --git a/peridot/builder/v1/workflow/build.go b/peridot/builder/v1/workflow/build.go index 0e5f303..f1243c9 100644 --- a/peridot/builder/v1/workflow/build.go +++ b/peridot/builder/v1/workflow/build.go @@ -621,7 +621,7 @@ func (c *Controller) BuildWorkflow(ctx workflow.Context, req *peridotpb.SubmitBu return nil, err } - subtask, err := c.db.GetTask(uploadSRPMResult.Subtask.ID.String(), utils.Pointer(project.ID.String())) + subtask, err := c.db.GetTask(uploadSRPMResult.Subtask.ID.String(), utils.Pointer[string](project.ID.String())) if err != nil { setInternalError(errorDetails, err) return nil, err @@ -730,7 +730,7 @@ func (c *Controller) BuildWorkflow(ctx workflow.Context, req *peridotpb.SubmitBu if result.Skip { continue } - subtask, err := c.db.GetTask(result.Subtask.ID.String(), utils.Pointer(project.ID.String())) + subtask, err := c.db.GetTask(result.Subtask.ID.String(), utils.Pointer[string](project.ID.String())) if err != nil { ret.err = fmt.Errorf("failed to get task: %s", err) return diff --git a/peridot/builder/v1/workflow/hashed_repositories.go b/peridot/builder/v1/workflow/hashed_repositories.go index 0c287cf..6c2d23c 100644 --- a/peridot/builder/v1/workflow/hashed_repositories.go +++ b/peridot/builder/v1/workflow/hashed_repositories.go @@ -183,6 +183,9 @@ func (c *Controller) CreateHashedRepositoriesActivity(req *peridotpb.CreateHashe } var primaryXml []byte err = decompressWithGz(primaryXmlGz, &primaryXml) + if err != nil { + return nil, fmt.Errorf("decompress primary xml: %w", err) + } primaryXml = []byte(strings.ReplaceAll(string(primaryXml), "rpm:", "rpm_")) if err != nil { return nil, fmt.Errorf("could not decompress primary.xml: %v", err) diff --git a/peridot/builder/v1/workflow/import.go b/peridot/builder/v1/workflow/import.go index 49019ed..4b9963a 100644 --- a/peridot/builder/v1/workflow/import.go +++ b/peridot/builder/v1/workflow/import.go @@ -56,7 +56,6 @@ import ( "github.com/sirupsen/logrus" "github.com/spf13/viper" "github.com/xanzy/go-gitlab" - "go.temporal.io/sdk/activity" "go.temporal.io/sdk/temporal" "go.temporal.io/sdk/workflow" "google.golang.org/grpc/codes" @@ -605,12 +604,8 @@ func (c *Controller) ImportPackageWorkflow(ctx workflow.Context, req *peridotpb. } func (c *Controller) PackageSrcGitActivity(ctx context.Context, packageName string, project *models.Project, parentTaskId string) (*peridotpb.PackageSrcGitResponse, error) { - go func() { - for { - activity.RecordHeartbeat(ctx) - time.Sleep(4 * time.Second) - } - }() + stopChan := makeHeartbeat(ctx, 4*time.Second) + defer func() { stopChan <- true }() task, err := c.db.CreateTask(nil, "noarch", peridotpb.TaskType_TASK_TYPE_IMPORT_SRC_GIT, utils.StringP(project.ID.String()), &parentTaskId) if err != nil { @@ -681,6 +676,7 @@ func (c *Controller) PackageSrcGitActivity(ctx context.Context, packageName stri if err != nil { return nil, err } + tarBts := buf.Bytes() h := sha256.New() _, err = h.Write(buf.Bytes()) @@ -691,7 +687,16 @@ func (c *Controller) PackageSrcGitActivity(ctx context.Context, packageName stri sum := hex.EncodeToString(sumBytes) name := fmt.Sprintf("%s.tar.gz", elem.Name()) - _, err = c.storage.PutObjectBytes(sum, buf.Bytes()) + f, err := w.Filesystem.OpenFile(filepath.Join("SOURCES", name), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644) + if err != nil { + return nil, err + } + _, err = io.Copy(f, bytes.NewReader(tarBts)) + if err != nil { + return nil, err + } + + _, err = c.storage.PutObjectBytes(sum, tarBts) if err != nil { return nil, err } @@ -707,12 +712,8 @@ func (c *Controller) PackageSrcGitActivity(ctx context.Context, packageName stri } func (c *Controller) UpdateDistGitForSrcGitActivity(ctx context.Context, packageName string, project *models.Project, packageRes *peridotpb.PackageSrcGitResponse) (*peridotpb.ImportRevision, error) { - go func() { - for { - activity.RecordHeartbeat(ctx) - time.Sleep(4 * time.Second) - } - }() + stopChan := makeHeartbeat(ctx, 4*time.Second) + defer func() { stopChan <- true }() task, err := c.db.CreateTask(nil, "noarch", peridotpb.TaskType_TASK_TYPE_IMPORT_SRC_GIT_TO_DIST_GIT, utils.StringP(project.ID.String()), &packageRes.TaskId) if err != nil { @@ -751,6 +752,7 @@ func (c *Controller) UpdateDistGitForSrcGitActivity(ctx context.Context, package } } } + _ = srcW.Filesystem.Remove(".gitlab-ci.yml") // Try checking out the dist-git repo createRepo := false @@ -1048,12 +1050,8 @@ func (c *Controller) srpmprocToImportRevisions(project *models.Project, pkg stri // Matches current Rocky workflow with distrobuild+srpmrpoc. // This activity also uses srpmproc as a library instead of a CLI tool func (c *Controller) UpstreamDistGitActivity(ctx context.Context, greq *UpstreamDistGitActivityRequest) (*UpstreamDistGitActivityResponse, error) { - go func() { - for { - activity.RecordHeartbeat(ctx) - time.Sleep(4 * time.Second) - } - }() + stopChan := makeHeartbeat(ctx, 4*time.Second) + defer func() { stopChan <- true }() project := greq.Project parentTaskId := greq.ParentTaskId diff --git a/peridot/builder/v1/workflow/infrastructure.go b/peridot/builder/v1/workflow/infrastructure.go index 4b21012..b3a0fb4 100644 --- a/peridot/builder/v1/workflow/infrastructure.go +++ b/peridot/builder/v1/workflow/infrastructure.go @@ -310,12 +310,8 @@ func (c *Controller) DestroyWorkerWorkflow(ctx workflow.Context, req *ProvisionW // DeleteK8sPodActivity deletes the pod that hosts the ephemeral worker func (c *Controller) DeleteK8sPodActivity(ctx context.Context, req *ProvisionWorkerRequest, task *models.Task) error { - go func() { - for { - activity.RecordHeartbeat(ctx) - time.Sleep(3 * time.Second) - } - }() + stopChan := makeHeartbeat(ctx, 3*time.Second) + defer func() { stopChan <- true }() // creates the in-cluster config config, err := rest.InClusterConfig() @@ -449,12 +445,8 @@ func (c *Controller) IngestLogsActivity(ctx context.Context, podName string, tas // CreateK8sPodActivity creates a new pod in the same namespace // with the specified container (the container has to contain peridotbuilder) func (c *Controller) CreateK8sPodActivity(ctx context.Context, req *ProvisionWorkerRequest, task *models.Task, imageArch string) (string, error) { - go func() { - for { - activity.RecordHeartbeat(ctx) - time.Sleep(3 * time.Second) - } - }() + stopChan := makeHeartbeat(ctx, 3*time.Second) + defer func() { stopChan <- true }() task.Status = peridotpb.TaskStatus_TASK_STATUS_FAILED diff --git a/peridot/builder/v1/workflow/rpmimport.go b/peridot/builder/v1/workflow/rpmimport.go index 9e54d80..5cfc442 100644 --- a/peridot/builder/v1/workflow/rpmimport.go +++ b/peridot/builder/v1/workflow/rpmimport.go @@ -37,7 +37,6 @@ import ( "database/sql" "fmt" "github.com/cavaliergopher/rpm" - "go.temporal.io/sdk/activity" "go.temporal.io/sdk/temporal" "go.temporal.io/sdk/workflow" "google.golang.org/grpc/codes" @@ -262,12 +261,8 @@ func (c *Controller) RpmLookasideBatchImportWorkflow(ctx workflow.Context, req * } func (c *Controller) RpmImportActivity(ctx context.Context, req *peridotpb.RpmImportRequest, taskID string, setTaskStatus bool, stage1 *RpmImportActivityTaskStage1) (*RpmImportActivityTaskStage1, error) { - go func() { - for { - activity.RecordHeartbeat(ctx) - time.Sleep(4 * time.Second) - } - }() + stopChan := makeHeartbeat(ctx, 4*time.Second) + defer func() { stopChan <- true }() var buf bytes.Buffer bts, err := c.storage.ReadObject(req.Rpms) diff --git a/peridot/builder/v1/workflow/srpm.go b/peridot/builder/v1/workflow/srpm.go index bf0300c..4da0284 100644 --- a/peridot/builder/v1/workflow/srpm.go +++ b/peridot/builder/v1/workflow/srpm.go @@ -53,7 +53,6 @@ import ( "github.com/go-git/go-git/v5/config" "github.com/go-git/go-git/v5/plumbing" "github.com/rocky-linux/srpmproc/pkg/srpmproc" - "go.temporal.io/sdk/activity" "google.golang.org/protobuf/types/known/anypb" "google.golang.org/protobuf/types/known/wrapperspb" "peridot.resf.org/peridot/db/models" @@ -326,12 +325,8 @@ func (c *Controller) uploadArtifact(projectId string, parentTaskId string, fileP } func (c *Controller) BuildSRPMActivity(ctx context.Context, upstreamPrefix string, scmHash string, projectId string, packageName string, packageVersion *models.PackageVersion, task *models.Task, extraOptions *peridotpb.ExtraBuildOptions) error { - go func() { - for { - activity.RecordHeartbeat(ctx) - time.Sleep(30 * time.Second) - } - }() + stopChan := makeHeartbeat(ctx, 30*time.Second) + defer func() { stopChan <- true }() err := c.db.SetTaskStatus(task.ID.String(), peridotpb.TaskStatus_TASK_STATUS_RUNNING) if err != nil { @@ -513,12 +508,8 @@ type UploadActivityResult struct { } func (c *Controller) UploadSRPMActivity(ctx context.Context, projectId string, parentTaskId string) (*UploadActivityResult, error) { - go func() { - for { - activity.RecordHeartbeat(ctx) - time.Sleep(4 * time.Second) - } - }() + stopChan := makeHeartbeat(ctx, 4*time.Second) + defer func() { stopChan <- true }() srpmFilePath, err := findSrpm() if err != nil { diff --git a/peridot/builder/v1/workflow/workflow.go b/peridot/builder/v1/workflow/workflow.go index 1e8f66f..7c317e5 100644 --- a/peridot/builder/v1/workflow/workflow.go +++ b/peridot/builder/v1/workflow/workflow.go @@ -31,10 +31,12 @@ package workflow import ( + "context" "fmt" "github.com/aws/aws-sdk-go/service/dynamodb" "github.com/go-git/go-billy/v5/osfs" "github.com/sirupsen/logrus" + "go.temporal.io/sdk/activity" "go.temporal.io/sdk/client" "google.golang.org/genproto/googleapis/rpc/errdetails" "google.golang.org/protobuf/proto" @@ -47,6 +49,7 @@ import ( "peridot.resf.org/peridot/plugin" "peridot.resf.org/peridot/rpmbuild" "peridot.resf.org/utils" + "time" ) const ( @@ -256,3 +259,21 @@ func (c *Controller) logToMon(lines []string, taskId string, parentTaskId string return nil } + +// makeHeartBeat provides a mechanism to start and stop heartbeats to a Temporal Activity. +func makeHeartbeat(ctx context.Context, pulseFrequency time.Duration, details ...interface{}) chan<- bool { + stopChan := make(chan bool, 1) + go func() { + stop := false + for !stop { + activity.RecordHeartbeat(ctx, details...) + time.Sleep(pulseFrequency) + select { + case stop = <-stopChan: + default: + } + } + }() + + return stopChan +} diff --git a/peridot/builder/v1/workflow/yumrepofs.go b/peridot/builder/v1/workflow/yumrepofs.go index 77a2f07..19056ea 100644 --- a/peridot/builder/v1/workflow/yumrepofs.go +++ b/peridot/builder/v1/workflow/yumrepofs.go @@ -44,8 +44,8 @@ import ( "github.com/gobwas/glob" "github.com/google/uuid" "github.com/rocky-linux/srpmproc/modulemd" + "github.com/sirupsen/logrus" "github.com/spf13/viper" - "go.temporal.io/sdk/activity" "go.temporal.io/sdk/temporal" "go.temporal.io/sdk/workflow" "google.golang.org/protobuf/encoding/protojson" @@ -93,15 +93,17 @@ type CompiledGlobFilter struct { } type CachedRepo struct { - Arch string - Repo *models.Repository - PrimaryRoot *yummeta.PrimaryRoot - FilelistsRoot *yummeta.FilelistsRoot - OtherRoot *yummeta.OtherRoot - Modulemd []*modulemd.ModuleMd - GroupsXml string - DefaultsYaml []byte - ModuleDefaults []*modulemd.Defaults + Arch string + Repo *models.Repository + PrimaryRoot *yummeta.PrimaryRoot + FilelistsRoot *yummeta.FilelistsRoot + OtherRoot *yummeta.OtherRoot + Modulemd []*modulemd.ModuleMd + GroupsXml string + DefaultsYaml []byte + ModuleDefaults []*modulemd.Defaults + UpdateInfoB64 string + UpdateInfoDataEntry *yummeta.RepoMdData } type Cache struct { @@ -331,6 +333,7 @@ func GenerateArchMapForArtifacts(artifacts models.TaskArtifacts, project *models } } else { if composetools.IsDebugPackage(name) { + logrus.Printf("checking if %s is a debug package", name) arch = arch + "-debug" } artifactArchMap[arch] = append(artifactArchMap[arch], &newArtifact) @@ -465,12 +468,8 @@ func (c *Controller) RepoUpdaterWorkflow(ctx workflow.Context, req *UpdateRepoRe } func (c *Controller) RequestKeykeeperSignActivity(ctx context.Context, buildId string, keyName string) (string, error) { - go func() { - for { - activity.RecordHeartbeat(ctx) - time.Sleep(4 * time.Second) - } - }() + stopChan := makeHeartbeat(ctx, 4*time.Second) + defer func() { stopChan <- true }() task, err := c.keykeeper.SignArtifacts(ctx, &keykeeperpb.SignArtifactsRequest{ BuildId: buildId, @@ -487,12 +486,8 @@ func (c *Controller) RequestKeykeeperSignActivity(ctx context.Context, buildId s } func (c *Controller) UpdateRepoActivity(ctx context.Context, req *UpdateRepoRequest, task *models.Task, gpgId *string, signTaskIds []string) (*yumrepofspb.UpdateRepoTask, error) { - go func() { - for { - activity.RecordHeartbeat(ctx) - time.Sleep(4 * time.Second) - } - }() + stopChan := makeHeartbeat(ctx, 4*time.Second) + defer func() { stopChan <- true }() signArtifactsTasks := &keykeeperpb.BatchSignArtifactsTask{ Tasks: []*keykeeperpb.SignArtifactsTask{}, @@ -908,6 +903,11 @@ func (c *Controller) UpdateRepoActivity(ctx context.Context, req *UpdateRepoRequ }) } + // Add updateinfo if not empty + if repo.UpdateInfoDataEntry != nil { + repomdRoot.Data = append(repomdRoot.Data, repo.UpdateInfoDataEntry) + } + var newRepoMd []byte err = xmlMarshal(repomdRoot, &newRepoMd) if err != nil { @@ -930,7 +930,7 @@ func (c *Controller) UpdateRepoActivity(ctx context.Context, req *UpdateRepoRequ PrimaryXml: newPrimaryGzB64, FilelistsXml: newFilelistsGzB64, OtherXml: newOtherGzB64, - UpdateinfoXml: "", + UpdateinfoXml: repo.UpdateInfoB64, ModuleDefaultsYaml: defaultsYamlB64, ModulesYaml: newModulesGzB64, GroupsXml: newGroupsGzB64, @@ -1158,6 +1158,8 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest idArchNoDebug := fmt.Sprintf("%s-%s", repo.Name, noDebugArch) var currentRevision *models.RepositoryRevision var groupsXml string + var updateInfoXml string + var updateInfoDataEntry *yummeta.RepoMdData if cache.Repos[idArchNoDebug] != nil { c.log.Infof("found cache for %s", idArchNoDebug) if cache.Repos[idArchNoDebug].Modulemd != nil { @@ -1176,6 +1178,8 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest otherRoot = *cache.Repos[idArch].OtherRoot } groupsXml = cache.Repos[idArch].GroupsXml + updateInfoXml = cache.Repos[idArch].UpdateInfoB64 + updateInfoDataEntry = cache.Repos[idArch].UpdateInfoDataEntry } else { c.log.Infof("no cache for %s", idArch) var noDebugRevision *models.RepositoryRevision @@ -1243,6 +1247,26 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest } } groupsXml = currentRevision.GroupsXml + updateInfoXml = currentRevision.UpdateinfoXml + + if updateInfoXml != "" { + repomdXml, err := base64.StdEncoding.DecodeString(currentRevision.RepomdXml) + if err != nil { + return nil, fmt.Errorf("decode repomd xml: %w", err) + } + var repomdRoot yummeta.RepoMdRoot + err = xml.Unmarshal(repomdXml, &repomdRoot) + if err != nil { + return nil, fmt.Errorf("could not unmarshal repomd.xml: %v", err) + } + + for _, data := range repomdRoot.Data { + if data.Type == "updateinfo" { + updateInfoDataEntry = data + break + } + } + } } if noDebugRevision != nil { if noDebugRevision.ModulesYaml != "" { @@ -1300,15 +1324,17 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest name = nvr[1] } - noDebugInfoName := strings.ReplaceAll(strings.ReplaceAll(name, "-debuginfo", ""), "-debugsource", "") + noDebugInfoName := strings.ReplaceAll(strings.ReplaceAll(strings.ReplaceAll(name, "-debuginfo-common", ""), "-debugsource", ""), "-debuginfo", "") archName := fmt.Sprintf("%s.%s", name, artifact.Arch) if strings.HasSuffix(arch, "-debug") { archName = fmt.Sprintf("%s.%s", noDebugInfoName, artifact.Arch) } shouldAdd := !req.Delete + // If debug-info common package, then it should be added + // if arch has "-debug" suffix + // If repo has a list for inclusion, then the artifact has to pass that first if arch != "src" && moduleStream == nil && !req.Delete { - // If repo has a list for inclusion, then the artifact has to pass that first if len(repo.IncludeFilter) > 0 { // If the artifact isn't forced, it should be in the include list or additional multilib list if !artifact.Forced && !utils.StrContains(archName, repo.IncludeFilter) && !utils.StrContains(noDebugInfoName, repo.AdditionalMultilib) { @@ -1495,16 +1521,15 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest var deleteIds []string if (!req.NoDeletePrevious || moduleStream != nil) || req.Delete { for _, pkg := range primaryRoot.Packages { - shouldAdd := !req.Delete - if !req.Delete { - for _, artifact := range currentActiveArtifacts { - noRpmName := strings.TrimSuffix(filepath.Base(artifact.Name), ".rpm") - if filepath.Base(artifact.Name) == filepath.Base(pkg.Location.Href) && !utils.StrContains(noRpmName, changes.ModifiedPackages) && !utils.StrContains(noRpmName, changes.AddedPackages) && !utils.StrContains(noRpmName, skipDeleteArtifacts) { - shouldAdd = false - if req.NoDeleteInChain { - if utils.StrContains(noRpmName, cache.NoDeleteChain) { - shouldAdd = true - } + shouldAdd := true + + for _, artifact := range currentActiveArtifacts { + noRpmName := strings.TrimSuffix(filepath.Base(artifact.Name), ".rpm") + if filepath.Base(artifact.Name) == filepath.Base(pkg.Location.Href) && !utils.StrContains(noRpmName, changes.ModifiedPackages) && !utils.StrContains(noRpmName, changes.AddedPackages) && !utils.StrContains(noRpmName, skipDeleteArtifacts) { + shouldAdd = false + if req.NoDeleteInChain { + if utils.StrContains(noRpmName, cache.NoDeleteChain) { + shouldAdd = true } } } @@ -1513,6 +1538,7 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest if utils.StrContains(noRpmNamePkg, changes.RemovedPackages) { shouldAdd = false } + if !shouldAdd { c.log.Infof("deleting %s", pkg.Location.Href) deleteIds = append(deleteIds, pkg.Checksum.Value) @@ -1601,15 +1627,17 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest nRepo := repo cache.Repos[idArch] = &CachedRepo{ - Arch: arch, - Repo: &nRepo, - PrimaryRoot: &primaryRoot, - FilelistsRoot: &filelistsRoot, - OtherRoot: &otherRoot, - Modulemd: modulesRoot, - DefaultsYaml: defaultsYaml, - ModuleDefaults: moduleDefaults, - GroupsXml: groupsXml, + Arch: arch, + Repo: &nRepo, + PrimaryRoot: &primaryRoot, + FilelistsRoot: &filelistsRoot, + OtherRoot: &otherRoot, + Modulemd: modulesRoot, + DefaultsYaml: defaultsYaml, + ModuleDefaults: moduleDefaults, + GroupsXml: groupsXml, + UpdateInfoB64: updateInfoXml, + UpdateInfoDataEntry: updateInfoDataEntry, } if strings.HasSuffix(arch, "-debug") || arch == "src" { cache.Repos[idArch].Modulemd = nil diff --git a/peridot/cmd/v1/yumrepofsupdater/main.go b/peridot/cmd/v1/yumrepofsupdater/main.go index 5d3b2a2..34f9570 100644 --- a/peridot/cmd/v1/yumrepofsupdater/main.go +++ b/peridot/cmd/v1/yumrepofsupdater/main.go @@ -102,6 +102,10 @@ func mn(_ *cobra.Command, _ []string) { w.Worker.RegisterActivity(w.WorkflowController.UpdateRepoActivity) w.Worker.RegisterActivity(w.WorkflowController.RequestKeykeeperSignActivity) + // Updateinfo + w.Worker.RegisterWorkflow(w.WorkflowController.UpdateInfoWorkflow) + w.Worker.RegisterActivity(w.WorkflowController.SetUpdateInfoActivity) + var wg sync.WaitGroup wg.Add(2) diff --git a/peridot/composetools/rpm.go b/peridot/composetools/rpm.go index 2fb80aa..f4afecc 100644 --- a/peridot/composetools/rpm.go +++ b/peridot/composetools/rpm.go @@ -46,7 +46,7 @@ var ( ) var ( - debugSuffixes = []string{"-debuginfo", "-debugsource"} + debugSuffixes = []string{"-debuginfo", "-debugsource", "-debuginfo-common"} soGlob = glob.MustCompile("*.so.*") gtk2EnginesGlob = glob.MustCompile("/usr/lib*/gtk-2.0/*/engines") gtk2ModulesGlob = glob.MustCompile("/usr/lib*/gtk-2.0/*/modules") diff --git a/peridot/db/psql/project.go b/peridot/db/psql/project.go index 5ec214d..d16d954 100644 --- a/peridot/db/psql/project.go +++ b/peridot/db/psql/project.go @@ -32,7 +32,6 @@ package serverpsql import ( "fmt" - "github.com/jmoiron/sqlx/types" "github.com/lib/pq" "google.golang.org/protobuf/encoding/protojson" @@ -78,7 +77,9 @@ func (a *Access) ListProjects(filters *peridotpb.ProjectFilters) (ret models.Pro branch_suffix, git_make_public, vendor_macro, - packager_macro + packager_macro, + srpm_stage_packages, + build_stage_packages from projects where ($1 :: uuid is null or id = $1 :: uuid) @@ -350,8 +351,7 @@ func (a *Access) SetBuildRootPackages(projectId string, srpmPackages pq.StringAr _, err := a.query.Exec( ` - update projects set srpm_stage_packages = $2, build_stage_packages = $3 - where id = $1 + update projects set srpm_stage_packages = $2, build_stage_packages = $3 where id = $1 `, projectId, srpmPackages, diff --git a/peridot/proto/v1/task.proto b/peridot/proto/v1/task.proto index 1246160..190ace4 100644 --- a/peridot/proto/v1/task.proto +++ b/peridot/proto/v1/task.proto @@ -74,6 +74,7 @@ enum TaskType { TASK_TYPE_LOOKASIDE_FILE_UPLOAD = 18; TASK_TYPE_RPM_LOOKASIDE_BATCH_IMPORT = 19; TASK_TYPE_CLONE_SWAP = 20; + TASK_TYPE_UPDATEINFO = 21; } enum TaskStatus { diff --git a/peridot/yumrepofs/v1/blob.go b/peridot/yumrepofs/v1/blob.go index 343e56d..5499bf4 100644 --- a/peridot/yumrepofs/v1/blob.go +++ b/peridot/yumrepofs/v1/blob.go @@ -112,6 +112,8 @@ func (s *Server) GetBlob(ctx context.Context, req *yumrepofspb.GetBlobRequest) ( dataB64 = revision.GroupsXml case "MODULES": dataB64 = revision.ModulesYaml + case "UPDATEINFO": + dataB64 = revision.UpdateinfoXml default: return nil, ErrInvalidBlob } diff --git a/yarn.lock b/yarn.lock index 5021d54..56ee9a4 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2117,10 +2117,10 @@ mkdirp "^1.0.4" rimraf "^3.0.2" -"@ory/hydra-client@^1.10.6": - version "1.11.8" - resolved "https://registry.yarnpkg.com/@ory/hydra-client/-/hydra-client-1.11.8.tgz#f163861e4c8b401e98b7b53092ea24eb80ab814c" - integrity sha512-fVTPjB/JWUrsI1uGnXf0eiYDClnH9fepGmJ+z1eg6b0EH5AQoaNc7nBU9QpEj/8pQjYy0p5hWEvOXGHWo9fJtw== +"@ory/hydra-client@^2.0.2": + version "2.0.2" + resolved "https://registry.yarnpkg.com/@ory/hydra-client/-/hydra-client-2.0.2.tgz#fd4e91f37dc2feaae9683a2620ab8d293aa1bc7c" + integrity sha512-1er8AHxZgOUGhipaTYeiZJVrPfCdiyycDGA2i93xdz4M/22FekC3E6www0+GKjbA1ifST3J1aIhssqel0ksFPA== dependencies: axios "^0.21.4"