rocky-linux
/
peridot
mirror of https://github.com/rocky-linux/peridot.git
4
0
Fork 0
Code Issues Projects Releases Activity

Yumrepofsupdater now supports updateinfo.xml, and delete works better. Heartbeats should no longer cause memory leaks

This commit is contained in:
Mustafa Gezen 2023-02-17 19:27:33 +01:00
parent c2b607522a
commit 2470a9df7f
19 changed files with 180 additions and 136 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
ci/resfdeploy.jsonnet
View File

@ -274,7 +274,7 @@ local manifestYamlStream = function (value, indent_array_in_object=false, c_docu
]),
[svcVsDr]:
manifestYamlStream(
[kubernetes.define_service(
([kubernetes.define_service(
metadata {
name: srv.name,
annotations: {
@ -286,8 +286,8 @@ local manifestYamlStream = function (value, indent_array_in_object=false, c_docu
portName=srv.portName,
selector=metadata.name,
env=mappings.get_env_from_svc(srv.name)
) for srv in services] +
if istio_mode then [] else [if std.objectHas(srv, 'expose') && srv.expose then kubernetes.define_ingress(
) for srv in services]) +
(if istio_mode then [] else [if std.objectHas(srv, 'expose') && srv.expose then kubernetes.define_ingress(
metadata {
name: srv.name,
annotations: ingress_annotations + {
@ -301,8 +301,8 @@ local manifestYamlStream = function (value, indent_array_in_object=false, c_docu
host=if helm_mode then '{{ .Values.%s.ingressHost }}' % srv.portName else mappings.get(srv.name, user),
port=srv.port,
srvName=srv.name + '-service',
) else null for srv in services] +
if !istio_mode then [] else [kubernetes.define_virtual_service(metadata { name: srv.name + '-internal' }, {
) else null for srv in services]) +
(if !istio_mode then [] else [kubernetes.define_virtual_service(metadata { name: srv.name + '-internal' }, {
hosts: [vshost(srv)],
gateways: [],
http: [
@ -318,8 +318,8 @@ local manifestYamlStream = function (value, indent_array_in_object=false, c_docu
} + (if std.objectHas(info, 'internal_route_options') then info.internal_route_options else {})],
},
],
},) for srv in services] +
if !istio_mode then [] else [if std.objectHas(srv, 'expose') && srv.expose then kubernetes.define_virtual_service(
},) for srv in services]) +
(if !istio_mode then [] else [if std.objectHas(srv, 'expose') && srv.expose then kubernetes.define_virtual_service(
metadata {
name: srv.name,
annotations: {
@ -343,8 +343,8 @@ local manifestYamlStream = function (value, indent_array_in_object=false, c_docu
},
],
}
) else null for srv in services] +
if !istio_mode then [] else [{
) else null for srv in services]) +
(if !istio_mode then [] else [{
apiVersion: 'security.istio.io/v1beta1',
kind: 'RequestAuthentication',
metadata: metadata {
@ -364,8 +364,8 @@ local manifestYamlStream = function (value, indent_array_in_object=false, c_docu
fromHeaders: [{ name: 'x-goog-iap-jwt-assertion' }],
}] else [],
},
} for srv in services] +
if !istio_mode then [] else [{
} for srv in services]) +
(if !istio_mode then [] else [{
apiVersion: 'security.istio.io/v1beta1',
kind: 'AuthorizationPolicy',
metadata: metadata {
@ -389,8 +389,8 @@ local manifestYamlStream = function (value, indent_array_in_object=false, c_docu
}]
}],
},
} for srv in services] +
if !istio_mode then [] else [kubernetes.define_destination_rule(metadata { name: srv.name }, {
} for srv in services]) +
(if !istio_mode then [] else [kubernetes.define_destination_rule(metadata { name: srv.name }, {
host: vshost(srv),
trafficPolicy: {
tls: {
@ -406,7 +406,7 @@ local manifestYamlStream = function (value, indent_array_in_object=false, c_docu
},
},
],
},) for srv in services]
},) for srv in services])
),
[if std.objectHas(info, 'custom_job_items') then custom else null]:
manifestYamlStream(if std.objectHas(info, 'custom_job_items') then info.custom_job_items(metadata, extra_info) else [{}]),

4
hack/push_peridot
View File

@ -46,3 +46,7 @@ targets=(
for target in "${targets[@]}"; do
br "$target" amd64
done
for target in "${targets[@]}"; do
GIT_COMMIT="$(git rev-parse HEAD)-arm64" br "$target" arm64
done

2
package.json
View File

@ -34,7 +34,7 @@
"@mui/styles": "^5.2.3",
"@mui/system": "^5.10.6",
"@mui/x-data-grid": "^5.2.0",
"@ory/hydra-client": "^1.10.6",
"@ory/hydra-client": "^2.0.2",
"@pmmmwh/react-refresh-webpack-plugin": "^0.5.1",
"@tailwindcss/forms": "^0.5.3",
"@types/classnames": "^2.2.11",

4
peridot/builder/v1/workflow/BUILD.bazel
View File

@ -13,12 +13,14 @@ go_library(
"rpmimport.go",
"srpm.go",
"sync.go",
"updateinfo.go",
"workflow.go",
"yumrepofs.go",
],
importpath = "peridot.resf.org/peridot/builder/v1/workflow",
visibility = ["//visibility:public"],
deps = [
"//apollo/proto/v1:pb",
"//apollo/rpmutils",
"//peridot/composetools",
"//peridot/db",
@ -26,10 +28,12 @@ go_library(
"//peridot/lookaside",
"//peridot/plugin",
"//peridot/proto/v1:pb",
"//peridot/proto/v1/admin:pb",
"//peridot/proto/v1/keykeeper:pb",
"//peridot/proto/v1/yumrepofs:pb",
"//peridot/rpmbuild",
"//peridot/yummeta",
"//publisher/updateinfo",
"//servicecatalog",
"//utils",
"//vendor/cirello.io/dynamolock",

29
peridot/builder/v1/workflow/arch.go
View File

@ -50,7 +50,6 @@ import (
"time"
"github.com/google/uuid"
"go.temporal.io/sdk/activity"
"google.golang.org/protobuf/types/known/wrapperspb"
"peridot.resf.org/apollo/rpmutils"
"peridot.resf.org/peridot/db/models"
@ -368,7 +367,7 @@ func (c *Controller) buildMacros(project *models.Project, packageVersion *models
}
func (c *Controller) setBuildMacros(project *models.Project, packageVersion *models.PackageVersion) error {
err := os.Remove("/etc/rpm/macros.dist")
err := os.RemoveAll("/etc/rpm/macros.dist")
if err != nil && !os.IsNotExist(err) {
return err
}
@ -474,6 +473,7 @@ config_opts['releasever'] = '{majorVersion}'
config_opts['package_manager'] = 'dnf'
config_opts['extra_chroot_dirs'] = [ '/run/lock' ]
config_opts['rpmbuild_command'] = '{rpmbuildCommand}'
{additionalVendorConfig}
config_opts['plugin_conf']['ccache_enable'] = False
config_opts['plugin_conf']['root_cache_enable'] = False
@ -557,6 +557,14 @@ config_opts['dnf.conf'] = """
yumConfig := c.yumConfig(project)
additionalVendorConfig := ""
if project.TargetVendor == "suse" {
additionalVendorConfig = `config_opts['useradd'] = '/usr/sbin/useradd -o -m -u {{chrootuid}} -g {{chrootgid}} -d {{chroothome}} {{chrootuser}}'
config_opts['ssl_ca_bundle_path'] = '/var/lib/ca-certificates/ca-bundle.pem'
config_opts['package_manager_max_attempts'] = 4
config_opts['package_manager_attempt_delay'] = 20`
}
rpmbuildNetworking := "False"
if extra.EnableNetworking {
rpmbuildNetworking = "True"
@ -574,6 +582,7 @@ config_opts['dnf.conf'] = """
"{targetVendor}", project.TargetVendor,
"{moduleSetupCommands}", strings.Join(moduleSetupCommands, ","),
"{rpmbuildNetworking}", rpmbuildNetworking,
"{additionalVendorConfig}", additionalVendorConfig,
).Replace(mockConfig)
return rendered, nil
@ -597,12 +606,8 @@ func (c *Controller) writeMockConfig(project *models.Project, packageVersion *mo
// Current implementation is broken for modules
// todo(mustafa): Evaluate if we can skip chroot again
func (c *Controller) BuildArchActivity(ctx context.Context, projectId string, packageName string, disableChecks bool, packageVersion *models.PackageVersion, uploadSRPMResult *UploadActivityResult, task *models.Task, arch string, extraOptions *peridotpb.ExtraBuildOptions) error {
go func() {
for {
activity.RecordHeartbeat(ctx)
time.Sleep(10 * time.Second)
}
}()
stopChan := makeHeartbeat(ctx, 10*time.Second)
defer func() { stopChan <- true }()
err := c.db.SetTaskStatus(task.ID.String(), peridotpb.TaskStatus_TASK_STATUS_RUNNING)
if err != nil {
@ -732,12 +737,8 @@ func (c *Controller) BuildArchActivity(ctx context.Context, projectId string, pa
}
func (c *Controller) UploadArchActivity(ctx context.Context, projectId string, parentTaskId string) ([]*UploadActivityResult, error) {
go func() {
for {
activity.RecordHeartbeat(ctx)
time.Sleep(4 * time.Second)
}
}()
stopChan := makeHeartbeat(ctx, 4*time.Second)
defer func() { stopChan <- true }()
rpms, err := findRpms()
if err != nil {

4
peridot/builder/v1/workflow/build.go
View File

@ -621,7 +621,7 @@ func (c *Controller) BuildWorkflow(ctx workflow.Context, req *peridotpb.SubmitBu
return nil, err
}
subtask, err := c.db.GetTask(uploadSRPMResult.Subtask.ID.String(), utils.Pointer(project.ID.String()))
subtask, err := c.db.GetTask(uploadSRPMResult.Subtask.ID.String(), utils.Pointer[string](project.ID.String()))
if err != nil {
setInternalError(errorDetails, err)
return nil, err
@ -730,7 +730,7 @@ func (c *Controller) BuildWorkflow(ctx workflow.Context, req *peridotpb.SubmitBu
if result.Skip {
continue
}
subtask, err := c.db.GetTask(result.Subtask.ID.String(), utils.Pointer(project.ID.String()))
subtask, err := c.db.GetTask(result.Subtask.ID.String(), utils.Pointer[string](project.ID.String()))
if err != nil {
ret.err = fmt.Errorf("failed to get task: %s", err)
return

3
peridot/builder/v1/workflow/hashed_repositories.go
View File

@ -183,6 +183,9 @@ func (c *Controller) CreateHashedRepositoriesActivity(req *peridotpb.CreateHashe
}
var primaryXml []byte
err = decompressWithGz(primaryXmlGz, &primaryXml)
if err != nil {
return nil, fmt.Errorf("decompress primary xml: %w", err)
}
primaryXml = []byte(strings.ReplaceAll(string(primaryXml), "rpm:", "rpm_"))
if err != nil {
return nil, fmt.Errorf("could not decompress primary.xml: %v", err)

38
peridot/builder/v1/workflow/import.go
View File

@ -56,7 +56,6 @@ import (
"github.com/sirupsen/logrus"
"github.com/spf13/viper"
"github.com/xanzy/go-gitlab"
"go.temporal.io/sdk/activity"
"go.temporal.io/sdk/temporal"
"go.temporal.io/sdk/workflow"
"google.golang.org/grpc/codes"
@ -605,12 +604,8 @@ func (c *Controller) ImportPackageWorkflow(ctx workflow.Context, req *peridotpb.
}
func (c *Controller) PackageSrcGitActivity(ctx context.Context, packageName string, project *models.Project, parentTaskId string) (*peridotpb.PackageSrcGitResponse, error) {
go func() {
for {
activity.RecordHeartbeat(ctx)
time.Sleep(4 * time.Second)
}
}()
stopChan := makeHeartbeat(ctx, 4*time.Second)
defer func() { stopChan <- true }()
task, err := c.db.CreateTask(nil, "noarch", peridotpb.TaskType_TASK_TYPE_IMPORT_SRC_GIT, utils.StringP(project.ID.String()), &parentTaskId)
if err != nil {
@ -681,6 +676,7 @@ func (c *Controller) PackageSrcGitActivity(ctx context.Context, packageName stri
if err != nil {
return nil, err
}
tarBts := buf.Bytes()
h := sha256.New()
_, err = h.Write(buf.Bytes())
@ -691,7 +687,16 @@ func (c *Controller) PackageSrcGitActivity(ctx context.Context, packageName stri
sum := hex.EncodeToString(sumBytes)
name := fmt.Sprintf("%s.tar.gz", elem.Name())
_, err = c.storage.PutObjectBytes(sum, buf.Bytes())
f, err := w.Filesystem.OpenFile(filepath.Join("SOURCES", name), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
if err != nil {
return nil, err
}
_, err = io.Copy(f, bytes.NewReader(tarBts))
if err != nil {
return nil, err
}
_, err = c.storage.PutObjectBytes(sum, tarBts)
if err != nil {
return nil, err
}
@ -707,12 +712,8 @@ func (c *Controller) PackageSrcGitActivity(ctx context.Context, packageName stri
}
func (c *Controller) UpdateDistGitForSrcGitActivity(ctx context.Context, packageName string, project *models.Project, packageRes *peridotpb.PackageSrcGitResponse) (*peridotpb.ImportRevision, error) {
go func() {
for {
activity.RecordHeartbeat(ctx)
time.Sleep(4 * time.Second)
}
}()
stopChan := makeHeartbeat(ctx, 4*time.Second)
defer func() { stopChan <- true }()
task, err := c.db.CreateTask(nil, "noarch", peridotpb.TaskType_TASK_TYPE_IMPORT_SRC_GIT_TO_DIST_GIT, utils.StringP(project.ID.String()), &packageRes.TaskId)
if err != nil {
@ -751,6 +752,7 @@ func (c *Controller) UpdateDistGitForSrcGitActivity(ctx context.Context, package
}
}
}
_ = srcW.Filesystem.Remove(".gitlab-ci.yml")
// Try checking out the dist-git repo
createRepo := false
@ -1048,12 +1050,8 @@ func (c *Controller) srpmprocToImportRevisions(project *models.Project, pkg stri
// Matches current Rocky workflow with distrobuild+srpmrpoc.
// This activity also uses srpmproc as a library instead of a CLI tool
func (c *Controller) UpstreamDistGitActivity(ctx context.Context, greq *UpstreamDistGitActivityRequest) (*UpstreamDistGitActivityResponse, error) {
go func() {
for {
activity.RecordHeartbeat(ctx)
time.Sleep(4 * time.Second)
}
}()
stopChan := makeHeartbeat(ctx, 4*time.Second)
defer func() { stopChan <- true }()
project := greq.Project
parentTaskId := greq.ParentTaskId

16
peridot/builder/v1/workflow/infrastructure.go
View File

@ -310,12 +310,8 @@ func (c *Controller) DestroyWorkerWorkflow(ctx workflow.Context, req *ProvisionW
// DeleteK8sPodActivity deletes the pod that hosts the ephemeral worker
func (c *Controller) DeleteK8sPodActivity(ctx context.Context, req *ProvisionWorkerRequest, task *models.Task) error {
go func() {
for {
activity.RecordHeartbeat(ctx)
time.Sleep(3 * time.Second)
}
}()
stopChan := makeHeartbeat(ctx, 3*time.Second)
defer func() { stopChan <- true }()
// creates the in-cluster config
config, err := rest.InClusterConfig()
@ -449,12 +445,8 @@ func (c *Controller) IngestLogsActivity(ctx context.Context, podName string, tas
// CreateK8sPodActivity creates a new pod in the same namespace
// with the specified container (the container has to contain peridotbuilder)
func (c *Controller) CreateK8sPodActivity(ctx context.Context, req *ProvisionWorkerRequest, task *models.Task, imageArch string) (string, error) {
go func() {
for {
activity.RecordHeartbeat(ctx)
time.Sleep(3 * time.Second)
}
}()
stopChan := makeHeartbeat(ctx, 3*time.Second)
defer func() { stopChan <- true }()
task.Status = peridotpb.TaskStatus_TASK_STATUS_FAILED

9
peridot/builder/v1/workflow/rpmimport.go
View File

@ -37,7 +37,6 @@ import (
"database/sql"
"fmt"
"github.com/cavaliergopher/rpm"
"go.temporal.io/sdk/activity"
"go.temporal.io/sdk/temporal"
"go.temporal.io/sdk/workflow"
"google.golang.org/grpc/codes"
@ -262,12 +261,8 @@ func (c *Controller) RpmLookasideBatchImportWorkflow(ctx workflow.Context, req *
}
func (c *Controller) RpmImportActivity(ctx context.Context, req *peridotpb.RpmImportRequest, taskID string, setTaskStatus bool, stage1 *RpmImportActivityTaskStage1) (*RpmImportActivityTaskStage1, error) {
go func() {
for {
activity.RecordHeartbeat(ctx)
time.Sleep(4 * time.Second)
}
}()
stopChan := makeHeartbeat(ctx, 4*time.Second)
defer func() { stopChan <- true }()
var buf bytes.Buffer
bts, err := c.storage.ReadObject(req.Rpms)

17
peridot/builder/v1/workflow/srpm.go
View File

@ -53,7 +53,6 @@ import (
"github.com/go-git/go-git/v5/config"
"github.com/go-git/go-git/v5/plumbing"
"github.com/rocky-linux/srpmproc/pkg/srpmproc"
"go.temporal.io/sdk/activity"
"google.golang.org/protobuf/types/known/anypb"
"google.golang.org/protobuf/types/known/wrapperspb"
"peridot.resf.org/peridot/db/models"
@ -326,12 +325,8 @@ func (c *Controller) uploadArtifact(projectId string, parentTaskId string, fileP
}
func (c *Controller) BuildSRPMActivity(ctx context.Context, upstreamPrefix string, scmHash string, projectId string, packageName string, packageVersion *models.PackageVersion, task *models.Task, extraOptions *peridotpb.ExtraBuildOptions) error {
go func() {
for {
activity.RecordHeartbeat(ctx)
time.Sleep(30 * time.Second)
}
}()
stopChan := makeHeartbeat(ctx, 30*time.Second)
defer func() { stopChan <- true }()
err := c.db.SetTaskStatus(task.ID.String(), peridotpb.TaskStatus_TASK_STATUS_RUNNING)
if err != nil {
@ -513,12 +508,8 @@ type UploadActivityResult struct {
}
func (c *Controller) UploadSRPMActivity(ctx context.Context, projectId string, parentTaskId string) (*UploadActivityResult, error) {
go func() {
for {
activity.RecordHeartbeat(ctx)
time.Sleep(4 * time.Second)
}
}()
stopChan := makeHeartbeat(ctx, 4*time.Second)
defer func() { stopChan <- true }()
srpmFilePath, err := findSrpm()
if err != nil {

21
peridot/builder/v1/workflow/workflow.go
View File

@ -31,10 +31,12 @@
package workflow
import (
"context"
"fmt"
"github.com/aws/aws-sdk-go/service/dynamodb"
"github.com/go-git/go-billy/v5/osfs"
"github.com/sirupsen/logrus"
"go.temporal.io/sdk/activity"
"go.temporal.io/sdk/client"
"google.golang.org/genproto/googleapis/rpc/errdetails"
"google.golang.org/protobuf/proto"
@ -47,6 +49,7 @@ import (
"peridot.resf.org/peridot/plugin"
"peridot.resf.org/peridot/rpmbuild"
"peridot.resf.org/utils"
"time"
)
const (
@ -256,3 +259,21 @@ func (c *Controller) logToMon(lines []string, taskId string, parentTaskId string
return nil
}
// makeHeartBeat provides a mechanism to start and stop heartbeats to a Temporal Activity.
func makeHeartbeat(ctx context.Context, pulseFrequency time.Duration, details ...interface{}) chan<- bool {
stopChan := make(chan bool, 1)
go func() {
stop := false
for !stop {
activity.RecordHeartbeat(ctx, details...)
time.Sleep(pulseFrequency)
select {
case stop = <-stopChan:
default:
}
}
}()
return stopChan
}

116
peridot/builder/v1/workflow/yumrepofs.go
View File

@ -44,8 +44,8 @@ import (
"github.com/gobwas/glob"
"github.com/google/uuid"
"github.com/rocky-linux/srpmproc/modulemd"
"github.com/sirupsen/logrus"
"github.com/spf13/viper"
"go.temporal.io/sdk/activity"
"go.temporal.io/sdk/temporal"
"go.temporal.io/sdk/workflow"
"google.golang.org/protobuf/encoding/protojson"
@ -93,15 +93,17 @@ type CompiledGlobFilter struct {
}
type CachedRepo struct {
Arch string
Repo *models.Repository
PrimaryRoot *yummeta.PrimaryRoot
FilelistsRoot *yummeta.FilelistsRoot
OtherRoot *yummeta.OtherRoot
Modulemd []*modulemd.ModuleMd
GroupsXml string
DefaultsYaml []byte
ModuleDefaults []*modulemd.Defaults
Arch string
Repo *models.Repository
PrimaryRoot *yummeta.PrimaryRoot
FilelistsRoot *yummeta.FilelistsRoot
OtherRoot *yummeta.OtherRoot
Modulemd []*modulemd.ModuleMd
GroupsXml string
DefaultsYaml []byte
ModuleDefaults []*modulemd.Defaults
UpdateInfoB64 string
UpdateInfoDataEntry *yummeta.RepoMdData
}
type Cache struct {
@ -331,6 +333,7 @@ func GenerateArchMapForArtifacts(artifacts models.TaskArtifacts, project *models
}
} else {
if composetools.IsDebugPackage(name) {
logrus.Printf("checking if %s is a debug package", name)
arch = arch + "-debug"
}
artifactArchMap[arch] = append(artifactArchMap[arch], &newArtifact)
@ -465,12 +468,8 @@ func (c *Controller) RepoUpdaterWorkflow(ctx workflow.Context, req *UpdateRepoRe
}
func (c *Controller) RequestKeykeeperSignActivity(ctx context.Context, buildId string, keyName string) (string, error) {
go func() {
for {
activity.RecordHeartbeat(ctx)
time.Sleep(4 * time.Second)
}
}()
stopChan := makeHeartbeat(ctx, 4*time.Second)
defer func() { stopChan <- true }()
task, err := c.keykeeper.SignArtifacts(ctx, &keykeeperpb.SignArtifactsRequest{
BuildId: buildId,
@ -487,12 +486,8 @@ func (c *Controller) RequestKeykeeperSignActivity(ctx context.Context, buildId s
}
func (c *Controller) UpdateRepoActivity(ctx context.Context, req *UpdateRepoRequest, task *models.Task, gpgId *string, signTaskIds []string) (*yumrepofspb.UpdateRepoTask, error) {
go func() {
for {
activity.RecordHeartbeat(ctx)
time.Sleep(4 * time.Second)
}
}()
stopChan := makeHeartbeat(ctx, 4*time.Second)
defer func() { stopChan <- true }()
signArtifactsTasks := &keykeeperpb.BatchSignArtifactsTask{
Tasks: []*keykeeperpb.SignArtifactsTask{},
@ -908,6 +903,11 @@ func (c *Controller) UpdateRepoActivity(ctx context.Context, req *UpdateRepoRequ
})
}
// Add updateinfo if not empty
if repo.UpdateInfoDataEntry != nil {
repomdRoot.Data = append(repomdRoot.Data, repo.UpdateInfoDataEntry)
}
var newRepoMd []byte
err = xmlMarshal(repomdRoot, &newRepoMd)
if err != nil {
@ -930,7 +930,7 @@ func (c *Controller) UpdateRepoActivity(ctx context.Context, req *UpdateRepoRequ
PrimaryXml: newPrimaryGzB64,
FilelistsXml: newFilelistsGzB64,
OtherXml: newOtherGzB64,
UpdateinfoXml: "",
UpdateinfoXml: repo.UpdateInfoB64,
ModuleDefaultsYaml: defaultsYamlB64,
ModulesYaml: newModulesGzB64,
GroupsXml: newGroupsGzB64,
@ -1158,6 +1158,8 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest
idArchNoDebug := fmt.Sprintf("%s-%s", repo.Name, noDebugArch)
var currentRevision *models.RepositoryRevision
var groupsXml string
var updateInfoXml string
var updateInfoDataEntry *yummeta.RepoMdData
if cache.Repos[idArchNoDebug] != nil {
c.log.Infof("found cache for %s", idArchNoDebug)
if cache.Repos[idArchNoDebug].Modulemd != nil {
@ -1176,6 +1178,8 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest
otherRoot = *cache.Repos[idArch].OtherRoot
}
groupsXml = cache.Repos[idArch].GroupsXml
updateInfoXml = cache.Repos[idArch].UpdateInfoB64
updateInfoDataEntry = cache.Repos[idArch].UpdateInfoDataEntry
} else {
c.log.Infof("no cache for %s", idArch)
var noDebugRevision *models.RepositoryRevision
@ -1243,6 +1247,26 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest
}
}
groupsXml = currentRevision.GroupsXml
updateInfoXml = currentRevision.UpdateinfoXml
if updateInfoXml != "" {
repomdXml, err := base64.StdEncoding.DecodeString(currentRevision.RepomdXml)
if err != nil {
return nil, fmt.Errorf("decode repomd xml: %w", err)
}
var repomdRoot yummeta.RepoMdRoot
err = xml.Unmarshal(repomdXml, &repomdRoot)
if err != nil {
return nil, fmt.Errorf("could not unmarshal repomd.xml: %v", err)
}
for _, data := range repomdRoot.Data {
if data.Type == "updateinfo" {
updateInfoDataEntry = data
break
}
}
}
}
if noDebugRevision != nil {
if noDebugRevision.ModulesYaml != "" {
@ -1300,15 +1324,17 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest
name = nvr[1]
}
noDebugInfoName := strings.ReplaceAll(strings.ReplaceAll(name, "-debuginfo", ""), "-debugsource", "")
noDebugInfoName := strings.ReplaceAll(strings.ReplaceAll(strings.ReplaceAll(name, "-debuginfo-common", ""), "-debugsource", ""), "-debuginfo", "")
archName := fmt.Sprintf("%s.%s", name, artifact.Arch)
if strings.HasSuffix(arch, "-debug") {
archName = fmt.Sprintf("%s.%s", noDebugInfoName, artifact.Arch)
}
shouldAdd := !req.Delete
// If debug-info common package, then it should be added
// if arch has "-debug" suffix
// If repo has a list for inclusion, then the artifact has to pass that first
if arch != "src" && moduleStream == nil && !req.Delete {
// If repo has a list for inclusion, then the artifact has to pass that first
if len(repo.IncludeFilter) > 0 {
// If the artifact isn't forced, it should be in the include list or additional multilib list
if !artifact.Forced && !utils.StrContains(archName, repo.IncludeFilter) && !utils.StrContains(noDebugInfoName, repo.AdditionalMultilib) {
@ -1495,16 +1521,15 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest
var deleteIds []string
if (!req.NoDeletePrevious || moduleStream != nil) || req.Delete {
for _, pkg := range primaryRoot.Packages {
shouldAdd := !req.Delete
if !req.Delete {
for _, artifact := range currentActiveArtifacts {
noRpmName := strings.TrimSuffix(filepath.Base(artifact.Name), ".rpm")
if filepath.Base(artifact.Name) == filepath.Base(pkg.Location.Href) && !utils.StrContains(noRpmName, changes.ModifiedPackages) && !utils.StrContains(noRpmName, changes.AddedPackages) && !utils.StrContains(noRpmName, skipDeleteArtifacts) {
shouldAdd = false
if req.NoDeleteInChain {
if utils.StrContains(noRpmName, cache.NoDeleteChain) {
shouldAdd = true
}
shouldAdd := true
for _, artifact := range currentActiveArtifacts {
noRpmName := strings.TrimSuffix(filepath.Base(artifact.Name), ".rpm")
if filepath.Base(artifact.Name) == filepath.Base(pkg.Location.Href) && !utils.StrContains(noRpmName, changes.ModifiedPackages) && !utils.StrContains(noRpmName, changes.AddedPackages) && !utils.StrContains(noRpmName, skipDeleteArtifacts) {
shouldAdd = false
if req.NoDeleteInChain {
if utils.StrContains(noRpmName, cache.NoDeleteChain) {
shouldAdd = true
}
}
}
@ -1513,6 +1538,7 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest
if utils.StrContains(noRpmNamePkg, changes.RemovedPackages) {
shouldAdd = false
}
if !shouldAdd {
c.log.Infof("deleting %s", pkg.Location.Href)
deleteIds = append(deleteIds, pkg.Checksum.Value)
@ -1601,15 +1627,17 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest
nRepo := repo
cache.Repos[idArch] = &CachedRepo{
Arch: arch,
Repo: &nRepo,
PrimaryRoot: &primaryRoot,
FilelistsRoot: &filelistsRoot,
OtherRoot: &otherRoot,
Modulemd: modulesRoot,
DefaultsYaml: defaultsYaml,
ModuleDefaults: moduleDefaults,
GroupsXml: groupsXml,
Arch: arch,
Repo: &nRepo,
PrimaryRoot: &primaryRoot,
FilelistsRoot: &filelistsRoot,
OtherRoot: &otherRoot,
Modulemd: modulesRoot,
DefaultsYaml: defaultsYaml,
ModuleDefaults: moduleDefaults,
GroupsXml: groupsXml,
UpdateInfoB64: updateInfoXml,
UpdateInfoDataEntry: updateInfoDataEntry,
}
if strings.HasSuffix(arch, "-debug") || arch == "src" {
cache.Repos[idArch].Modulemd = nil

4
peridot/cmd/v1/yumrepofsupdater/main.go
View File

@ -102,6 +102,10 @@ func mn(_ *cobra.Command, _ []string) {
w.Worker.RegisterActivity(w.WorkflowController.UpdateRepoActivity)
w.Worker.RegisterActivity(w.WorkflowController.RequestKeykeeperSignActivity)
// Updateinfo
w.Worker.RegisterWorkflow(w.WorkflowController.UpdateInfoWorkflow)
w.Worker.RegisterActivity(w.WorkflowController.SetUpdateInfoActivity)
var wg sync.WaitGroup
wg.Add(2)

2
peridot/composetools/rpm.go
View File

@ -46,7 +46,7 @@ var (
)
var (
debugSuffixes = []string{"-debuginfo", "-debugsource"}
debugSuffixes = []string{"-debuginfo", "-debugsource", "-debuginfo-common"}
soGlob = glob.MustCompile("*.so.*")
gtk2EnginesGlob = glob.MustCompile("/usr/lib*/gtk-2.0/*/engines")
gtk2ModulesGlob = glob.MustCompile("/usr/lib*/gtk-2.0/*/modules")

8
peridot/db/psql/project.go
View File

@ -32,7 +32,6 @@ package serverpsql
import (
"fmt"
"github.com/jmoiron/sqlx/types"
"github.com/lib/pq"
"google.golang.org/protobuf/encoding/protojson"
@ -78,7 +77,9 @@ func (a *Access) ListProjects(filters *peridotpb.ProjectFilters) (ret models.Pro
branch_suffix,
git_make_public,
vendor_macro,
packager_macro
packager_macro,
srpm_stage_packages,
build_stage_packages
from projects
where
($1 :: uuid is null or id = $1 :: uuid)
@ -350,8 +351,7 @@ func (a *Access) SetBuildRootPackages(projectId string, srpmPackages pq.StringAr
_, err := a.query.Exec(
`
update projects set srpm_stage_packages = $2, build_stage_packages = $3
where id = $1
update projects set srpm_stage_packages = $2, build_stage_packages = $3 where id = $1
`,
projectId,
srpmPackages,

1
peridot/proto/v1/task.proto
View File

@ -74,6 +74,7 @@ enum TaskType {
TASK_TYPE_LOOKASIDE_FILE_UPLOAD = 18;
TASK_TYPE_RPM_LOOKASIDE_BATCH_IMPORT = 19;
TASK_TYPE_CLONE_SWAP = 20;
TASK_TYPE_UPDATEINFO = 21;
}
enum TaskStatus {

2
peridot/yumrepofs/v1/blob.go
View File

@ -112,6 +112,8 @@ func (s *Server) GetBlob(ctx context.Context, req *yumrepofspb.GetBlobRequest) (
dataB64 = revision.GroupsXml
case "MODULES":
dataB64 = revision.ModulesYaml
case "UPDATEINFO":
dataB64 = revision.UpdateinfoXml
default:
return nil, ErrInvalidBlob
}

8
yarn.lock
View File

@ -2117,10 +2117,10 @@
mkdirp "^1.0.4"
rimraf "^3.0.2"
"@ory/hydra-client@^1.10.6":
version "1.11.8"
resolved "https://registry.yarnpkg.com/@ory/hydra-client/-/hydra-client-1.11.8.tgz#f163861e4c8b401e98b7b53092ea24eb80ab814c"
integrity sha512-fVTPjB/JWUrsI1uGnXf0eiYDClnH9fepGmJ+z1eg6b0EH5AQoaNc7nBU9QpEj/8pQjYy0p5hWEvOXGHWo9fJtw==
"@ory/hydra-client@^2.0.2":
version "2.0.2"
resolved "https://registry.yarnpkg.com/@ory/hydra-client/-/hydra-client-2.0.2.tgz#fd4e91f37dc2feaae9683a2620ab8d293aa1bc7c"
integrity sha512-1er8AHxZgOUGhipaTYeiZJVrPfCdiyycDGA2i93xdz4M/22FekC3E6www0+GKjbA1ifST3J1aIhssqel0ksFPA==
dependencies:
axios "^0.21.4"