From be2b871fffa9c6dcce42f16b5c19e79856d931c8 Mon Sep 17 00:00:00 2001 From: Mustafa Gezen Date: Sat, 19 Nov 2022 21:08:54 +0100 Subject: [PATCH] Strictly use module side repo during module builds and target only current project setting branches --- peridot/builder/v1/workflow/build.go | 5 ++++- peridot/builder/v1/workflow/module.go | 16 +++++++++++++++- .../cmd/v1/yumrepofsupdater/ci/deploy.jsonnet | 2 +- peridot/db/psql/build.go | 3 +-- peridot/proto/v1/build.proto | 1 + 5 files changed, 22 insertions(+), 5 deletions(-) diff --git a/peridot/builder/v1/workflow/build.go b/peridot/builder/v1/workflow/build.go index de8cc4c..0e5f303 100644 --- a/peridot/builder/v1/workflow/build.go +++ b/peridot/builder/v1/workflow/build.go @@ -478,12 +478,15 @@ func (c *Controller) BuildWorkflow(ctx workflow.Context, req *peridotpb.SubmitBu if extraOptions.ExtraYumrepofsRepos == nil { extraOptions.ExtraYumrepofsRepos = []*peridotpb.ExtraYumrepofsRepo{} } + if extraOptions.ExcludePackages == nil { + extraOptions.ExcludePackages = []string{} + } extraOptions.ExtraYumrepofsRepos = append(extraOptions.ExtraYumrepofsRepos, &peridotpb.ExtraYumrepofsRepo{ Name: repo.Name, ModuleHotfixes: true, IgnoreExclude: true, }) - extraOptions.ExcludePackages = excludes + extraOptions.ExcludePackages = append(extraOptions.ExcludePackages, excludes...) var buildIds []string for _, build := range buildNvrs { diff --git a/peridot/builder/v1/workflow/module.go b/peridot/builder/v1/workflow/module.go index f8f8b2c..a0f4790 100644 --- a/peridot/builder/v1/workflow/module.go +++ b/peridot/builder/v1/workflow/module.go @@ -49,6 +49,8 @@ import ( "google.golang.org/protobuf/types/known/wrapperspb" "gopkg.in/yaml.v3" "io/ioutil" + "path/filepath" + "peridot.resf.org/apollo/rpmutils" "peridot.resf.org/peridot/composetools" "peridot.resf.org/peridot/db/models" peridotpb "peridot.resf.org/peridot/pb" @@ -230,6 +232,9 @@ func (c *Controller) BuildModuleWorkflow(ctx workflow.Context, req *peridotpb.Su if len(req.Branches) > 0 && !utils.StrContains(revision.ScmBranchName, req.Branches) { continue } + if !strings.HasPrefix(revision.ScmBranchName, fmt.Sprintf("%s%d%s-stream", project.TargetBranchPrefix, project.MajorVersion, project.BranchSuffix.String)) { + continue + } if branchIndex[revision.ScmBranchName] { continue } @@ -539,6 +544,8 @@ func (c *Controller) BuildModuleStreamWorkflow(ctx workflow.Context, req *perido { Name: repo.Name, ModuleHotfixes: true, + Priority: -1, + IgnoreExclude: true, }, } } @@ -549,8 +556,8 @@ func (c *Controller) BuildModuleStreamWorkflow(ctx workflow.Context, req *perido // Building all project architectures is the default and cannot currently be overridden by the MD. // The MD can't override generated values such as repository or cache either yet. // Name specified by the component is also currently ignored and the key is forcefully used. - // We are not respecting platform or buildrequires at all since we don't have an active registry yet. // Whatever is available in the latest revision of yumrepofs for the project is what's used (including external repos). + var nonY1Excludes []string for _, buildOrder := range buildOrders { var futures []FutureContext for _, component := range buildOrderIndex[buildOrder] { @@ -590,6 +597,7 @@ func (c *Controller) BuildModuleStreamWorkflow(ctx workflow.Context, req *perido BuildBatchId: streamBuildOptions.BuildBatchId, Modules: buildRequiresModules, ForceDist: streamBuildOptions.Dist, + ExcludePackages: nonY1Excludes, } task, err := c.db.CreateTask(nil, "noarch", peridotpb.TaskType_TASK_TYPE_BUILD, &req.ProjectId, &parentTaskId) @@ -614,6 +622,12 @@ func (c *Controller) BuildModuleStreamWorkflow(ctx workflow.Context, req *perido return nil, err } buildTask.Builds = append(buildTask.Builds, &btask) + for _, a := range btask.Artifacts { + match := rpmutils.NVR().FindStringSubmatch(filepath.Base(a.Name)) + if !utils.StrContains(match[1], nonY1Excludes) { + nonY1Excludes = append(nonY1Excludes, match[1]) + } + } if repo != nil { yumrepoCtx := workflow.WithChildOptions(ctx, workflow.ChildWorkflowOptions{ diff --git a/peridot/cmd/v1/yumrepofsupdater/ci/deploy.jsonnet b/peridot/cmd/v1/yumrepofsupdater/ci/deploy.jsonnet index 03081d1..817f99a 100644 --- a/peridot/cmd/v1/yumrepofsupdater/ci/deploy.jsonnet +++ b/peridot/cmd/v1/yumrepofsupdater/ci/deploy.jsonnet @@ -24,7 +24,7 @@ resfdeploy.new({ cpu: '2', memory: '15G', }, - node_pool_request: { + node_pool_request: if kubernetes.prod() then { key: 'peridot.rockylinux.org/workflow-tolerates-arch', value: 'amd64', }, diff --git a/peridot/db/psql/build.go b/peridot/db/psql/build.go index ff39b49..0afaceb 100644 --- a/peridot/db/psql/build.go +++ b/peridot/db/psql/build.go @@ -498,8 +498,7 @@ func (a *Access) GetBuildIDsByPackageNameAndBranchName(name string, branchName s inner join project_package_versions ppv on ppv.package_version_id = b.package_version_id inner join import_revisions ir on ir.package_version_id = b.package_version_id where - b.project_id = $3 - and p.name = $1 + p.name = $1 and ir.scm_branch_name like '%-stream-' || $2 order by b.created_at asc `, diff --git a/peridot/proto/v1/build.proto b/peridot/proto/v1/build.proto index 3e86502..89f8cd0 100644 --- a/peridot/proto/v1/build.proto +++ b/peridot/proto/v1/build.proto @@ -297,6 +297,7 @@ message ExtraYumrepofsRepo { string name = 1; bool module_hotfixes = 2; bool ignore_exclude = 3; + int32 priority = 4; } // These options can be used to customize the behavior of the service