rocky-linux/peridot
4
0
Fork 0
mirror of https://github.com/rocky-linux/peridot.git synced 2024-12-21 02:08:29 +00:00
Code Issues Projects Releases Activity

Merge pull request #71 from mstg/yumrepofs-speedup

Browse Source 
Multiple changes related to build, modules and cloning
This commit is contained in:
resf-prow[bot] 2022-12-08 06:23:43 +00:00 committed by GitHub
commit a2fb251227
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
26 changed files with 842 additions and 207 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
apollo/rpmutils/BUILD.bazel
View File

@ -5,4 +5,5 @@ go_library(
srcs = ["regex.go"],
importpath = "peridot.resf.org/apollo/rpmutils",
visibility = ["//visibility:public"],
deps = ["//vendor/github.com/rocky-linux/srpmproc/pkg/rpmutils"],
)

9
apollo/rpmutils/regex.go
View File

@ -30,7 +30,10 @@
package rpmutils
import "regexp"
import (
"github.com/rocky-linux/srpmproc/pkg/rpmutils"
"regexp"
)
var (
nvr *regexp.Regexp
@ -45,14 +48,14 @@ var (
func NVR() *regexp.Regexp {
if nvr == nil {
nvr = regexp.MustCompile("^(\\S+)-([\\w~%.+]+)-(\\w+(?:\\.[\\w+]+)+?)(?:\\.(\\w+))?(?:\\.rpm)?$")
nvr = regexp.MustCompile("^(\\S+)-([\\w~%.+]+)-(\\w+(?:\\.[\\w~%+]+)+?)(?:\\.(\\w+))?(?:\\.rpm)?$")
}
return nvr
}
func NVRNoArch() *regexp.Regexp {
if nvrNoArch == nil {
nvrNoArch = regexp.MustCompile("^(\\S+)-([\\w~%.+]+)-(\\w+(?:\\.[\\w+]+)+?)(?:\\.rpm)?$")
nvrNoArch = rpmutils.Nvr
}
return nvrNoArch
}

30
ci/kubernetes.jsonnet
View File

@ -228,7 +228,7 @@ local dev() = stage == '-dev';
},
},
],
affinity: if !std.objectHas(deployment, 'no_anti_affinity') || !deployment.no_anti_affinity then {
affinity: (if !std.objectHas(deployment, 'no_anti_affinity') || !deployment.no_anti_affinity then {
podAntiAffinity: {
preferredDuringSchedulingIgnoredDuringExecution: [
{
@ -267,7 +267,33 @@ local dev() = stage == '-dev';
},
],
},
},
} else {}) + (if deployment.node_pool_request != null then {
nodeAffinity: {
requiredDuringSchedulingIgnoredDuringExecution: {
nodeSelectorTerms: [
{
matchExpressions: [
{
key: deployment.node_pool_request.key,
operator: 'In',
values: [
deployment.node_pool_request.value,
],
},
],
},
],
},
},
} else {}),
tolerations: if deployment.node_pool_request != null then [
{
key: deployment.node_pool_request.key,
operator: 'Equal',
value: deployment.node_pool_request.value,
effect: 'NoSchedule',
},
] else [],
restartPolicy: 'Always',
imagePullSecrets: if std.objectHas(deployment, 'imagePullSecrets') && deployment.imagePullSecrets != null then if std.type(deployment.imagePullSecrets) == 'string' then deployment.imagePullSecrets else [
{

1
ci/resfdeploy.jsonnet
View File

@ -267,6 +267,7 @@ local manifestYamlStream = function (value, indent_array_in_object=false, c_docu
limits: if std.objectHas(info, 'limits') then info.limits,
requests: if std.objectHas(info, 'requests') then info.requests,
args: if std.objectHas(info, 'args') then info.args else [],
node_pool_request: if std.objectHas(info, 'node_pool_request') then info.node_pool_request else null,
serviceAccount: sa_name,
},
),

13
go.mod
View File

@ -26,18 +26,18 @@ require (
github.com/gocolly/colly/v2 v2.1.0
github.com/gogo/status v1.1.0
github.com/google/uuid v1.3.0
github.com/gorilla/feeds v1.1.1 // indirect
github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
github.com/gorilla/feeds v1.1.1
github.com/grpc-ecosystem/go-grpc-middleware v1.3.0
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0
github.com/grpc-ecosystem/grpc-gateway/v2 v2.6.0
github.com/imdario/mergo v0.3.11 // indirect
github.com/jmoiron/sqlx v1.3.4
github.com/lib/pq v1.10.2
github.com/ory/hydra-client-go v1.10.6
github.com/pelletier/go-toml v1.8.1 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pkg/errors v0.9.1
github.com/prometheus/client_golang v1.13.0
github.com/rocky-linux/srpmproc v0.4.2
github.com/rocky-linux/srpmproc v0.4.3
github.com/sirupsen/logrus v1.8.1
github.com/spf13/cobra v1.1.3
github.com/spf13/pflag v1.0.5
@ -50,7 +50,6 @@ require (
go.temporal.io/sdk v1.13.1
golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d // indirect
golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2 // indirect
google.golang.org/genproto v0.0.0-20211104193956-4c6863e31247
google.golang.org/grpc v1.44.0
@ -61,7 +60,7 @@ require (
k8s.io/apimachinery v0.22.1
k8s.io/client-go v0.22.1
openapi.peridot.resf.org/peridotopenapi v0.0.0-00010101000000-000000000000
peridot.resf.org/apollo/pb v0.0.0-00010101000000-000000000000 // indirect
peridot.resf.org/apollo/pb v0.0.0-00010101000000-000000000000
peridot.resf.org/common v0.0.0-00010101000000-000000000000
peridot.resf.org/obsidian/pb v0.0.0-00010101000000-000000000000
peridot.resf.org/peridot/keykeeper/pb v0.0.0-00010101000000-000000000000

11
go.sum
View File

@ -111,7 +111,6 @@ github.com/authzed/grpcutil v0.0.0-20211115181027-063820eb2511 h1:3/LcA84F8rSMZ8
github.com/authzed/grpcutil v0.0.0-20211115181027-063820eb2511/go.mod h1:rqjY3zyK/YP7NID9+B2BdIRRkvnK+cdf9/qya/zaFZE=
github.com/aws/aws-sdk-go v1.34.13/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
github.com/aws/aws-sdk-go v1.34.28/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48=
github.com/aws/aws-sdk-go v1.36.12 h1:YJpKFEMbqEoo+incs5qMe61n1JH3o4O1IMkMexLzJG8=
github.com/aws/aws-sdk-go v1.36.12/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
github.com/aws/aws-sdk-go v1.44.129 h1:yld8Rc8OCahLtenY1mnve4w1jVeBu/rSiscGzodaDOs=
github.com/aws/aws-sdk-go v1.44.129/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
@ -533,6 +532,7 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxv
github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs=
github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA=
@ -663,12 +663,8 @@ github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/robfig/cron v1.2.0 h1:ZjScXvvxeQ63Dbyxy76Fj3AT3Ut0aKsyd2/tl3DTMuQ=
github.com/robfig/cron v1.2.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k=
github.com/rocky-linux/srpmproc v0.3.16 h1:kxJEiQsZ0DcMhX0vY482n82XvjPiP2WifxI3NYuyLLM=
github.com/rocky-linux/srpmproc v0.3.16/go.mod h1:vWZzxPTfxh4pmfr5Mw20FyrqyKsbGHzDwOlN+W5EMpw=
github.com/rocky-linux/srpmproc v0.4.1 h1:qcq7bGLplKbu+dSKQ9VBwcTao3OqPNb6rdKz58MCFLA=
github.com/rocky-linux/srpmproc v0.4.1/go.mod h1:x8Z2wqhV2JqRnYMhYz3thOQkfsSWjJkyX8DVGDPOb48=
github.com/rocky-linux/srpmproc v0.4.2 h1:U8SnYPxYtJ2XIAnrcEvxx+PsHCi4uQsfIOIC754MKas=
github.com/rocky-linux/srpmproc v0.4.2/go.mod h1:x8Z2wqhV2JqRnYMhYz3thOQkfsSWjJkyX8DVGDPOb48=
github.com/rocky-linux/srpmproc v0.4.3 h1:PnfuOQbGZEwOEr1u7Notz9Pm+Ol38Tu7mo8/63rgeaE=
github.com/rocky-linux/srpmproc v0.4.3/go.mod h1:x8Z2wqhV2JqRnYMhYz3thOQkfsSWjJkyX8DVGDPOb48=
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
@ -985,7 +981,6 @@ golang.org/x/sys v0.0.0-20210910150752-751e447fb3d0/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20211110154304-99a53858aa08/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a h1:dGzPydgVsqGcTRVwiLJ1jVbufYwmzD3LfVPLKsKg+0k=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U=
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

2
peridot/builder/v1/workflow/BUILD.bazel
View File

@ -5,6 +5,7 @@ go_library(
srcs = [
"arch.go",
"build.go",
"clone_swap.go",
"hashed_repositories.go",
"import.go",
"infrastructure.go",
@ -46,6 +47,7 @@ go_library(
"//vendor/github.com/go-git/go-git/v5/storage/memory",
"//vendor/github.com/gobwas/glob",
"//vendor/github.com/google/uuid",
"//vendor/github.com/pkg/errors",
"//vendor/github.com/rocky-linux/srpmproc/modulemd",
"//vendor/github.com/rocky-linux/srpmproc/pb",
"//vendor/github.com/rocky-linux/srpmproc/pkg/data",

11
peridot/builder/v1/workflow/arch.go
View File

@ -395,11 +395,16 @@ obsoletes=1
gpgcheck=0
assumeyes=1
keepcache=1
best=1
syslog_ident=peridotbuilder
syslog_device=
metadata_expire=0
install_weak_deps=0
protected_packages=
reposdir=/dev/null
logfile=/var/log/yum.log
mdpolicy=group:primary
metadata_expire=0
user_agent=peridotbuilder`
switch project.TargetVendor {
@ -535,12 +540,6 @@ config_opts['module_setup_commands'] = [{moduleSetupCommands}]
mockConfig += `
config_opts['dnf.conf'] = """
{yumConfig}
reposdir=/dev/null
cachedir=/var/cache/yum
logfile=/var/log/yum.log
mdpolicy=group:primary
metadata_expire=0
`
repos, err := c.repos(project.ID.String(), arch, extra)

85
peridot/builder/v1/workflow/build.go
View File

@ -413,10 +413,16 @@ func (c *Controller) BuildWorkflow(ctx workflow.Context, req *peridotpb.SubmitBu
return nil, err
}
// If there is a parent task, we need to use that ID as the parent task ID
taskID := task.ID.String()
if task.ParentTaskId.Valid {
taskID = task.ParentTaskId.String
}
// Create a side repo if the build request specifies side NVRs
// Packages specified here will be excluded from the main repo
if len(req.SideNvrs) > 0 {
var buildNvrs []*models.Build
var buildNvrs []models.Build
var excludes []string
for _, sideNvr := range req.SideNvrs {
if !rpmutils.NVRNoArch().MatchString(sideNvr) {
@ -429,7 +435,7 @@ func (c *Controller) BuildWorkflow(ctx workflow.Context, req *peridotpb.SubmitBu
nvrVersion := nvrMatch[2]
nvrRelease := nvrMatch[3]
buildNvr, err := c.db.GetBuildByPackageNameAndVersionAndRelease(nvrName, nvrVersion, nvrRelease, req.ProjectId)
buildNvrsFromBuild, err := c.db.GetBuildByPackageNameAndVersionAndRelease(nvrName, nvrVersion, nvrRelease)
if err != nil {
if err == sql.ErrNoRows {
err = fmt.Errorf("side NVR %s not found in project %s", sideNvr, req.ProjectId)
@ -441,21 +447,23 @@ func (c *Controller) BuildWorkflow(ctx workflow.Context, req *peridotpb.SubmitBu
return nil, err
}
artifacts, err := c.db.GetArtifactsForBuild(buildNvr.ID.String())
if err != nil {
err = fmt.Errorf("could not get artifacts for build %s: %v", buildNvr.ID, err)
setInternalError(errorDetails, err)
return nil, err
}
for _, buildNvr := range buildNvrsFromBuild {
artifacts, err := c.db.GetArtifactsForBuild(buildNvr.ID.String())
if err != nil {
err = fmt.Errorf("could not get artifacts for build %s: %v", buildNvr.ID, err)
setInternalError(errorDetails, err)
return nil, err
}
for _, artifact := range artifacts {
artifactName := strings.TrimPrefix(artifact.Name, fmt.Sprintf("%s/", buildNvr.TaskId))
if rpmutils.NVR().MatchString(artifactName) {
excludes = append(excludes, rpmutils.NVR().FindStringSubmatch(artifactName)[1])
for _, artifact := range artifacts {
artifactName := strings.TrimPrefix(artifact.Name, fmt.Sprintf("%s/", buildNvr.TaskId))
if rpmutils.NVR().MatchString(artifactName) {
excludes = append(excludes, rpmutils.NVR().FindStringSubmatch(artifactName)[1])
}
}
}
buildNvrs = append(buildNvrs, buildNvr)
buildNvrs = append(buildNvrs, buildNvrsFromBuild...)
}
repo, err := c.db.CreateRepositoryWithPackages(uuid.New().String(), req.ProjectId, true, []string{})
@ -470,40 +478,41 @@ func (c *Controller) BuildWorkflow(ctx workflow.Context, req *peridotpb.SubmitBu
if extraOptions.ExtraYumrepofsRepos == nil {
extraOptions.ExtraYumrepofsRepos = []*peridotpb.ExtraYumrepofsRepo{}
}
if extraOptions.ExcludePackages == nil {
extraOptions.ExcludePackages = []string{}
}
extraOptions.ExtraYumrepofsRepos = append(extraOptions.ExtraYumrepofsRepos, &peridotpb.ExtraYumrepofsRepo{
Name: repo.Name,
ModuleHotfixes: true,
IgnoreExclude: true,
})
extraOptions.ExcludePackages = excludes
extraOptions.ExcludePackages = append(extraOptions.ExcludePackages, excludes...)
for _, buildNvr := range buildNvrs {
yumrepoCtx := workflow.WithChildOptions(ctx, workflow.ChildWorkflowOptions{
TaskQueue: "yumrepofs",
})
updateRepoRequest := &UpdateRepoRequest{
ProjectID: req.ProjectId,
TaskID: &buildNvr.TaskId,
BuildIDs: []string{buildNvr.ID.String()},
Delete: false,
ForceRepoId: repo.ID.String(),
ForceNonModular: true,
DisableSigning: true,
DisableSetActive: true,
}
updateRepoTask := &yumrepofspb.UpdateRepoTask{}
err = workflow.ExecuteChildWorkflow(yumrepoCtx, c.RepoUpdaterWorkflow, updateRepoRequest).Get(ctx, updateRepoTask)
if err != nil {
return nil, err
}
var buildIds []string
for _, build := range buildNvrs {
buildIds = append(buildIds, build.ID.String())
}
yumrepoCtx := workflow.WithChildOptions(ctx, workflow.ChildWorkflowOptions{
TaskQueue: "yumrepofs",
})
updateRepoRequest := &UpdateRepoRequest{
ProjectID: req.ProjectId,
TaskID: &taskID,
BuildIDs: buildIds,
Delete: false,
ForceRepoId: repo.ID.String(),
ForceNonModular: true,
DisableSigning: true,
DisableSetActive: true,
}
updateRepoTask := &yumrepofspb.UpdateRepoTask{}
err = workflow.ExecuteChildWorkflow(yumrepoCtx, c.RepoUpdaterWorkflow, updateRepoRequest).Get(ctx, updateRepoTask)
if err != nil {
return nil, err
}
}
// If there is a parent task, we need to use that ID as the parent task ID
taskID := task.ID.String()
if task.ParentTaskId.Valid {
taskID = task.ParentTaskId.String
}
buildID := extraOptions.ReusableBuildId
if buildID == "" {
err = errors.New("reusable build id not found")

260
peridot/builder/v1/workflow/clone_swap.go Normal file
View File

@ -0,0 +1,260 @@
// Copyright (c) All respective contributors to the Peridot Project. All rights reserved.
// Copyright (c) 2021-2022 Rocky Enterprise Software Foundation, Inc. All rights reserved.
// Copyright (c) 2021-2022 Ctrl IQ, Inc. All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice,
// this list of conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice,
// this list of conditions and the following disclaimer in the documentation
// and/or other materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its contributors
// may be used to endorse or promote products derived from this software without
// specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
// POSSIBILITY OF SUCH DAMAGE.
package workflow
import (
"context"
"database/sql"
"fmt"
"github.com/google/uuid"
"github.com/pkg/errors"
"go.temporal.io/sdk/temporal"
"go.temporal.io/sdk/workflow"
"google.golang.org/protobuf/types/known/wrapperspb"
"peridot.resf.org/peridot/db/models"
peridotpb "peridot.resf.org/peridot/pb"
yumrepofspb "peridot.resf.org/peridot/yumrepofs/pb"
"peridot.resf.org/utils"
"time"
)
type CloneSwapStep1 struct {
Batches [][]string
}
func (c *Controller) getSingleProject(projectId string) (*models.Project, error) {
projects, err := c.db.ListProjects(&peridotpb.ProjectFilters{
Id: wrapperspb.String(projectId),
})
if err != nil {
return nil, errors.Wrap(err, "could not list projects")
}
if len(projects) == 0 {
return nil, errors.New("no projects found")
}
p := projects[0]
return &p, nil
}
func (c *Controller) CloneSwapWorkflow(ctx workflow.Context, req *peridotpb.CloneSwapRequest, task *models.Task) (*peridotpb.CloneSwapTask, error) {
var ret peridotpb.CloneSwapTask
deferTask, errorDetails, err := c.commonCreateTask(task, &ret)
defer deferTask()
if err != nil {
return nil, err
}
var step1 CloneSwapStep1
syncCtx := workflow.WithActivityOptions(ctx, workflow.ActivityOptions{
ScheduleToStartTimeout: 25 * time.Minute,
StartToCloseTimeout: 60 * time.Minute,
TaskQueue: c.mainQueue,
RetryPolicy: &temporal.RetryPolicy{
MaximumAttempts: 1,
},
})
err = workflow.ExecuteActivity(syncCtx, c.CloneSwapActivity, req, task).Get(ctx, &step1)
if err != nil {
setActivityError(errorDetails, err)
return nil, err
}
// We're going to create a workflow for each batch of builds.
for _, batch := range step1.Batches {
yumrepoCtx := workflow.WithChildOptions(ctx, workflow.ChildWorkflowOptions{
TaskQueue: "yumrepofs",
})
taskID := task.ID.String()
updateRepoRequest := &UpdateRepoRequest{
ProjectID: req.TargetProjectId.Value,
BuildIDs: batch,
Delete: false,
TaskID: &taskID,
NoDeletePrevious: true,
}
updateRepoTask := &yumrepofspb.UpdateRepoTask{}
err = workflow.ExecuteChildWorkflow(yumrepoCtx, c.RepoUpdaterWorkflow, updateRepoRequest).Get(yumrepoCtx, updateRepoTask)
if err != nil {
setActivityError(errorDetails, err)
return nil, err
}
}
var flattenedBuilds []string
for _, batch := range step1.Batches {
flattenedBuilds = append(flattenedBuilds, batch...)
}
task.Status = peridotpb.TaskStatus_TASK_STATUS_SUCCEEDED
ret.TargetProjectId = req.TargetProjectId.Value
ret.SrcProjectId = req.SrcProjectId.Value
ret.BuildIdsLayered = flattenedBuilds
return &ret, nil
}
func (c *Controller) CloneSwapActivity(ctx context.Context, req *peridotpb.CloneSwapRequest, task *models.Task) (*CloneSwapStep1, error) {
srcProject, err := c.getSingleProject(req.SrcProjectId.Value)
if err != nil {
return nil, err
}
targetProject, err := c.getSingleProject(req.TargetProjectId.Value)
if err != nil {
return nil, err
}
// We're going to fetch all repositories in the source project, and then
// copy them to the target project.
srcRepos, err := c.db.FindRepositoriesForProject(srcProject.ID.String(), nil, false)
if err != nil {
return nil, err
}
beginTx, err := c.db.Begin()
if err != nil {
return nil, errors.Wrap(err, "could not begin transaction")
}
tx := c.db.UseTransaction(beginTx)
for _, srcRepo := range srcRepos {
_ = c.logToMon(
[]string{fmt.Sprintf("Processing %s/%s", srcProject.Name, srcRepo.Name)},
task.ID.String(),
utils.NullStringToEmptyString(task.ParentTaskId),
)
// Check if the repo exists in the target project.
dbRepo, err := tx.GetRepository(nil, &srcRepo.Name, &req.TargetProjectId.Value)
if err != nil && err != sql.ErrNoRows {
return nil, errors.Wrap(err, "could not get repository")
}
if dbRepo == nil {
// The repo doesn't exist in the target project, so we need to create it.
dbRepo, err = tx.CreateRepositoryWithPackages(srcRepo.Name, req.TargetProjectId.Value, false, srcRepo.Packages)
if err != nil {
return nil, errors.Wrap(err, "could not create repository")
}
_ = c.logToMon(
[]string{fmt.Sprintf("Created %s/%s", targetProject.Name, dbRepo.Name)},
task.ID.String(),
utils.NullStringToEmptyString(task.ParentTaskId),
)
}
allArches := append(srcProject.Archs, "src")
for _, a := range allArches {
allArches = append(allArches, a+"-debug")
}
for _, arch := range allArches {
srcRepoLatestRevision, err := tx.GetLatestActiveRepositoryRevision(srcRepo.ID.String(), arch)
if err != nil && err != sql.ErrNoRows {
return nil, errors.Wrap(err, "could not get latest active repository revision")
}
if srcRepoLatestRevision == nil {
_ = c.logToMon(
[]string{fmt.Sprintf("Skipping %s/%s/%s because it has no active revisions", srcProject.Name, srcRepo.Name, arch)},
task.ID.String(),
utils.NullStringToEmptyString(task.ParentTaskId),
)
continue
}
id := uuid.New()
_, err = tx.CreateRevisionForRepository(
id.String(),
dbRepo.ID.String(),
arch,
srcRepoLatestRevision.RepomdXml,
srcRepoLatestRevision.PrimaryXml,
srcRepoLatestRevision.FilelistsXml,
srcRepoLatestRevision.OtherXml,
srcRepoLatestRevision.UpdateinfoXml,
srcRepoLatestRevision.ModuleDefaultsYaml,
srcRepoLatestRevision.ModulesYaml,
srcRepoLatestRevision.GroupsXml,
srcRepoLatestRevision.UrlMappings.String(),
)
if err != nil {
return nil, errors.Wrap(err, "could not create repository revision")
}
_ = c.logToMon(
[]string{fmt.Sprintf("Created revision %s for %s/%s/%s", id.String(), targetProject.Name, srcRepo.Name, arch)},
task.ID.String(),
utils.NullStringToEmptyString(task.ParentTaskId),
)
}
}
// Now let's get all succeeded builds for the target project.
builds, err := tx.GetSuccessfulBuildIDsAsc(req.TargetProjectId.Value)
if err != nil {
return nil, errors.Wrap(err, "could not list builds")
}
// We're creating batches of 200 builds to process at a time.
var syncBatches [][]string
for i := 0; i < len(builds); i += 200 {
end := i + 200
if end > len(builds) {
end = len(builds)
}
syncBatches = append(syncBatches, builds[i:end])
}
_ = c.logToMon(
[]string{
fmt.Sprintf("Created %d batches", len(syncBatches)),
"Following builds will be synced:",
},
task.ID.String(),
utils.NullStringToEmptyString(task.ParentTaskId),
)
for _, id := range builds {
_ = c.logToMon(
[]string{fmt.Sprintf("\t* %s", id)},
task.ID.String(),
utils.NullStringToEmptyString(task.ParentTaskId),
)
}
err = beginTx.Commit()
if err != nil {
return nil, errors.Wrap(err, "could not commit transaction")
}
return &CloneSwapStep1{
Batches: syncBatches,
}, nil
}

173
peridot/builder/v1/workflow/module.go
View File

@ -49,6 +49,8 @@ import (
"google.golang.org/protobuf/types/known/wrapperspb"
"gopkg.in/yaml.v3"
"io/ioutil"
"path/filepath"
"peridot.resf.org/apollo/rpmutils"
"peridot.resf.org/peridot/composetools"
"peridot.resf.org/peridot/db/models"
peridotpb "peridot.resf.org/peridot/pb"
@ -230,6 +232,9 @@ func (c *Controller) BuildModuleWorkflow(ctx workflow.Context, req *peridotpb.Su
if len(req.Branches) > 0 && !utils.StrContains(revision.ScmBranchName, req.Branches) {
continue
}
if !strings.HasPrefix(revision.ScmBranchName, fmt.Sprintf("%s%d%s-stream", project.TargetBranchPrefix, project.MajorVersion, project.BranchSuffix.String)) {
continue
}
if branchIndex[revision.ScmBranchName] {
continue
}
@ -539,6 +544,8 @@ func (c *Controller) BuildModuleStreamWorkflow(ctx workflow.Context, req *perido
{
Name: repo.Name,
ModuleHotfixes: true,
Priority: -1,
IgnoreExclude: true,
},
}
}
@ -549,8 +556,8 @@ func (c *Controller) BuildModuleStreamWorkflow(ctx workflow.Context, req *perido
// Building all project architectures is the default and cannot currently be overridden by the MD.
// The MD can't override generated values such as repository or cache either yet.
// Name specified by the component is also currently ignored and the key is forcefully used.
// We are not respecting platform or buildrequires at all since we don't have an active registry yet.
// Whatever is available in the latest revision of yumrepofs for the project is what's used (including external repos).
var nonY1Excludes []string
for _, buildOrder := range buildOrders {
var futures []FutureContext
for _, component := range buildOrderIndex[buildOrder] {
@ -578,6 +585,7 @@ func (c *Controller) BuildModuleStreamWorkflow(ctx workflow.Context, req *perido
},
ScmHash: wrapperspb.String(component.Ref),
DisableChecks: req.DisableChecks,
SideNvrs: req.SideNvrs,
}
extraOptions := &peridotpb.ExtraBuildOptions{
DisableYumrepofsUpdates: true,
@ -589,6 +597,7 @@ func (c *Controller) BuildModuleStreamWorkflow(ctx workflow.Context, req *perido
BuildBatchId: streamBuildOptions.BuildBatchId,
Modules: buildRequiresModules,
ForceDist: streamBuildOptions.Dist,
ExcludePackages: nonY1Excludes,
}
task, err := c.db.CreateTask(nil, "noarch", peridotpb.TaskType_TASK_TYPE_BUILD, &req.ProjectId, &parentTaskId)
@ -613,6 +622,12 @@ func (c *Controller) BuildModuleStreamWorkflow(ctx workflow.Context, req *perido
return nil, err
}
buildTask.Builds = append(buildTask.Builds, &btask)
for _, a := range btask.Artifacts {
match := rpmutils.NVR().FindStringSubmatch(filepath.Base(a.Name))
if !utils.StrContains(match[1], nonY1Excludes) {
nonY1Excludes = append(nonY1Excludes, match[1])
}
}
if repo != nil {
yumrepoCtx := workflow.WithChildOptions(ctx, workflow.ChildWorkflowOptions{
@ -726,11 +741,12 @@ func (c *Controller) BuildModuleStreamWorkflow(ctx workflow.Context, req *perido
// Generate a modulemd for each arch
for _, arch := range streamBuildOptions.Project.Archs {
err := fillInRpmArtifactsForModuleMd(md, streamBuildOptions, buildTask, artifactPrimaryIndex, arch, licenses, false)
newMd := copyModuleMd(*md)
err := fillInRpmArtifactsForModuleMd(newMd, streamBuildOptions, buildTask, artifactPrimaryIndex, arch, licenses, false)
if err != nil {
return nil, err
}
err = fillInRpmArtifactsForModuleMd(md, streamBuildOptions, buildTask, artifactPrimaryIndex, arch, licenses, true)
err = fillInRpmArtifactsForModuleMd(newMd, streamBuildOptions, buildTask, artifactPrimaryIndex, arch, licenses, true)
if err != nil {
return nil, err
}
@ -793,8 +809,153 @@ func doesRpmPassFilter(artifact *ArtifactIndex, md *modulemd.ModuleMd, arch stri
return true
}
func copyModuleMd(md modulemd.ModuleMd) *modulemd.ModuleMd {
ret := modulemd.ModuleMd{
Document: md.Document,
Version: md.Version,
Data: &modulemd.Data{
Name: md.Data.Name,
Stream: md.Data.Stream,
Version: md.Data.Version,
StaticContext: md.Data.StaticContext,
Context: md.Data.Context,
Arch: md.Data.Arch,
Summary: md.Data.Summary,
Description: md.Data.Description,
ServiceLevels: map[modulemd.ServiceLevelType]*modulemd.ServiceLevel{},
License: &modulemd.License{},
Xmd: map[string]map[string]string{},
Dependencies: []*modulemd.Dependencies{},
References: &modulemd.References{},
Profiles: map[string]*modulemd.Profile{},
Profile: map[string]*modulemd.Profile{},
API: &modulemd.API{},
Filter: &modulemd.API{},
BuildOpts: &modulemd.BuildOpts{},
Components: &modulemd.Components{},
Artifacts: &modulemd.Artifacts{},
},
}
if md.Data.ServiceLevels != nil {
for k, v := range md.Data.ServiceLevels {
c := *v
ret.Data.ServiceLevels[k] = &c
}
} else {
ret.Data.ServiceLevels = nil
}
if md.Data.License != nil {
c := *md.Data.License
ret.Data.License = &c
} else {
ret.Data.License = nil
}
if md.Data.Xmd != nil {
for k, v := range md.Data.Xmd {
c := map[string]string{}
for k2, v2 := range v {
c[k2] = v2
}
ret.Data.Xmd[k] = c
}
} else {
ret.Data.Xmd = nil
}
if md.Data.Dependencies != nil {
for _, v := range md.Data.Dependencies {
c := *v
ret.Data.Dependencies = append(ret.Data.Dependencies, &c)
}
}
if md.Data.References != nil {
c := *md.Data.References
ret.Data.References = &c
} else {
ret.Data.References = nil
}
if md.Data.Profiles != nil {
for k, v := range md.Data.Profiles {
c := *v
ret.Data.Profiles[k] = &c
}
} else {
ret.Data.Profiles = nil
}
if md.Data.Profile != nil {
for k, v := range md.Data.Profile {
c := *v
ret.Data.Profile[k] = &c
}
} else {
ret.Data.Profile = nil
}
if md.Data.API != nil {
c := *md.Data.API
ret.Data.API = &c
} else {
ret.Data.API = nil
}
if md.Data.Filter != nil {
c := *md.Data.Filter
ret.Data.Filter = &c
} else {
ret.Data.Filter = nil
}
if md.Data.BuildOpts != nil {
c := *md.Data.BuildOpts
if md.Data.BuildOpts.Rpms != nil {
rpms := *md.Data.BuildOpts.Rpms
c.Rpms = &rpms
}
ret.Data.BuildOpts = &c
} else {
ret.Data.BuildOpts = nil
}
if md.Data.Components != nil {
c := *md.Data.Components
if md.Data.Components.Rpms != nil {
rpms := map[string]*modulemd.ComponentRPM{}
for k, v := range md.Data.Components.Rpms {
x := *v
rpms[k] = &x
}
c.Rpms = rpms
}
if md.Data.Components.Modules != nil {
modules := map[string]*modulemd.ComponentModule{}
for k, v := range md.Data.Components.Modules {
x := *v
modules[k] = &x
}
c.Modules = modules
}
ret.Data.Components = &c
} else {
ret.Data.Components = nil
}
if md.Data.Artifacts != nil {
c := *md.Data.Artifacts
if md.Data.Artifacts.RpmMap != nil {
rpmMap := map[string]map[string]*modulemd.ArtifactsRPMMap{}
for k, v := range md.Data.Artifacts.RpmMap {
x := map[string]*modulemd.ArtifactsRPMMap{}
for k2, v2 := range v {
y := *v2
x[k2] = &y
}
rpmMap[k] = x
}
}
ret.Data.Artifacts = &c
} else {
ret.Data.Artifacts = nil
}
return &ret
}
func fillInRpmArtifactsForModuleMd(md *modulemd.ModuleMd, streamBuildOptions *ModuleStreamBuildOptions, buildTask *peridotpb.ModuleStream, artifactPrimaryIndex map[string]ArtifactIndex, arch string, licenses []string, devel bool) error {
newMd := *md
newMd := copyModuleMd(*md)
// Set version, context, arch and licenses
newMd.Data.Version = streamBuildOptions.Version
newMd.Data.Context = streamBuildOptions.Context
@ -973,8 +1134,8 @@ func fillInRpmArtifactsForModuleMd(md *modulemd.ModuleMd, streamBuildOptions *Mo
streamName := streamBuildOptions.Stream
if devel {
newMd.Data.Name = newMd.Data.Name + "-devel"
streamName += "-devel"
newMd.Data.Name = streamBuildOptions.Name + "-devel"
streamName = streamBuildOptions.Stream + "-devel"
}
yamlBytes, err := yaml.Marshal(&newMd)

9
peridot/builder/v1/workflow/rpmimport.go
View File

@ -323,20 +323,13 @@ func (c *Controller) RpmImportActivity(ctx context.Context, req *peridotpb.RpmIm
var nvr string
for _, rpmObj := range rpms {
realNvr := rpmObj.String()
if rpmObj.SourceRPM() == "" && rpmObj.Architecture() == "i686" {
realNvr = strings.ReplaceAll(realNvr, ".i686", ".src")
}
if nvr == "" {
nvr = rpmObj.SourceRPM()
if nvr == "" && rpmObj.Architecture() == "i686" {
if nvr == "" {
nvr = realNvr
}
break
} else {
if nvr != rpmObj.SourceRPM() && nvr != fmt.Sprintf("%s.rpm", realNvr) {
return nil, fmt.Errorf("only include RPMs from one package")
}
}
}
if !rpmutils.NVR().MatchString(nvr) {

4
peridot/builder/v1/workflow/sync.go
View File

@ -926,7 +926,7 @@ func (c *Controller) SyncCatalogActivity(req *peridotpb.SyncCatalogRequest) (*pe
if utils.StrContains(newPackage, newBuildPackages) {
continue
}
dbIDs, err := c.db.GetLatestBuildIdsByPackageName(newPackage, req.ProjectId.Value)
dbIDs, err := c.db.GetLatestBuildIdsByPackageName(newPackage, &req.ProjectId.Value)
if err != nil {
if err == sql.ErrNoRows {
continue
@ -949,7 +949,7 @@ func (c *Controller) SyncCatalogActivity(req *peridotpb.SyncCatalogRequest) (*pe
if utils.StrContains(newPackage, newBuildPackages) {
continue
}
dbIDs, err := c.db.GetLatestBuildIdsByPackageName(newPackage, req.ProjectId.Value)
dbIDs, err := c.db.GetLatestBuildIdsByPackageName(newPackage, &req.ProjectId.Value)
if err != nil {
if err == sql.ErrNoRows {
continue

211
peridot/builder/v1/workflow/yumrepofs.go
View File

@ -75,16 +75,16 @@ var (
)
type UpdateRepoRequest struct {
ProjectID string `json:"projectId"`
BuildIDs []string `json:"buildId"`
TaskID *string `json:"taskId"`
ForceRepoId string `json:"forceRepoId"`
// todo(mustafa): Add support for deleting packages
Delete bool `json:"delete"`
ForceNonModular bool `json:"forceNonModular"`
DisableSigning bool `json:"disableSigning"`
DisableSetActive bool `json:"disableSetActive"`
NoDeletePrevious bool `json:"noDeletePrevious"`
ProjectID string `json:"projectId"`
BuildIDs []string `json:"buildId"`
TaskID *string `json:"taskId"`
ForceRepoId string `json:"forceRepoId"`
Delete bool `json:"delete"`
ForceNonModular bool `json:"forceNonModular"`
DisableSigning bool `json:"disableSigning"`
DisableSetActive bool `json:"disableSetActive"`
NoDeletePrevious bool `json:"noDeletePrevious"`
NoDeleteInChain bool `json:"noDeleteInChain"`
}
type CompiledGlobFilter struct {
@ -105,8 +105,9 @@ type CachedRepo struct {
}
type Cache struct {
GlobFilters map[string]*CompiledGlobFilter
Repos map[string]*CachedRepo
GlobFilters map[string]*CompiledGlobFilter
Repos map[string]*CachedRepo
NoDeleteChain []string
}
// Chain multiple errors and stop processing if any error is returned
@ -379,7 +380,7 @@ func (c *Controller) generateIndexedModuleDefaults(projectId string) (map[string
}
}
index[fmt.Sprintf("module:%s:%s", moduleDefault.Name, moduleDefault.Stream)] = document
index[moduleDefault.Name] = document
}
return index, nil
@ -445,7 +446,7 @@ func (c *Controller) RepoUpdaterWorkflow(ctx workflow.Context, req *UpdateRepoRe
updateRepoCtx := workflow.WithActivityOptions(ctx, workflow.ActivityOptions{
ScheduleToStartTimeout: 25 * time.Hour,
StartToCloseTimeout: 30 * time.Hour,
HeartbeatTimeout: 3 * time.Minute,
HeartbeatTimeout: 30 * time.Second,
TaskQueue: c.mainQueue,
// Yumrepofs is locking for a short period so let's not wait too long to retry
RetryPolicy: &temporal.RetryPolicy{
@ -695,6 +696,7 @@ func (c *Controller) UpdateRepoActivity(ctx context.Context, req *UpdateRepoRequ
updateRepoTask.Changes = reducedChanges
for _, repo := range cache.Repos {
c.log.Infof("processing repo %s - %s", repo.Repo.Name, repo.Repo.ID.String())
primaryRoot := repo.PrimaryRoot
filelistsRoot := repo.FilelistsRoot
otherRoot := repo.OtherRoot
@ -951,7 +953,7 @@ func (c *Controller) UpdateRepoActivity(ctx context.Context, req *UpdateRepoRequ
// todo(mustafa): Convert to request struct
func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest, errorDetails *peridotpb.TaskErrorDetails, packageName string, buildId string, moduleStream *peridotpb.ModuleStream, gpgId *string, signArtifactsTasks *keykeeperpb.BatchSignArtifactsTask, cache *Cache) (*yumrepofspb.UpdateRepoTask, error) {
build, err := tx.GetBuild(req.ProjectID, buildId)
build, err := c.db.GetBuildByID(buildId)
if err != nil {
c.log.Errorf("error getting build: %v", err)
return nil, err
@ -970,7 +972,7 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest
}
project := projects[0]
artifacts, err := tx.GetArtifactsForBuild(buildId)
artifacts, err := c.db.GetArtifactsForBuild(buildId)
if err != nil {
setInternalError(errorDetails, err)
return nil, fmt.Errorf("failed to get artifacts for build: %v", err)
@ -980,13 +982,13 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest
var skipDeleteArtifacts []string
if moduleStream == nil {
// Get currently active artifacts
latestBuilds, err := tx.GetLatestBuildIdsByPackageName(build.PackageName, project.ID.String())
latestBuilds, err := c.db.GetLatestBuildIdsByPackageName(build.PackageName, nil)
if err != nil {
setInternalError(errorDetails, err)
return nil, fmt.Errorf("failed to get latest build ids: %v", err)
}
for _, latestBuild := range latestBuilds {
buildArtifacts, err := tx.GetArtifactsForBuild(latestBuild)
buildArtifacts, err := c.db.GetArtifactsForBuild(latestBuild)
if err != nil {
setInternalError(errorDetails, err)
return nil, fmt.Errorf("failed to get artifacts for build: %v", err)
@ -995,13 +997,13 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest
}
} else {
// Get currently active artifacts
latestBuilds, err := tx.GetLatestBuildsByPackageNameAndBranchName(build.PackageName, moduleStream.ImportRevision.ScmBranchName, project.ID.String())
latestBuilds, err := c.db.GetBuildIDsByPackageNameAndBranchName(build.PackageName, moduleStream.Stream)
if err != nil {
setInternalError(errorDetails, err)
return nil, fmt.Errorf("failed to get latest build ids: %v", err)
}
for _, latestBuild := range latestBuilds {
buildArtifacts, err := tx.GetArtifactsForBuild(latestBuild)
buildArtifacts, err := c.db.GetArtifactsForBuild(latestBuild)
if err != nil {
setInternalError(errorDetails, err)
return nil, fmt.Errorf("failed to get artifacts for build: %v", err)
@ -1011,15 +1013,17 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest
}
// Get artifacts to skip deletion
for _, artifact := range artifacts {
skipDeleteArtifacts = append(skipDeleteArtifacts, strings.TrimSuffix(filepath.Base(artifact.Name), ".rpm"))
if !req.Delete && moduleStream == nil {
for _, artifact := range artifacts {
skipDeleteArtifacts = append(skipDeleteArtifacts, strings.TrimSuffix(filepath.Base(artifact.Name), ".rpm"))
}
}
var repos models.Repositories
if req.ForceRepoId != "" {
var err error
repo, err := tx.GetRepository(&req.ForceRepoId, nil, nil)
repo, err := c.db.GetRepository(&req.ForceRepoId, nil, nil)
if err != nil {
setInternalError(errorDetails, err)
return nil, fmt.Errorf("failed to get repo: %v", err)
@ -1027,7 +1031,7 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest
repos = models.Repositories{*repo}
} else {
var err error
repos, err = tx.FindRepositoriesForPackage(req.ProjectID, packageName, false)
repos, err = c.db.FindRepositoriesForPackage(req.ProjectID, packageName, false)
if err != nil {
setInternalError(errorDetails, err)
return nil, fmt.Errorf("failed to find repo: %v", err)
@ -1041,47 +1045,17 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest
}
for _, repo := range repos {
c.log.Infof("repo: %s, buildId: %s", repo.ID.String(), buildId)
artifactArchMap, err := GenerateArchMapForArtifacts(artifacts, &project, &repo)
if err != nil {
setInternalError(errorDetails, err)
return nil, err
}
var moduleDefaults []*modulemd.Defaults
if defaultsIndex != nil {
for module, defaults := range defaultsIndex {
if utils.StrContains(module, repo.Packages) || len(repo.Packages) == 0 {
moduleDefaults = append(moduleDefaults, &*defaults)
}
}
}
var defaultsYaml []byte
if len(moduleDefaults) > 0 {
var defaultsBuf bytes.Buffer
_, _ = defaultsBuf.WriteString("---\n")
defaultsEncoder := yaml.NewEncoder(&defaultsBuf)
for _, def := range moduleDefaults {
err := defaultsEncoder.Encode(def)
if err != nil {
return nil, fmt.Errorf("failed to encode defaults: %v", err)
}
}
err = defaultsEncoder.Close()
if err != nil {
return nil, fmt.Errorf("failed to close defaults encoder: %v", err)
}
defaultsYaml = defaultsBuf.Bytes()
}
c.log.Infof("generated arch map for build id %s", buildId)
var compiledExcludeGlobs []*CompiledGlobFilter
for _, excludeGlob := range repo.ExcludeFilter {
if cache.GlobFilters[excludeGlob] != nil {
compiledExcludeGlobs = append(compiledExcludeGlobs, cache.GlobFilters[excludeGlob])
continue
}
var arch string
var globVal string
if globFilterArchRegex.MatchString(excludeGlob) {
@ -1100,10 +1074,10 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest
Glob: g,
}
compiledExcludeGlobs = append(compiledExcludeGlobs, globFilter)
cache.GlobFilters[excludeGlob] = globFilter
}
for arch, archArtifacts := range artifactArchMap {
c.log.Infof("arch: %s, buildId: %s", arch, buildId)
noDebugArch := strings.TrimSuffix(arch, "-debug")
var streamDocument *modulemd.ModuleMd
@ -1185,11 +1159,13 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest
var currentRevision *models.RepositoryRevision
var groupsXml string
if cache.Repos[idArchNoDebug] != nil {
c.log.Infof("found cache for %s", idArchNoDebug)
if cache.Repos[idArchNoDebug].Modulemd != nil {
modulesRoot = cache.Repos[idArchNoDebug].Modulemd
}
}
if cache.Repos[idArch] != nil {
c.log.Infof("found cache for %s", idArch)
if cache.Repos[idArch].PrimaryRoot != nil {
primaryRoot = *cache.Repos[idArch].PrimaryRoot
}
@ -1201,16 +1177,18 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest
}
groupsXml = cache.Repos[idArch].GroupsXml
} else {
c.log.Infof("no cache for %s", idArch)
var noDebugRevision *models.RepositoryRevision
currentRevision, err = tx.GetLatestActiveRepositoryRevision(repo.ID.String(), arch)
currentRevision, err = c.db.GetLatestActiveRepositoryRevision(repo.ID.String(), arch)
if err != nil {
if err != sql.ErrNoRows {
return nil, fmt.Errorf("failed to get latest active repository revision: %v", err)
}
}
if strings.HasSuffix(arch, "-debug") {
noDebugRevision, err = tx.GetLatestActiveRepositoryRevision(repo.ID.String(), noDebugArch)
if strings.HasSuffix(arch, "-debug") && moduleStream != nil {
c.log.Infof("arch has debug and module stream is not nil")
noDebugRevision, err = c.db.GetLatestActiveRepositoryRevision(repo.ID.String(), noDebugArch)
if err != nil {
if err != sql.ErrNoRows {
return nil, fmt.Errorf("failed to get latest active repository revision: %v", err)
@ -1221,6 +1199,7 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest
}
if currentRevision != nil {
c.log.Infof("current revision is not nil")
if currentRevision.PrimaryXml != "" {
var primaryXmlGz []byte
var primaryXml []byte
@ -1303,22 +1282,13 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest
}
}
moduleArtifacts := map[string]bool{}
if moduleStream != nil && len(modulesRoot) > 0 {
for _, md := range modulesRoot {
if md.Data.Name == moduleStream.Name && md.Data.Stream == moduleStream.Stream {
for _, artifact := range md.Data.Artifacts.Rpms {
moduleArtifacts[artifact] = true
}
}
}
}
c.log.Infof("processing %d artifacts", len(archArtifacts))
for _, artifact := range archArtifacts {
// This shouldn't happen
if !artifact.Metadata.Valid {
continue
}
c.log.Infof("processing artifact %s", artifact.Name)
var name string
base := strings.TrimSuffix(filepath.Base(artifact.Name), ".rpm")
@ -1336,8 +1306,8 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest
archName = fmt.Sprintf("%s.%s", noDebugInfoName, artifact.Arch)
}
shouldAdd := true
if arch != "src" && moduleStream == nil {
shouldAdd := !req.Delete
if arch != "src" && moduleStream == nil && !req.Delete {
// If repo has a list for inclusion, then the artifact has to pass that first
if len(repo.IncludeFilter) > 0 {
// If the artifact isn't forced, it should be in the include list or additional multilib list
@ -1348,7 +1318,7 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest
// Check if it matches any exclude filter
for _, excludeFilter := range compiledExcludeGlobs {
if excludeFilter.Arch != "" && excludeFilter.Arch != strings.TrimSuffix(arch, "-debug") {
if excludeFilter.Arch != "" && excludeFilter.Arch != noDebugArch {
continue
}
if excludeFilter.Glob.Match(noDebugInfoName) || excludeFilter.Glob.Match(archName) {
@ -1356,14 +1326,10 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest
}
}
}
c.log.Infof("should add %s: %v", artifact.Name, shouldAdd)
baseNoRpm := strings.Replace(filepath.Base(artifact.Name), ".rpm", "", 1)
if !shouldAdd {
changes.RemovedPackages = append(changes.RemovedPackages, baseNoRpm)
continue
}
var anyMetadata anypb.Any
err := protojson.Unmarshal(artifact.Metadata.JSONText, &anyMetadata)
if err != nil {
@ -1388,6 +1354,8 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest
return nil, err
}
c.log.Infof("unmarshalled metadata for %s", artifact.Name)
if gpgId != nil {
newObjectKey := fmt.Sprintf("%s/%s/%s", filepath.Dir(artifact.Name), *gpgId, filepath.Base(artifact.Name))
@ -1430,7 +1398,10 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest
break
}
} else {
// If a module stream, search for a module entry
if !strings.Contains(primaryPackage.Version.Rel, ".module+") {
continue
}
for _, streamArtifact := range artifacts {
var rpmName string
var rpmVersion string
@ -1457,13 +1428,28 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest
}
}
}
if !shouldAdd {
if primaryIndex != nil {
changes.RemovedPackages = append(changes.RemovedPackages, baseNoRpm)
}
continue
}
if primaryIndex != nil {
changes.ModifiedPackages = append(changes.ModifiedPackages, baseNoRpm)
c.log.Infof("found primary index %d", *primaryIndex)
if !utils.StrContains(baseNoRpm, changes.ModifiedPackages) && !utils.StrContains(baseNoRpm, changes.AddedPackages) {
changes.ModifiedPackages = append(changes.ModifiedPackages, baseNoRpm)
}
primaryRoot.Packages[*primaryIndex] = pkgPrimary.Packages[0]
} else {
changes.AddedPackages = append(changes.AddedPackages, baseNoRpm)
c.log.Infof("did not find primary index")
if !utils.StrContains(baseNoRpm, changes.AddedPackages) && !utils.StrContains(baseNoRpm, changes.ModifiedPackages) {
changes.AddedPackages = append(changes.AddedPackages, baseNoRpm)
}
primaryRoot.Packages = append(primaryRoot.Packages, pkgPrimary.Packages[0])
}
cache.NoDeleteChain = append(cache.NoDeleteChain, baseNoRpm)
var filelistsIndex *int
if pkgId != nil {
@ -1497,6 +1483,7 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest
otherRoot.Packages = append(otherRoot.Packages, pkgOther.Packages[0])
}
}
c.log.Infof("processed %d artifacts", len(archArtifacts))
// First let's delete older artifacts
// Instead of doing re-slicing, let's just not add anything matching the artifacts
@ -1506,16 +1493,28 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest
var nFilelists []*yummeta.FilelistsPackage
var nOther []*yummeta.OtherPackage
var deleteIds []string
if !req.NoDeletePrevious || moduleStream != nil {
if (!req.NoDeletePrevious || moduleStream != nil) || req.Delete {
for _, pkg := range primaryRoot.Packages {
shouldAdd := true
for _, artifact := range currentActiveArtifacts {
noRpmName := strings.TrimSuffix(filepath.Base(artifact.Name), ".rpm")
if filepath.Base(artifact.Name) == filepath.Base(pkg.Location.Href) && !utils.StrContains(noRpmName, changes.ModifiedPackages) && !utils.StrContains(noRpmName, changes.AddedPackages) && !utils.StrContains(noRpmName, skipDeleteArtifacts) {
shouldAdd = false
shouldAdd := !req.Delete
if !req.Delete {
for _, artifact := range currentActiveArtifacts {
noRpmName := strings.TrimSuffix(filepath.Base(artifact.Name), ".rpm")
if filepath.Base(artifact.Name) == filepath.Base(pkg.Location.Href) && !utils.StrContains(noRpmName, changes.ModifiedPackages) && !utils.StrContains(noRpmName, changes.AddedPackages) && !utils.StrContains(noRpmName, skipDeleteArtifacts) {
shouldAdd = false
if req.NoDeleteInChain {
if utils.StrContains(noRpmName, cache.NoDeleteChain) {
shouldAdd = true
}
}
}
}
}
noRpmNamePkg := strings.TrimSuffix(filepath.Base(pkg.Location.Href), ".rpm")
if utils.StrContains(noRpmNamePkg, changes.RemovedPackages) {
shouldAdd = false
}
if !shouldAdd {
c.log.Infof("deleting %s", pkg.Location.Href)
deleteIds = append(deleteIds, pkg.Checksum.Value)
}
}
@ -1554,6 +1553,7 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest
var moduleIndex *int
for i, moduleMd := range modulesRoot {
if moduleMd.Data.Name == moduleStream.Name && moduleMd.Data.Stream == moduleStream.Stream {
c.log.Infof("found existing module entry for %s:%s", moduleStream.Name, moduleStream.Stream)
moduleIndex = &*&i
break
}
@ -1562,11 +1562,43 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest
changes.ModifiedModules = append(changes.ModifiedModules, fmt.Sprintf("%s:%s", moduleStream.Name, moduleStream.Stream))
modulesRoot[*moduleIndex] = streamDocument
} else {
c.log.Infof("adding new module entry for %s:%s", moduleStream.Name, moduleStream.Stream)
changes.AddedModules = append(changes.AddedModules, fmt.Sprintf("%s:%s", moduleStream.Name, moduleStream.Stream))
modulesRoot = append(modulesRoot, streamDocument)
}
}
var moduleDefaults []*modulemd.Defaults
if defaultsIndex != nil {
for module, defaults := range defaultsIndex {
for _, rootModule := range modulesRoot {
if module == rootModule.Data.Name {
moduleDefaults = append(moduleDefaults, defaults)
break
}
}
}
}
var defaultsYaml []byte
if len(moduleDefaults) > 0 {
var defaultsBuf bytes.Buffer
_, _ = defaultsBuf.WriteString("---\n")
defaultsEncoder := yaml.NewEncoder(&defaultsBuf)
for _, def := range moduleDefaults {
err := defaultsEncoder.Encode(def)
if err != nil {
return nil, fmt.Errorf("failed to encode defaults: %v", err)
}
}
err = defaultsEncoder.Close()
if err != nil {
return nil, fmt.Errorf("failed to close defaults encoder: %v", err)
}
defaultsYaml = defaultsBuf.Bytes()
}
nRepo := repo
cache.Repos[idArch] = &CachedRepo{
Arch: arch,
@ -1582,6 +1614,7 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest
if strings.HasSuffix(arch, "-debug") || arch == "src" {
cache.Repos[idArch].Modulemd = nil
}
c.log.Infof("set cache for %s", idArch)
repoTask.Changes = append(repoTask.Changes, changes)
}
@ -1595,5 +1628,7 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest
}
}
c.log.Infof("finished processing %d artifacts", len(artifacts))
return repoTask, nil
}

2
peridot/cmd/v1/peridotephemeral/main.go
View File

@ -106,6 +106,8 @@ func mn(_ *cobra.Command, _ []string) {
w.Worker.RegisterWorkflow(w.WorkflowController.RpmImportWorkflow)
w.Worker.RegisterWorkflow(w.WorkflowController.RpmLookasideBatchImportWorkflow)
w.Worker.RegisterWorkflow(w.WorkflowController.CreateHashedRepositoriesWorkflow)
w.Worker.RegisterWorkflow(w.WorkflowController.CloneSwapWorkflow)
w.Worker.RegisterActivity(w.WorkflowController.CloneSwapActivity)
}
w.Worker.RegisterWorkflow(w.WorkflowController.ProvisionWorkerWorkflow)
w.Worker.RegisterWorkflow(w.WorkflowController.DestroyWorkerWorkflow)

12
peridot/cmd/v1/yumrepofsupdater/ci/deploy.jsonnet
View File

@ -6,7 +6,7 @@ local utils = import 'ci/utils.jsonnet';
resfdeploy.new({
name: 'yumrepofsupdater',
replicas: if kubernetes.prod() then 4 else 1,
replicas: if kubernetes.prod() then 1 else 1,
dbname: 'peridot',
backend: true,
migrate: true,
@ -21,12 +21,12 @@ resfdeploy.new({
value: db.dsn_legacy('peridot', false, 'yumrepofsupdater'),
},
requests: if kubernetes.prod() then {
cpu: '0.5',
memory: '1G',
cpu: '2',
memory: '15G',
},
limits: if kubernetes.prod() then {
cpu: '3',
memory: '64G',
node_pool_request: if kubernetes.prod() then {
key: 'peridot.rockylinux.org/workflow-tolerates-arch',
value: 'amd64',
},
service_account_options: {
annotations: {

10
peridot/db/db.go
View File

@ -53,9 +53,11 @@ type Access interface {
GetBuildCount() (int64, error)
CreateBuildBatch(projectId string) (string, error)
AttachBuildToBatch(buildId string, batchId string) error
ListBuilds(projectId string, page int32, limit int32) (models.Builds, error)
ListBuilds(filters *peridotpb.BuildFilters, projectId string, page int32, limit int32) (models.Builds, error)
GetSuccessfulBuildIDsAsc(projectId string) ([]string, error)
BuildCountInProject(projectId string) (int64, error)
GetBuild(projectId string, buildId string) (*models.Build, error)
GetBuildByID(buildId string) (*models.Build, error)
GetBuildByTaskIdAndPackageId(taskId string, packageId string) (*models.Build, error)
GetBuildBatch(projectId string, batchId string, batchFilter *peridotpb.BatchFilter, page int32, limit int32) (models.Builds, error)
ListBuildBatches(projectId string, batchId *string, page int32, limit int32) (models.BuildBatches, error)
@ -64,9 +66,9 @@ type Access interface {
LockNVRA(nvra string) error
UnlockNVRA(nvra string) error
NVRAExists(nvra string) (bool, error)
GetBuildByPackageNameAndVersionAndRelease(name string, version string, release string, projectId string) (*models.Build, error)
GetLatestBuildIdsByPackageName(name string, projectId string) ([]string, error)
GetLatestBuildsByPackageNameAndBranchName(name string, branchName string, projectId string) ([]string, error)
GetBuildByPackageNameAndVersionAndRelease(name string, version string, release string) (models.Builds, error)
GetLatestBuildIdsByPackageName(name string, projectId *string) ([]string, error)
GetBuildIDsByPackageNameAndBranchName(name string, branchName string) ([]string, error)
GetActiveBuildIdsByTaskArtifactGlob(taskArtifactGlob string, projectId string) ([]string, error)
GetAllBuildIdsByPackageName(name string, projectId string) ([]string, error)

102
peridot/db/psql/build.go
View File

@ -108,7 +108,11 @@ func (a *Access) AttachBuildToBatch(buildId string, batchId string) error {
return nil
}
func (a *Access) ListBuilds(projectId string, page int32, limit int32) (models.Builds, error) {
func (a *Access) ListBuilds(filters *peridotpb.BuildFilters, projectId string, page int32, limit int32) (models.Builds, error) {
if filters == nil {
filters = &peridotpb.BuildFilters{}
}
var ret models.Builds
err := a.query.Select(
&ret,
@ -128,11 +132,14 @@ func (a *Access) ListBuilds(projectId string, page int32, limit int32) (models.B
from builds b
inner join tasks t on t.id = b.task_id
inner join packages p on p.id = b.package_id
where b.project_id = $1
where
b.project_id = $1
and ($2 :: int is null or $2 :: int = 0 or t.status = $2 :: int)
order by b.created_at desc
limit $2 offset $3
limit $3 offset $4
`,
projectId,
filters.Status,
limit,
utils.GetOffset(page, limit),
)
@ -143,6 +150,29 @@ func (a *Access) ListBuilds(projectId string, page int32, limit int32) (models.B
return ret, nil
}
func (a *Access) GetSuccessfulBuildIDsAsc(projectId string) ([]string, error) {
var ret []string
err := a.query.Select(
&ret,
`
select
b.id
from builds b
inner join tasks t on t.id = b.task_id
where
b.project_id = $1
and t.status = 3
order by b.created_at asc
`,
projectId,
)
if err != nil {
return nil, err
}
return ret, nil
}
func (a *Access) BuildCountInProject(projectId string) (int64, error) {
var count int64
err := a.query.Get(&count, "select count(*) from imports where project_id = $1", projectId)
@ -186,6 +216,38 @@ func (a *Access) GetBuild(projectId string, buildId string) (*models.Build, erro
return &ret, nil
}
func (a *Access) GetBuildByID(buildId string) (*models.Build, error) {
var ret models.Build
err := a.query.Get(
&ret,
`
select
b.id,
b.created_at,
b.package_id,
p.name as package_name,
b.package_version_id,
b.task_id,
b.project_id,
t.status as task_status,
t.response as task_response,
t.metadata as task_metadata
from builds b
inner join tasks t on t.id = b.task_id
inner join packages p on p.id = b.package_id
where
b.id = $1
order by b.created_at desc
`,
buildId,
)
if err != nil {
return nil, err
}
return &ret, nil
}
func (a *Access) GetBuildByTaskIdAndPackageId(taskId string, packageId string) (*models.Build, error) {
var ret models.Build
err := a.query.Get(
@ -358,9 +420,9 @@ func (a *Access) NVRAExists(nvra string) (bool, error) {
return false, nil
}
func (a *Access) GetBuildByPackageNameAndVersionAndRelease(name string, version string, release string, projectId string) (*models.Build, error) {
var ret models.Build
err := a.query.Get(
func (a *Access) GetBuildByPackageNameAndVersionAndRelease(name string, version string, release string) (models.Builds, error) {
var ret models.Builds
err := a.query.Select(
&ret,
`
select
@ -379,15 +441,12 @@ func (a *Access) GetBuildByPackageNameAndVersionAndRelease(name string, version
inner join packages p on p.id = b.package_id
inner join package_versions pv on pv.id = b.package_version_id
where
b.project_id = $1
and p.name = $2
and pv.version = $3
and pv.release = $4
p.name = $1
and pv.version = $2
and pv.release = $3
and t.status = 3
order by b.created_at desc
limit 1
`,
projectId,
name,
version,
release,
@ -396,10 +455,10 @@ func (a *Access) GetBuildByPackageNameAndVersionAndRelease(name string, version
return nil, err
}
return &ret, nil
return ret, nil
}
func (a *Access) GetLatestBuildIdsByPackageName(name string, projectId string) ([]string, error) {
func (a *Access) GetLatestBuildIdsByPackageName(name string, projectId *string) ([]string, error) {
var ret []string
err := a.query.Select(
&ret,
@ -412,10 +471,8 @@ func (a *Access) GetLatestBuildIdsByPackageName(name string, projectId string) (
inner join project_package_versions ppv on ppv.package_version_id = b.package_version_id
where
p.name = $2
and t.status = 3
and ppv.active_in_repo = true
and ppv.project_id = b.project_id
and b.project_id = $1
and ($1 :: uuid is null or b.project_id = $1 :: uuid)
order by b.created_at asc
`,
projectId,
@ -428,7 +485,7 @@ func (a *Access) GetLatestBuildIdsByPackageName(name string, projectId string) (
return ret, nil
}
func (a *Access) GetLatestBuildsByPackageNameAndBranchName(name string, branchName string, projectId string) ([]string, error) {
func (a *Access) GetBuildIDsByPackageNameAndBranchName(name string, branchName string) ([]string, error) {
var ret []string
err := a.query.Select(
&ret,
@ -441,17 +498,12 @@ func (a *Access) GetLatestBuildsByPackageNameAndBranchName(name string, branchNa
inner join project_package_versions ppv on ppv.package_version_id = b.package_version_id
inner join import_revisions ir on ir.package_version_id = b.package_version_id
where
b.project_id = $3
and p.name = $1
and ppv.active_in_repo = true
and ppv.project_id = b.project_id
and ir.scm_branch_name = $2
and t.status = 3
p.name = $1
and ir.scm_branch_name like '%-stream-' || $2
order by b.created_at asc
`,
name,
branchName,
projectId,
)
if err != nil {
return nil, err

2
peridot/impl/v1/build.go
View File

@ -57,7 +57,7 @@ func (s *Server) ListBuilds(ctx context.Context, req *peridotpb.ListBuildsReques
page := utils.MinPage(req.Page)
limit := utils.MinLimit(req.Limit)
builds, err := s.db.ListBuilds(req.ProjectId, page, limit)
builds, err := s.db.ListBuilds(req.Filters, req.ProjectId, page, limit)
if err != nil {
s.log.Errorf("could not list builds: %v", err)
return nil, utils.CouldNotRetrieveObjects

68
peridot/impl/v1/project.go
View File

@ -448,3 +448,71 @@ func (s *Server) LookasideFileUpload(ctx context.Context, req *peridotpb.Lookasi
Digest: sha256Sum,
}, nil
}
func (s *Server) CloneSwap(ctx context.Context, req *peridotpb.CloneSwapRequest) (*peridotpb.AsyncTask, error) {
if err := req.ValidateAll(); err != nil {
return nil, err
}
if err := s.checkPermission(ctx, ObjectProject, req.TargetProjectId.Value, PermissionManage); err != nil {
return nil, err
}
if err := s.checkPermission(ctx, ObjectProject, req.SrcProjectId.Value, PermissionView); err != nil {
return nil, err
}
user, err := utils.UserFromContext(ctx)
if err != nil {
return nil, err
}
rollback := true
beginTx, err := s.db.Begin()
if err != nil {
s.log.Error(err)
return nil, utils.InternalError
}
defer func() {
if rollback {
_ = beginTx.Rollback()
}
}()
tx := s.db.UseTransaction(beginTx)
task, err := tx.CreateTask(user, "noarch", peridotpb.TaskType_TASK_TYPE_CLONE_SWAP, &req.TargetProjectId.Value, nil)
if err != nil {
s.log.Errorf("could not create task: %v", err)
return nil, utils.InternalError
}
taskProto, err := task.ToProto(false)
if err != nil {
return nil, status.Errorf(codes.Internal, "could not marshal task: %v", err)
}
rollback = false
err = beginTx.Commit()
if err != nil {
return nil, status.Error(codes.Internal, "could not save, try again")
}
_, err = s.temporal.ExecuteWorkflow(
context.Background(),
client.StartWorkflowOptions{
ID: task.ID.String(),
TaskQueue: MainTaskQueue,
},
s.temporalWorker.WorkflowController.CloneSwapWorkflow,
req,
task,
)
if err != nil {
s.log.Errorf("could not start workflow: %v", err)
_ = s.db.SetTaskStatus(task.ID.String(), peridotpb.TaskStatus_TASK_STATUS_FAILED)
return nil, err
}
return &peridotpb.AsyncTask{
TaskId: task.ID.String(),
Subtasks: []*peridotpb.Subtask{taskProto},
Done: false,
}, nil
}

1
peridot/proto/v1/build.proto
View File

@ -297,6 +297,7 @@ message ExtraYumrepofsRepo {
string name = 1;
bool module_hotfixes = 2;
bool ignore_exclude = 3;
int32 priority = 4;
}
// These options can be used to customize the behavior of the service

18
peridot/proto/v1/project.proto
View File

@ -83,6 +83,13 @@ service ProjectService {
};
}
rpc CloneSwap(CloneSwapRequest) returns (resf.peridot.v1.AsyncTask) {
option (google.api.http) = {
post: "/v1/projects/{target_project_id=*}/cloneswap"
body: "*"
};
}
rpc ListExternalRepositories(ListExternalRepositoriesRequest) returns (ListExternalRepositoriesResponse) {
option (google.api.http) = {
get: "/v1/projects/{project_id=*}/external_repositories"
@ -311,6 +318,17 @@ message CreateHashedRepositoriesTask {
repeated string repo_revisions = 1;
}
message CloneSwapRequest {
google.protobuf.StringValue target_project_id = 1 [(validate.rules).message.required = true];
google.protobuf.StringValue src_project_id = 2 [(validate.rules).message.required = true];
}
message CloneSwapTask {
string target_project_id = 1;
string src_project_id = 2;
repeated string build_ids_layered = 3;
}
message LookasideFileUploadRequest {
string file = 1 [(validate.rules).string.min_bytes = 1];
}

1
peridot/proto/v1/task.proto
View File

@ -73,6 +73,7 @@ enum TaskType {
TASK_TYPE_CREATE_HASHED_REPOSITORIES = 17;
TASK_TYPE_LOOKASIDE_FILE_UPLOAD = 18;
TASK_TYPE_RPM_LOOKASIDE_BATCH_IMPORT = 19;
TASK_TYPE_CLONE_SWAP = 20;
}
enum TaskStatus {

8
utils/pointer.go
View File

@ -121,6 +121,14 @@ func NullStringToPointer(s sql.NullString) *string {
return &s.String
}
func NullStringToEmptyString(s sql.NullString) string {
if !s.Valid {
return ""
}
return s.String
}
func Int64(i int64) *int64 {
return &i
}

2
vendor/github.com/rocky-linux/srpmproc/pkg/rpmutils/regex.go generated vendored
View File

@ -3,4 +3,4 @@ package rpmutils
import "regexp"
// Nvr is a regular expression that matches a NVR.
var Nvr = regexp.MustCompile("^(\\S+)-([\\w~%.+]+)-(\\w+(?:\\.[\\w+]+)+?)(?:\\.rpm)?$")
var Nvr = regexp.MustCompile("^(\\S+)-([\\w~%.+]+)-(\\w+(?:\\.[\\w~%+]+)+?)(?:\\.rpm)?$")

3
vendor/modules.txt vendored
View File

@ -453,7 +453,7 @@ github.com/prometheus/procfs/internal/util
github.com/rivo/uniseg
# github.com/robfig/cron v1.2.0
github.com/robfig/cron
# github.com/rocky-linux/srpmproc v0.4.2
# github.com/rocky-linux/srpmproc v0.4.3
## explicit
github.com/rocky-linux/srpmproc/modulemd
github.com/rocky-linux/srpmproc/pb
@ -636,7 +636,6 @@ golang.org/x/sys/plan9
golang.org/x/sys/unix
golang.org/x/sys/windows
# golang.org/x/term v0.0.0-20210927222741-03fcf44c2211
## explicit
golang.org/x/term
# golang.org/x/text v0.3.7
golang.org/x/text/encoding