rocky-linux/peridot
4
0
Fork 0
mirror of https://github.com/rocky-linux/peridot.git synced 2024-12-21 02:08:29 +00:00
Code Issues Projects Releases Activity

Merge pull request #99 from PaulCharlton/issues/pcharlton/roc-1394

Browse Source 
allow peridot project to specify a build pool type in additional to build pool architecture
This commit is contained in:
resf-prow[bot] 2023-02-18 00:38:50 +00:00 committed by GitHub
commit fe05de365a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 60 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@ -43,6 +43,9 @@ vendor/peridot.resf.org
.ijwb/.idea/**/dictionaries .ijwb/.idea/**/dictionaries
.ijwb/.idea/**/shelf .ijwb/.idea/**/shelf
# top-level IDE configuration - user specific contents
/.idea/**
# AWS User-specific # AWS User-specific
.ijwb/.idea/**/aws.xml .ijwb/.idea/**/aws.xml

46
peridot/builder/v1/workflow/infrastructure.go
View File

@ -61,13 +61,15 @@ import (
) )
type ProvisionWorkerRequest struct { type ProvisionWorkerRequest struct {
TaskId string `json:"taskId"` TaskId string `json:"taskId"`
ParentTaskId sql.NullString `json:"parentTaskId"` ParentTaskId sql.NullString `json:"parentTaskId"`
Purpose string `json:"purpose"` Purpose string `json:"purpose"`
Arch string `json:"arch"` Arch string `json:"arch"`
ProjectId string `json:"projectId"` ImageArch string `json:"imageArch"`
HighResource bool `json:"highResource"` BuildPoolType string `json:"buildPoolType"`
Privileged bool `json:"privileged"` ProjectId string `json:"projectId"`
HighResource bool `json:"highResource"`
Privileged bool `json:"privileged"`
} }
func archToGoArch(arch string) string { func archToGoArch(arch string) string {
@ -93,6 +95,14 @@ func goArchToArch(arch string) string {
} }
} }
func buildPoolArch(goArch string, req *ProvisionWorkerRequest) (result string) {
result = goArch
if len(req.BuildPoolType) > 0 {
result = result + "-" + req.BuildPoolType
}
return result
}
func (c *Controller) genNameWorker(buildID, purpose string) string { func (c *Controller) genNameWorker(buildID, purpose string) string {
return strings.ReplaceAll(fmt.Sprintf("pb-%s-%s", buildID, purpose), "_", "-") return strings.ReplaceAll(fmt.Sprintf("pb-%s-%s", buildID, purpose), "_", "-")
} }
@ -171,7 +181,13 @@ func (c *Controller) provisionWorker(ctx workflow.Context, req *ProvisionWorkerR
ctx = workflow.WithChildOptions(ctx, workflow.ChildWorkflowOptions{ ctx = workflow.WithChildOptions(ctx, workflow.ChildWorkflowOptions{
TaskQueue: queue, TaskQueue: queue,
}) })
err := workflow.ExecuteChildWorkflow(ctx, c.ProvisionWorkerWorkflow, req, queue, imageArch).Get(ctx, &podName) req.ImageArch = imageArch
if project.BuildPoolType.Valid {
req.BuildPoolType = project.BuildPoolType.String
} else {
req.BuildPoolType = ""
}
err := workflow.ExecuteChildWorkflow(ctx, c.ProvisionWorkerWorkflow, req, queue).Get(ctx, &podName)
if err != nil { if err != nil {
var applicationErr *temporal.ApplicationError var applicationErr *temporal.ApplicationError
if errors.As(err, &applicationErr) { if errors.As(err, &applicationErr) {
@ -200,7 +216,7 @@ func (c *Controller) provisionWorker(ctx workflow.Context, req *ProvisionWorkerR
// ProvisionWorkerWorkflow provisions a new job specific container using // ProvisionWorkerWorkflow provisions a new job specific container using
// the provided ephemeral provisioner. // the provided ephemeral provisioner.
// Returns an identifier // Returns an identifier
func (c *Controller) ProvisionWorkerWorkflow(ctx workflow.Context, req *ProvisionWorkerRequest, queue string, imageArch string) (string, error) { func (c *Controller) ProvisionWorkerWorkflow(ctx workflow.Context, req *ProvisionWorkerRequest, queue string) (string, error) {
var task models.Task var task models.Task
taskSideEffect := workflow.SideEffect(ctx, func(ctx workflow.Context) interface{} { taskSideEffect := workflow.SideEffect(ctx, func(ctx workflow.Context) interface{} {
var projectId *string var projectId *string
@ -247,7 +263,7 @@ func (c *Controller) ProvisionWorkerWorkflow(ctx workflow.Context, req *Provisio
ctx = workflow.WithActivityOptions(ctx, options) ctx = workflow.WithActivityOptions(ctx, options)
var podName string var podName string
if err := workflow.ExecuteActivity(ctx, c.CreateK8sPodActivity, req, task, imageArch).Get(ctx, &podName); err != nil { if err := workflow.ExecuteActivity(ctx, c.CreateK8sPodActivity, req, task).Get(ctx, &podName); err != nil {
return "", err return "", err
} }
@ -444,7 +460,7 @@ func (c *Controller) IngestLogsActivity(ctx context.Context, podName string, tas
// CreateK8sPodActivity creates a new pod in the same namespace // CreateK8sPodActivity creates a new pod in the same namespace
// with the specified container (the container has to contain peridotbuilder) // with the specified container (the container has to contain peridotbuilder)
func (c *Controller) CreateK8sPodActivity(ctx context.Context, req *ProvisionWorkerRequest, task *models.Task, imageArch string) (string, error) { func (c *Controller) CreateK8sPodActivity(ctx context.Context, req *ProvisionWorkerRequest, task *models.Task) (string, error) {
stopChan := makeHeartbeat(ctx, 3*time.Second) stopChan := makeHeartbeat(ctx, 3*time.Second)
defer func() { stopChan <- true }() defer func() { stopChan <- true }()
@ -465,7 +481,9 @@ func (c *Controller) CreateK8sPodActivity(ctx context.Context, req *ProvisionWor
production = "true" production = "true"
} }
imageArch := req.ImageArch
goArch := archToGoArch(imageArch) goArch := archToGoArch(imageArch)
nodePoolArch := buildPoolArch(goArch, req)
_ = c.logToMon([]string{ _ = c.logToMon([]string{
fmt.Sprintf("Creating worker for purpose %s", req.Purpose), fmt.Sprintf("Creating worker for purpose %s", req.Purpose),
@ -566,7 +584,7 @@ func (c *Controller) CreateK8sPodActivity(ctx context.Context, req *ProvisionWor
"peridot.rockylinux.org/managed-by": "peridotephemeral", "peridot.rockylinux.org/managed-by": "peridotephemeral",
"peridot.rockylinux.org/task-id": req.TaskId, "peridot.rockylinux.org/task-id": req.TaskId,
"peridot.rockylinux.org/name": name, "peridot.rockylinux.org/name": name,
"peridot.rockylinux.org/workflow-tolerates-arch": goArch, "peridot.rockylinux.org/workflow-tolerates-arch": nodePoolArch,
// todo(mustafa): Implement janitor (cron workflow?) // todo(mustafa): Implement janitor (cron workflow?)
"janitor.peridot.rockylinux.org/allow-cleanup": "yes", "janitor.peridot.rockylinux.org/allow-cleanup": "yes",
"janitor.peridot.rockylinux.org/cleanup-timeout": "864000s", "janitor.peridot.rockylinux.org/cleanup-timeout": "864000s",
@ -656,7 +674,7 @@ func (c *Controller) CreateK8sPodActivity(ctx context.Context, req *ProvisionWor
{ {
Key: "peridot.rockylinux.org/workflow-tolerates-arch", Key: "peridot.rockylinux.org/workflow-tolerates-arch",
Operator: v1.TolerationOpEqual, Operator: v1.TolerationOpEqual,
Value: goArch, Value: nodePoolArch,
Effect: v1.TaintEffectNoSchedule, Effect: v1.TaintEffectNoSchedule,
}, },
}, },
@ -721,7 +739,7 @@ func (c *Controller) CreateK8sPodActivity(ctx context.Context, req *ProvisionWor
{ {
Key: "peridot.rockylinux.org/workflow-tolerates-arch", Key: "peridot.rockylinux.org/workflow-tolerates-arch",
Operator: "In", Operator: "In",
Values: []string{goArch}, Values: []string{nodePoolArch},
}, },
}, },
}, },

12
peridot/builder/v1/workflow/module.go
View File

@ -120,12 +120,12 @@ type ArtifactIndex struct {
// Peridot doesn't need to generate and install arbitrary macro RPMs to define macros. // Peridot doesn't need to generate and install arbitrary macro RPMs to define macros.
// For that reason, we can define macros in the container we create. // For that reason, we can define macros in the container we create.
// Module components requires the following macros: // Module components requires the following macros:
// * %dist -> as described above // - %dist -> as described above
// * %_module_build -> the module increment (iteration) // - %_module_build -> the module increment (iteration)
// * %_module_name -> the module name // - %_module_name -> the module name
// * %_module_stream -> the module stream // - %_module_stream -> the module stream
// * %_module_version -> generated version (as describe above) // - %_module_version -> generated version (as describe above)
// * %_module_context -> generated context (calculate sha1 of the buildrequires section) # todo(mustafa): Currently the yaml content is used to calculate the context // - %_module_context -> generated context (calculate sha1 of the buildrequires section) # todo(mustafa): Currently the yaml content is used to calculate the context
// //
// The macros above will be written to the following file: /etc/rpm/macros.zz-module // The macros above will be written to the following file: /etc/rpm/macros.zz-module
// This is to ensure that the macros are applied last. // This is to ensure that the macros are applied last.

1
peridot/db/models/project.go
View File

@ -64,6 +64,7 @@ type Project struct {
AdditionalVendor string `json:"additionalVendor" db:"additional_vendor"` AdditionalVendor string `json:"additionalVendor" db:"additional_vendor"`
Archs pq.StringArray `json:"archs" db:"archs"` Archs pq.StringArray `json:"archs" db:"archs"`
BuildPoolType sql.NullString `json:"buildPoolType" db:"build_pool_type"`
FollowImportDist bool `json:"followImportDist" db:"follow_import_dist"` FollowImportDist bool `json:"followImportDist" db:"follow_import_dist"`
BranchSuffix sql.NullString `json:"branchSuffix" db:"branch_suffix"` BranchSuffix sql.NullString `json:"branchSuffix" db:"branch_suffix"`
GitMakePublic bool `json:"gitMakePublic" db:"git_make_public"` GitMakePublic bool `json:"gitMakePublic" db:"git_make_public"`

23
peridot/db/psql/project.go
View File

@ -73,6 +73,7 @@ func (a *Access) ListProjects(filters *peridotpb.ProjectFilters) (ret models.Pro
target_vendor, target_vendor,
additional_vendor, additional_vendor,
archs, archs,
build_pool_type,
follow_import_dist, follow_import_dist,
branch_suffix, branch_suffix,
git_make_public, git_make_public,
@ -200,6 +201,7 @@ func (a *Access) CreateProject(project *peridotpb.Project) (*models.Project, err
TargetVendor: project.TargetVendor, TargetVendor: project.TargetVendor,
AdditionalVendor: project.AdditionalVendor.Value, AdditionalVendor: project.AdditionalVendor.Value,
Archs: project.Archs, Archs: project.Archs,
BuildPoolType: utils.StringValueToNullString(project.BuildPoolType),
FollowImportDist: project.FollowImportDist, FollowImportDist: project.FollowImportDist,
BranchSuffix: utils.StringValueToNullString(project.BranchSuffix), BranchSuffix: utils.StringValueToNullString(project.BranchSuffix),
GitMakePublic: project.GitMakePublic, GitMakePublic: project.GitMakePublic,
@ -213,8 +215,9 @@ func (a *Access) CreateProject(project *peridotpb.Project) (*models.Project, err
insert into projects insert into projects
(name, major_version, dist_tag_override, target_gitlab_host, target_prefix, (name, major_version, dist_tag_override, target_gitlab_host, target_prefix,
target_branch_prefix, source_git_host, source_prefix, source_branch_prefix, cdn_url, target_branch_prefix, source_git_host, source_prefix, source_branch_prefix, cdn_url,
stream_mode, target_vendor, additional_vendor, archs, follow_import_dist, branch_suffix, git_make_public, vendor_macro, packager_macro) stream_mode, target_vendor, additional_vendor, archs, build_pool_type,
values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19) follow_import_dist, branch_suffix, git_make_public, vendor_macro, packager_macro)
values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20)
returning id, created_at, updated_at returning id, created_at, updated_at
`, `,
ret.Name, ret.Name,
@ -231,6 +234,7 @@ func (a *Access) CreateProject(project *peridotpb.Project) (*models.Project, err
ret.TargetVendor, ret.TargetVendor,
ret.AdditionalVendor, ret.AdditionalVendor,
ret.Archs, ret.Archs,
ret.BuildPoolType,
ret.FollowImportDist, ret.FollowImportDist,
ret.BranchSuffix, ret.BranchSuffix,
ret.GitMakePublic, ret.GitMakePublic,
@ -264,6 +268,7 @@ func (a *Access) UpdateProject(id string, project *peridotpb.Project) (*models.P
TargetVendor: project.TargetVendor, TargetVendor: project.TargetVendor,
AdditionalVendor: project.AdditionalVendor.Value, AdditionalVendor: project.AdditionalVendor.Value,
Archs: project.Archs, Archs: project.Archs,
BuildPoolType: utils.StringValueToNullString(project.BuildPoolType),
FollowImportDist: project.FollowImportDist, FollowImportDist: project.FollowImportDist,
BranchSuffix: utils.StringValueToNullString(project.BranchSuffix), BranchSuffix: utils.StringValueToNullString(project.BranchSuffix),
GitMakePublic: project.GitMakePublic, GitMakePublic: project.GitMakePublic,
@ -289,13 +294,14 @@ func (a *Access) UpdateProject(id string, project *peridotpb.Project) (*models.P
target_vendor = $12, target_vendor = $12,
additional_vendor = $13, additional_vendor = $13,
archs = $14, archs = $14,
follow_import_dist = $15, build_pool_type = $15,
branch_suffix = $16, follow_import_dist = $16,
git_make_public = $17, branch_suffix = $17,
vendor_macro = $18, git_make_public = $18,
packager_macro = $19, vendor_macro = $19,
packager_macro = $20,
updated_at = now() updated_at = now()
where id = $20 where id = $21
returning id, created_at, updated_at returning id, created_at, updated_at
`, `,
ret.Name, ret.Name,
@ -312,6 +318,7 @@ func (a *Access) UpdateProject(id string, project *peridotpb.Project) (*models.P
ret.TargetVendor, ret.TargetVendor,
ret.AdditionalVendor, ret.AdditionalVendor,
ret.Archs, ret.Archs,
ret.BuildPoolType,
ret.FollowImportDist, ret.FollowImportDist,
ret.BranchSuffix, ret.BranchSuffix,
ret.GitMakePublic, ret.GitMakePublic,

0
peridot/migrate/20230217215146_add_build_pool_type_to_projects.down.sql Normal file
View File

0
peridot/migrate/20230217215146_add_build_pool_type_to_projects.up.sql Normal file
View File

3
peridot/proto/v1/project.proto
View File

@ -174,6 +174,9 @@ message Project {
// Packager macro is what gets inserted as the packager in the RPM // Packager macro is what gets inserted as the packager in the RPM
google.protobuf.StringValue packager_macro = 22; google.protobuf.StringValue packager_macro = 22;
// specify a build pool type in additional to build pool architecture
google.protobuf.StringValue build_pool_type = 23;
} }
// A repository is a yum repository that yumrepofs maintains // A repository is a yum repository that yumrepofs maintains