Merge pull request #99 from PaulCharlton/issues/pcharlton/roc-1394

allow peridot project to specify a build pool type in additional to build pool architecture

.gitignore

@@ -43,6 +43,9 @@ vendor/peridot.resf.org
 .ijwb/.idea/**/dictionaries
 .ijwb/.idea/**/shelf
 
+# top-level IDE configuration - user specific contents
+/.idea/**
+
 # AWS User-specific
 .ijwb/.idea/**/aws.xml
 

peridot/builder/v1/workflow/infrastructure.go

@@ -61,13 +61,15 @@ import (
 )
 
 type ProvisionWorkerRequest struct {
-	TaskId       string         `json:"taskId"`
-	ParentTaskId sql.NullString `json:"parentTaskId"`
-	Purpose      string         `json:"purpose"`
-	Arch         string         `json:"arch"`
-	ProjectId    string         `json:"projectId"`
-	HighResource bool           `json:"highResource"`
-	Privileged   bool           `json:"privileged"`
+	TaskId        string         `json:"taskId"`
+	ParentTaskId  sql.NullString `json:"parentTaskId"`
+	Purpose       string         `json:"purpose"`
+	Arch          string         `json:"arch"`
+	ImageArch     string         `json:"imageArch"`
+	BuildPoolType string         `json:"buildPoolType"`
+	ProjectId     string         `json:"projectId"`
+	HighResource  bool           `json:"highResource"`
+	Privileged    bool           `json:"privileged"`
 }
 
 func archToGoArch(arch string) string {
@@ -93,6 +95,14 @@ func goArchToArch(arch string) string {
 	}
 }
 
+func buildPoolArch(goArch string, req *ProvisionWorkerRequest) (result string) {
+	result = goArch
+	if len(req.BuildPoolType) > 0 {
+		result = result + "-" + req.BuildPoolType
+	}
+	return result
+}
+
 func (c *Controller) genNameWorker(buildID, purpose string) string {
 	return strings.ReplaceAll(fmt.Sprintf("pb-%s-%s", buildID, purpose), "_", "-")
 }
@@ -171,7 +181,13 @@ func (c *Controller) provisionWorker(ctx workflow.Context, req *ProvisionWorkerR
 	ctx = workflow.WithChildOptions(ctx, workflow.ChildWorkflowOptions{
 		TaskQueue: queue,
 	})
-	err := workflow.ExecuteChildWorkflow(ctx, c.ProvisionWorkerWorkflow, req, queue, imageArch).Get(ctx, &podName)
+	req.ImageArch = imageArch
+	if project.BuildPoolType.Valid {
+		req.BuildPoolType = project.BuildPoolType.String
+	} else {
+		req.BuildPoolType = ""
+	}
+	err := workflow.ExecuteChildWorkflow(ctx, c.ProvisionWorkerWorkflow, req, queue).Get(ctx, &podName)
 	if err != nil {
 		var applicationErr *temporal.ApplicationError
 		if errors.As(err, &applicationErr) {
@@ -200,7 +216,7 @@ func (c *Controller) provisionWorker(ctx workflow.Context, req *ProvisionWorkerR
 // ProvisionWorkerWorkflow provisions a new job specific container using
 // the provided ephemeral provisioner.
 // Returns an identifier
-func (c *Controller) ProvisionWorkerWorkflow(ctx workflow.Context, req *ProvisionWorkerRequest, queue string, imageArch string) (string, error) {
+func (c *Controller) ProvisionWorkerWorkflow(ctx workflow.Context, req *ProvisionWorkerRequest, queue string) (string, error) {
 	var task models.Task
 	taskSideEffect := workflow.SideEffect(ctx, func(ctx workflow.Context) interface{} {
 		var projectId *string
@@ -247,7 +263,7 @@ func (c *Controller) ProvisionWorkerWorkflow(ctx workflow.Context, req *Provisio
 	ctx = workflow.WithActivityOptions(ctx, options)
 
 	var podName string
-	if err := workflow.ExecuteActivity(ctx, c.CreateK8sPodActivity, req, task, imageArch).Get(ctx, &podName); err != nil {
+	if err := workflow.ExecuteActivity(ctx, c.CreateK8sPodActivity, req, task).Get(ctx, &podName); err != nil {
 		return "", err
 	}
 
@@ -444,7 +460,7 @@ func (c *Controller) IngestLogsActivity(ctx context.Context, podName string, tas
 
 // CreateK8sPodActivity creates a new pod in the same namespace
 // with the specified container (the container has to contain peridotbuilder)
-func (c *Controller) CreateK8sPodActivity(ctx context.Context, req *ProvisionWorkerRequest, task *models.Task, imageArch string) (string, error) {
+func (c *Controller) CreateK8sPodActivity(ctx context.Context, req *ProvisionWorkerRequest, task *models.Task) (string, error) {
 	stopChan := makeHeartbeat(ctx, 3*time.Second)
 	defer func() { stopChan <- true }()
 
@@ -465,7 +481,9 @@ func (c *Controller) CreateK8sPodActivity(ctx context.Context, req *ProvisionWor
 		production = "true"
 	}
 
+	imageArch := req.ImageArch
 	goArch := archToGoArch(imageArch)
+	nodePoolArch := buildPoolArch(goArch, req)
 
 	_ = c.logToMon([]string{
 		fmt.Sprintf("Creating worker for purpose %s", req.Purpose),
@@ -566,7 +584,7 @@ func (c *Controller) CreateK8sPodActivity(ctx context.Context, req *ProvisionWor
 				"peridot.rockylinux.org/managed-by":              "peridotephemeral",
 				"peridot.rockylinux.org/task-id":                 req.TaskId,
 				"peridot.rockylinux.org/name":                    name,
-				"peridot.rockylinux.org/workflow-tolerates-arch": goArch,
+				"peridot.rockylinux.org/workflow-tolerates-arch": nodePoolArch,
 				// todo(mustafa): Implement janitor (cron workflow?)
 				"janitor.peridot.rockylinux.org/allow-cleanup":   "yes",
 				"janitor.peridot.rockylinux.org/cleanup-timeout": "864000s",
@@ -656,7 +674,7 @@ func (c *Controller) CreateK8sPodActivity(ctx context.Context, req *ProvisionWor
 				{
 					Key:      "peridot.rockylinux.org/workflow-tolerates-arch",
 					Operator: v1.TolerationOpEqual,
-					Value:    goArch,
+					Value:    nodePoolArch,
 					Effect:   v1.TaintEffectNoSchedule,
 				},
 			},
@@ -721,7 +739,7 @@ func (c *Controller) CreateK8sPodActivity(ctx context.Context, req *ProvisionWor
 								{
 									Key:      "peridot.rockylinux.org/workflow-tolerates-arch",
 									Operator: "In",
-									Values:   []string{goArch},
+									Values:   []string{nodePoolArch},
 								},
 							},
 						},

peridot/builder/v1/workflow/module.go

@@ -120,12 +120,12 @@ type ArtifactIndex struct {
 // Peridot doesn't need to generate and install arbitrary macro RPMs to define macros.
 // For that reason, we can define macros in the container we create.
 // Module components requires the following macros:
-//   * %dist -> as described above
-//   * %_module_build -> the module increment (iteration)
-//   * %_module_name -> the module name
-//   * %_module_stream -> the module stream
-//   * %_module_version -> generated version (as describe above)
-//   * %_module_context -> generated context (calculate sha1 of the buildrequires section) # todo(mustafa): Currently the yaml content is used to calculate the context
+//   - %dist -> as described above
+//   - %_module_build -> the module increment (iteration)
+//   - %_module_name -> the module name
+//   - %_module_stream -> the module stream
+//   - %_module_version -> generated version (as describe above)
+//   - %_module_context -> generated context (calculate sha1 of the buildrequires section) # todo(mustafa): Currently the yaml content is used to calculate the context
 //
 // The macros above will be written to the following file: /etc/rpm/macros.zz-module
 // This is to ensure that the macros are applied last.

peridot/db/models/project.go

@@ -64,6 +64,7 @@ type Project struct {
 	AdditionalVendor   string         `json:"additionalVendor" db:"additional_vendor"`
 
 	Archs            pq.StringArray `json:"archs" db:"archs"`
+	BuildPoolType    sql.NullString `json:"buildPoolType" db:"build_pool_type"`
 	FollowImportDist bool           `json:"followImportDist" db:"follow_import_dist"`
 	BranchSuffix     sql.NullString `json:"branchSuffix" db:"branch_suffix"`
 	GitMakePublic    bool           `json:"gitMakePublic" db:"git_make_public"`

peridot/db/psql/project.go

@@ -73,6 +73,7 @@ func (a *Access) ListProjects(filters *peridotpb.ProjectFilters) (ret models.Pro
 			target_vendor,
 			additional_vendor,
 			archs,
+            build_pool_type,
 			follow_import_dist,
 			branch_suffix,
 			git_make_public,
@@ -200,6 +201,7 @@ func (a *Access) CreateProject(project *peridotpb.Project) (*models.Project, err
 		TargetVendor:       project.TargetVendor,
 		AdditionalVendor:   project.AdditionalVendor.Value,
 		Archs:              project.Archs,
+		BuildPoolType:      utils.StringValueToNullString(project.BuildPoolType),
 		FollowImportDist:   project.FollowImportDist,
 		BranchSuffix:       utils.StringValueToNullString(project.BranchSuffix),
 		GitMakePublic:      project.GitMakePublic,
@@ -213,8 +215,9 @@ func (a *Access) CreateProject(project *peridotpb.Project) (*models.Project, err
 		insert into projects
 		(name, major_version, dist_tag_override, target_gitlab_host, target_prefix,
 		target_branch_prefix, source_git_host, source_prefix, source_branch_prefix, cdn_url,
-		stream_mode, target_vendor, additional_vendor, archs, follow_import_dist, branch_suffix, git_make_public, vendor_macro, packager_macro)
-		values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19)
+		stream_mode, target_vendor, additional_vendor, archs, build_pool_type,
+        follow_import_dist, branch_suffix, git_make_public, vendor_macro, packager_macro)
+		values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20)
 		returning id, created_at, updated_at
 		`,
 		ret.Name,
@@ -231,6 +234,7 @@ func (a *Access) CreateProject(project *peridotpb.Project) (*models.Project, err
 		ret.TargetVendor,
 		ret.AdditionalVendor,
 		ret.Archs,
+		ret.BuildPoolType,
 		ret.FollowImportDist,
 		ret.BranchSuffix,
 		ret.GitMakePublic,
@@ -264,6 +268,7 @@ func (a *Access) UpdateProject(id string, project *peridotpb.Project) (*models.P
 		TargetVendor:       project.TargetVendor,
 		AdditionalVendor:   project.AdditionalVendor.Value,
 		Archs:              project.Archs,
+		BuildPoolType:      utils.StringValueToNullString(project.BuildPoolType),
 		FollowImportDist:   project.FollowImportDist,
 		BranchSuffix:       utils.StringValueToNullString(project.BranchSuffix),
 		GitMakePublic:      project.GitMakePublic,
@@ -289,13 +294,14 @@ func (a *Access) UpdateProject(id string, project *peridotpb.Project) (*models.P
 			target_vendor = $12,
 			additional_vendor = $13,
 			archs = $14,
-			follow_import_dist = $15,
-			branch_suffix = $16,
-			git_make_public = $17,
-            vendor_macro = $18,
-            packager_macro = $19,
+			build_pool_type = $15,
+			follow_import_dist = $16,
+			branch_suffix = $17,
+			git_make_public = $18,
+            vendor_macro = $19,
+            packager_macro = $20,
 			updated_at = now()
-		where id = $20
+		where id = $21
 		returning id, created_at, updated_at
 		`,
 		ret.Name,
@@ -312,6 +318,7 @@ func (a *Access) UpdateProject(id string, project *peridotpb.Project) (*models.P
 		ret.TargetVendor,
 		ret.AdditionalVendor,
 		ret.Archs,
+		ret.BuildPoolType,
 		ret.FollowImportDist,
 		ret.BranchSuffix,
 		ret.GitMakePublic,

peridot/proto/v1/project.proto

@@ -174,6 +174,9 @@ message Project {
 
   // Packager macro is what gets inserted as the packager in the RPM
   google.protobuf.StringValue packager_macro = 22;
+
+  // specify a build pool type in additional to build pool architecture
+  google.protobuf.StringValue build_pool_type = 23;
 }
 
 // A repository is a yum repository that yumrepofs maintains