From f19044b3d2af166c95802cac10d776b35c04f4fd Mon Sep 17 00:00:00 2001 From: mystic knight Date: Fri, 17 Feb 2023 12:55:49 -1000 Subject: [PATCH 1/3] allow peridot project to specify a build pool type in additional to build pool architecture Signed-off-by: mystic knight --- .gitignore | 3 ++ peridot/builder/v1/workflow/infrastructure.go | 46 +++++++++++++------ peridot/builder/v1/workflow/module.go | 12 ++--- peridot/db/models/project.go | 1 + peridot/db/psql/project.go | 23 ++++++---- ...6_add_build_pool_type_to_projects.down.sql | 0 ...146_add_build_pool_type_to_projects.up.sql | 0 peridot/proto/v1/project.proto | 3 ++ 8 files changed, 60 insertions(+), 28 deletions(-) create mode 100644 peridot/migrate/20230217215146_add_build_pool_type_to_projects.down.sql create mode 100644 peridot/migrate/20230217215146_add_build_pool_type_to_projects.up.sql diff --git a/.gitignore b/.gitignore index a0e6db6..ddf5330 100644 --- a/.gitignore +++ b/.gitignore @@ -43,6 +43,9 @@ vendor/peridot.resf.org .ijwb/.idea/**/dictionaries .ijwb/.idea/**/shelf +# top-level IDE configuration - user specific contents +/.idea/** + # AWS User-specific .ijwb/.idea/**/aws.xml diff --git a/peridot/builder/v1/workflow/infrastructure.go b/peridot/builder/v1/workflow/infrastructure.go index b3a0fb4..e8a411d 100644 --- a/peridot/builder/v1/workflow/infrastructure.go +++ b/peridot/builder/v1/workflow/infrastructure.go @@ -61,13 +61,15 @@ import ( ) type ProvisionWorkerRequest struct { - TaskId string `json:"taskId"` - ParentTaskId sql.NullString `json:"parentTaskId"` - Purpose string `json:"purpose"` - Arch string `json:"arch"` - ProjectId string `json:"projectId"` - HighResource bool `json:"highResource"` - Privileged bool `json:"privileged"` + TaskId string `json:"taskId"` + ParentTaskId sql.NullString `json:"parentTaskId"` + Purpose string `json:"purpose"` + Arch string `json:"arch"` + ImageArch string `json:"imageArch"` + BuildPoolType string `json:"buildPoolType"` + ProjectId string `json:"projectId"` + HighResource bool `json:"highResource"` + Privileged bool `json:"privileged"` } func archToGoArch(arch string) string { @@ -93,6 +95,14 @@ func goArchToArch(arch string) string { } } +func buildPoolArch(goArch string, req *ProvisionWorkerRequest) (result string) { + result = goArch + if len(req.BuildPoolType) > 0 { + result = result + "-" + req.BuildPoolType + } + return result +} + func (c *Controller) genNameWorker(buildID, purpose string) string { return strings.ReplaceAll(fmt.Sprintf("pb-%s-%s", buildID, purpose), "_", "-") } @@ -171,7 +181,13 @@ func (c *Controller) provisionWorker(ctx workflow.Context, req *ProvisionWorkerR ctx = workflow.WithChildOptions(ctx, workflow.ChildWorkflowOptions{ TaskQueue: queue, }) - err := workflow.ExecuteChildWorkflow(ctx, c.ProvisionWorkerWorkflow, req, queue, imageArch).Get(ctx, &podName) + req.ImageArch = imageArch + if project.BuildPoolType.Valid { + req.BuildPoolType = project.BuildPoolType.String + } else { + req.BuildPoolType = "" + } + err := workflow.ExecuteChildWorkflow(ctx, c.ProvisionWorkerWorkflow, req, queue).Get(ctx, &podName) if err != nil { var applicationErr *temporal.ApplicationError if errors.As(err, &applicationErr) { @@ -200,7 +216,7 @@ func (c *Controller) provisionWorker(ctx workflow.Context, req *ProvisionWorkerR // ProvisionWorkerWorkflow provisions a new job specific container using // the provided ephemeral provisioner. // Returns an identifier -func (c *Controller) ProvisionWorkerWorkflow(ctx workflow.Context, req *ProvisionWorkerRequest, queue string, imageArch string) (string, error) { +func (c *Controller) ProvisionWorkerWorkflow(ctx workflow.Context, req *ProvisionWorkerRequest, queue string) (string, error) { var task models.Task taskSideEffect := workflow.SideEffect(ctx, func(ctx workflow.Context) interface{} { var projectId *string @@ -247,7 +263,7 @@ func (c *Controller) ProvisionWorkerWorkflow(ctx workflow.Context, req *Provisio ctx = workflow.WithActivityOptions(ctx, options) var podName string - if err := workflow.ExecuteActivity(ctx, c.CreateK8sPodActivity, req, task, imageArch).Get(ctx, &podName); err != nil { + if err := workflow.ExecuteActivity(ctx, c.CreateK8sPodActivity, req, task).Get(ctx, &podName); err != nil { return "", err } @@ -444,7 +460,7 @@ func (c *Controller) IngestLogsActivity(ctx context.Context, podName string, tas // CreateK8sPodActivity creates a new pod in the same namespace // with the specified container (the container has to contain peridotbuilder) -func (c *Controller) CreateK8sPodActivity(ctx context.Context, req *ProvisionWorkerRequest, task *models.Task, imageArch string) (string, error) { +func (c *Controller) CreateK8sPodActivity(ctx context.Context, req *ProvisionWorkerRequest, task *models.Task) (string, error) { stopChan := makeHeartbeat(ctx, 3*time.Second) defer func() { stopChan <- true }() @@ -465,7 +481,9 @@ func (c *Controller) CreateK8sPodActivity(ctx context.Context, req *ProvisionWor production = "true" } + imageArch := req.ImageArch goArch := archToGoArch(imageArch) + nodePoolArch := buildPoolArch(goArch, req) _ = c.logToMon([]string{ fmt.Sprintf("Creating worker for purpose %s", req.Purpose), @@ -566,7 +584,7 @@ func (c *Controller) CreateK8sPodActivity(ctx context.Context, req *ProvisionWor "peridot.rockylinux.org/managed-by": "peridotephemeral", "peridot.rockylinux.org/task-id": req.TaskId, "peridot.rockylinux.org/name": name, - "peridot.rockylinux.org/workflow-tolerates-arch": goArch, + "peridot.rockylinux.org/workflow-tolerates-arch": nodePoolArch, // todo(mustafa): Implement janitor (cron workflow?) "janitor.peridot.rockylinux.org/allow-cleanup": "yes", "janitor.peridot.rockylinux.org/cleanup-timeout": "864000s", @@ -656,7 +674,7 @@ func (c *Controller) CreateK8sPodActivity(ctx context.Context, req *ProvisionWor { Key: "peridot.rockylinux.org/workflow-tolerates-arch", Operator: v1.TolerationOpEqual, - Value: goArch, + Value: nodePoolArch, Effect: v1.TaintEffectNoSchedule, }, }, @@ -721,7 +739,7 @@ func (c *Controller) CreateK8sPodActivity(ctx context.Context, req *ProvisionWor { Key: "peridot.rockylinux.org/workflow-tolerates-arch", Operator: "In", - Values: []string{goArch}, + Values: []string{nodePoolArch}, }, }, }, diff --git a/peridot/builder/v1/workflow/module.go b/peridot/builder/v1/workflow/module.go index a0f4790..715fd0a 100644 --- a/peridot/builder/v1/workflow/module.go +++ b/peridot/builder/v1/workflow/module.go @@ -120,12 +120,12 @@ type ArtifactIndex struct { // Peridot doesn't need to generate and install arbitrary macro RPMs to define macros. // For that reason, we can define macros in the container we create. // Module components requires the following macros: -// * %dist -> as described above -// * %_module_build -> the module increment (iteration) -// * %_module_name -> the module name -// * %_module_stream -> the module stream -// * %_module_version -> generated version (as describe above) -// * %_module_context -> generated context (calculate sha1 of the buildrequires section) # todo(mustafa): Currently the yaml content is used to calculate the context +// - %dist -> as described above +// - %_module_build -> the module increment (iteration) +// - %_module_name -> the module name +// - %_module_stream -> the module stream +// - %_module_version -> generated version (as describe above) +// - %_module_context -> generated context (calculate sha1 of the buildrequires section) # todo(mustafa): Currently the yaml content is used to calculate the context // // The macros above will be written to the following file: /etc/rpm/macros.zz-module // This is to ensure that the macros are applied last. diff --git a/peridot/db/models/project.go b/peridot/db/models/project.go index fd56484..16218ce 100644 --- a/peridot/db/models/project.go +++ b/peridot/db/models/project.go @@ -64,6 +64,7 @@ type Project struct { AdditionalVendor string `json:"additionalVendor" db:"additional_vendor"` Archs pq.StringArray `json:"archs" db:"archs"` + BuildPoolType sql.NullString `json:"buildPoolType" db:"build_pool_type"` FollowImportDist bool `json:"followImportDist" db:"follow_import_dist"` BranchSuffix sql.NullString `json:"branchSuffix" db:"branch_suffix"` GitMakePublic bool `json:"gitMakePublic" db:"git_make_public"` diff --git a/peridot/db/psql/project.go b/peridot/db/psql/project.go index d16d954..f57d74b 100644 --- a/peridot/db/psql/project.go +++ b/peridot/db/psql/project.go @@ -73,6 +73,7 @@ func (a *Access) ListProjects(filters *peridotpb.ProjectFilters) (ret models.Pro target_vendor, additional_vendor, archs, + build_pool_type, follow_import_dist, branch_suffix, git_make_public, @@ -200,6 +201,7 @@ func (a *Access) CreateProject(project *peridotpb.Project) (*models.Project, err TargetVendor: project.TargetVendor, AdditionalVendor: project.AdditionalVendor.Value, Archs: project.Archs, + BuildPoolType: project.BuildPoolType, FollowImportDist: project.FollowImportDist, BranchSuffix: utils.StringValueToNullString(project.BranchSuffix), GitMakePublic: project.GitMakePublic, @@ -213,8 +215,9 @@ func (a *Access) CreateProject(project *peridotpb.Project) (*models.Project, err insert into projects (name, major_version, dist_tag_override, target_gitlab_host, target_prefix, target_branch_prefix, source_git_host, source_prefix, source_branch_prefix, cdn_url, - stream_mode, target_vendor, additional_vendor, archs, follow_import_dist, branch_suffix, git_make_public, vendor_macro, packager_macro) - values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19) + stream_mode, target_vendor, additional_vendor, archs, build_pool_type, + follow_import_dist, branch_suffix, git_make_public, vendor_macro, packager_macro) + values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20) returning id, created_at, updated_at `, ret.Name, @@ -231,6 +234,7 @@ func (a *Access) CreateProject(project *peridotpb.Project) (*models.Project, err ret.TargetVendor, ret.AdditionalVendor, ret.Archs, + ret.BuildPoolType, ret.FollowImportDist, ret.BranchSuffix, ret.GitMakePublic, @@ -264,6 +268,7 @@ func (a *Access) UpdateProject(id string, project *peridotpb.Project) (*models.P TargetVendor: project.TargetVendor, AdditionalVendor: project.AdditionalVendor.Value, Archs: project.Archs, + BuildPoolType: project.BuildPoolType, FollowImportDist: project.FollowImportDist, BranchSuffix: utils.StringValueToNullString(project.BranchSuffix), GitMakePublic: project.GitMakePublic, @@ -289,13 +294,14 @@ func (a *Access) UpdateProject(id string, project *peridotpb.Project) (*models.P target_vendor = $12, additional_vendor = $13, archs = $14, - follow_import_dist = $15, - branch_suffix = $16, - git_make_public = $17, - vendor_macro = $18, - packager_macro = $19, + build_pool_type = $15, + follow_import_dist = $16, + branch_suffix = $17, + git_make_public = $18, + vendor_macro = $19, + packager_macro = $20, updated_at = now() - where id = $20 + where id = $21 returning id, created_at, updated_at `, ret.Name, @@ -312,6 +318,7 @@ func (a *Access) UpdateProject(id string, project *peridotpb.Project) (*models.P ret.TargetVendor, ret.AdditionalVendor, ret.Archs, + ret.BuildPoolType, ret.FollowImportDist, ret.BranchSuffix, ret.GitMakePublic, diff --git a/peridot/migrate/20230217215146_add_build_pool_type_to_projects.down.sql b/peridot/migrate/20230217215146_add_build_pool_type_to_projects.down.sql new file mode 100644 index 0000000..e69de29 diff --git a/peridot/migrate/20230217215146_add_build_pool_type_to_projects.up.sql b/peridot/migrate/20230217215146_add_build_pool_type_to_projects.up.sql new file mode 100644 index 0000000..e69de29 diff --git a/peridot/proto/v1/project.proto b/peridot/proto/v1/project.proto index 0084d51..c512f9f 100644 --- a/peridot/proto/v1/project.proto +++ b/peridot/proto/v1/project.proto @@ -174,6 +174,9 @@ message Project { // Packager macro is what gets inserted as the packager in the RPM google.protobuf.StringValue packager_macro = 22; + + // specify a build pool type in additional to build pool architecture + google.protobuf.StringValue build_pool_type = 23; } // A repository is a yum repository that yumrepofs maintains From fdc848b8476f74707e6eb6436035e7c6a1ce3e7b Mon Sep 17 00:00:00 2001 From: mystic knight Date: Fri, 17 Feb 2023 14:01:50 -1000 Subject: [PATCH 2/3] allow peridot project to specify a build pool type in additional to build pool architecture Signed-off-by: mystic knight --- peridot/db/psql/project.go | 396 ++++++++++++++++++------------------- 1 file changed, 198 insertions(+), 198 deletions(-) diff --git a/peridot/db/psql/project.go b/peridot/db/psql/project.go index f57d74b..599e842 100644 --- a/peridot/db/psql/project.go +++ b/peridot/db/psql/project.go @@ -31,30 +31,30 @@ package serverpsql import ( - "fmt" - "github.com/jmoiron/sqlx/types" - "github.com/lib/pq" - "google.golang.org/protobuf/encoding/protojson" - "google.golang.org/protobuf/proto" - "google.golang.org/protobuf/types/known/anypb" - "peridot.resf.org/peridot/db/models" - peridotpb "peridot.resf.org/peridot/pb" - "peridot.resf.org/utils" + "fmt" + "github.com/jmoiron/sqlx/types" + "github.com/lib/pq" + "google.golang.org/protobuf/encoding/protojson" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" + "peridot.resf.org/peridot/db/models" + peridotpb "peridot.resf.org/peridot/pb" + "peridot.resf.org/utils" ) func (a *Access) ListProjects(filters *peridotpb.ProjectFilters) (ret models.Projects, err error) { - if filters == nil { - filters = &peridotpb.ProjectFilters{} - } + if filters == nil { + filters = &peridotpb.ProjectFilters{} + } - var ids pq.StringArray = nil - if filters.Ids != nil { - ids = filters.Ids - } + var ids pq.StringArray = nil + if filters.Ids != nil { + ids = filters.Ids + } - err = a.query.Select( - &ret, - ` + err = a.query.Select( + &ret, + ` select id, created_at, @@ -88,19 +88,19 @@ func (a *Access) ListProjects(filters *peridotpb.ProjectFilters) (ret models.Pro and ($3 :: uuid[] is null or id = any($3 :: uuid[])) order by created_at desc `, - utils.StringValueP(filters.Id), - utils.StringValueP(filters.Name), - ids, - ) + utils.StringValueP(filters.Id), + utils.StringValueP(filters.Name), + ids, + ) - return ret, err + return ret, err } func (a *Access) GetProjectKeys(projectId string) (*models.ProjectKey, error) { - var ret models.ProjectKey - err := a.query.Get( - &ret, - ` + var ret models.ProjectKey + err := a.query.Get( + &ret, + ` select id, created_at, @@ -110,21 +110,21 @@ func (a *Access) GetProjectKeys(projectId string) (*models.ProjectKey, error) { from project_keys where project_id = $1 `, - projectId, - ) - if err != nil { - return nil, err - } + projectId, + ) + if err != nil { + return nil, err + } - return &ret, nil + return &ret, nil } // GetProjectModuleConfiguration returns the module configurations for the given project. func (a *Access) GetProjectModuleConfiguration(projectId string) (*peridotpb.ModuleConfiguration, error) { - var ret types.JSONText - err := a.query.Get( - &ret, - ` + var ret types.JSONText + err := a.query.Get( + &ret, + ` select proto from project_module_configuration @@ -132,86 +132,86 @@ func (a *Access) GetProjectModuleConfiguration(projectId string) (*peridotpb.Mod project_id = $1 and active = true `, - projectId, - ) - if err != nil { - return nil, err - } + projectId, + ) + if err != nil { + return nil, err + } - anyPb := &anypb.Any{} - err = protojson.Unmarshal(ret, anyPb) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal module configuration (protojson): %v", err) - } + anyPb := &anypb.Any{} + err = protojson.Unmarshal(ret, anyPb) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal module configuration (protojson): %v", err) + } - pb := &peridotpb.ModuleConfiguration{} - err = anypb.UnmarshalTo(anyPb, pb, proto.UnmarshalOptions{}) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal module configuration: %v", err) - } + pb := &peridotpb.ModuleConfiguration{} + err = anypb.UnmarshalTo(anyPb, pb, proto.UnmarshalOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal module configuration: %v", err) + } - return pb, nil + return pb, nil } func (a *Access) CreateProjectModuleConfiguration(projectId string, config *peridotpb.ModuleConfiguration) error { - anyPb, err := anypb.New(config) - if err != nil { - return fmt.Errorf("failed to marshal module configuration: %v", err) - } + anyPb, err := anypb.New(config) + if err != nil { + return fmt.Errorf("failed to marshal module configuration: %v", err) + } - protoJson, err := protojson.Marshal(anyPb) - if err != nil { - return fmt.Errorf("failed to marshal module configuration (protojson): %v", err) - } + protoJson, err := protojson.Marshal(anyPb) + if err != nil { + return fmt.Errorf("failed to marshal module configuration (protojson): %v", err) + } - _, err = a.query.Exec( - ` + _, err = a.query.Exec( + ` insert into project_module_configuration (project_id, proto, active) values ($1, $2, true) on conflict (project_id) do update set proto = $2, active = true `, - projectId, - protoJson, - ) - if err != nil { - return err - } + projectId, + protoJson, + ) + if err != nil { + return err + } - return nil + return nil } func (a *Access) CreateProject(project *peridotpb.Project) (*models.Project, error) { - if err := project.ValidateAll(); err != nil { - return nil, err - } + if err := project.ValidateAll(); err != nil { + return nil, err + } - ret := models.Project{ - Name: project.Name.Value, - MajorVersion: int(project.MajorVersion.Value), - DistTagOverride: utils.StringValueToNullString(project.DistTag), - TargetGitlabHost: project.TargetGitlabHost.Value, - TargetPrefix: project.TargetPrefix.Value, - TargetBranchPrefix: project.TargetBranchPrefix.Value, - SourceGitHost: utils.StringValueToNullString(project.SourceGitHost), - SourcePrefix: utils.StringValueToNullString(project.SourcePrefix), - SourceBranchPrefix: utils.StringValueToNullString(project.SourceBranchPrefix), - CdnUrl: utils.StringValueToNullString(project.CdnUrl), - StreamMode: project.StreamMode, - TargetVendor: project.TargetVendor, - AdditionalVendor: project.AdditionalVendor.Value, - Archs: project.Archs, - BuildPoolType: project.BuildPoolType, - FollowImportDist: project.FollowImportDist, - BranchSuffix: utils.StringValueToNullString(project.BranchSuffix), - GitMakePublic: project.GitMakePublic, - VendorMacro: utils.StringValueToNullString(project.VendorMacro), - PackagerMacro: utils.StringValueToNullString(project.PackagerMacro), - } + ret := models.Project{ + Name: project.Name.Value, + MajorVersion: int(project.MajorVersion.Value), + DistTagOverride: utils.StringValueToNullString(project.DistTag), + TargetGitlabHost: project.TargetGitlabHost.Value, + TargetPrefix: project.TargetPrefix.Value, + TargetBranchPrefix: project.TargetBranchPrefix.Value, + SourceGitHost: utils.StringValueToNullString(project.SourceGitHost), + SourcePrefix: utils.StringValueToNullString(project.SourcePrefix), + SourceBranchPrefix: utils.StringValueToNullString(project.SourceBranchPrefix), + CdnUrl: utils.StringValueToNullString(project.CdnUrl), + StreamMode: project.StreamMode, + TargetVendor: project.TargetVendor, + AdditionalVendor: project.AdditionalVendor.Value, + Archs: project.Archs, + BuildPoolType: project.BuildPoolType, + FollowImportDist: project.FollowImportDist, + BranchSuffix: utils.StringValueToNullString(project.BranchSuffix), + GitMakePublic: project.GitMakePublic, + VendorMacro: utils.StringValueToNullString(project.VendorMacro), + PackagerMacro: utils.StringValueToNullString(project.PackagerMacro), + } - err := a.query.Get( - &ret, - ` + err := a.query.Get( + &ret, + ` insert into projects (name, major_version, dist_tag_override, target_gitlab_host, target_prefix, target_branch_prefix, source_git_host, source_prefix, source_branch_prefix, cdn_url, @@ -220,65 +220,65 @@ func (a *Access) CreateProject(project *peridotpb.Project) (*models.Project, err values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20) returning id, created_at, updated_at `, - ret.Name, - ret.MajorVersion, - ret.DistTagOverride, - ret.TargetGitlabHost, - ret.TargetPrefix, - ret.TargetBranchPrefix, - ret.SourceGitHost, - ret.SourcePrefix, - ret.SourceBranchPrefix, - ret.CdnUrl, - ret.StreamMode, - ret.TargetVendor, - ret.AdditionalVendor, - ret.Archs, - ret.BuildPoolType, - ret.FollowImportDist, - ret.BranchSuffix, - ret.GitMakePublic, - ret.VendorMacro, - ret.PackagerMacro, - ) - if err != nil { - return nil, err - } + ret.Name, + ret.MajorVersion, + ret.DistTagOverride, + ret.TargetGitlabHost, + ret.TargetPrefix, + ret.TargetBranchPrefix, + ret.SourceGitHost, + ret.SourcePrefix, + ret.SourceBranchPrefix, + ret.CdnUrl, + ret.StreamMode, + ret.TargetVendor, + ret.AdditionalVendor, + ret.Archs, + ret.BuildPoolType, + ret.FollowImportDist, + ret.BranchSuffix, + ret.GitMakePublic, + ret.VendorMacro, + ret.PackagerMacro, + ) + if err != nil { + return nil, err + } - return &ret, nil + return &ret, nil } func (a *Access) UpdateProject(id string, project *peridotpb.Project) (*models.Project, error) { - if err := project.ValidateAll(); err != nil { - return nil, err - } + if err := project.ValidateAll(); err != nil { + return nil, err + } - ret := models.Project{ - Name: project.Name.Value, - MajorVersion: int(project.MajorVersion.Value), - DistTagOverride: utils.StringValueToNullString(project.DistTag), - TargetGitlabHost: project.TargetGitlabHost.Value, - TargetPrefix: project.TargetPrefix.Value, - TargetBranchPrefix: project.TargetBranchPrefix.Value, - SourceGitHost: utils.StringValueToNullString(project.SourceGitHost), - SourcePrefix: utils.StringValueToNullString(project.SourcePrefix), - SourceBranchPrefix: utils.StringValueToNullString(project.SourceBranchPrefix), - CdnUrl: utils.StringValueToNullString(project.CdnUrl), - StreamMode: project.StreamMode, - TargetVendor: project.TargetVendor, - AdditionalVendor: project.AdditionalVendor.Value, - Archs: project.Archs, - BuildPoolType: project.BuildPoolType, - FollowImportDist: project.FollowImportDist, - BranchSuffix: utils.StringValueToNullString(project.BranchSuffix), - GitMakePublic: project.GitMakePublic, - VendorMacro: utils.StringValueToNullString(project.VendorMacro), - PackagerMacro: utils.StringValueToNullString(project.PackagerMacro), - } + ret := models.Project{ + Name: project.Name.Value, + MajorVersion: int(project.MajorVersion.Value), + DistTagOverride: utils.StringValueToNullString(project.DistTag), + TargetGitlabHost: project.TargetGitlabHost.Value, + TargetPrefix: project.TargetPrefix.Value, + TargetBranchPrefix: project.TargetBranchPrefix.Value, + SourceGitHost: utils.StringValueToNullString(project.SourceGitHost), + SourcePrefix: utils.StringValueToNullString(project.SourcePrefix), + SourceBranchPrefix: utils.StringValueToNullString(project.SourceBranchPrefix), + CdnUrl: utils.StringValueToNullString(project.CdnUrl), + StreamMode: project.StreamMode, + TargetVendor: project.TargetVendor, + AdditionalVendor: project.AdditionalVendor.Value, + Archs: project.Archs, + BuildPoolType: project.BuildPoolType.Value, + FollowImportDist: project.FollowImportDist, + BranchSuffix: utils.StringValueToNullString(project.BranchSuffix), + GitMakePublic: project.GitMakePublic, + VendorMacro: utils.StringValueToNullString(project.VendorMacro), + PackagerMacro: utils.StringValueToNullString(project.PackagerMacro), + } - err := a.query.Get( - &ret, - ` + err := a.query.Get( + &ret, + ` update projects set name = $1, major_version = $2, @@ -304,65 +304,65 @@ func (a *Access) UpdateProject(id string, project *peridotpb.Project) (*models.P where id = $21 returning id, created_at, updated_at `, - ret.Name, - ret.MajorVersion, - ret.DistTagOverride, - ret.TargetGitlabHost, - ret.TargetPrefix, - ret.TargetBranchPrefix, - ret.SourceGitHost, - ret.SourcePrefix, - ret.SourceBranchPrefix, - ret.CdnUrl, - ret.StreamMode, - ret.TargetVendor, - ret.AdditionalVendor, - ret.Archs, - ret.BuildPoolType, - ret.FollowImportDist, - ret.BranchSuffix, - ret.GitMakePublic, - ret.VendorMacro, - ret.PackagerMacro, - id, - ) - if err != nil { - return nil, err - } + ret.Name, + ret.MajorVersion, + ret.DistTagOverride, + ret.TargetGitlabHost, + ret.TargetPrefix, + ret.TargetBranchPrefix, + ret.SourceGitHost, + ret.SourcePrefix, + ret.SourceBranchPrefix, + ret.CdnUrl, + ret.StreamMode, + ret.TargetVendor, + ret.AdditionalVendor, + ret.Archs, + ret.BuildPoolType, + ret.FollowImportDist, + ret.BranchSuffix, + ret.GitMakePublic, + ret.VendorMacro, + ret.PackagerMacro, + id, + ) + if err != nil { + return nil, err + } - return &ret, nil + return &ret, nil } func (a *Access) SetProjectKeys(projectId string, username string, password string) error { - _, err := a.query.Exec( - ` + _, err := a.query.Exec( + ` insert into project_keys (project_id, gitlab_username, gitlab_secret) values ($1, $2, $3) on conflict (project_id) do update set gitlab_username = $2, gitlab_secret = $3 `, - projectId, - username, - password, - ) - return err + projectId, + username, + password, + ) + return err } func (a *Access) SetBuildRootPackages(projectId string, srpmPackages pq.StringArray, buildPackages pq.StringArray) error { - if srpmPackages == nil { - srpmPackages = pq.StringArray{} - } - if buildPackages == nil { - buildPackages = pq.StringArray{} - } + if srpmPackages == nil { + srpmPackages = pq.StringArray{} + } + if buildPackages == nil { + buildPackages = pq.StringArray{} + } - _, err := a.query.Exec( - ` + _, err := a.query.Exec( + ` update projects set srpm_stage_packages = $2, build_stage_packages = $3 where id = $1 `, - projectId, - srpmPackages, - buildPackages, - ) - return err + projectId, + srpmPackages, + buildPackages, + ) + return err } From e9e3cb54e71ddbfe2b98a3971edfc351e32828b8 Mon Sep 17 00:00:00 2001 From: mystic knight Date: Fri, 17 Feb 2023 14:11:35 -1000 Subject: [PATCH 3/3] allow peridot project to specify a build pool type in additional to build pool architecture Signed-off-by: mystic knight --- peridot/db/psql/project.go | 396 ++++++++++++++++++------------------- 1 file changed, 198 insertions(+), 198 deletions(-) diff --git a/peridot/db/psql/project.go b/peridot/db/psql/project.go index 599e842..0613059 100644 --- a/peridot/db/psql/project.go +++ b/peridot/db/psql/project.go @@ -31,30 +31,30 @@ package serverpsql import ( - "fmt" - "github.com/jmoiron/sqlx/types" - "github.com/lib/pq" - "google.golang.org/protobuf/encoding/protojson" - "google.golang.org/protobuf/proto" - "google.golang.org/protobuf/types/known/anypb" - "peridot.resf.org/peridot/db/models" - peridotpb "peridot.resf.org/peridot/pb" - "peridot.resf.org/utils" + "fmt" + "github.com/jmoiron/sqlx/types" + "github.com/lib/pq" + "google.golang.org/protobuf/encoding/protojson" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" + "peridot.resf.org/peridot/db/models" + peridotpb "peridot.resf.org/peridot/pb" + "peridot.resf.org/utils" ) func (a *Access) ListProjects(filters *peridotpb.ProjectFilters) (ret models.Projects, err error) { - if filters == nil { - filters = &peridotpb.ProjectFilters{} - } + if filters == nil { + filters = &peridotpb.ProjectFilters{} + } - var ids pq.StringArray = nil - if filters.Ids != nil { - ids = filters.Ids - } + var ids pq.StringArray = nil + if filters.Ids != nil { + ids = filters.Ids + } - err = a.query.Select( - &ret, - ` + err = a.query.Select( + &ret, + ` select id, created_at, @@ -88,19 +88,19 @@ func (a *Access) ListProjects(filters *peridotpb.ProjectFilters) (ret models.Pro and ($3 :: uuid[] is null or id = any($3 :: uuid[])) order by created_at desc `, - utils.StringValueP(filters.Id), - utils.StringValueP(filters.Name), - ids, - ) + utils.StringValueP(filters.Id), + utils.StringValueP(filters.Name), + ids, + ) - return ret, err + return ret, err } func (a *Access) GetProjectKeys(projectId string) (*models.ProjectKey, error) { - var ret models.ProjectKey - err := a.query.Get( - &ret, - ` + var ret models.ProjectKey + err := a.query.Get( + &ret, + ` select id, created_at, @@ -110,21 +110,21 @@ func (a *Access) GetProjectKeys(projectId string) (*models.ProjectKey, error) { from project_keys where project_id = $1 `, - projectId, - ) - if err != nil { - return nil, err - } + projectId, + ) + if err != nil { + return nil, err + } - return &ret, nil + return &ret, nil } // GetProjectModuleConfiguration returns the module configurations for the given project. func (a *Access) GetProjectModuleConfiguration(projectId string) (*peridotpb.ModuleConfiguration, error) { - var ret types.JSONText - err := a.query.Get( - &ret, - ` + var ret types.JSONText + err := a.query.Get( + &ret, + ` select proto from project_module_configuration @@ -132,86 +132,86 @@ func (a *Access) GetProjectModuleConfiguration(projectId string) (*peridotpb.Mod project_id = $1 and active = true `, - projectId, - ) - if err != nil { - return nil, err - } + projectId, + ) + if err != nil { + return nil, err + } - anyPb := &anypb.Any{} - err = protojson.Unmarshal(ret, anyPb) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal module configuration (protojson): %v", err) - } + anyPb := &anypb.Any{} + err = protojson.Unmarshal(ret, anyPb) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal module configuration (protojson): %v", err) + } - pb := &peridotpb.ModuleConfiguration{} - err = anypb.UnmarshalTo(anyPb, pb, proto.UnmarshalOptions{}) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal module configuration: %v", err) - } + pb := &peridotpb.ModuleConfiguration{} + err = anypb.UnmarshalTo(anyPb, pb, proto.UnmarshalOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal module configuration: %v", err) + } - return pb, nil + return pb, nil } func (a *Access) CreateProjectModuleConfiguration(projectId string, config *peridotpb.ModuleConfiguration) error { - anyPb, err := anypb.New(config) - if err != nil { - return fmt.Errorf("failed to marshal module configuration: %v", err) - } + anyPb, err := anypb.New(config) + if err != nil { + return fmt.Errorf("failed to marshal module configuration: %v", err) + } - protoJson, err := protojson.Marshal(anyPb) - if err != nil { - return fmt.Errorf("failed to marshal module configuration (protojson): %v", err) - } + protoJson, err := protojson.Marshal(anyPb) + if err != nil { + return fmt.Errorf("failed to marshal module configuration (protojson): %v", err) + } - _, err = a.query.Exec( - ` + _, err = a.query.Exec( + ` insert into project_module_configuration (project_id, proto, active) values ($1, $2, true) on conflict (project_id) do update set proto = $2, active = true `, - projectId, - protoJson, - ) - if err != nil { - return err - } + projectId, + protoJson, + ) + if err != nil { + return err + } - return nil + return nil } func (a *Access) CreateProject(project *peridotpb.Project) (*models.Project, error) { - if err := project.ValidateAll(); err != nil { - return nil, err - } + if err := project.ValidateAll(); err != nil { + return nil, err + } - ret := models.Project{ - Name: project.Name.Value, - MajorVersion: int(project.MajorVersion.Value), - DistTagOverride: utils.StringValueToNullString(project.DistTag), - TargetGitlabHost: project.TargetGitlabHost.Value, - TargetPrefix: project.TargetPrefix.Value, - TargetBranchPrefix: project.TargetBranchPrefix.Value, - SourceGitHost: utils.StringValueToNullString(project.SourceGitHost), - SourcePrefix: utils.StringValueToNullString(project.SourcePrefix), - SourceBranchPrefix: utils.StringValueToNullString(project.SourceBranchPrefix), - CdnUrl: utils.StringValueToNullString(project.CdnUrl), - StreamMode: project.StreamMode, - TargetVendor: project.TargetVendor, - AdditionalVendor: project.AdditionalVendor.Value, - Archs: project.Archs, - BuildPoolType: project.BuildPoolType, - FollowImportDist: project.FollowImportDist, - BranchSuffix: utils.StringValueToNullString(project.BranchSuffix), - GitMakePublic: project.GitMakePublic, - VendorMacro: utils.StringValueToNullString(project.VendorMacro), - PackagerMacro: utils.StringValueToNullString(project.PackagerMacro), - } + ret := models.Project{ + Name: project.Name.Value, + MajorVersion: int(project.MajorVersion.Value), + DistTagOverride: utils.StringValueToNullString(project.DistTag), + TargetGitlabHost: project.TargetGitlabHost.Value, + TargetPrefix: project.TargetPrefix.Value, + TargetBranchPrefix: project.TargetBranchPrefix.Value, + SourceGitHost: utils.StringValueToNullString(project.SourceGitHost), + SourcePrefix: utils.StringValueToNullString(project.SourcePrefix), + SourceBranchPrefix: utils.StringValueToNullString(project.SourceBranchPrefix), + CdnUrl: utils.StringValueToNullString(project.CdnUrl), + StreamMode: project.StreamMode, + TargetVendor: project.TargetVendor, + AdditionalVendor: project.AdditionalVendor.Value, + Archs: project.Archs, + BuildPoolType: utils.StringValueToNullString(project.BuildPoolType), + FollowImportDist: project.FollowImportDist, + BranchSuffix: utils.StringValueToNullString(project.BranchSuffix), + GitMakePublic: project.GitMakePublic, + VendorMacro: utils.StringValueToNullString(project.VendorMacro), + PackagerMacro: utils.StringValueToNullString(project.PackagerMacro), + } - err := a.query.Get( - &ret, - ` + err := a.query.Get( + &ret, + ` insert into projects (name, major_version, dist_tag_override, target_gitlab_host, target_prefix, target_branch_prefix, source_git_host, source_prefix, source_branch_prefix, cdn_url, @@ -220,65 +220,65 @@ func (a *Access) CreateProject(project *peridotpb.Project) (*models.Project, err values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20) returning id, created_at, updated_at `, - ret.Name, - ret.MajorVersion, - ret.DistTagOverride, - ret.TargetGitlabHost, - ret.TargetPrefix, - ret.TargetBranchPrefix, - ret.SourceGitHost, - ret.SourcePrefix, - ret.SourceBranchPrefix, - ret.CdnUrl, - ret.StreamMode, - ret.TargetVendor, - ret.AdditionalVendor, - ret.Archs, - ret.BuildPoolType, - ret.FollowImportDist, - ret.BranchSuffix, - ret.GitMakePublic, - ret.VendorMacro, - ret.PackagerMacro, - ) - if err != nil { - return nil, err - } + ret.Name, + ret.MajorVersion, + ret.DistTagOverride, + ret.TargetGitlabHost, + ret.TargetPrefix, + ret.TargetBranchPrefix, + ret.SourceGitHost, + ret.SourcePrefix, + ret.SourceBranchPrefix, + ret.CdnUrl, + ret.StreamMode, + ret.TargetVendor, + ret.AdditionalVendor, + ret.Archs, + ret.BuildPoolType, + ret.FollowImportDist, + ret.BranchSuffix, + ret.GitMakePublic, + ret.VendorMacro, + ret.PackagerMacro, + ) + if err != nil { + return nil, err + } - return &ret, nil + return &ret, nil } func (a *Access) UpdateProject(id string, project *peridotpb.Project) (*models.Project, error) { - if err := project.ValidateAll(); err != nil { - return nil, err - } + if err := project.ValidateAll(); err != nil { + return nil, err + } - ret := models.Project{ - Name: project.Name.Value, - MajorVersion: int(project.MajorVersion.Value), - DistTagOverride: utils.StringValueToNullString(project.DistTag), - TargetGitlabHost: project.TargetGitlabHost.Value, - TargetPrefix: project.TargetPrefix.Value, - TargetBranchPrefix: project.TargetBranchPrefix.Value, - SourceGitHost: utils.StringValueToNullString(project.SourceGitHost), - SourcePrefix: utils.StringValueToNullString(project.SourcePrefix), - SourceBranchPrefix: utils.StringValueToNullString(project.SourceBranchPrefix), - CdnUrl: utils.StringValueToNullString(project.CdnUrl), - StreamMode: project.StreamMode, - TargetVendor: project.TargetVendor, - AdditionalVendor: project.AdditionalVendor.Value, - Archs: project.Archs, - BuildPoolType: project.BuildPoolType.Value, - FollowImportDist: project.FollowImportDist, - BranchSuffix: utils.StringValueToNullString(project.BranchSuffix), - GitMakePublic: project.GitMakePublic, - VendorMacro: utils.StringValueToNullString(project.VendorMacro), - PackagerMacro: utils.StringValueToNullString(project.PackagerMacro), - } + ret := models.Project{ + Name: project.Name.Value, + MajorVersion: int(project.MajorVersion.Value), + DistTagOverride: utils.StringValueToNullString(project.DistTag), + TargetGitlabHost: project.TargetGitlabHost.Value, + TargetPrefix: project.TargetPrefix.Value, + TargetBranchPrefix: project.TargetBranchPrefix.Value, + SourceGitHost: utils.StringValueToNullString(project.SourceGitHost), + SourcePrefix: utils.StringValueToNullString(project.SourcePrefix), + SourceBranchPrefix: utils.StringValueToNullString(project.SourceBranchPrefix), + CdnUrl: utils.StringValueToNullString(project.CdnUrl), + StreamMode: project.StreamMode, + TargetVendor: project.TargetVendor, + AdditionalVendor: project.AdditionalVendor.Value, + Archs: project.Archs, + BuildPoolType: utils.StringValueToNullString(project.BuildPoolType), + FollowImportDist: project.FollowImportDist, + BranchSuffix: utils.StringValueToNullString(project.BranchSuffix), + GitMakePublic: project.GitMakePublic, + VendorMacro: utils.StringValueToNullString(project.VendorMacro), + PackagerMacro: utils.StringValueToNullString(project.PackagerMacro), + } - err := a.query.Get( - &ret, - ` + err := a.query.Get( + &ret, + ` update projects set name = $1, major_version = $2, @@ -304,65 +304,65 @@ func (a *Access) UpdateProject(id string, project *peridotpb.Project) (*models.P where id = $21 returning id, created_at, updated_at `, - ret.Name, - ret.MajorVersion, - ret.DistTagOverride, - ret.TargetGitlabHost, - ret.TargetPrefix, - ret.TargetBranchPrefix, - ret.SourceGitHost, - ret.SourcePrefix, - ret.SourceBranchPrefix, - ret.CdnUrl, - ret.StreamMode, - ret.TargetVendor, - ret.AdditionalVendor, - ret.Archs, - ret.BuildPoolType, - ret.FollowImportDist, - ret.BranchSuffix, - ret.GitMakePublic, - ret.VendorMacro, - ret.PackagerMacro, - id, - ) - if err != nil { - return nil, err - } + ret.Name, + ret.MajorVersion, + ret.DistTagOverride, + ret.TargetGitlabHost, + ret.TargetPrefix, + ret.TargetBranchPrefix, + ret.SourceGitHost, + ret.SourcePrefix, + ret.SourceBranchPrefix, + ret.CdnUrl, + ret.StreamMode, + ret.TargetVendor, + ret.AdditionalVendor, + ret.Archs, + ret.BuildPoolType, + ret.FollowImportDist, + ret.BranchSuffix, + ret.GitMakePublic, + ret.VendorMacro, + ret.PackagerMacro, + id, + ) + if err != nil { + return nil, err + } - return &ret, nil + return &ret, nil } func (a *Access) SetProjectKeys(projectId string, username string, password string) error { - _, err := a.query.Exec( - ` + _, err := a.query.Exec( + ` insert into project_keys (project_id, gitlab_username, gitlab_secret) values ($1, $2, $3) on conflict (project_id) do update set gitlab_username = $2, gitlab_secret = $3 `, - projectId, - username, - password, - ) - return err + projectId, + username, + password, + ) + return err } func (a *Access) SetBuildRootPackages(projectId string, srpmPackages pq.StringArray, buildPackages pq.StringArray) error { - if srpmPackages == nil { - srpmPackages = pq.StringArray{} - } - if buildPackages == nil { - buildPackages = pq.StringArray{} - } + if srpmPackages == nil { + srpmPackages = pq.StringArray{} + } + if buildPackages == nil { + buildPackages = pq.StringArray{} + } - _, err := a.query.Exec( - ` + _, err := a.query.Exec( + ` update projects set srpm_stage_packages = $2, build_stage_packages = $3 where id = $1 `, - projectId, - srpmPackages, - buildPackages, - ) - return err + projectId, + srpmPackages, + buildPackages, + ) + return err }