From b22060dd522095d6475cc5367240f0ac6719c3b1 Mon Sep 17 00:00:00 2001 From: Neil Hanlon Date: Mon, 19 Dec 2022 15:27:40 -0500 Subject: [PATCH] support modular enable/disable for scoped packages Signed-off-by: Neil Hanlon --- peridot/builder/v1/workflow/arch.go | 18 +++++--- peridot/builder/v1/workflow/srpm.go | 42 ++++++++++++++++--- peridot/builder/v1/workflow/sync.go | 2 +- peridot/db/db.go | 2 +- peridot/db/models/package.go | 12 +++--- peridot/db/psql/package.go | 18 +++++--- ...-dependencies-for-scoped-packages.down.sql | 3 ++ ...ar-dependencies-for-scoped-packages.up.sql | 5 +++ peridot/proto/v1/build.proto | 3 ++ peridot/proto/v1/catalog.proto | 2 + 10 files changed, 83 insertions(+), 24 deletions(-) create mode 100644 peridot/migrate/20221219144131_support-modular-dependencies-for-scoped-packages.down.sql create mode 100644 peridot/migrate/20221219144131_support-modular-dependencies-for-scoped-packages.up.sql diff --git a/peridot/builder/v1/workflow/arch.go b/peridot/builder/v1/workflow/arch.go index 62c3e6b..1f0e15d 100644 --- a/peridot/builder/v1/workflow/arch.go +++ b/peridot/builder/v1/workflow/arch.go @@ -535,6 +535,9 @@ config_opts['module_setup_commands'] = [{moduleSetupCommands}] for _, module := range extra.Modules { moduleSetupCommands = append(moduleSetupCommands, fmt.Sprintf("('enable', '%s')", module)) } + for _, module := range extra.DisabledModules { + moduleSetupCommands = append(moduleSetupCommands, fmt.Sprintf("('disable', '%s')", module)) + } mockConfig += "\n" mockConfig += ` @@ -671,14 +674,17 @@ func (c *Controller) BuildArchActivity(ctx context.Context, projectId string, pa pkgGroup = project.BuildStagePackages } - if pkgEo != nil { - if len(pkgEo.DependsOn) != 0 { - for _, pkg := range pkgEo.DependsOn { - pkgGroup = append(pkgGroup, pkg) - } - } + var enableModules []string + var disableModules []string + err = ParsePackageExtraOptions(pkgEo, &pkgGroup, &enableModules, &disableModules) + + if err != nil { + c.log.Infof("no extra options to process for package") } + extraOptions.DisabledModules = disableModules + extraOptions.Modules = enableModules + hostArch := os.Getenv("REAL_BUILD_ARCH") err = c.writeMockConfig(&project, packageVersion, extraOptions, arch, hostArch, pkgGroup) if err != nil { diff --git a/peridot/builder/v1/workflow/srpm.go b/peridot/builder/v1/workflow/srpm.go index 2774c36..bf0300c 100644 --- a/peridot/builder/v1/workflow/srpm.go +++ b/peridot/builder/v1/workflow/srpm.go @@ -421,14 +421,18 @@ func (c *Controller) BuildSRPMActivity(ctx context.Context, upstreamPrefix strin if len(project.SrpmStagePackages) != 0 { pkgGroup = project.SrpmStagePackages } - if pkgEo != nil { - if len(pkgEo.DependsOn) != 0 { - for _, pkg := range pkgEo.DependsOn { - pkgGroup = append(pkgGroup, pkg) - } - } + + var enableModules []string + var disableModules []string + err = ParsePackageExtraOptions(pkgEo, &pkgGroup, &enableModules, &disableModules) + + if err != nil { + c.log.Infof("no extra options to process for package") } + extraOptions.DisabledModules = disableModules + extraOptions.Modules = enableModules + hostArch := os.Getenv("REAL_BUILD_ARCH") extraOptions.EnableNetworking = true err = c.writeMockConfig(&project, packageVersion, extraOptions, "noarch", hostArch, pkgGroup) @@ -474,6 +478,32 @@ func (c *Controller) BuildSRPMActivity(ctx context.Context, upstreamPrefix strin return nil } +func ParsePackageExtraOptions(pkgEo *models.ExtraOptions, pkgGroup *[]string, enableModules *[]string, disableModules *[]string) error { + + if pkgEo == nil { + return fmt.Errorf("no extra options to parse for package") + } + + if len(pkgEo.DependsOn) != 0 { + for _, pkg := range pkgEo.DependsOn { + *pkgGroup = append(*pkgGroup, pkg) + } + } + + if len(pkgEo.EnableModule) != 0 { + for _, pkg := range pkgEo.EnableModule { + *enableModules = append(*enableModules, pkg) + } + } + + if len(pkgEo.DisableModule) != 0 { + for _, pkg := range pkgEo.DisableModule { + *disableModules = append(*disableModules, pkg) + } + } + return nil +} + type UploadActivityResult struct { ObjectName string `json:"objectName"` Subtask *models.Task `json:"subtask"` diff --git a/peridot/builder/v1/workflow/sync.go b/peridot/builder/v1/workflow/sync.go index 4177bba..f734b89 100644 --- a/peridot/builder/v1/workflow/sync.go +++ b/peridot/builder/v1/workflow/sync.go @@ -526,7 +526,7 @@ func processGroupInstallScopedPackageOptions(tx peridotdb.Access, req *peridotpb } for _, dbPkg := range dbPkgs { - err = tx.SetGroupInstallOptionsForPackage(req.ProjectId.Value, dbPkg.Name, scopedPackage.DependsOn) + err = tx.SetGroupInstallOptionsForPackage(req.ProjectId.Value, dbPkg.Name, scopedPackage.DependsOn, scopedPackage.EnableModule, scopedPackage.DisableModule) if err != nil { return nil, fmt.Errorf("failed to set scoped package options for package %s", scopedPackage.Name) } diff --git a/peridot/db/db.go b/peridot/db/db.go index 2719a2a..f86a8df 100644 --- a/peridot/db/db.go +++ b/peridot/db/db.go @@ -101,7 +101,7 @@ type Access interface { GetPackageID(name string) (string, error) SetExtraOptionsForPackage(projectId string, packageName string, withFlags pq.StringArray, withoutFlags pq.StringArray) error GetExtraOptionsForPackage(projectId string, packageName string) (*models.ExtraOptions, error) - SetGroupInstallOptionsForPackage(projectId string, packageName string, dependsOn pq.StringArray) error + SetGroupInstallOptionsForPackage(projectId string, packageName string, dependsOn pq.StringArray, enableModule pq.StringArray, disableModule pq.StringArray) error SetPackageType(projectId string, packageName string, packageType peridotpb.PackageType) error CreateTask(user *utils.ContextUser, arch string, taskType peridotpb.TaskType, projectId *string, parentTaskId *string) (*models.Task, error) diff --git a/peridot/db/models/package.go b/peridot/db/models/package.go index 23077a7..7421a68 100644 --- a/peridot/db/models/package.go +++ b/peridot/db/models/package.go @@ -73,11 +73,13 @@ type ExtraOptions struct { CreatedAt time.Time `json:"createdAt" db:"created_at"` UpdatedAt sql.NullTime `json:"updatedAt" db:"updated_at"` - ProjectId string `json:"projectId" db:"project_id"` - PackageName string `json:"packageName" db:"package_name"` - WithFlags pq.StringArray `json:"withFlags" db:"with_flags"` - WithoutFlags pq.StringArray `json:"withoutFlags" db:"without_flags"` - DependsOn pq.StringArray `json:"dependsOn" db:"depends_on"` + ProjectId string `json:"projectId" db:"project_id"` + PackageName string `json:"packageName" db:"package_name"` + WithFlags pq.StringArray `json:"withFlags" db:"with_flags"` + WithoutFlags pq.StringArray `json:"withoutFlags" db:"without_flags"` + DependsOn pq.StringArray `json:"dependsOn" db:"depends_on"` + EnableModule pq.StringArray `json:"enableModule" db:"enable_module"` + DisableModule pq.StringArray `json:"disableModule" db:"disable_module"` } func (p *Package) ToProto() *peridotpb.Package { diff --git a/peridot/db/psql/package.go b/peridot/db/psql/package.go index 8551842..4cc97a6 100644 --- a/peridot/db/psql/package.go +++ b/peridot/db/psql/package.go @@ -335,7 +335,7 @@ func (a *Access) GetExtraOptionsForPackage(projectId string, packageName string) var ret models.ExtraOptions err := a.query.Get( &ret, - "select id, created_at, updated_at, project_id, package_name, with_flags, without_flags, depends_on from extra_package_options where project_id = $1 and package_name = $2", + "select id, created_at, updated_at, project_id, package_name, with_flags, without_flags, depends_on, enable_module, disable_module from extra_package_options where project_id = $1 and package_name = $2", projectId, packageName, ) @@ -345,16 +345,24 @@ func (a *Access) GetExtraOptionsForPackage(projectId string, packageName string) return &ret, nil } -func (a *Access) SetGroupInstallOptionsForPackage(projectId string, packageName string, dependsOn pq.StringArray) error { +func (a *Access) SetGroupInstallOptionsForPackage(projectId string, packageName string, dependsOn pq.StringArray, enableModule pq.StringArray, disableModule pq.StringArray) error { + //NOTE(nhanlon) - 2022-12-19 there is probably a better way to default these? if dependsOn == nil { dependsOn = pq.StringArray{} } + if enableModule == nil { + enableModule = pq.StringArray{} + } + if disableModule == nil { + disableModule = pq.StringArray{} + } + _, err := a.query.Exec( ` - insert into extra_package_options (project_id, package_name, depends_on) - values ($1, $2, $3) + insert into extra_package_options (project_id, package_name, depends_on, enable_module, disable_module) + values ($1, $2, $3, $4, $5) on conflict on constraint extra_package_options_uniq do - update set depends_on = $3, updated_at = now() + update set depends_on = $3, enable_module = $4, disable_module = $5, updated_at = now() `, projectId, packageName, diff --git a/peridot/migrate/20221219144131_support-modular-dependencies-for-scoped-packages.down.sql b/peridot/migrate/20221219144131_support-modular-dependencies-for-scoped-packages.down.sql new file mode 100644 index 0000000..3c002d5 --- /dev/null +++ b/peridot/migrate/20221219144131_support-modular-dependencies-for-scoped-packages.down.sql @@ -0,0 +1,3 @@ +ALTER TABLE + IF EXISTS extra_package_options DROP COLUMN IF EXISTS enable_module, + IF EXISTS extra_package_options DROP COLUMN IF EXISTS disable_module; diff --git a/peridot/migrate/20221219144131_support-modular-dependencies-for-scoped-packages.up.sql b/peridot/migrate/20221219144131_support-modular-dependencies-for-scoped-packages.up.sql new file mode 100644 index 0000000..10863c7 --- /dev/null +++ b/peridot/migrate/20221219144131_support-modular-dependencies-for-scoped-packages.up.sql @@ -0,0 +1,5 @@ +ALTER TABLE + IF EXISTS extra_package_options +ADD + COLUMN IF NOT EXISTS enable_module text [] not null default array [] :: text [], + COLUMN IF NOT EXISTS disable_module text [] not null default array [] :: text []; diff --git a/peridot/proto/v1/build.proto b/peridot/proto/v1/build.proto index 89f8cd0..220056c 100644 --- a/peridot/proto/v1/build.proto +++ b/peridot/proto/v1/build.proto @@ -324,6 +324,9 @@ message ExtraBuildOptions { // Modules to enable during build repeated string modules = 6; + // Modules to be disabled during build + repeated string disabled_modules = 10; + // Packages to exclude from all repositories not marked with ignore_exclude repeated string exclude_packages = 7; diff --git a/peridot/proto/v1/catalog.proto b/peridot/proto/v1/catalog.proto index bac5b61..2d28d61 100644 --- a/peridot/proto/v1/catalog.proto +++ b/peridot/proto/v1/catalog.proto @@ -61,6 +61,8 @@ message CatalogExtraPackageOptions { message CatalogGroupInstallScopedPackage { string name = 1 [(validate.rules).string.min_bytes = 1]; repeated string depends_on = 2 [(validate.rules).repeated = {unique: true}]; + repeated string enable_module = 3 [(validate.rules).repeated = {unique: true}]; + repeated string disable_module = 4 [(validate.rules).repeated = {unique: true}]; } message CatalogGroupInstallOption {