diff --git a/peridot/builder/v1/workflow/arch.go b/peridot/builder/v1/workflow/arch.go index 8cebb28..4be20f9 100644 --- a/peridot/builder/v1/workflow/arch.go +++ b/peridot/builder/v1/workflow/arch.go @@ -36,9 +36,6 @@ import ( "database/sql" "errors" "fmt" - "github.com/google/uuid" - "go.temporal.io/sdk/activity" - "google.golang.org/protobuf/types/known/wrapperspb" "io" "io/fs" "io/ioutil" @@ -46,21 +43,26 @@ import ( "os" "os/exec" "path/filepath" - "peridot.resf.org/peridot/db/models" - peridotpb "peridot.resf.org/peridot/pb" - "peridot.resf.org/peridot/rpmbuild" - "peridot.resf.org/secparse/rpmutils" - "peridot.resf.org/servicecatalog" "regexp" "strconv" "strings" "sync" "time" + + "github.com/google/uuid" + "go.temporal.io/sdk/activity" + "google.golang.org/protobuf/types/known/wrapperspb" + "peridot.resf.org/peridot/db/models" + peridotpb "peridot.resf.org/peridot/pb" + "peridot.resf.org/peridot/rpmbuild" + "peridot.resf.org/secparse/rpmutils" + "peridot.resf.org/servicecatalog" ) var ( releaseDistRegex = regexp.MustCompile(".+\\.(el[^. \\t\\n]+)") - BuildPkgGroup = []string{ + // defaults are for el9 + DefaultBuildPkgGroup = []string{ "bash", "bzip2", "coreutils", @@ -83,7 +85,8 @@ var ( "which", "xz", } - SrpmBuildPkgGroup = []string{ + // defaults are for EL9 + DefaultSrpmBuildPkgGroup = []string{ "bash", "glibc-minimal-langpack", "gnupg2", @@ -659,8 +662,20 @@ func (c *Controller) BuildArchActivity(ctx context.Context, projectId string, pa return err } + var pkgGroup = DefaultBuildPkgGroup + + if len(project.BuildStagePackages) != 0 { + pkgGroup = project.BuildStagePackages + } + + if len(pkgEo.DependsOn) != 0 { + for _, pkg := range pkgEo.DependsOn { + pkgGroup = append(pkgGroup, pkg) + } + } + hostArch := os.Getenv("REAL_BUILD_ARCH") - err = c.writeMockConfig(&project, packageVersion, extraOptions, arch, hostArch, BuildPkgGroup) + err = c.writeMockConfig(&project, packageVersion, extraOptions, arch, hostArch, pkgGroup) if err != nil { return fmt.Errorf("could not write mock config: %v", err) } diff --git a/peridot/builder/v1/workflow/srpm.go b/peridot/builder/v1/workflow/srpm.go index dae9b6e..b23d149 100644 --- a/peridot/builder/v1/workflow/srpm.go +++ b/peridot/builder/v1/workflow/srpm.go @@ -39,6 +39,14 @@ import ( "encoding/hex" "errors" "fmt" + "io" + "io/fs" + "os" + "os/exec" + "path/filepath" + "strings" + "time" + "github.com/cavaliergopher/rpm" "github.com/go-git/go-billy/v5/osfs" "github.com/go-git/go-git/v5" @@ -48,16 +56,9 @@ import ( "go.temporal.io/sdk/activity" "google.golang.org/protobuf/types/known/anypb" "google.golang.org/protobuf/types/known/wrapperspb" - "io" - "io/fs" - "os" - "os/exec" - "path/filepath" "peridot.resf.org/peridot/db/models" peridotpb "peridot.resf.org/peridot/pb" "peridot.resf.org/peridot/rpmbuild" - "strings" - "time" ) func gitlabify(str string) string { @@ -415,9 +416,20 @@ func (c *Controller) BuildSRPMActivity(ctx context.Context, upstreamPrefix strin return fmt.Errorf("could not find spec file: %v", err) } + var pkgGroup = DefaultSrpmBuildPkgGroup + + if len(project.SrpmStagePackages) != 0 { + pkgGroup = project.SrpmStagePackages + } + if len(pkgEo.DependsOn) != 0 { + for _, pkg := range pkgEo.DependsOn { + pkgGroup = append(pkgGroup, pkg) + } + } + hostArch := os.Getenv("REAL_BUILD_ARCH") extraOptions.EnableNetworking = true - err = c.writeMockConfig(&project, packageVersion, extraOptions, "noarch", hostArch, SrpmBuildPkgGroup) + err = c.writeMockConfig(&project, packageVersion, extraOptions, "noarch", hostArch, pkgGroup) if err != nil { return fmt.Errorf("could not write mock config: %v", err) } diff --git a/peridot/builder/v1/workflow/sync.go b/peridot/builder/v1/workflow/sync.go index f065eaa..8825204 100644 --- a/peridot/builder/v1/workflow/sync.go +++ b/peridot/builder/v1/workflow/sync.go @@ -35,6 +35,15 @@ import ( "encoding/base64" "encoding/xml" "fmt" + "github.com/gobwas/glob" + "io/ioutil" + "os" + "path" + "path/filepath" + "regexp" + "strings" + "time" + "github.com/go-git/go-billy/v5" "github.com/go-git/go-billy/v5/memfs" "github.com/go-git/go-git/v5" @@ -47,19 +56,12 @@ import ( "go.temporal.io/sdk/workflow" "google.golang.org/protobuf/encoding/prototext" "google.golang.org/protobuf/types/known/wrapperspb" - "io/ioutil" - "os" - "path" - "path/filepath" peridotdb "peridot.resf.org/peridot/db" "peridot.resf.org/peridot/db/models" peridotpb "peridot.resf.org/peridot/pb" "peridot.resf.org/peridot/yummeta" yumrepofspb "peridot.resf.org/peridot/yumrepofs/pb" "peridot.resf.org/utils" - "regexp" - "strings" - "time" ) var ( @@ -450,6 +452,97 @@ func kindCatalogSync(tx peridotdb.Access, req *peridotpb.SyncCatalogRequest, cat return &ret, nil } +func processGroupInstallScopedPackageOptions(tx peridotdb.Access, req *peridotpb.SyncCatalogRequest, groupInstallOptionSet *peridotpb.CatalogGroupInstallOption) (scopedPackages *peridotpb.CatalogGroupInstallScopedPackage, err error) { + // handle scoped packages relationships on packages for injection into build root + for _, scopedPackage := range groupInstallOptionSet.ScopedPackage { + filters := &peridotpb.PackageFilters{NameExact: wrapperspb.String(scopedPackage.Name)} + isGlob := false + if strings.HasPrefix(scopedPackage.Name, "*") || strings.HasSuffix(scopedPackage.Name, "*") { + filters.Name = wrapperspb.String(strings.TrimSuffix(strings.TrimPrefix(scopedPackage.Name, "*"), "*")) + filters.NameExact = nil + isGlob = true + } + + pkgs, err := tx.GetPackagesInProject(filters, req.ProjectId.Value, 0, -1) + if err != nil { + return nil, fmt.Errorf("failed to get package %s: %w", scopedPackage.Name, err) + } + if len(pkgs) == 0 { + return nil, fmt.Errorf("package %s not found in project %s (scoped package)", scopedPackage.Name, req.ProjectId.Value) + } + + var dbPkgs []models.Package + if isGlob { + for _, p := range pkgs { + g, err := glob.Compile(scopedPackage.Name) + if err != nil { + return nil, fmt.Errorf("failed to compile glob %s: %w", scopedPackage.Name, err) + } + if g.Match(p.Name) { + dbPkgs = append(dbPkgs, p) + } + } + } else { + if scopedPackage.Name != pkgs[0].Name { + return nil, fmt.Errorf("package %s not found in project %s (cannot set extra options, not glob)", scopedPackage.Name, req.ProjectId.Value) + } + dbPkgs = append(dbPkgs, pkgs[0]) + } + if len(dbPkgs) == 0 { + return nil, fmt.Errorf("package %s not found in project %s (cannot set extra options, glob)", scopedPackage.Name, req.ProjectId.Value) + } + + for _, dbPkg := range dbPkgs { + err = tx.SetGroupInstallOptionsForPackage(req.ProjectId.Value, dbPkg.Name, scopedPackage.DependsOn) + if err != nil { + return nil, fmt.Errorf("failed to set scoped package options for package %s", scopedPackage.Name) + } + } + } + return scopedPackages, nil +} + +func processGroupInstallOptionSet(groupInstallOptionSet *peridotpb.CatalogGroupInstallOption) (packages []string, err error) { + for _, name := range groupInstallOptionSet.Name { + packages = append(packages, name) + } + if len(packages) == 0 { + return nil, fmt.Errorf("failed to parse packages from GroupInstall options") + } + + return packages, nil +} + +func kindCatalogGroupInstallOptions(tx peridotdb.Access, req *peridotpb.SyncCatalogRequest, groupInstallOptions []*peridotpb.CatalogGroupInstallOptions) (*peridotpb.KindCatalogGroupInstallOptions, error) { + ret := &peridotpb.KindCatalogGroupInstallOptions{} + + for _, groupInstallOption := range groupInstallOptions { + + // Proces scoped packages + scopedPackages, err := processGroupInstallScopedPackageOptions(tx, req, groupInstallOption.Srpm) + if err != nil { + return nil, fmt.Errorf("failed to parse srpm groupinstall options: %s", err.Error()) + } + ret.ScopedPackage = append(ret.ScopedPackage, scopedPackages) + + // Process build root packages + srpmPackages, err := processGroupInstallOptionSet(groupInstallOption.Srpm) + if err != nil { + return nil, fmt.Errorf("failed to parse srpm groupinstall options: %w", err) + } + buildPackages, err := processGroupInstallOptionSet(groupInstallOption.Build) + if err != nil { + return nil, fmt.Errorf("failed to parse build groupinstall options: %w", err) + } + err = tx.SetBuildRootPackages(req.ProjectId.Value, srpmPackages, buildPackages) + + ret.SrpmPackages = append(ret.SrpmPackages, srpmPackages...) + ret.BuildPackages = append(ret.BuildPackages, buildPackages...) + } + + return ret, nil +} + func kindCatalogExtraOptions(tx peridotdb.Access, req *peridotpb.SyncCatalogRequest, extraOptions []*peridotpb.CatalogExtraOptions) (*peridotpb.KindCatalogExtraOptions, error) { ret := &peridotpb.KindCatalogExtraOptions{} @@ -704,6 +797,7 @@ func (c *Controller) SyncCatalogActivity(req *peridotpb.SyncCatalogRequest) (*pe var catalogs []*peridotpb.CatalogSync var extraOptions []*peridotpb.CatalogExtraOptions + var groupInstallOptions []*peridotpb.CatalogGroupInstallOptions files, err := recursiveSearchBillyFs(w.Filesystem, ".", ".cfg") if err != nil { @@ -745,6 +839,13 @@ func (c *Controller) SyncCatalogActivity(req *peridotpb.SyncCatalogRequest) (*pe return nil, fmt.Errorf("failed to parse kind resf.peridot.v1.CatalogExtraOptions: %w", err) } extraOptions = append(extraOptions, ce1) + case "resf.peridot.v1.CatalogGroupInstallOptions": + cg1 := &peridotpb.CatalogGroupInstallOptions{} + err = prototext.Unmarshal(bts, cg1) + if err != nil { + return nil, fmt.Errorf("failed to parse kind resf.peridot.v1.CatalogGroupInstallOptions: %w", err) + } + groupInstallOptions = append(groupInstallOptions, cg1) default: return nil, fmt.Errorf("unknown format %s", format) } @@ -768,6 +869,12 @@ func (c *Controller) SyncCatalogActivity(req *peridotpb.SyncCatalogRequest) (*pe } ret.ExtraOptions = resKindCatalogExtraOptions + resKindCatalogGroupInstallOptions, err := kindCatalogGroupInstallOptions(tx, req, groupInstallOptions) + if err != nil { + return nil, fmt.Errorf("failed to process kind CatalogSyncGroupInstallOptions: %w", err) + } + ret.GroupInstallOptions = resKindCatalogGroupInstallOptions + var buildIDs []string var newBuildPackages []string for _, newPackage := range ret.CatalogSync.NewPackages { diff --git a/peridot/db/db.go b/peridot/db/db.go index 17b89e5..a620ba0 100644 --- a/peridot/db/db.go +++ b/peridot/db/db.go @@ -45,6 +45,7 @@ type Access interface { CreateProject(project *peridotpb.Project) (*models.Project, error) UpdateProject(id string, project *peridotpb.Project) (*models.Project, error) SetProjectKeys(projectId string, username string, password string) error + SetBuildRootPackages(projectId string, srpmPackages []string, buildPackages []string) error CreateBuild(packageId string, packageVersionId string, taskId string, projectId string) (*models.Build, error) GetArtifactsForBuild(buildId string) (models.TaskArtifacts, error) @@ -96,6 +97,7 @@ type Access interface { GetPackageID(name string) (string, error) SetExtraOptionsForPackage(projectId string, packageName string, withFlags pq.StringArray, withoutFlags pq.StringArray) error GetExtraOptionsForPackage(projectId string, packageName string) (*models.ExtraOptions, error) + SetGroupInstallOptionsForPackage(projectId string, packageName string, dependsOn pq.StringArray) error CreateTask(user *utils.ContextUser, arch string, taskType peridotpb.TaskType, projectId *string, parentTaskId *string) (*models.Task, error) SetTaskStatus(id string, status peridotpb.TaskStatus) error diff --git a/peridot/db/models/package.go b/peridot/db/models/package.go index e2a1925..23077a7 100644 --- a/peridot/db/models/package.go +++ b/peridot/db/models/package.go @@ -77,6 +77,7 @@ type ExtraOptions struct { PackageName string `json:"packageName" db:"package_name"` WithFlags pq.StringArray `json:"withFlags" db:"with_flags"` WithoutFlags pq.StringArray `json:"withoutFlags" db:"without_flags"` + DependsOn pq.StringArray `json:"dependsOn" db:"depends_on"` } func (p *Package) ToProto() *peridotpb.Package { diff --git a/peridot/db/models/project.go b/peridot/db/models/project.go index 756cb01..fe6c8da 100644 --- a/peridot/db/models/project.go +++ b/peridot/db/models/project.go @@ -70,6 +70,9 @@ type Project struct { VendorMacro sql.NullString `json:"vendorMacro" db:"vendor_macro"` PackagerMacro sql.NullString `json:"packagerMacro" db:"packager_macro"` + + SrpmStagePackages pq.StringArray `json:"srpmStagePackages" db:"srpm_stage_packages"` + BuildStagePackages pq.StringArray `json:"buildStagePackages" db:"build_stage_packages"` } type Projects []Project diff --git a/peridot/db/psql/package.go b/peridot/db/psql/package.go index 1db767d..949571c 100644 --- a/peridot/db/psql/package.go +++ b/peridot/db/psql/package.go @@ -335,7 +335,7 @@ func (a *Access) GetExtraOptionsForPackage(projectId string, packageName string) var ret models.ExtraOptions err := a.query.Get( &ret, - "select id, created_at, updated_at, project_id, package_name, with_flags, without_flags from extra_package_options where project_id = $1 and package_name = $2", + "select id, created_at, updated_at, project_id, package_name, with_flags, without_flags, depends_on from extra_package_options where project_id = $1 and package_name = $2", projectId, packageName, ) @@ -344,3 +344,21 @@ func (a *Access) GetExtraOptionsForPackage(projectId string, packageName string) } return &ret, nil } + +func (a *Access) SetGroupInstallOptionsForPackage(projectId string, packageName string, dependsOn pq.StringArray) error { + if dependsOn == nil { + dependsOn = pq.StringArray{} + } + _, err := a.query.Exec( + ` + insert into extra_package_options (project_id, package_name, depends_on) + values ($1, $2, $3) + on conflict on constraint extra_package_options_uniq do + update set depends_on = $3, updated_at = now() + `, + projectId, + packageName, + dependsOn, + ) + return err +} diff --git a/peridot/db/psql/project.go b/peridot/db/psql/project.go index 73ada62..b2a93a4 100644 --- a/peridot/db/psql/project.go +++ b/peridot/db/psql/project.go @@ -310,3 +310,16 @@ func (a *Access) SetProjectKeys(projectId string, username string, password stri ) return err } + +func (a *Access) SetBuildRootPackages(projectId string, srpmPackages []string, buildPackages []string) error { + _, err := a.query.Exec( + ` + update projects set srpm_stage_packages = $2, build_stage_packages = $3 + where project_id = $1 + `, + projectId, + srpmPackages, + buildPackages, + ) + return err +} diff --git a/peridot/migrate/20221002094519_add-scl-support-and-permit-per-project-buildroot-packages.down.sql b/peridot/migrate/20221002094519_add-scl-support-and-permit-per-project-buildroot-packages.down.sql new file mode 100644 index 0000000..224b5c1 --- /dev/null +++ b/peridot/migrate/20221002094519_add-scl-support-and-permit-per-project-buildroot-packages.down.sql @@ -0,0 +1,6 @@ +ALTER TABLE + IF EXISTS extra_package_options DROP COLUMN IF EXISTS depends_on; + +ALTER TABLE + IF EXISTS projects DROP COLUMN IF EXISTS srpm_stage_packages, + DROP COLUMN IF EXISTS build_stage_packages; diff --git a/peridot/migrate/20221002094519_add-scl-support-and-permit-per-project-buildroot-packages.up.sql b/peridot/migrate/20221002094519_add-scl-support-and-permit-per-project-buildroot-packages.up.sql new file mode 100644 index 0000000..df7f473 --- /dev/null +++ b/peridot/migrate/20221002094519_add-scl-support-and-permit-per-project-buildroot-packages.up.sql @@ -0,0 +1,11 @@ +ALTER TABLE + IF EXISTS extra_package_options +ADD + COLUMN IF NOT EXISTS depends_on text [] not null default array [] :: text []; + +ALTER TABLE + IF EXISTS projects +ADD + COLUMN IF NOT EXISTS srpm_stage_packages text [] not null default array [] :: text [], +ADD + COLUMN IF NOT EXISTS build_stage_packages text [] not null default array [] :: text []; diff --git a/peridot/proto/v1/catalog.proto b/peridot/proto/v1/catalog.proto index 609a2c8..cad5440 100644 --- a/peridot/proto/v1/catalog.proto +++ b/peridot/proto/v1/catalog.proto @@ -55,6 +55,23 @@ message CatalogExtraPackageOptions { repeated string without = 3 [(validate.rules).repeated = {unique: true}]; } +message CatalogGroupInstallScopedPackage { + string name = 1 [(validate.rules).string.min_bytes = 1]; + repeated string depends_on = 2 [(validate.rules).repeated = {unique: true}]; +} + +message CatalogGroupInstallOption { + // list of all packages required to be installed in the build root per project + repeated string name = 1 [(validate.rules).repeated.items.string.min_bytes = 1]; + // Scoped packages allow for dynamically injecting build requirements into the build root e.g. when building SCLs + repeated CatalogGroupInstallScopedPackage scoped_package = 2; +} + +message CatalogGroupInstallOptions { + CatalogGroupInstallOption srpm = 1; + CatalogGroupInstallOption build = 2; +} + message CatalogExtraOptions { repeated CatalogExtraPackageOptions package_options = 1; } @@ -71,8 +88,15 @@ message KindCatalogExtraOptions { repeated string modified_packages = 1; } +message KindCatalogGroupInstallOptions { + repeated string srpm_packages = 1; + repeated string build_packages = 2; + repeated CatalogGroupInstallScopedPackage scoped_package = 3; +} + message SyncCatalogTask { KindCatalogSync catalog_sync = 1; KindCatalogExtraOptions extra_options = 2; + KindCatalogGroupInstallOptions group_install_options = 4; repeated string reprocess_build_ids = 3; } diff --git a/utils/pointer.go b/utils/pointer.go index 5bc8f26..237f867 100644 --- a/utils/pointer.go +++ b/utils/pointer.go @@ -119,5 +119,6 @@ func Bool(b bool) *bool { } func Pointer[T any](t T) *T { - return &t + s := t + return &s }