[WIP] Break out packages to per-project settings defined by catalog import

* Add support for Software Collections by way of dymanic package
  dependency injection

peridot/builder/v1/workflow/arch.go

@@ -36,9 +36,6 @@ import (
 	"database/sql"
 	"errors"
 	"fmt"
-	"github.com/google/uuid"
-	"go.temporal.io/sdk/activity"
-	"google.golang.org/protobuf/types/known/wrapperspb"
 	"io"
 	"io/fs"
 	"io/ioutil"
@@ -46,21 +43,26 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
-	"peridot.resf.org/peridot/db/models"
-	peridotpb "peridot.resf.org/peridot/pb"
-	"peridot.resf.org/peridot/rpmbuild"
-	"peridot.resf.org/secparse/rpmutils"
-	"peridot.resf.org/servicecatalog"
 	"regexp"
 	"strconv"
 	"strings"
 	"sync"
 	"time"
+
+	"github.com/google/uuid"
+	"go.temporal.io/sdk/activity"
+	"google.golang.org/protobuf/types/known/wrapperspb"
+	"peridot.resf.org/peridot/db/models"
+	peridotpb "peridot.resf.org/peridot/pb"
+	"peridot.resf.org/peridot/rpmbuild"
+	"peridot.resf.org/secparse/rpmutils"
+	"peridot.resf.org/servicecatalog"
 )
 
 var (
 	releaseDistRegex = regexp.MustCompile(".+\\.(el[^. \\t\\n]+)")
-	BuildPkgGroup    = []string{
+	// defaults are for el9
+	DefaultBuildPkgGroup = []string{
 		"bash",
 		"bzip2",
 		"coreutils",
@@ -83,7 +85,8 @@ var (
 		"which",
 		"xz",
 	}
-	SrpmBuildPkgGroup = []string{
+	// defaults are for EL9
+	DefaultSrpmBuildPkgGroup = []string{
 		"bash",
 		"glibc-minimal-langpack",
 		"gnupg2",
@@ -659,8 +662,20 @@ func (c *Controller) BuildArchActivity(ctx context.Context, projectId string, pa
 		return err
 	}
 
+  var pkgGroup []string = DefaultBuildPkgGroup
+
+  if len(project.BuildStagePackages) != 0 {
+    pkgGroup = project.BuildStagePackages
+  }
+
+  if len(pkgEo.DependsOn) != 0 {
+    for _, pkg := range pkgEo.DependsOn {
+      pkgGroup = append(pkgGroup, pkg)
+    }
+  }
+
 	hostArch := os.Getenv("REAL_BUILD_ARCH")
-	err = c.writeMockConfig(&project, packageVersion, extraOptions, arch, hostArch, BuildPkgGroup)
+	err = c.writeMockConfig(&project, packageVersion, extraOptions, arch, hostArch, pkgGroup)
 	if err != nil {
 		return fmt.Errorf("could not write mock config: %v", err)
 	}

peridot/builder/v1/workflow/srpm.go

@@ -39,6 +39,14 @@ import (
 	"encoding/hex"
 	"errors"
 	"fmt"
+	"io"
+	"io/fs"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"time"
+
 	"github.com/cavaliergopher/rpm"
 	"github.com/go-git/go-billy/v5/osfs"
 	"github.com/go-git/go-git/v5"
@@ -48,16 +56,9 @@ import (
 	"go.temporal.io/sdk/activity"
 	"google.golang.org/protobuf/types/known/anypb"
 	"google.golang.org/protobuf/types/known/wrapperspb"
-	"io"
-	"io/fs"
-	"os"
-	"os/exec"
-	"path/filepath"
 	"peridot.resf.org/peridot/db/models"
 	peridotpb "peridot.resf.org/peridot/pb"
 	"peridot.resf.org/peridot/rpmbuild"
-	"strings"
-	"time"
 )
 
 func gitlabify(str string) string {
@@ -415,9 +416,20 @@ func (c *Controller) BuildSRPMActivity(ctx context.Context, upstreamPrefix strin
 		return fmt.Errorf("could not find spec file: %v", err)
 	}
 
+  var pkgGroup []string = DefaultSrpmBuildPkgGroup
+
+  if len(project.SrpmStagePackages) != 0 {
+    pkgGroup = project.SrpmStagePackages
+  }
+  if len(pkgEo.DependsOn) != 0 {
+    for _, pkg := range pkgEo.DependsOn {
+      pkgGroup = append(pkgGroup, pkg)
+    }
+  }
+
 	hostArch := os.Getenv("REAL_BUILD_ARCH")
 	extraOptions.EnableNetworking = true
-	err = c.writeMockConfig(&project, packageVersion, extraOptions, "noarch", hostArch, SrpmBuildPkgGroup)
+	err = c.writeMockConfig(&project, packageVersion, extraOptions, "noarch", hostArch, pkgGroup)
 	if err != nil {
 		return fmt.Errorf("could not write mock config: %v", err)
 	}

peridot/builder/v1/workflow/sync.go

@@ -35,6 +35,14 @@ import (
 	"encoding/base64"
 	"encoding/xml"
 	"fmt"
+	"io/ioutil"
+	"os"
+	"path"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"time"
+
 	"github.com/go-git/go-billy/v5"
 	"github.com/go-git/go-billy/v5/memfs"
 	"github.com/go-git/go-git/v5"
@@ -47,19 +55,12 @@ import (
 	"go.temporal.io/sdk/workflow"
 	"google.golang.org/protobuf/encoding/prototext"
 	"google.golang.org/protobuf/types/known/wrapperspb"
-	"io/ioutil"
-	"os"
-	"path"
-	"path/filepath"
 	peridotdb "peridot.resf.org/peridot/db"
 	"peridot.resf.org/peridot/db/models"
 	peridotpb "peridot.resf.org/peridot/pb"
 	"peridot.resf.org/peridot/yummeta"
 	yumrepofspb "peridot.resf.org/peridot/yumrepofs/pb"
 	"peridot.resf.org/utils"
-	"regexp"
-	"strings"
-	"time"
 )
 
 var (
@@ -450,6 +451,66 @@ func kindCatalogSync(tx peridotdb.Access, req *peridotpb.SyncCatalogRequest, cat
 	return &ret, nil
 }
 
+func processGroupInstallScopedPackageOptions(tx peridotdb.Access, req *peridotpb.SyncCatalogRequest, groupInstallOptionSet *peridotpb.CatalogGroupInstallOption) (scopedPackages *peridotpb.CatalogGroupInstallScopedPackage, err error) {
+	// handle scoped packages relationships on packages for injection into build root
+	for _, scopedPackage := range groupInstallOptionSet.ScopedPackage {
+		// Search by name to allow globbing to support pkg-*
+		pkgs, err := tx.GetPackagesInProject(&peridotpb.PackageFilters{Name: wrapperspb.String(scopedPackage.Name)}, req.ProjectId.Value, 0, 1)
+		if err != nil {
+			return nil, fmt.Errorf("failed to get package %s: %w", scopedPackage.Name, err)
+		}
+		if len(pkgs) == 0 {
+			return nil, fmt.Errorf("package %s not found in project %s (cannot set scoped package options)", scopedPackage.Name, req.ProjectId.Value)
+		}
+		err = tx.SetGroupInstallOptionsForPackage(req.ProjectId.Value, scopedPackage.Name, scopedPackage.DependsOn)
+		if err != nil {
+			return nil, fmt.Errorf("failed to set scoped package options for package %s", scopedPackage.Name)
+		}
+	}
+	return scopedPackages, nil
+}
+
+func processGroupInstallOptionSet(groupInstallOptionSet *peridotpb.CatalogGroupInstallOption) (packages []string, err error) {
+	for _, name := range groupInstallOptionSet.Name {
+		packages = append(packages, name)
+	}
+	if len(packages) == 0 {
+		return nil, fmt.Errorf("failed to parse packages from GroupInstall options")
+	}
+
+	return packages, nil
+}
+
+func kindCatalogGroupInstallOptions(tx peridotdb.Access, req *peridotpb.SyncCatalogRequest, groupInstallOptions []*peridotpb.CatalogGroupInstallOptions) (*peridotpb.KindCatalogGroupInstallOptions, error) {
+	ret := &peridotpb.KindCatalogGroupInstallOptions{}
+
+	for _, groupInstallOption := range groupInstallOptions {
+
+		// Proces scoped packages
+		scopedPackages, err := processGroupInstallScopedPackageOptions(tx, req, groupInstallOption.Srpm)
+		if err != nil {
+			return nil, fmt.Errorf("failed to parse srpm groupinstall options")
+		}
+		ret.ScopedPackage = append(ret.ScopedPackage, scopedPackages)
+
+		// Process build root packages
+		srpmPackages, err := processGroupInstallOptionSet(groupInstallOption.Srpm)
+		if err != nil {
+      return nil, fmt.Errorf("failed to parse srpm groupinstall options: %w", err)
+		}
+		buildPackages, err := processGroupInstallOptionSet(groupInstallOption.Build)
+		if err != nil {
+      return nil, fmt.Errorf("failed to parse build groupinstall options: %w", err)
+		}
+		err = tx.SetBuildRootPackages(req.ProjectId.Value, srpmPackages, buildPackages)
+    
+		ret.SrpmPackages = append(ret.SrpmPackages, srpmPackages...)
+		ret.BuildPackages = append(ret.BuildPackages, buildPackages...)
+	}
+
+	return ret, nil
+}
+
 func kindCatalogExtraOptions(tx peridotdb.Access, req *peridotpb.SyncCatalogRequest, extraOptions []*peridotpb.CatalogExtraOptions) (*peridotpb.KindCatalogExtraOptions, error) {
 	ret := &peridotpb.KindCatalogExtraOptions{}
 
@@ -704,6 +765,7 @@ func (c *Controller) SyncCatalogActivity(req *peridotpb.SyncCatalogRequest) (*pe
 
 	var catalogs []*peridotpb.CatalogSync
 	var extraOptions []*peridotpb.CatalogExtraOptions
+	var groupInstallOptions []*peridotpb.CatalogGroupInstallOptions
 
 	files, err := recursiveSearchBillyFs(w.Filesystem, ".", ".cfg")
 	if err != nil {
@@ -745,6 +807,13 @@ func (c *Controller) SyncCatalogActivity(req *peridotpb.SyncCatalogRequest) (*pe
 				return nil, fmt.Errorf("failed to parse kind resf.peridot.v1.CatalogExtraOptions: %w", err)
 			}
 			extraOptions = append(extraOptions, ce1)
+		case "resf.peridot.v1.CatalogGroupInstallOptions":
+			cg1 := &peridotpb.CatalogGroupInstallOptions{}
+			err = prototext.Unmarshal(bts, cg1)
+			if err != nil {
+				return nil, fmt.Errorf("failed to parse kind resf.peridot.v1.CatalogExtraOptions: %w", err)
+			}
+			groupInstallOptions = append(groupInstallOptions, cg1)
 		default:
 			return nil, fmt.Errorf("unknown format %s", format)
 		}
@@ -768,6 +837,12 @@ func (c *Controller) SyncCatalogActivity(req *peridotpb.SyncCatalogRequest) (*pe
 	}
 	ret.ExtraOptions = resKindCatalogExtraOptions
 
+	resKindCatalogGroupInstallOptions, err := kindCatalogGroupInstallOptions(tx, req, groupInstallOptions)
+	if err != nil {
+		return nil, fmt.Errorf("failed to process kind CatalogSyncGroupInstallOptions: %w", err)
+	}
+	ret.GroupInstallOptions = resKindCatalogGroupInstallOptions
+
 	var buildIDs []string
 	var newBuildPackages []string
 	for _, newPackage := range ret.CatalogSync.NewPackages {

peridot/db/db.go

@@ -45,6 +45,7 @@ type Access interface {
 	CreateProject(project *peridotpb.Project) (*models.Project, error)
 	UpdateProject(id string, project *peridotpb.Project) (*models.Project, error)
 	SetProjectKeys(projectId string, username string, password string) error
+  SetBuildRootPackages(projectId string, srpmPackages []string, buildPackages []string) error
 
 	CreateBuild(packageId string, packageVersionId string, taskId string, projectId string) (*models.Build, error)
 	GetArtifactsForBuild(buildId string) (models.TaskArtifacts, error)
@@ -96,6 +97,7 @@ type Access interface {
 	GetPackageID(name string) (string, error)
 	SetExtraOptionsForPackage(projectId string, packageName string, withFlags pq.StringArray, withoutFlags pq.StringArray) error
 	GetExtraOptionsForPackage(projectId string, packageName string) (*models.ExtraOptions, error)
+  SetGroupInstallOptionsForPackage(projectId string, packageName string, dependsOn pq.StringArray) error
 
 	CreateTask(user *utils.ContextUser, arch string, taskType peridotpb.TaskType, projectId *string, parentTaskId *string) (*models.Task, error)
 	SetTaskStatus(id string, status peridotpb.TaskStatus) error

peridot/db/models/package.go

@@ -77,6 +77,7 @@ type ExtraOptions struct {
 	PackageName  string         `json:"packageName" db:"package_name"`
 	WithFlags    pq.StringArray `json:"withFlags" db:"with_flags"`
 	WithoutFlags pq.StringArray `json:"withoutFlags" db:"without_flags"`
+	DependsOn    pq.StringArray `json:"dependsOn" db:"depends_on"`
 }
 
 func (p *Package) ToProto() *peridotpb.Package {

peridot/db/models/project.go

@@ -70,6 +70,9 @@ type Project struct {
 
 	VendorMacro   sql.NullString `json:"vendorMacro" db:"vendor_macro"`
 	PackagerMacro sql.NullString `json:"packagerMacro" db:"packager_macro"`
+
+  SrpmStagePackages  pq.StringArray `json:"srpmStagePackages" db:"srpm_stage_packages"`
+  BuildStagePackages  pq.StringArray `json:"buildStagePackages" db:"build_stage_packages"`
 }
 
 type Projects []Project

peridot/db/psql/package.go

@@ -344,3 +344,21 @@ func (a *Access) GetExtraOptionsForPackage(projectId string, packageName string)
 	}
 	return &ret, nil
 }
+
+func (a *Access) SetGroupInstallOptionsForPackage(projectId string, packageName string, dependsOn pq.StringArray) error {
+	if dependsOn == nil {
+		dependsOn = pq.StringArray{}
+	}
+	_, err := a.query.Exec(
+		`
+        insert into extra_package_options (project_id, package_name, depends_on)
+        values ($1, $2, $3)
+        on conflict on constraint extra_package_options_uniq do
+            update set depends_on = $3, updated_at = now()
+        `,
+		projectId,
+		packageName,
+    dependsOn,
+	)
+	return err
+}

peridot/db/psql/project.go

@@ -310,3 +310,16 @@ func (a *Access) SetProjectKeys(projectId string, username string, password stri
 	)
 	return err
 }
+
+func (a *Access) SetBuildRootPackages(projectId string, srpmPackages []string, buildPackages []string) error {
+	_, err := a.query.Exec(
+		`
+		update projects set srpm_stage_packages = $2, build_stage_packages = $3
+    where project_id = $1
+		`,
+		projectId,
+		srpmPackages,
+		buildPackages,
+	)
+	return err
+}

peridot/migrate/20221002094519_add-scl-support-and-permit-per-project-buildroot-packages.down.sql

@@ -0,0 +1,6 @@
+ALTER TABLE
+  IF EXISTS extra_package_options DROP COLUMN IF EXISTS depends_on;
+
+ALTER TABLE
+  IF EXISTS projects DROP COLUMN IF EXISTS srpm_stage_packages,
+  DROP COLUMN IF EXISTS build_stage_packages;

peridot/migrate/20221002094519_add-scl-support-and-permit-per-project-buildroot-packages.up.sql

@@ -0,0 +1,11 @@
+ALTER TABLE
+  IF EXISTS extra_package_options
+ADD
+  COLUMN IF NOT EXISTS depends_on text [] not null default array [] :: text [];
+
+ALTER TABLE
+  IF EXISTS projects
+ADD
+  COLUMN IF NOT EXISTS srpm_stage_packages text [] not null default array [] :: text [],
+ADD
+  COLUMN IF NOT EXISTS build_stage_packages text [] not null default array [] :: text [];

peridot/proto/v1/catalog.proto

@@ -55,6 +55,23 @@ message CatalogExtraPackageOptions {
   repeated string without = 3 [(validate.rules).repeated = {unique: true}];
 }
 
+message CatalogGroupInstallScopedPackage {
+    string name = 1 [(validate.rules).string.min_bytes = 1];
+    repeated string depends_on = 2 [(validate.rules).repeated = {unique: true}];
+}
+
+message CatalogGroupInstallOption {
+  // list of all packages required to be installed in the build root per project
+  repeated string name = 1 [(validate.rules).repeated.items.string.min_bytes = 1];
+  // Scoped packages allow for dynamically injecting build requirements into the build root e.g. when building SCLs
+  repeated CatalogGroupInstallScopedPackage scoped_package = 2;
+}
+
+message CatalogGroupInstallOptions {
+  CatalogGroupInstallOption srpm  = 1;
+  CatalogGroupInstallOption build = 2;
+}
+
 message CatalogExtraOptions {
   repeated CatalogExtraPackageOptions package_options = 1;
 }
@@ -71,8 +88,15 @@ message KindCatalogExtraOptions {
   repeated string modified_packages = 1;
 }
 
+message KindCatalogGroupInstallOptions {
+  repeated string srpm_packages  = 1;
+  repeated string build_packages = 2;
+  repeated CatalogGroupInstallScopedPackage scoped_package = 3;
+}
+
 message SyncCatalogTask {
   KindCatalogSync catalog_sync = 1;
   KindCatalogExtraOptions extra_options = 2;
+  KindCatalogGroupInstallOptions group_install_options = 4;
   repeated string reprocess_build_ids = 3;
 }