rocky-linux/peridot
4
0
Fork 0
mirror of https://github.com/rocky-linux/peridot.git synced 2024-11-18 03:11:24 +00:00
Code Issues Projects Releases Activity

[WIP] Break out packages to per-project settings defined by catalog import

Browse Source 
* Add support for Software Collections by way of dymanic package
  dependency injection
This commit is contained in:
Neil Hanlon 2022-10-01 15:19:24 -04:00
parent f57052758e
commit 949ab935e6
Signed by: neil
GPG Key ID: 705BC21EC3C70F34
11 changed files with 206 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
peridot/builder/v1/workflow/arch.go
View File

@ -36,9 +36,6 @@ import (
"database/sql"
"errors"
"fmt"
"github.com/google/uuid"
"go.temporal.io/sdk/activity"
"google.golang.org/protobuf/types/known/wrapperspb"
"io"
"io/fs"
"io/ioutil"
@ -46,21 +43,26 @@ import (
"os"
"os/exec"
"path/filepath"
"peridot.resf.org/peridot/db/models"
peridotpb "peridot.resf.org/peridot/pb"
"peridot.resf.org/peridot/rpmbuild"
"peridot.resf.org/secparse/rpmutils"
"peridot.resf.org/servicecatalog"
"regexp"
"strconv"
"strings"
"sync"
"time"
"github.com/google/uuid"
"go.temporal.io/sdk/activity"
"google.golang.org/protobuf/types/known/wrapperspb"
"peridot.resf.org/peridot/db/models"
peridotpb "peridot.resf.org/peridot/pb"
"peridot.resf.org/peridot/rpmbuild"
"peridot.resf.org/secparse/rpmutils"
"peridot.resf.org/servicecatalog"
)
var (
releaseDistRegex = regexp.MustCompile(".+\\.(el[^. \\t\\n]+)")
BuildPkgGroup = []string{
// defaults are for el9
DefaultBuildPkgGroup = []string{
"bash",
"bzip2",
"coreutils",
@ -83,7 +85,8 @@ var (
"which",
"xz",
}
SrpmBuildPkgGroup = []string{
// defaults are for EL9
DefaultSrpmBuildPkgGroup = []string{
"bash",
"glibc-minimal-langpack",
"gnupg2",
@ -659,8 +662,20 @@ func (c *Controller) BuildArchActivity(ctx context.Context, projectId string, pa
return err
}
var pkgGroup []string = DefaultBuildPkgGroup
if len(project.BuildStagePackages) != 0 {
pkgGroup = project.BuildStagePackages
}
if len(pkgEo.DependsOn) != 0 {
for _, pkg := range pkgEo.DependsOn {
pkgGroup = append(pkgGroup, pkg)
}
}
hostArch := os.Getenv("REAL_BUILD_ARCH")
err = c.writeMockConfig(&project, packageVersion, extraOptions, arch, hostArch, BuildPkgGroup)
err = c.writeMockConfig(&project, packageVersion, extraOptions, arch, hostArch, pkgGroup)
if err != nil {
return fmt.Errorf("could not write mock config: %v", err)
}

28
peridot/builder/v1/workflow/srpm.go
View File

@ -39,6 +39,14 @@ import (
"encoding/hex"
"errors"
"fmt"
"io"
"io/fs"
"os"
"os/exec"
"path/filepath"
"strings"
"time"
"github.com/cavaliergopher/rpm"
"github.com/go-git/go-billy/v5/osfs"
"github.com/go-git/go-git/v5"
@ -48,16 +56,9 @@ import (
"go.temporal.io/sdk/activity"
"google.golang.org/protobuf/types/known/anypb"
"google.golang.org/protobuf/types/known/wrapperspb"
"io"
"io/fs"
"os"
"os/exec"
"path/filepath"
"peridot.resf.org/peridot/db/models"
peridotpb "peridot.resf.org/peridot/pb"
"peridot.resf.org/peridot/rpmbuild"
"strings"
"time"
)
func gitlabify(str string) string {
@ -415,9 +416,20 @@ func (c *Controller) BuildSRPMActivity(ctx context.Context, upstreamPrefix strin
return fmt.Errorf("could not find spec file: %v", err)
}
var pkgGroup []string = DefaultSrpmBuildPkgGroup
if len(project.SrpmStagePackages) != 0 {
pkgGroup = project.SrpmStagePackages
}
if len(pkgEo.DependsOn) != 0 {
for _, pkg := range pkgEo.DependsOn {
pkgGroup = append(pkgGroup, pkg)
}
}
hostArch := os.Getenv("REAL_BUILD_ARCH")
extraOptions.EnableNetworking = true
err = c.writeMockConfig(&project, packageVersion, extraOptions, "noarch", hostArch, SrpmBuildPkgGroup)
err = c.writeMockConfig(&project, packageVersion, extraOptions, "noarch", hostArch, pkgGroup)
if err != nil {
return fmt.Errorf("could not write mock config: %v", err)
}

89
peridot/builder/v1/workflow/sync.go
View File

@ -35,6 +35,14 @@ import (
"encoding/base64"
"encoding/xml"
"fmt"
"io/ioutil"
"os"
"path"
"path/filepath"
"regexp"
"strings"
"time"
"github.com/go-git/go-billy/v5"
"github.com/go-git/go-billy/v5/memfs"
"github.com/go-git/go-git/v5"
@ -47,19 +55,12 @@ import (
"go.temporal.io/sdk/workflow"
"google.golang.org/protobuf/encoding/prototext"
"google.golang.org/protobuf/types/known/wrapperspb"
"io/ioutil"
"os"
"path"
"path/filepath"
peridotdb "peridot.resf.org/peridot/db"
"peridot.resf.org/peridot/db/models"
peridotpb "peridot.resf.org/peridot/pb"
"peridot.resf.org/peridot/yummeta"
yumrepofspb "peridot.resf.org/peridot/yumrepofs/pb"
"peridot.resf.org/utils"
"regexp"
"strings"
"time"
)
var (
@ -450,6 +451,66 @@ func kindCatalogSync(tx peridotdb.Access, req *peridotpb.SyncCatalogRequest, cat
return &ret, nil
}
func processGroupInstallScopedPackageOptions(tx peridotdb.Access, req *peridotpb.SyncCatalogRequest, groupInstallOptionSet *peridotpb.CatalogGroupInstallOption) (scopedPackages *peridotpb.CatalogGroupInstallScopedPackage, err error) {
// handle scoped packages relationships on packages for injection into build root
for _, scopedPackage := range groupInstallOptionSet.ScopedPackage {
// Search by name to allow globbing to support pkg-*
pkgs, err := tx.GetPackagesInProject(&peridotpb.PackageFilters{Name: wrapperspb.String(scopedPackage.Name)}, req.ProjectId.Value, 0, 1)
if err != nil {
return nil, fmt.Errorf("failed to get package %s: %w", scopedPackage.Name, err)
}
if len(pkgs) == 0 {
return nil, fmt.Errorf("package %s not found in project %s (cannot set scoped package options)", scopedPackage.Name, req.ProjectId.Value)
}
err = tx.SetGroupInstallOptionsForPackage(req.ProjectId.Value, scopedPackage.Name, scopedPackage.DependsOn)
if err != nil {
return nil, fmt.Errorf("failed to set scoped package options for package %s", scopedPackage.Name)
}
}
return scopedPackages, nil
}
func processGroupInstallOptionSet(groupInstallOptionSet *peridotpb.CatalogGroupInstallOption) (packages []string, err error) {
for _, name := range groupInstallOptionSet.Name {
packages = append(packages, name)
}
if len(packages) == 0 {
return nil, fmt.Errorf("failed to parse packages from GroupInstall options")
}
return packages, nil
}
func kindCatalogGroupInstallOptions(tx peridotdb.Access, req *peridotpb.SyncCatalogRequest, groupInstallOptions []*peridotpb.CatalogGroupInstallOptions) (*peridotpb.KindCatalogGroupInstallOptions, error) {
ret := &peridotpb.KindCatalogGroupInstallOptions{}
for _, groupInstallOption := range groupInstallOptions {
// Proces scoped packages
scopedPackages, err := processGroupInstallScopedPackageOptions(tx, req, groupInstallOption.Srpm)
if err != nil {
return nil, fmt.Errorf("failed to parse srpm groupinstall options")
}
ret.ScopedPackage = append(ret.ScopedPackage, scopedPackages)
// Process build root packages
srpmPackages, err := processGroupInstallOptionSet(groupInstallOption.Srpm)
if err != nil {
return nil, fmt.Errorf("failed to parse srpm groupinstall options: %w", err)
}
buildPackages, err := processGroupInstallOptionSet(groupInstallOption.Build)
if err != nil {
return nil, fmt.Errorf("failed to parse build groupinstall options: %w", err)
}
err = tx.SetBuildRootPackages(req.ProjectId.Value, srpmPackages, buildPackages)
ret.SrpmPackages = append(ret.SrpmPackages, srpmPackages...)
ret.BuildPackages = append(ret.BuildPackages, buildPackages...)
}
return ret, nil
}
func kindCatalogExtraOptions(tx peridotdb.Access, req *peridotpb.SyncCatalogRequest, extraOptions []*peridotpb.CatalogExtraOptions) (*peridotpb.KindCatalogExtraOptions, error) {
ret := &peridotpb.KindCatalogExtraOptions{}
@ -704,6 +765,7 @@ func (c *Controller) SyncCatalogActivity(req *peridotpb.SyncCatalogRequest) (*pe
var catalogs []*peridotpb.CatalogSync
var extraOptions []*peridotpb.CatalogExtraOptions
var groupInstallOptions []*peridotpb.CatalogGroupInstallOptions
files, err := recursiveSearchBillyFs(w.Filesystem, ".", ".cfg")
if err != nil {
@ -745,6 +807,13 @@ func (c *Controller) SyncCatalogActivity(req *peridotpb.SyncCatalogRequest) (*pe
return nil, fmt.Errorf("failed to parse kind resf.peridot.v1.CatalogExtraOptions: %w", err)
}
extraOptions = append(extraOptions, ce1)
case "resf.peridot.v1.CatalogGroupInstallOptions":
cg1 := &peridotpb.CatalogGroupInstallOptions{}
err = prototext.Unmarshal(bts, cg1)
if err != nil {
return nil, fmt.Errorf("failed to parse kind resf.peridot.v1.CatalogExtraOptions: %w", err)
}
groupInstallOptions = append(groupInstallOptions, cg1)
default:
return nil, fmt.Errorf("unknown format %s", format)
}
@ -768,6 +837,12 @@ func (c *Controller) SyncCatalogActivity(req *peridotpb.SyncCatalogRequest) (*pe
}
ret.ExtraOptions = resKindCatalogExtraOptions
resKindCatalogGroupInstallOptions, err := kindCatalogGroupInstallOptions(tx, req, groupInstallOptions)
if err != nil {
return nil, fmt.Errorf("failed to process kind CatalogSyncGroupInstallOptions: %w", err)
}
ret.GroupInstallOptions = resKindCatalogGroupInstallOptions
var buildIDs []string
var newBuildPackages []string
for _, newPackage := range ret.CatalogSync.NewPackages {

2
peridot/db/db.go
View File

@ -45,6 +45,7 @@ type Access interface {
CreateProject(project *peridotpb.Project) (*models.Project, error)
UpdateProject(id string, project *peridotpb.Project) (*models.Project, error)
SetProjectKeys(projectId string, username string, password string) error
SetBuildRootPackages(projectId string, srpmPackages []string, buildPackages []string) error
CreateBuild(packageId string, packageVersionId string, taskId string, projectId string) (*models.Build, error)
GetArtifactsForBuild(buildId string) (models.TaskArtifacts, error)
@ -96,6 +97,7 @@ type Access interface {
GetPackageID(name string) (string, error)
SetExtraOptionsForPackage(projectId string, packageName string, withFlags pq.StringArray, withoutFlags pq.StringArray) error
GetExtraOptionsForPackage(projectId string, packageName string) (*models.ExtraOptions, error)
SetGroupInstallOptionsForPackage(projectId string, packageName string, dependsOn pq.StringArray) error
CreateTask(user *utils.ContextUser, arch string, taskType peridotpb.TaskType, projectId *string, parentTaskId *string) (*models.Task, error)
SetTaskStatus(id string, status peridotpb.TaskStatus) error

1
peridot/db/models/package.go
View File

@ -77,6 +77,7 @@ type ExtraOptions struct {
PackageName string `json:"packageName" db:"package_name"`
WithFlags pq.StringArray `json:"withFlags" db:"with_flags"`
WithoutFlags pq.StringArray `json:"withoutFlags" db:"without_flags"`
DependsOn pq.StringArray `json:"dependsOn" db:"depends_on"`
}
func (p *Package) ToProto() *peridotpb.Package {

3
peridot/db/models/project.go
View File

@ -70,6 +70,9 @@ type Project struct {
VendorMacro sql.NullString `json:"vendorMacro" db:"vendor_macro"`
PackagerMacro sql.NullString `json:"packagerMacro" db:"packager_macro"`
SrpmStagePackages pq.StringArray `json:"srpmStagePackages" db:"srpm_stage_packages"`
BuildStagePackages pq.StringArray `json:"buildStagePackages" db:"build_stage_packages"`
}
type Projects []Project

18
peridot/db/psql/package.go
View File

@ -344,3 +344,21 @@ func (a *Access) GetExtraOptionsForPackage(projectId string, packageName string)
}
return &ret, nil
}
func (a *Access) SetGroupInstallOptionsForPackage(projectId string, packageName string, dependsOn pq.StringArray) error {
if dependsOn == nil {
dependsOn = pq.StringArray{}
}
_, err := a.query.Exec(
`
insert into extra_package_options (project_id, package_name, depends_on)
values ($1, $2, $3)
on conflict on constraint extra_package_options_uniq do
update set depends_on = $3, updated_at = now()
`,
projectId,
packageName,
dependsOn,
)
return err
}

13
peridot/db/psql/project.go
View File

@ -310,3 +310,16 @@ func (a *Access) SetProjectKeys(projectId string, username string, password stri
)
return err
}
func (a *Access) SetBuildRootPackages(projectId string, srpmPackages []string, buildPackages []string) error {
_, err := a.query.Exec(
`
update projects set srpm_stage_packages = $2, build_stage_packages = $3
where project_id = $1
`,
projectId,
srpmPackages,
buildPackages,
)
return err
}

6
peridot/migrate/20221002094519_add-scl-support-and-permit-per-project-buildroot-packages.down.sql Normal file
View File

@ -0,0 +1,6 @@
ALTER TABLE
IF EXISTS extra_package_options DROP COLUMN IF EXISTS depends_on;
ALTER TABLE
IF EXISTS projects DROP COLUMN IF EXISTS srpm_stage_packages,
DROP COLUMN IF EXISTS build_stage_packages;

11
peridot/migrate/20221002094519_add-scl-support-and-permit-per-project-buildroot-packages.up.sql Normal file
View File

@ -0,0 +1,11 @@
ALTER TABLE
IF EXISTS extra_package_options
ADD
COLUMN IF NOT EXISTS depends_on text [] not null default array [] :: text [];
ALTER TABLE
IF EXISTS projects
ADD
COLUMN IF NOT EXISTS srpm_stage_packages text [] not null default array [] :: text [],
ADD
COLUMN IF NOT EXISTS build_stage_packages text [] not null default array [] :: text [];

24
peridot/proto/v1/catalog.proto
View File

@ -55,6 +55,23 @@ message CatalogExtraPackageOptions {
repeated string without = 3 [(validate.rules).repeated = {unique: true}];
}
message CatalogGroupInstallScopedPackage {
string name = 1 [(validate.rules).string.min_bytes = 1];
repeated string depends_on = 2 [(validate.rules).repeated = {unique: true}];
}
message CatalogGroupInstallOption {
// list of all packages required to be installed in the build root per project
repeated string name = 1 [(validate.rules).repeated.items.string.min_bytes = 1];
// Scoped packages allow for dynamically injecting build requirements into the build root e.g. when building SCLs
repeated CatalogGroupInstallScopedPackage scoped_package = 2;
}
message CatalogGroupInstallOptions {
CatalogGroupInstallOption srpm = 1;
CatalogGroupInstallOption build = 2;
}
message CatalogExtraOptions {
repeated CatalogExtraPackageOptions package_options = 1;
}
@ -71,8 +88,15 @@ message KindCatalogExtraOptions {
repeated string modified_packages = 1;
}
message KindCatalogGroupInstallOptions {
repeated string srpm_packages = 1;
repeated string build_packages = 2;
repeated CatalogGroupInstallScopedPackage scoped_package = 3;
}
message SyncCatalogTask {
KindCatalogSync catalog_sync = 1;
KindCatalogExtraOptions extra_options = 2;
KindCatalogGroupInstallOptions group_install_options = 4;
repeated string reprocess_build_ids = 3;
}