From d62a16923f955f3c286bf34bf5e9248ba9b338c3 Mon Sep 17 00:00:00 2001 From: Mustafa Gezen Date: Fri, 4 Nov 2022 03:25:09 +0100 Subject: [PATCH] Support modulemd v3 and module sync --- modulemd/BUILD.bazel | 10 ++- modulemd/modulemd.go | 72 ++++++++++++++++++- modulemd/v3.go | 62 ++++++++++++++++ peridot/builder/v1/workflow/BUILD.bazel | 2 +- peridot/builder/v1/workflow/arch.go | 4 ++ peridot/builder/v1/workflow/module.go | 66 ++++++++++++++++- peridot/builder/v1/workflow/sync.go | 49 +++++++++++++ peridot/db/db.go | 3 + peridot/db/psql/build.go | 31 ++++++++ peridot/db/psql/package.go | 13 ++++ peridot/db/psql/project.go | 28 ++++++++ peridot/impl/v1/build.go | 34 +++++---- ...d_unique_in_module_configurations.down.sql | 33 +++++++++ ..._id_unique_in_module_configurations.up.sql | 33 +++++++++ ..._unique_builds_task_id_project_id.down.sql | 33 +++++++++ ...op_unique_builds_task_id_project_id.up.sql | 33 +++++++++ peridot/proto/v1/build.proto | 3 + peridot/proto/v1/catalog.proto | 4 ++ peridot/ui/src/components/ProjectPackages.tsx | 1 + 19 files changed, 490 insertions(+), 24 deletions(-) create mode 100644 modulemd/v3.go create mode 100644 peridot/migrate/20221103054935_make_project_id_unique_in_module_configurations.down.sql create mode 100644 peridot/migrate/20221103054935_make_project_id_unique_in_module_configurations.up.sql create mode 100644 peridot/migrate/20221103070624_drop_unique_builds_task_id_project_id.down.sql create mode 100644 peridot/migrate/20221103070624_drop_unique_builds_task_id_project_id.up.sql diff --git a/modulemd/BUILD.bazel b/modulemd/BUILD.bazel index 55c9d20..947beef 100644 --- a/modulemd/BUILD.bazel +++ b/modulemd/BUILD.bazel @@ -2,8 +2,14 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library") go_library( name = "modulemd", - srcs = ["modulemd.go"], + srcs = [ + "modulemd.go", + "v3.go", + ], importpath = "peridot.resf.org/modulemd", visibility = ["//visibility:public"], - deps = ["//vendor/gopkg.in/yaml.v3:yaml_v3"], + deps = [ + "//vendor/github.com/go-git/go-billy/v5:go-billy", + "//vendor/gopkg.in/yaml.v3:yaml_v3", + ], ) diff --git a/modulemd/modulemd.go b/modulemd/modulemd.go index 02cf565..7d64ae6 100644 --- a/modulemd/modulemd.go +++ b/modulemd/modulemd.go @@ -32,6 +32,7 @@ package modulemd import ( "fmt" + "github.com/go-git/go-billy/v5" "gopkg.in/yaml.v3" ) @@ -152,6 +153,11 @@ type ModuleMd struct { Data *Data `yaml:"data,omitempty"` } +type DetectVersionDocument struct { + Document string `yaml:"document,omitempty"` + Version int `yaml:"version,omitempty"` +} + type DefaultsData struct { Module string `yaml:"module,omitempty"` Stream string `yaml:"stream,omitempty"` @@ -165,11 +171,71 @@ type Defaults struct { } func Parse(input []byte) (*ModuleMd, error) { - var ret ModuleMd - err := yaml.Unmarshal(input, &ret) + var detect DetectVersionDocument + err := yaml.Unmarshal(input, &detect) if err != nil { - return nil, fmt.Errorf("error parsing modulemd: %s", err) + return nil, fmt.Errorf("error detecting document version: %s", err) + } + + var ret ModuleMd + + if detect.Version == 2 { + err = yaml.Unmarshal(input, &ret) + if err != nil { + return nil, fmt.Errorf("error parsing modulemd: %s", err) + } + } else if detect.Version == 3 { + var v3 V3 + err = yaml.Unmarshal(input, &v3) + if err != nil { + return nil, fmt.Errorf("error parsing modulemd: %s", err) + } + + ret = ModuleMd{ + Document: v3.Document, + Version: v3.Version, + Data: &Data{ + Name: v3.Data.Name, + Stream: v3.Data.Stream, + Summary: v3.Data.Summary, + Description: v3.Data.Description, + License: &License{ + Module: v3.Data.License, + }, + Xmd: v3.Data.Xmd, + References: v3.Data.References, + Profiles: v3.Data.Profiles, + Profile: v3.Data.Profile, + API: v3.Data.API, + Filter: v3.Data.Filter, + BuildOpts: &BuildOpts{ + Rpms: v3.Data.Configurations[0].BuildOpts.Rpms, + Arches: v3.Data.Configurations[0].BuildOpts.Arches, + }, + Components: v3.Data.Components, + }, + } } return &ret, nil } + +func (m *ModuleMd) Marshal(fs billy.Filesystem, path string) error { + bts, err := yaml.Marshal(m) + if err != nil { + return err + } + + _ = fs.Remove(path) + f, err := fs.Create(path) + if err != nil { + return err + } + _, err = f.Write(bts) + if err != nil { + return err + } + _ = f.Close() + + return nil +} diff --git a/modulemd/v3.go b/modulemd/v3.go new file mode 100644 index 0000000..a97f074 --- /dev/null +++ b/modulemd/v3.go @@ -0,0 +1,62 @@ +// Copyright (c) All respective contributors to the Peridot Project. All rights reserved. +// Copyright (c) 2021-2022 Rocky Enterprise Software Foundation, Inc. All rights reserved. +// Copyright (c) 2021-2022 Ctrl IQ, Inc. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors +// may be used to endorse or promote products derived from this software without +// specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +package modulemd + +type V3 struct { + Document string `yaml:"document,omitempty"` + Version int `yaml:"version,omitempty"` + Data *V3Data `yaml:"data,omitempty"` +} + +type Configurations struct { + Context string `yaml:"context,omitempty"` + Platform string `yaml:"platform,omitempty"` + BuildRequires map[string][]string `yaml:"buildrequires,omitempty"` + Requires map[string][]string `yaml:"requires,omitempty"` + BuildOpts *BuildOpts `yaml:"buildopts,omitempty"` +} + +type V3Data struct { + Name string `yaml:"name,omitempty"` + Stream string `yaml:"stream,omitempty"` + Summary string `yaml:"summary,omitempty"` + Description string `yaml:"description,omitempty"` + License []string `yaml:"license,omitempty"` + Xmd map[string]map[string]string `yaml:"xmd,omitempty"` + Configurations []*Configurations `yaml:"configurations,omitempty"` + References *References `yaml:"references,omitempty"` + Profiles map[string]*Profile `yaml:"profiles,omitempty"` + Profile map[string]*Profile `yaml:"profile,omitempty"` + API *API `yaml:"api,omitempty"` + Filter *API `yaml:"filter,omitempty"` + Demodularized *API `yaml:"demodularized,omitempty"` + Components *Components `yaml:"components,omitempty"` +} diff --git a/peridot/builder/v1/workflow/BUILD.bazel b/peridot/builder/v1/workflow/BUILD.bazel index da031fd..a70c93a 100644 --- a/peridot/builder/v1/workflow/BUILD.bazel +++ b/peridot/builder/v1/workflow/BUILD.bazel @@ -19,7 +19,6 @@ go_library( visibility = ["//visibility:public"], deps = [ "//apollo/rpmutils", - "//modulemd", "//peridot/composetools", "//peridot/db", "//peridot/db/models", @@ -47,6 +46,7 @@ go_library( "//vendor/github.com/go-git/go-git/v5/storage/memory", "//vendor/github.com/gobwas/glob", "//vendor/github.com/google/uuid", + "//vendor/github.com/rocky-linux/srpmproc/modulemd", "//vendor/github.com/rocky-linux/srpmproc/pb", "//vendor/github.com/rocky-linux/srpmproc/pkg/data", "//vendor/github.com/rocky-linux/srpmproc/pkg/srpmproc", diff --git a/peridot/builder/v1/workflow/arch.go b/peridot/builder/v1/workflow/arch.go index 7b125c9..26a3721 100644 --- a/peridot/builder/v1/workflow/arch.go +++ b/peridot/builder/v1/workflow/arch.go @@ -455,6 +455,10 @@ func (c *Controller) mockConfig(project *models.Project, packageVersion *models. } buildMacros := c.buildMacros(project, packageVersion) + if extra != nil && extra.ForceDist != "" { + buildMacros["%dist"] = "." + extra.ForceDist + } + mockConfig := ` config_opts['root'] = '{additionalVendor}-{majorVersion}-{hostArch}' config_opts['target_arch'] = '{arch}' diff --git a/peridot/builder/v1/workflow/module.go b/peridot/builder/v1/workflow/module.go index 7516fb8..08eda27 100644 --- a/peridot/builder/v1/workflow/module.go +++ b/peridot/builder/v1/workflow/module.go @@ -40,6 +40,7 @@ import ( "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/storage/memory" "github.com/google/uuid" + "github.com/rocky-linux/srpmproc/modulemd" "go.temporal.io/sdk/workflow" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -48,7 +49,6 @@ import ( "google.golang.org/protobuf/types/known/wrapperspb" "gopkg.in/yaml.v3" "io/ioutil" - "peridot.resf.org/modulemd" "peridot.resf.org/peridot/composetools" "peridot.resf.org/peridot/db/models" peridotpb "peridot.resf.org/peridot/pb" @@ -223,13 +223,18 @@ func (c *Controller) BuildModuleWorkflow(ctx workflow.Context, req *peridotpb.Su return nil, err } + branchIndex := map[string]bool{} var streamRevisions models.ImportRevisions for _, revision := range importRevisions { if revision.Modular { if len(req.Branches) > 0 && !utils.StrContains(revision.ScmBranchName, req.Branches) { continue } + if branchIndex[revision.ScmBranchName] { + continue + } streamRevisions = append(streamRevisions, revision) + branchIndex[revision.ScmBranchName] = true } } @@ -247,8 +252,14 @@ func (c *Controller) BuildModuleWorkflow(ctx workflow.Context, req *peridotpb.Su repoUrl := fmt.Sprintf("%s/modules/%s", upstreamPrefix, gitlabify(pkg.Name)) + authenticator, err := c.getAuthenticator(req.ProjectId) + if err != nil { + setInternalError(errorDetails, err) + return nil, err + } r, err := git.Clone(storer, worktree, &git.CloneOptions{ - URL: repoUrl, + URL: repoUrl, + Auth: authenticator, }) if err != nil { newErr := fmt.Errorf("failed to clone module repo: %s", err) @@ -291,13 +302,61 @@ func (c *Controller) BuildModuleWorkflow(ctx workflow.Context, req *peridotpb.Su } // Parse yaml content to module metadata - moduleMd, err := modulemd.Parse(yamlContent) + moduleMdNotBackwardsCompatible, err := modulemd.Parse(yamlContent) if err != nil { newErr := fmt.Errorf("could not parse yaml file from modules repo in branch %s: %v", revision.ScmBranchName, err) setActivityError(errorDetails, newErr) return nil, newErr } + var moduleMd *modulemd.ModuleMd + if moduleMdNotBackwardsCompatible.V2 != nil { + moduleMd = moduleMdNotBackwardsCompatible.V2 + } else if moduleMdNotBackwardsCompatible.V3 != nil { + v3 := moduleMdNotBackwardsCompatible.V3 + moduleMd = &modulemd.ModuleMd{ + Document: "modulemd", + Version: 2, + Data: &modulemd.Data{ + Name: v3.Data.Name, + Stream: v3.Data.Stream, + Summary: v3.Data.Summary, + Description: v3.Data.Description, + ServiceLevels: nil, + License: &modulemd.License{ + Module: v3.Data.License, + }, + Xmd: v3.Data.Xmd, + References: v3.Data.References, + Profiles: v3.Data.Profiles, + Profile: v3.Data.Profile, + API: v3.Data.API, + Filter: v3.Data.Filter, + BuildOpts: nil, + Components: v3.Data.Components, + Artifacts: nil, + }, + } + if len(v3.Data.Configurations) > 0 { + cfg := v3.Data.Configurations[0] + if cfg.BuildOpts != nil { + moduleMd.Data.BuildOpts = &modulemd.BuildOpts{ + Rpms: cfg.BuildOpts.Rpms, + Arches: cfg.BuildOpts.Arches, + } + moduleMd.Data.Dependencies = []*modulemd.Dependencies{ + { + BuildRequires: cfg.BuildRequires, + Requires: cfg.Requires, + }, + } + } + } + } + if moduleMd.Data.Name == "" { + moduleMd.Data.Name = pkg.Name + } + // Invalid modulemd in repo if moduleMd.Data == nil || moduleMd.Data.Components == nil { setActivityError(errorDetails, ErrInvalidModule) @@ -526,6 +585,7 @@ func (c *Controller) BuildModuleStreamWorkflow(ctx workflow.Context, req *perido ExtraYumrepofsRepos: extraRepos, BuildBatchId: streamBuildOptions.BuildBatchId, Modules: buildRequiresModules, + ForceDist: streamBuildOptions.Dist, } task, err := c.db.CreateTask(nil, "noarch", peridotpb.TaskType_TASK_TYPE_BUILD, &req.ProjectId, &parentTaskId) diff --git a/peridot/builder/v1/workflow/sync.go b/peridot/builder/v1/workflow/sync.go index 8825204..d0d92ca 100644 --- a/peridot/builder/v1/workflow/sync.go +++ b/peridot/builder/v1/workflow/sync.go @@ -195,6 +195,13 @@ func kindCatalogSync(tx peridotdb.Access, req *peridotpb.SyncCatalogRequest, cat // perl.aarch64 -> perl nvrIndex := map[string]string{} for _, catalog := range catalogs { + if catalog.ModuleConfiguration != nil { + if ret.ModuleConfiguration != nil { + return nil, fmt.Errorf("multiple module configurations found") + } + ret.ModuleConfiguration = catalog.ModuleConfiguration + } + for _, pkg := range catalog.Package { for _, repo := range pkg.Repository { if repoIndex[repo.Name] == nil { @@ -222,6 +229,22 @@ func kindCatalogSync(tx peridotdb.Access, req *peridotpb.SyncCatalogRequest, cat Type: pkg.Type, }) } + for _, moduleStream := range repo.ModuleStream { + modulePkg := fmt.Sprintf("module:%s:%s", pkg.Name, moduleStream) + alreadyExists := false + for _, p := range repoIndex[repo.Name].Packages { + if p.Name == modulePkg { + alreadyExists = true + break + } + } + if !alreadyExists { + repoIndex[repo.Name].Packages = append(repoIndex[repo.Name].Packages, RepoSyncPackage{ + Name: modulePkg, + Type: pkg.Type, + }) + } + } for _, inf := range repo.IncludeFilter { nvrIndex[inf] = pkg.Name if repoIndex[repo.Name].IncludeFilter[pkg.Name] == nil { @@ -321,6 +344,16 @@ func kindCatalogSync(tx peridotdb.Access, req *peridotpb.SyncCatalogRequest, cat for _, repo := range repoIndex { for _, pkg := range repo.Packages { + // Skip if it starts with module: as it's a module stream + if strings.HasPrefix(pkg.Name, "module:") { + continue + } + + // Always refresh type, expensive but necessary + if err := tx.SetPackageType(req.ProjectId.Value, pkg.Name, pkg.Type); err != nil { + return nil, fmt.Errorf("failed to update package type: %w", err) + } + // Skip if already in project if packageExistsIndex[pkg.Name] { continue @@ -857,6 +890,14 @@ func (c *Controller) SyncCatalogActivity(req *peridotpb.SyncCatalogRequest) (*pe } ret.CatalogSync = resKindCatalogSync + // Set module configuration if it exists + if resKindCatalogSync.ModuleConfiguration != nil { + err := tx.CreateProjectModuleConfiguration(req.ProjectId.Value, resKindCatalogSync.ModuleConfiguration) + if err != nil { + return nil, fmt.Errorf("failed to create project module configuration: %w", err) + } + } + // Check if we have comps err = checkApplyComps(w, tx, req.ProjectId.Value) if err != nil { @@ -878,6 +919,10 @@ func (c *Controller) SyncCatalogActivity(req *peridotpb.SyncCatalogRequest) (*pe var buildIDs []string var newBuildPackages []string for _, newPackage := range ret.CatalogSync.NewPackages { + // Skip module streams + if strings.Contains(newPackage, "module:") { + continue + } if utils.StrContains(newPackage, newBuildPackages) { continue } @@ -897,6 +942,10 @@ func (c *Controller) SyncCatalogActivity(req *peridotpb.SyncCatalogRequest) (*pe newBuildPackages = append(newBuildPackages, newPackage) } for _, newPackage := range ret.CatalogSync.ModifiedPackages { + // Skip module streams + if strings.Contains(newPackage, "module:") { + continue + } if utils.StrContains(newPackage, newBuildPackages) { continue } diff --git a/peridot/db/db.go b/peridot/db/db.go index c72591d..84da87a 100644 --- a/peridot/db/db.go +++ b/peridot/db/db.go @@ -42,6 +42,7 @@ type Access interface { ListProjects(filters *peridotpb.ProjectFilters) (models.Projects, error) GetProjectKeys(projectId string) (*models.ProjectKey, error) GetProjectModuleConfiguration(projectId string) (*peridotpb.ModuleConfiguration, error) + CreateProjectModuleConfiguration(projectId string, config *peridotpb.ModuleConfiguration) error CreateProject(project *peridotpb.Project) (*models.Project, error) UpdateProject(id string, project *peridotpb.Project) (*models.Project, error) SetProjectKeys(projectId string, username string, password string) error @@ -65,6 +66,7 @@ type Access interface { NVRAExists(nvra string) (bool, error) GetBuildByPackageNameAndVersionAndRelease(name string, version string, release string, projectId string) (*models.Build, error) GetLatestBuildIdsByPackageName(name string, projectId string) ([]string, error) + GetLatestBuildsByPackageNameAndPackageVersionID(name string, packageVersionId string, projectId string) ([]string, error) GetActiveBuildIdsByTaskArtifactGlob(taskArtifactGlob string, projectId string) ([]string, error) GetAllBuildIdsByPackageName(name string, projectId string) ([]string, error) @@ -98,6 +100,7 @@ type Access interface { SetExtraOptionsForPackage(projectId string, packageName string, withFlags pq.StringArray, withoutFlags pq.StringArray) error GetExtraOptionsForPackage(projectId string, packageName string) (*models.ExtraOptions, error) SetGroupInstallOptionsForPackage(projectId string, packageName string, dependsOn pq.StringArray) error + SetPackageType(projectId string, packageName string, packageType peridotpb.PackageType) error CreateTask(user *utils.ContextUser, arch string, taskType peridotpb.TaskType, projectId *string, parentTaskId *string) (*models.Task, error) SetTaskStatus(id string, status peridotpb.TaskStatus) error diff --git a/peridot/db/psql/build.go b/peridot/db/psql/build.go index f78663e..bb5fe0c 100644 --- a/peridot/db/psql/build.go +++ b/peridot/db/psql/build.go @@ -428,6 +428,37 @@ func (a *Access) GetLatestBuildIdsByPackageName(name string, projectId string) ( return ret, nil } +func (a *Access) GetLatestBuildsByPackageNameAndPackageVersionID(name string, packageVersionId string, projectId string) ([]string, error) { + var ret []string + err := a.query.Select( + &ret, + ` + select + b.id + from builds b + inner join tasks t on t.id = b.task_id + inner join packages p on p.id = b.package_id + inner join project_package_versions ppv on ppv.package_version_id = b.package_version_id + where + b.project_id = $1 + and p.name = $2 + and ppv.active_in_repo = true + and ppv.project_id = b.project_id + and b.package_version_id = $3 + and t.status = 3 + order by b.created_at asc + `, + projectId, + name, + packageVersionId, + ) + if err != nil { + return nil, err + } + + return ret, nil +} + func (a *Access) GetActiveBuildIdsByTaskArtifactGlob(taskArtifactGlob string, projectId string) ([]string, error) { var ret []string err := a.query.Select( diff --git a/peridot/db/psql/package.go b/peridot/db/psql/package.go index 949571c..c5a06ef 100644 --- a/peridot/db/psql/package.go +++ b/peridot/db/psql/package.go @@ -362,3 +362,16 @@ func (a *Access) SetGroupInstallOptionsForPackage(projectId string, packageName ) return err } + +func (a *Access) SetPackageType(projectId string, packageName string, packageType peridotpb.PackageType) error { + _, err := a.query.Exec( + ` + update project_packages set package_type_override = $3 + where project_id = $1 and package_id = (select id from packages where name = $2) + `, + projectId, + packageName, + packageType, + ) + return err +} diff --git a/peridot/db/psql/project.go b/peridot/db/psql/project.go index b2a93a4..f93acdb 100644 --- a/peridot/db/psql/project.go +++ b/peridot/db/psql/project.go @@ -150,6 +150,34 @@ func (a *Access) GetProjectModuleConfiguration(projectId string) (*peridotpb.Mod return pb, nil } +func (a *Access) CreateProjectModuleConfiguration(projectId string, config *peridotpb.ModuleConfiguration) error { + anyPb, err := anypb.New(config) + if err != nil { + return fmt.Errorf("failed to marshal module configuration: %v", err) + } + + protoJson, err := protojson.Marshal(anyPb) + if err != nil { + return fmt.Errorf("failed to marshal module configuration (protojson): %v", err) + } + + _, err = a.query.Exec( + ` + insert into project_module_configuration (project_id, proto, active) + values ($1, $2, true) + on conflict (project_id) do update + set proto = $2, active = true + `, + projectId, + protoJson, + ) + if err != nil { + return err + } + + return nil +} + func (a *Access) CreateProject(project *peridotpb.Project) (*models.Project, error) { if err := project.ValidateAll(); err != nil { return nil, err diff --git a/peridot/impl/v1/build.go b/peridot/impl/v1/build.go index 6acc7f1..ad86736 100644 --- a/peridot/impl/v1/build.go +++ b/peridot/impl/v1/build.go @@ -316,23 +316,11 @@ func (s *Server) SubmitBuild(ctx context.Context, req *peridotpb.SubmitBuildRequ return nil, errors.New("could not find upstream branch") } - build, err := tx.CreateBuild(pkg.ID.String(), importRevision.PackageVersionId, task.ID.String(), req.ProjectId) - if err != nil { - s.log.Errorf("could not create build: %v", err) - return nil, status.Error(codes.InvalidArgument, "could not create build") - } - taskProto, err := task.ToProto(true) if err != nil { return nil, status.Errorf(codes.Internal, "could not marshal task: %v", err) } - rollback = false - err = beginTx.Commit() - if err != nil { - return nil, status.Error(codes.Internal, "could not save, try again") - } - // Check if all branches are modular (that means it's only a module component/module) allStream := true for _, revision := range revisions { @@ -346,6 +334,12 @@ func (s *Server) SubmitBuild(ctx context.Context, req *peridotpb.SubmitBuildRequ } if (packageType == peridotpb.PackageType_PACKAGE_TYPE_MODULE_FORK || packageType == peridotpb.PackageType_PACKAGE_TYPE_NORMAL_FORK_MODULE || packageType == peridotpb.PackageType_PACKAGE_TYPE_MODULE_FORK_MODULE_COMPONENT) && req.ModuleVariant { + rollback = false + err = beginTx.Commit() + if err != nil { + return nil, status.Error(codes.Internal, "could not save, try again") + } + _, err = s.temporal.ExecuteWorkflow( context.Background(), client.StartWorkflowOptions{ @@ -356,9 +350,7 @@ func (s *Server) SubmitBuild(ctx context.Context, req *peridotpb.SubmitBuildRequ s.temporalWorker.WorkflowController.BuildModuleWorkflow, req, task, - &peridotpb.ExtraBuildOptions{ - ReusableBuildId: build.ID.String(), - }, + &peridotpb.ExtraBuildOptions{}, ) if err != nil { return nil, err @@ -366,6 +358,18 @@ func (s *Server) SubmitBuild(ctx context.Context, req *peridotpb.SubmitBuildRequ } if packageType != peridotpb.PackageType_PACKAGE_TYPE_MODULE_FORK && packageType != peridotpb.PackageType_PACKAGE_TYPE_MODULE_FORK_MODULE_COMPONENT && len(req.Branches) == 0 && !allStream && !req.ModuleVariant { + build, err := tx.CreateBuild(pkg.ID.String(), importRevision.PackageVersionId, task.ID.String(), req.ProjectId) + if err != nil { + s.log.Errorf("could not create build: %v", err) + return nil, status.Error(codes.InvalidArgument, "could not create build") + } + + rollback = false + err = beginTx.Commit() + if err != nil { + return nil, status.Error(codes.Internal, "could not save, try again") + } + _, err = s.temporal.ExecuteWorkflow( context.Background(), client.StartWorkflowOptions{ diff --git a/peridot/migrate/20221103054935_make_project_id_unique_in_module_configurations.down.sql b/peridot/migrate/20221103054935_make_project_id_unique_in_module_configurations.down.sql new file mode 100644 index 0000000..942b595 --- /dev/null +++ b/peridot/migrate/20221103054935_make_project_id_unique_in_module_configurations.down.sql @@ -0,0 +1,33 @@ +/* + * Copyright (c) All respective contributors to the Peridot Project. All rights reserved. + * Copyright (c) 2021-2022 Rocky Enterprise Software Foundation, Inc. All rights reserved. + * Copyright (c) 2021-2022 Ctrl IQ, Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +alter table project_module_configuration drop constraint project_id_uniq; diff --git a/peridot/migrate/20221103054935_make_project_id_unique_in_module_configurations.up.sql b/peridot/migrate/20221103054935_make_project_id_unique_in_module_configurations.up.sql new file mode 100644 index 0000000..2c62dd3 --- /dev/null +++ b/peridot/migrate/20221103054935_make_project_id_unique_in_module_configurations.up.sql @@ -0,0 +1,33 @@ +/* + * Copyright (c) All respective contributors to the Peridot Project. All rights reserved. + * Copyright (c) 2021-2022 Rocky Enterprise Software Foundation, Inc. All rights reserved. + * Copyright (c) 2021-2022 Ctrl IQ, Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +alter table project_module_configuration add constraint project_id_uniq unique (project_id); diff --git a/peridot/migrate/20221103070624_drop_unique_builds_task_id_project_id.down.sql b/peridot/migrate/20221103070624_drop_unique_builds_task_id_project_id.down.sql new file mode 100644 index 0000000..232cd45 --- /dev/null +++ b/peridot/migrate/20221103070624_drop_unique_builds_task_id_project_id.down.sql @@ -0,0 +1,33 @@ +/* + * Copyright (c) All respective contributors to the Peridot Project. All rights reserved. + * Copyright (c) 2021-2022 Rocky Enterprise Software Foundation, Inc. All rights reserved. + * Copyright (c) 2021-2022 Ctrl IQ, Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +alter table builds add constraint builds_task_id_package_id unique (task_id, package_id); diff --git a/peridot/migrate/20221103070624_drop_unique_builds_task_id_project_id.up.sql b/peridot/migrate/20221103070624_drop_unique_builds_task_id_project_id.up.sql new file mode 100644 index 0000000..0d2c319 --- /dev/null +++ b/peridot/migrate/20221103070624_drop_unique_builds_task_id_project_id.up.sql @@ -0,0 +1,33 @@ +/* + * Copyright (c) All respective contributors to the Peridot Project. All rights reserved. + * Copyright (c) 2021-2022 Rocky Enterprise Software Foundation, Inc. All rights reserved. + * Copyright (c) 2021-2022 Ctrl IQ, Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +alter table builds drop constraint builds_task_id_package_id; diff --git a/peridot/proto/v1/build.proto b/peridot/proto/v1/build.proto index 47ff78c..3e86502 100644 --- a/peridot/proto/v1/build.proto +++ b/peridot/proto/v1/build.proto @@ -328,6 +328,9 @@ message ExtraBuildOptions { // Whether to enable networking in rpmbuild bool enable_networking = 8; + + // Force a specific dist + string force_dist = 9; } message RpmImportRequest { diff --git a/peridot/proto/v1/catalog.proto b/peridot/proto/v1/catalog.proto index cad5440..bac5b61 100644 --- a/peridot/proto/v1/catalog.proto +++ b/peridot/proto/v1/catalog.proto @@ -8,6 +8,7 @@ import "validate/validate.proto"; import "google/api/annotations.proto"; import "peridot/proto/v1/task.proto"; import "peridot/proto/v1/package.proto"; +import "peridot/proto/v1/module.proto"; option go_package = "peridot.resf.org/peridot/pb;peridotpb"; @@ -15,6 +16,7 @@ message CatalogSyncRepository { string name = 1; repeated string include_filter = 2; repeated string multilib = 3; + repeated string module_stream = 4; } message CatalogSyncPackage { @@ -47,6 +49,7 @@ message CatalogSync { repeated string exclude_multilib_filter = 3; repeated GlobFilter exclude_filter = 4; repeated GlobFilter include_filter = 5; + resf.peridot.v1.ModuleConfiguration module_configuration = 6; } message CatalogExtraPackageOptions { @@ -82,6 +85,7 @@ message KindCatalogSync { repeated string new_repositories = 2; repeated string modified_repositories = 3; repeated string additional_nvr_globs = 5; + resf.peridot.v1.ModuleConfiguration module_configuration = 6; } message KindCatalogExtraOptions { diff --git a/peridot/ui/src/components/ProjectPackages.tsx b/peridot/ui/src/components/ProjectPackages.tsx index 70eb032..3bcef1b 100644 --- a/peridot/ui/src/components/ProjectPackages.tsx +++ b/peridot/ui/src/components/ProjectPackages.tsx @@ -94,6 +94,7 @@ const columns: GridColDef[] = [ )} {(params.row['type'] === V1PackageType.ModuleFork || + params.row['type'] === V1PackageType.NormalForkModule || params.row['type'] === V1PackageType.ModuleForkModuleComponent) && (