From e0a043afb36f89d8ff202f02a2dc7987665f9476 Mon Sep 17 00:00:00 2001 From: Mustafa Gezen Date: Mon, 14 Nov 2022 11:06:16 +0100 Subject: [PATCH] Module mode now correctly replaces all artifacts There was a bug in module mode where debuginfo+src would not get replaced correctly. That is now fixed. Additional changes to module artifact detection is also made after the build merge solution was implemented. Previously src.rpms could be collapsed because of multiple src.rpms in one build, leading to non-devel artifacts being placed in devel. That is also fixed with this. --- peridot/builder/v1/workflow/build.go | 9 +- peridot/builder/v1/workflow/module.go | 122 +++++++----- peridot/builder/v1/workflow/yumrepofs.go | 239 +++++++++++++++++------ peridot/db/db.go | 2 +- peridot/db/psql/build.go | 13 +- 5 files changed, 271 insertions(+), 114 deletions(-) diff --git a/peridot/builder/v1/workflow/build.go b/peridot/builder/v1/workflow/build.go index f5bd03f..8359347 100644 --- a/peridot/builder/v1/workflow/build.go +++ b/peridot/builder/v1/workflow/build.go @@ -739,6 +739,12 @@ func (c *Controller) BuildWorkflow(ctx workflow.Context, req *peridotpb.SubmitBu Arch: result.Arch, Metadata: metadata, }) + artifacts = append(artifacts, &peridotpb.TaskArtifact{ + TaskId: result.Subtask.ID.String(), + Name: result.ObjectName, + HashSha256: result.HashSha256, + Arch: result.Arch, + }) } }) } @@ -805,7 +811,7 @@ func (c *Controller) BuildWorkflow(ctx workflow.Context, req *peridotpb.SubmitBu submitBuildTask = peridotpb.SubmitBuildTask{ BuildId: buildID, - BuildTaskId: taskID, + BuildTaskId: task.ID.String(), PackageName: pkg.Name, ImportRevision: importRevision.ToProto(), Artifacts: artifacts, @@ -824,7 +830,6 @@ func (c *Controller) BuildWorkflow(ctx workflow.Context, req *peridotpb.SubmitBu setInternalError(errorDetails, err) return nil, err } - submitBuildTask.Artifacts = nil task.Status = peridotpb.TaskStatus_TASK_STATUS_RUNNING diff --git a/peridot/builder/v1/workflow/module.go b/peridot/builder/v1/workflow/module.go index 08eda27..c996288 100644 --- a/peridot/builder/v1/workflow/module.go +++ b/peridot/builder/v1/workflow/module.go @@ -442,27 +442,30 @@ func (c *Controller) BuildModuleWorkflow(ctx workflow.Context, req *peridotpb.Su task.Status = peridotpb.TaskStatus_TASK_STATUS_FAILED + var buildIDs []string for _, options := range streamBuildOptions { - yumrepoCtx := workflow.WithChildOptions(ctx, workflow.ChildWorkflowOptions{ - TaskQueue: "yumrepofs", - }) - taskID := task.ID.String() - updateRepoRequest := &UpdateRepoRequest{ - ProjectID: req.ProjectId, - BuildIDs: []string{options.BuildId}, - Delete: false, - TaskID: &taskID, - } - updateRepoTask := &yumrepofspb.UpdateRepoTask{} - err = workflow.ExecuteChildWorkflow(yumrepoCtx, c.RepoUpdaterWorkflow, updateRepoRequest).Get(ctx, updateRepoTask) - if err != nil { - setActivityError(errorDetails, err) - return nil, err - } - - moduleBuildTask.RepoChanges.Changes = append(moduleBuildTask.RepoChanges.Changes, updateRepoTask.Changes...) + buildIDs = append(buildIDs, options.BuildId) } + yumrepoCtx := workflow.WithChildOptions(ctx, workflow.ChildWorkflowOptions{ + TaskQueue: "yumrepofs", + }) + taskID := task.ID.String() + updateRepoRequest := &UpdateRepoRequest{ + ProjectID: req.ProjectId, + BuildIDs: buildIDs, + Delete: false, + TaskID: &taskID, + } + updateRepoTask := &yumrepofspb.UpdateRepoTask{} + err = workflow.ExecuteChildWorkflow(yumrepoCtx, c.RepoUpdaterWorkflow, updateRepoRequest).Get(ctx, updateRepoTask) + if err != nil { + setActivityError(errorDetails, err) + return nil, err + } + + moduleBuildTask.RepoChanges.Changes = append(moduleBuildTask.RepoChanges.Changes, updateRepoTask.Changes...) + task.Status = peridotpb.TaskStatus_TASK_STATUS_SUCCEEDED return moduleBuildTask, nil @@ -637,47 +640,55 @@ func (c *Controller) BuildModuleStreamWorkflow(ctx workflow.Context, req *perido // Create an index, so we don't have to re-unmarshal later on artifactPrimaryIndex := map[string]ArtifactIndex{} + artifacts, err := c.db.GetArtifactsForBuild(streamBuildOptions.BuildId) + if err != nil { + return nil, err + } + // Pre-warm SRPM NEVRAs srpmNevras := map[string]string{} - for _, build := range buildTask.Builds { - artifacts, err := c.db.GetArtifactsForBuild(build.BuildId) + for _, artifact := range artifacts { + if artifact.Arch != "src" { + continue + } + + rpmArtifactMetadata := &peridotpb.RpmArtifactMetadata{} + artifactMetadataAny := &anypb.Any{} + err = protojson.Unmarshal(artifact.Metadata.JSONText, artifactMetadataAny) + if err != nil { + return nil, err + } + err := artifactMetadataAny.UnmarshalTo(rpmArtifactMetadata) if err != nil { return nil, err } - for _, artifact := range artifacts { - if artifact.Arch != "src" { - continue - } - rpmArtifactMetadata := &peridotpb.RpmArtifactMetadata{} - artifactMetadataAny := &anypb.Any{} - err = protojson.Unmarshal(artifact.Metadata.JSONText, artifactMetadataAny) - if err != nil { - return nil, err - } - err := artifactMetadataAny.UnmarshalTo(rpmArtifactMetadata) - if err != nil { - return nil, err - } + var primary yummeta.PrimaryRoot + err = yummeta.UnmarshalPrimary(rpmArtifactMetadata.Primary, &primary) + if err != nil { + return nil, err + } - var primary yummeta.PrimaryRoot - err = yummeta.UnmarshalPrimary(rpmArtifactMetadata.Primary, &primary) - if err != nil { - return nil, err + for _, build := range buildTask.Builds { + if build.PackageName == primary.Packages[0].Name { + srpmNevras[build.PackageName] = composetools.GenNevraPrimaryPkg(primary.Packages[0]) } - - srpmNevras[build.PackageName] = composetools.GenNevraPrimaryPkg(primary.Packages[0]) } } // Group content licenses var licenses []string for _, build := range buildTask.Builds { - artifacts, err := c.db.GetArtifactsForBuild(build.BuildId) - if err != nil { - return nil, err + var buildTaskArtifactNames []string + for _, a := range build.Artifacts { + buildTaskArtifactNames = append(buildTaskArtifactNames, a.Name) } + for _, artifact := range artifacts { + if !utils.StrContains(artifact.Name, buildTaskArtifactNames) { + continue + } + rpmArtifactMetadata := &peridotpb.RpmArtifactMetadata{} artifactMetadataAny := &anypb.Any{} err = protojson.Unmarshal(artifact.Metadata.JSONText, artifactMetadataAny) @@ -791,12 +802,23 @@ func fillInRpmArtifactsForModuleMd(md *modulemd.ModuleMd, streamBuildOptions *Mo newMd.Data.License.Content = licenses // Set buildrequires platform to the one used + didSetPlatform := false + platform := streamBuildOptions.Configuration.Platform + platformReq := fmt.Sprintf("el%d.%d.%d", platform.Major, platform.Minor, platform.Patch) for _, dep := range newMd.Data.Dependencies { if dep.BuildRequires["platform"] != nil { - platform := streamBuildOptions.Configuration.Platform - dep.BuildRequires["platform"] = []string{fmt.Sprintf("el%d.%d.%d", platform.Major, platform.Minor, platform.Patch)} + dep.BuildRequires["platform"] = []string{platformReq} + didSetPlatform = true + break } } + if !didSetPlatform { + newMd.Data.Dependencies = append(newMd.Data.Dependencies, &modulemd.Dependencies{ + BuildRequires: map[string][]string{ + "platform": {platformReq}, + }, + }) + } // Set arch for components for _, component := range newMd.Data.Components.Rpms { @@ -855,7 +877,15 @@ func fillInRpmArtifactsForModuleMd(md *modulemd.ModuleMd, streamBuildOptions *Mo collected[nevra] = pkg } - for nevra, rpmObj := range collected { + // Sort collected by rpmObj.Name + var collectedNames []string + for name := range collected { + collectedNames = append(collectedNames, name) + } + sort.Strings(collectedNames) + + for _, nevra := range collectedNames { + rpmObj := collected[nevra] // Skip source RPMs for now as they're // only added if the main RPM is included if rpmObj.Arch == "src" { diff --git a/peridot/builder/v1/workflow/yumrepofs.go b/peridot/builder/v1/workflow/yumrepofs.go index 9f769fc..cd43acc 100644 --- a/peridot/builder/v1/workflow/yumrepofs.go +++ b/peridot/builder/v1/workflow/yumrepofs.go @@ -951,7 +951,7 @@ func (c *Controller) UpdateRepoActivity(ctx context.Context, req *UpdateRepoRequ // todo(mustafa): Convert to request struct func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest, errorDetails *peridotpb.TaskErrorDetails, packageName string, buildId string, moduleStream *peridotpb.ModuleStream, gpgId *string, signArtifactsTasks *keykeeperpb.BatchSignArtifactsTask, cache *Cache) (*yumrepofspb.UpdateRepoTask, error) { - build, err := c.db.GetBuild(req.ProjectID, buildId) + build, err := tx.GetBuild(req.ProjectID, buildId) if err != nil { c.log.Errorf("error getting build: %v", err) return nil, err @@ -970,33 +970,56 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest } project := projects[0] - artifacts, err := c.db.GetArtifactsForBuild(buildId) + artifacts, err := tx.GetArtifactsForBuild(buildId) if err != nil { setInternalError(errorDetails, err) return nil, fmt.Errorf("failed to get artifacts for build: %v", err) } var currentActiveArtifacts models.TaskArtifacts - // Get currently active artifacts - latestBuilds, err := c.db.GetLatestBuildsByPackageNameAndPackageVersionID(build.PackageName, build.PackageVersionId, project.ID.String()) - if err != nil { - setInternalError(errorDetails, err) - return nil, fmt.Errorf("failed to get latest build ids: %v", err) - } - for _, latestBuild := range latestBuilds { - buildArtifacts, err := c.db.GetArtifactsForBuild(latestBuild) + var skipDeleteArtifacts []string + if moduleStream == nil { + // Get currently active artifacts + latestBuilds, err := tx.GetLatestBuildIdsByPackageName(build.PackageName, project.ID.String()) if err != nil { setInternalError(errorDetails, err) - return nil, fmt.Errorf("failed to get artifacts for build: %v", err) + return nil, fmt.Errorf("failed to get latest build ids: %v", err) } - currentActiveArtifacts = append(currentActiveArtifacts, buildArtifacts...) + for _, latestBuild := range latestBuilds { + buildArtifacts, err := tx.GetArtifactsForBuild(latestBuild) + if err != nil { + setInternalError(errorDetails, err) + return nil, fmt.Errorf("failed to get artifacts for build: %v", err) + } + currentActiveArtifacts = append(currentActiveArtifacts, buildArtifacts...) + } + } else { + // Get currently active artifacts + latestBuilds, err := tx.GetLatestBuildsByPackageNameAndBranchName(build.PackageName, moduleStream.ImportRevision.ScmBranchName, project.ID.String()) + if err != nil { + setInternalError(errorDetails, err) + return nil, fmt.Errorf("failed to get latest build ids: %v", err) + } + for _, latestBuild := range latestBuilds { + buildArtifacts, err := tx.GetArtifactsForBuild(latestBuild) + if err != nil { + setInternalError(errorDetails, err) + return nil, fmt.Errorf("failed to get artifacts for build: %v", err) + } + currentActiveArtifacts = append(currentActiveArtifacts, buildArtifacts...) + } + } + + // Get artifacts to skip deletion + for _, artifact := range artifacts { + skipDeleteArtifacts = append(skipDeleteArtifacts, strings.TrimSuffix(filepath.Base(artifact.Name), ".rpm")) } var repos models.Repositories if req.ForceRepoId != "" { var err error - repo, err := c.db.GetRepository(&req.ForceRepoId, nil, nil) + repo, err := tx.GetRepository(&req.ForceRepoId, nil, nil) if err != nil { setInternalError(errorDetails, err) return nil, fmt.Errorf("failed to get repo: %v", err) @@ -1004,7 +1027,7 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest repos = models.Repositories{*repo} } else { var err error - repos, err = c.db.FindRepositoriesForPackage(req.ProjectID, packageName, false) + repos, err = tx.FindRepositoriesForPackage(req.ProjectID, packageName, false) if err != nil { setInternalError(errorDetails, err) return nil, fmt.Errorf("failed to find repo: %v", err) @@ -1018,6 +1041,7 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest } for _, repo := range repos { + c.log.Infof("processing repo: %v", repo.Name) artifactArchMap, err := GenerateArchMapForArtifacts(artifacts, &project, &repo) if err != nil { setInternalError(errorDetails, err) @@ -1081,6 +1105,63 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest } for arch, archArtifacts := range artifactArchMap { + c.log.Infof("processing arch %s", arch) + + noDebugArch := strings.TrimSuffix(arch, "-debug") + var streamDocument *modulemd.ModuleMd + + if moduleStream != nil { + var newArtifacts models.TaskArtifacts + + streamDocuments := moduleStream.ModuleStreamDocuments[noDebugArch] + if streamDocuments != nil { + streamDocumentNbc, err := modulemd.Parse(streamDocuments.Streams[moduleStream.Stream]) + if err != nil { + return nil, fmt.Errorf("failed to decode modulemd: %v", err) + } + streamDocument = streamDocumentNbc.V2 + + if arch != "src" { + for _, artifact := range archArtifacts { + for _, moduleArtifact := range streamDocument.Data.Artifacts.Rpms { + moduleArtifactNoEpoch := rpmutils.Epoch().ReplaceAllString(moduleArtifact, "") + if strings.TrimSuffix(filepath.Base(artifact.Name), ".rpm") == moduleArtifactNoEpoch { + newArtifacts = append(newArtifacts, *artifact) + } + } + } + + artifactArchMap2, err := GenerateArchMapForArtifacts(newArtifacts, &project, &repo) + if err != nil { + setInternalError(errorDetails, err) + return nil, err + } + c.log.Infof("artifactArchMap2: %v", artifactArchMap2) + archArtifacts = artifactArchMap2[arch] + } else { + // Remove duplicates for src + seen := make(map[string]bool) + for _, artifact := range archArtifacts { + if !seen[artifact.Name] { + newArtifacts = append(newArtifacts, *artifact) + seen[artifact.Name] = true + } + } + + artifactArchMap2, err := GenerateArchMapForArtifacts(newArtifacts, &project, &repo) + if err != nil { + setInternalError(errorDetails, err) + return nil, err + } + c.log.Infof("artifactArchMap2: %v", artifactArchMap2) + archArtifacts = artifactArchMap2[arch] + } + } + if strings.HasSuffix(arch, "-debug") || arch == "src" { + streamDocument = nil + } + } + changes := &yumrepofspb.RepositoryChange{ Name: fmt.Sprintf("%s-%s", repo.Name, arch), AddedPackages: []string{}, @@ -1104,9 +1185,15 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest } var modulesRoot []*modulemd.ModuleMd - idArch := fmt.Sprintf("%s-%s", repo.ID, arch) + idArch := fmt.Sprintf("%s-%s", repo.Name, arch) + idArchNoDebug := fmt.Sprintf("%s-%s", repo.Name, noDebugArch) var currentRevision *models.RepositoryRevision var groupsXml string + if cache.Repos[idArchNoDebug] != nil { + if cache.Repos[idArchNoDebug].Modulemd != nil { + modulesRoot = cache.Repos[idArchNoDebug].Modulemd + } + } if cache.Repos[idArch] != nil { if cache.Repos[idArch].PrimaryRoot != nil { primaryRoot = *cache.Repos[idArch].PrimaryRoot @@ -1117,17 +1204,27 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest if cache.Repos[idArch].OtherRoot != nil { otherRoot = *cache.Repos[idArch].OtherRoot } - if cache.Repos[idArch].Modulemd != nil { - modulesRoot = cache.Repos[idArch].Modulemd - } groupsXml = cache.Repos[idArch].GroupsXml } else { - currentRevision, err = c.db.GetLatestActiveRepositoryRevision(repo.ID.String(), arch) + var noDebugRevision *models.RepositoryRevision + + currentRevision, err = tx.GetLatestActiveRepositoryRevision(repo.ID.String(), arch) if err != nil { if err != sql.ErrNoRows { return nil, fmt.Errorf("failed to get latest active repository revision: %v", err) } } + if strings.HasSuffix(arch, "-debug") { + noDebugRevision, err = tx.GetLatestActiveRepositoryRevision(repo.ID.String(), noDebugArch) + if err != nil { + if err != sql.ErrNoRows { + return nil, fmt.Errorf("failed to get latest active repository revision: %v", err) + } + } + } else { + noDebugRevision = currentRevision + } + if currentRevision != nil { if currentRevision.PrimaryXml != "" { var primaryXmlGz []byte @@ -1171,11 +1268,14 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest return nil, err } } - if currentRevision.ModulesYaml != "" { + groupsXml = currentRevision.GroupsXml + } + if noDebugRevision != nil { + if noDebugRevision.ModulesYaml != "" { var modulesYamlGz []byte var modulesYaml []byte err := multiErrorCheck( - b64Decode(currentRevision.ModulesYaml, &modulesYamlGz), + b64Decode(noDebugRevision.ModulesYaml, &modulesYamlGz), decompressWithGz(modulesYamlGz, &modulesYaml), ) if err != nil { @@ -1205,7 +1305,6 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest modulesRoot = append(modulesRoot, &*&md) } } - groupsXml = currentRevision.GroupsXml } } @@ -1219,6 +1318,7 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest } } } + c.log.Infof("module artifacts: %v", moduleArtifacts) for _, artifact := range archArtifacts { // This shouldn't happen @@ -1243,7 +1343,7 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest } shouldAdd := true - if arch != "src" { + if arch != "src" && moduleStream == nil { // If repo has a list for inclusion, then the artifact has to pass that first if len(repo.IncludeFilter) > 0 { // If the artifact isn't forced, it should be in the include list or additional multilib list @@ -1326,16 +1426,7 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest var pkgId *string var primaryIndex *int for i, primaryPackage := range primaryRoot.Packages { - // We check module entries by artifact index from a previous revision - if moduleStream != nil { - isStreamArtifact := moduleArtifacts[strings.TrimSuffix(filepath.Base(artifact.Name), ".rpm")] - if isStreamArtifact { - ix := i - primaryIndex = &ix - pkgId = &primaryPackage.Checksum.Value - break - } - } else { + if moduleStream == nil { // If not a module stream, search for a non-module entry // Double check arch as well for multilib purposes if primaryPackage.Name == name && !strings.Contains(primaryPackage.Version.Rel, ".module+") && primaryPackage.Arch == artifact.Arch { @@ -1344,6 +1435,33 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest pkgId = &primaryPackage.Checksum.Value break } + } else { + // If a module stream, search for a module entry + for _, streamArtifact := range artifacts { + var rpmName string + var rpmVersion string + var rpmRelease string + rpmBase := strings.TrimSuffix(filepath.Base(streamArtifact.Name), ".rpm") + if rpmutils.NVRUnusualRelease().MatchString(rpmBase) { + nvr := rpmutils.NVRUnusualRelease().FindStringSubmatch(rpmBase) + rpmName = nvr[1] + rpmVersion = nvr[2] + rpmRelease = nvr[3] + } else if rpmutils.NVR().MatchString(rpmBase) { + nvr := rpmutils.NVR().FindStringSubmatch(rpmBase) + rpmName = nvr[1] + rpmVersion = nvr[2] + rpmRelease = nvr[3] + } + + if name == rpmName && primaryPackage.Name == name && primaryPackage.Version.Ver == rpmVersion && primaryPackage.Version.Rel == rpmRelease && primaryPackage.Arch == artifact.Arch { + c.log.Infof("Found module stream package: %s", rpmBase) + ix := i + primaryIndex = &ix + pkgId = &primaryPackage.Checksum.Value + break + } + } } } if primaryIndex != nil { @@ -1395,19 +1513,28 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest var nFilelists []*yummeta.FilelistsPackage var nOther []*yummeta.OtherPackage var deleteIds []string - if !req.NoDeletePrevious { + c.log.Infof("modified packages: %v", changes.ModifiedPackages) + c.log.Infof("added packages: %v", changes.AddedPackages) + if !req.NoDeletePrevious || moduleStream != nil { for _, pkg := range primaryRoot.Packages { shouldAdd := true for _, artifact := range currentActiveArtifacts { - if filepath.Base(artifact.Name) == filepath.Base(pkg.Location.Href) && !utils.StrContains(strings.TrimSuffix(filepath.Base(artifact.Name), ".rpm"), changes.ModifiedPackages) && !utils.StrContains(strings.TrimSuffix(filepath.Base(artifact.Name), ".rpm"), changes.AddedPackages) { + noRpmName := strings.TrimSuffix(filepath.Base(artifact.Name), ".rpm") + if filepath.Base(artifact.Name) == filepath.Base(pkg.Location.Href) && !utils.StrContains(noRpmName, changes.ModifiedPackages) && !utils.StrContains(noRpmName, changes.AddedPackages) && !utils.StrContains(noRpmName, skipDeleteArtifacts) { shouldAdd = false } } if !shouldAdd { deleteIds = append(deleteIds, pkg.Checksum.Value) + } + } + for _, pkg := range primaryRoot.Packages { + if utils.StrContains(pkg.Checksum.Value, deleteIds) { + c.log.Infof("Deleting package %s-%s-%s from primary metadata", pkg.Name, pkg.Version.Ver, pkg.Version.Rel) continue } - nPackages = append(nPackages, &*pkg) + + nPackages = append(nPackages, pkg) } for _, pkg := range filelistsRoot.Packages { if utils.StrContains(pkg.PkgId, deleteIds) { @@ -1432,31 +1559,22 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest otherRoot.PackageCount = len(filelistsRoot.Packages) // Module builds needs a few more steps - if moduleStream != nil { - streamDocument := moduleStream.ModuleStreamDocuments[arch] - if streamDocument != nil { - newEntryNbc, err := modulemd.Parse(streamDocument.Streams[moduleStream.Stream]) - if err != nil { - return nil, err - } - newEntry := newEntryNbc.V2 - - // If a previous entry exists, we need to overwrite that - var moduleIndex *int - for i, moduleMd := range modulesRoot { - if moduleMd.Data.Name == moduleStream.Name && moduleMd.Data.Stream == moduleStream.Stream { - moduleIndex = &*&i - break - } - } - if moduleIndex != nil { - changes.ModifiedModules = append(changes.ModifiedModules, fmt.Sprintf("%s:%s", moduleStream.Name, moduleStream.Stream)) - modulesRoot[*moduleIndex] = newEntry - } else { - changes.AddedModules = append(changes.AddedModules, fmt.Sprintf("%s:%s", moduleStream.Name, moduleStream.Stream)) - modulesRoot = append(modulesRoot, newEntry) + if moduleStream != nil && streamDocument != nil { + // If a previous entry exists, we need to overwrite that + var moduleIndex *int + for i, moduleMd := range modulesRoot { + if moduleMd.Data.Name == moduleStream.Name && moduleMd.Data.Stream == moduleStream.Stream { + moduleIndex = &*&i + break } } + if moduleIndex != nil { + changes.ModifiedModules = append(changes.ModifiedModules, fmt.Sprintf("%s:%s", moduleStream.Name, moduleStream.Stream)) + modulesRoot[*moduleIndex] = streamDocument + } else { + changes.AddedModules = append(changes.AddedModules, fmt.Sprintf("%s:%s", moduleStream.Name, moduleStream.Stream)) + modulesRoot = append(modulesRoot, streamDocument) + } } nRepo := repo @@ -1471,6 +1589,9 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest ModuleDefaults: moduleDefaults, GroupsXml: groupsXml, } + if strings.HasSuffix(arch, "-debug") || arch == "src" { + cache.Repos[idArch].Modulemd = nil + } repoTask.Changes = append(repoTask.Changes, changes) } diff --git a/peridot/db/db.go b/peridot/db/db.go index edd4d2f..0a21088 100644 --- a/peridot/db/db.go +++ b/peridot/db/db.go @@ -66,7 +66,7 @@ type Access interface { NVRAExists(nvra string) (bool, error) GetBuildByPackageNameAndVersionAndRelease(name string, version string, release string, projectId string) (*models.Build, error) GetLatestBuildIdsByPackageName(name string, projectId string) ([]string, error) - GetLatestBuildsByPackageNameAndPackageVersionID(name string, packageVersionId string, projectId string) ([]string, error) + GetLatestBuildsByPackageNameAndBranchName(name string, branchName string, projectId string) ([]string, error) GetActiveBuildIdsByTaskArtifactGlob(taskArtifactGlob string, projectId string) ([]string, error) GetAllBuildIdsByPackageName(name string, projectId string) ([]string, error) diff --git a/peridot/db/psql/build.go b/peridot/db/psql/build.go index bb5fe0c..6493ef1 100644 --- a/peridot/db/psql/build.go +++ b/peridot/db/psql/build.go @@ -428,7 +428,7 @@ func (a *Access) GetLatestBuildIdsByPackageName(name string, projectId string) ( return ret, nil } -func (a *Access) GetLatestBuildsByPackageNameAndPackageVersionID(name string, packageVersionId string, projectId string) ([]string, error) { +func (a *Access) GetLatestBuildsByPackageNameAndBranchName(name string, branchName string, projectId string) ([]string, error) { var ret []string err := a.query.Select( &ret, @@ -439,18 +439,19 @@ func (a *Access) GetLatestBuildsByPackageNameAndPackageVersionID(name string, pa inner join tasks t on t.id = b.task_id inner join packages p on p.id = b.package_id inner join project_package_versions ppv on ppv.package_version_id = b.package_version_id + inner join import_revisions ir on ir.package_version_id = b.package_version_id where - b.project_id = $1 - and p.name = $2 + b.project_id = $3 + and p.name = $1 and ppv.active_in_repo = true and ppv.project_id = b.project_id - and b.package_version_id = $3 + and ir.scm_branch_name = $2 and t.status = 3 order by b.created_at asc `, - projectId, name, - packageVersionId, + branchName, + projectId, ) if err != nil { return nil, err