Module mode now correctly replaces all artifacts

There was a bug in module mode where debuginfo+src would not get replaced correctly. That is now fixed. Additional changes to module artifact detection is also made after the build merge solution was implemented. Previously src.rpms could be collapsed because of multiple src.rpms in one build, leading to non-devel artifacts being placed in devel. That is also fixed with this.

peridot/builder/v1/workflow/build.go

@@ -739,6 +739,12 @@ func (c *Controller) BuildWorkflow(ctx workflow.Context, req *peridotpb.SubmitBu
 					Arch:       result.Arch,
 					Metadata:   metadata,
 				})
+				artifacts = append(artifacts, &peridotpb.TaskArtifact{
+					TaskId:     result.Subtask.ID.String(),
+					Name:       result.ObjectName,
+					HashSha256: result.HashSha256,
+					Arch:       result.Arch,
+				})
 			}
 		})
 	}
@@ -805,7 +811,7 @@ func (c *Controller) BuildWorkflow(ctx workflow.Context, req *peridotpb.SubmitBu
 
 	submitBuildTask = peridotpb.SubmitBuildTask{
 		BuildId:        buildID,
-		BuildTaskId:    taskID,
+		BuildTaskId:    task.ID.String(),
 		PackageName:    pkg.Name,
 		ImportRevision: importRevision.ToProto(),
 		Artifacts:      artifacts,
@@ -824,7 +830,6 @@ func (c *Controller) BuildWorkflow(ctx workflow.Context, req *peridotpb.SubmitBu
 		setInternalError(errorDetails, err)
 		return nil, err
 	}
-	submitBuildTask.Artifacts = nil
 
 	task.Status = peridotpb.TaskStatus_TASK_STATUS_RUNNING
 

peridot/builder/v1/workflow/module.go

@@ -442,27 +442,30 @@ func (c *Controller) BuildModuleWorkflow(ctx workflow.Context, req *peridotpb.Su
 
 	task.Status = peridotpb.TaskStatus_TASK_STATUS_FAILED
 
+	var buildIDs []string
 	for _, options := range streamBuildOptions {
-		yumrepoCtx := workflow.WithChildOptions(ctx, workflow.ChildWorkflowOptions{
+		buildIDs = append(buildIDs, options.BuildId)
-			TaskQueue: "yumrepofs",
-		})
-		taskID := task.ID.String()
-		updateRepoRequest := &UpdateRepoRequest{
-			ProjectID: req.ProjectId,
-			BuildIDs:  []string{options.BuildId},
-			Delete:    false,
-			TaskID:    &taskID,
-		}
-		updateRepoTask := &yumrepofspb.UpdateRepoTask{}
-		err = workflow.ExecuteChildWorkflow(yumrepoCtx, c.RepoUpdaterWorkflow, updateRepoRequest).Get(ctx, updateRepoTask)
-		if err != nil {
-			setActivityError(errorDetails, err)
-			return nil, err
-		}
-
-		moduleBuildTask.RepoChanges.Changes = append(moduleBuildTask.RepoChanges.Changes, updateRepoTask.Changes...)
 	}
 
+	yumrepoCtx := workflow.WithChildOptions(ctx, workflow.ChildWorkflowOptions{
+		TaskQueue: "yumrepofs",
+	})
+	taskID := task.ID.String()
+	updateRepoRequest := &UpdateRepoRequest{
+		ProjectID: req.ProjectId,
+		BuildIDs:  buildIDs,
+		Delete:    false,
+		TaskID:    &taskID,
+	}
+	updateRepoTask := &yumrepofspb.UpdateRepoTask{}
+	err = workflow.ExecuteChildWorkflow(yumrepoCtx, c.RepoUpdaterWorkflow, updateRepoRequest).Get(ctx, updateRepoTask)
+	if err != nil {
+		setActivityError(errorDetails, err)
+		return nil, err
+	}
+
+	moduleBuildTask.RepoChanges.Changes = append(moduleBuildTask.RepoChanges.Changes, updateRepoTask.Changes...)
+
 	task.Status = peridotpb.TaskStatus_TASK_STATUS_SUCCEEDED
 
 	return moduleBuildTask, nil
@@ -637,47 +640,55 @@ func (c *Controller) BuildModuleStreamWorkflow(ctx workflow.Context, req *perido
 	// Create an index, so we don't have to re-unmarshal later on
 	artifactPrimaryIndex := map[string]ArtifactIndex{}
 
+	artifacts, err := c.db.GetArtifactsForBuild(streamBuildOptions.BuildId)
+	if err != nil {
+		return nil, err
+	}
+
 	// Pre-warm SRPM NEVRAs
 	srpmNevras := map[string]string{}
-	for _, build := range buildTask.Builds {
+	for _, artifact := range artifacts {
-		artifacts, err := c.db.GetArtifactsForBuild(build.BuildId)
+		if artifact.Arch != "src" {
+			continue
+		}
+
+		rpmArtifactMetadata := &peridotpb.RpmArtifactMetadata{}
+		artifactMetadataAny := &anypb.Any{}
+		err = protojson.Unmarshal(artifact.Metadata.JSONText, artifactMetadataAny)
+		if err != nil {
+			return nil, err
+		}
+		err := artifactMetadataAny.UnmarshalTo(rpmArtifactMetadata)
 		if err != nil {
 			return nil, err
 		}
-		for _, artifact := range artifacts {
-			if artifact.Arch != "src" {
-				continue
-			}
 
-			rpmArtifactMetadata := &peridotpb.RpmArtifactMetadata{}
+		var primary yummeta.PrimaryRoot
-			artifactMetadataAny := &anypb.Any{}
+		err = yummeta.UnmarshalPrimary(rpmArtifactMetadata.Primary, &primary)
-			err = protojson.Unmarshal(artifact.Metadata.JSONText, artifactMetadataAny)
+		if err != nil {
-			if err != nil {
+			return nil, err
-				return nil, err
+		}
-			}
-			err := artifactMetadataAny.UnmarshalTo(rpmArtifactMetadata)
-			if err != nil {
-				return nil, err
-			}
 
-			var primary yummeta.PrimaryRoot
+		for _, build := range buildTask.Builds {
-			err = yummeta.UnmarshalPrimary(rpmArtifactMetadata.Primary, &primary)
+			if build.PackageName == primary.Packages[0].Name {
-			if err != nil {
+				srpmNevras[build.PackageName] = composetools.GenNevraPrimaryPkg(primary.Packages[0])
-				return nil, err
 			}
-
-			srpmNevras[build.PackageName] = composetools.GenNevraPrimaryPkg(primary.Packages[0])
 		}
 	}
 
 	// Group content licenses
 	var licenses []string
 	for _, build := range buildTask.Builds {
-		artifacts, err := c.db.GetArtifactsForBuild(build.BuildId)
+		var buildTaskArtifactNames []string
-		if err != nil {
+		for _, a := range build.Artifacts {
-			return nil, err
+			buildTaskArtifactNames = append(buildTaskArtifactNames, a.Name)
 		}
+
 		for _, artifact := range artifacts {
+			if !utils.StrContains(artifact.Name, buildTaskArtifactNames) {
+				continue
+			}
+
 			rpmArtifactMetadata := &peridotpb.RpmArtifactMetadata{}
 			artifactMetadataAny := &anypb.Any{}
 			err = protojson.Unmarshal(artifact.Metadata.JSONText, artifactMetadataAny)
@@ -791,12 +802,23 @@ func fillInRpmArtifactsForModuleMd(md *modulemd.ModuleMd, streamBuildOptions *Mo
 	newMd.Data.License.Content = licenses
 
 	// Set buildrequires platform to the one used
+	didSetPlatform := false
+	platform := streamBuildOptions.Configuration.Platform
+	platformReq := fmt.Sprintf("el%d.%d.%d", platform.Major, platform.Minor, platform.Patch)
 	for _, dep := range newMd.Data.Dependencies {
 		if dep.BuildRequires["platform"] != nil {
-			platform := streamBuildOptions.Configuration.Platform
+			dep.BuildRequires["platform"] = []string{platformReq}
-			dep.BuildRequires["platform"] = []string{fmt.Sprintf("el%d.%d.%d", platform.Major, platform.Minor, platform.Patch)}
+			didSetPlatform = true
+			break
 		}
 	}
+	if !didSetPlatform {
+		newMd.Data.Dependencies = append(newMd.Data.Dependencies, &modulemd.Dependencies{
+			BuildRequires: map[string][]string{
+				"platform": {platformReq},
+			},
+		})
+	}
 
 	// Set arch for components
 	for _, component := range newMd.Data.Components.Rpms {
@@ -855,7 +877,15 @@ func fillInRpmArtifactsForModuleMd(md *modulemd.ModuleMd, streamBuildOptions *Mo
 		collected[nevra] = pkg
 	}
 
-	for nevra, rpmObj := range collected {
+	// Sort collected by rpmObj.Name
+	var collectedNames []string
+	for name := range collected {
+		collectedNames = append(collectedNames, name)
+	}
+	sort.Strings(collectedNames)
+
+	for _, nevra := range collectedNames {
+		rpmObj := collected[nevra]
 		// Skip source RPMs for now as they're
 		// only added if the main RPM is included
 		if rpmObj.Arch == "src" {

peridot/builder/v1/workflow/yumrepofs.go

@@ -951,7 +951,7 @@ func (c *Controller) UpdateRepoActivity(ctx context.Context, req *UpdateRepoRequ
 
 // todo(mustafa): Convert to request struct
 func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest, errorDetails *peridotpb.TaskErrorDetails, packageName string, buildId string, moduleStream *peridotpb.ModuleStream, gpgId *string, signArtifactsTasks *keykeeperpb.BatchSignArtifactsTask, cache *Cache) (*yumrepofspb.UpdateRepoTask, error) {
-	build, err := c.db.GetBuild(req.ProjectID, buildId)
+	build, err := tx.GetBuild(req.ProjectID, buildId)
 	if err != nil {
 		c.log.Errorf("error getting build: %v", err)
 		return nil, err
@@ -970,33 +970,56 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest
 	}
 	project := projects[0]
 
-	artifacts, err := c.db.GetArtifactsForBuild(buildId)
+	artifacts, err := tx.GetArtifactsForBuild(buildId)
 	if err != nil {
 		setInternalError(errorDetails, err)
 		return nil, fmt.Errorf("failed to get artifacts for build: %v", err)
 	}
 
 	var currentActiveArtifacts models.TaskArtifacts
-	// Get currently active artifacts
+	var skipDeleteArtifacts []string
-	latestBuilds, err := c.db.GetLatestBuildsByPackageNameAndPackageVersionID(build.PackageName, build.PackageVersionId, project.ID.String())
+	if moduleStream == nil {
-	if err != nil {
+		// Get currently active artifacts
-		setInternalError(errorDetails, err)
+		latestBuilds, err := tx.GetLatestBuildIdsByPackageName(build.PackageName, project.ID.String())
-		return nil, fmt.Errorf("failed to get latest build ids: %v", err)
-	}
-	for _, latestBuild := range latestBuilds {
-		buildArtifacts, err := c.db.GetArtifactsForBuild(latestBuild)
 		if err != nil {
 			setInternalError(errorDetails, err)
-			return nil, fmt.Errorf("failed to get artifacts for build: %v", err)
+			return nil, fmt.Errorf("failed to get latest build ids: %v", err)
 		}
-		currentActiveArtifacts = append(currentActiveArtifacts, buildArtifacts...)
+		for _, latestBuild := range latestBuilds {
+			buildArtifacts, err := tx.GetArtifactsForBuild(latestBuild)
+			if err != nil {
+				setInternalError(errorDetails, err)
+				return nil, fmt.Errorf("failed to get artifacts for build: %v", err)
+			}
+			currentActiveArtifacts = append(currentActiveArtifacts, buildArtifacts...)
+		}
+	} else {
+		// Get currently active artifacts
+		latestBuilds, err := tx.GetLatestBuildsByPackageNameAndBranchName(build.PackageName, moduleStream.ImportRevision.ScmBranchName, project.ID.String())
+		if err != nil {
+			setInternalError(errorDetails, err)
+			return nil, fmt.Errorf("failed to get latest build ids: %v", err)
+		}
+		for _, latestBuild := range latestBuilds {
+			buildArtifacts, err := tx.GetArtifactsForBuild(latestBuild)
+			if err != nil {
+				setInternalError(errorDetails, err)
+				return nil, fmt.Errorf("failed to get artifacts for build: %v", err)
+			}
+			currentActiveArtifacts = append(currentActiveArtifacts, buildArtifacts...)
+		}
+	}
+
+	// Get artifacts to skip deletion
+	for _, artifact := range artifacts {
+		skipDeleteArtifacts = append(skipDeleteArtifacts, strings.TrimSuffix(filepath.Base(artifact.Name), ".rpm"))
 	}
 
 	var repos models.Repositories
 
 	if req.ForceRepoId != "" {
 		var err error
-		repo, err := c.db.GetRepository(&req.ForceRepoId, nil, nil)
+		repo, err := tx.GetRepository(&req.ForceRepoId, nil, nil)
 		if err != nil {
 			setInternalError(errorDetails, err)
 			return nil, fmt.Errorf("failed to get repo: %v", err)
@@ -1004,7 +1027,7 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest
 		repos = models.Repositories{*repo}
 	} else {
 		var err error
-		repos, err = c.db.FindRepositoriesForPackage(req.ProjectID, packageName, false)
+		repos, err = tx.FindRepositoriesForPackage(req.ProjectID, packageName, false)
 		if err != nil {
 			setInternalError(errorDetails, err)
 			return nil, fmt.Errorf("failed to find repo: %v", err)
@@ -1018,6 +1041,7 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest
 	}
 
 	for _, repo := range repos {
+		c.log.Infof("processing repo: %v", repo.Name)
 		artifactArchMap, err := GenerateArchMapForArtifacts(artifacts, &project, &repo)
 		if err != nil {
 			setInternalError(errorDetails, err)
@@ -1081,6 +1105,63 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest
 		}
 
 		for arch, archArtifacts := range artifactArchMap {
+			c.log.Infof("processing arch %s", arch)
+
+			noDebugArch := strings.TrimSuffix(arch, "-debug")
+			var streamDocument *modulemd.ModuleMd
+
+			if moduleStream != nil {
+				var newArtifacts models.TaskArtifacts
+
+				streamDocuments := moduleStream.ModuleStreamDocuments[noDebugArch]
+				if streamDocuments != nil {
+					streamDocumentNbc, err := modulemd.Parse(streamDocuments.Streams[moduleStream.Stream])
+					if err != nil {
+						return nil, fmt.Errorf("failed to decode modulemd: %v", err)
+					}
+					streamDocument = streamDocumentNbc.V2
+
+					if arch != "src" {
+						for _, artifact := range archArtifacts {
+							for _, moduleArtifact := range streamDocument.Data.Artifacts.Rpms {
+								moduleArtifactNoEpoch := rpmutils.Epoch().ReplaceAllString(moduleArtifact, "")
+								if strings.TrimSuffix(filepath.Base(artifact.Name), ".rpm") == moduleArtifactNoEpoch {
+									newArtifacts = append(newArtifacts, *artifact)
+								}
+							}
+						}
+
+						artifactArchMap2, err := GenerateArchMapForArtifacts(newArtifacts, &project, &repo)
+						if err != nil {
+							setInternalError(errorDetails, err)
+							return nil, err
+						}
+						c.log.Infof("artifactArchMap2: %v", artifactArchMap2)
+						archArtifacts = artifactArchMap2[arch]
+					} else {
+						// Remove duplicates for src
+						seen := make(map[string]bool)
+						for _, artifact := range archArtifacts {
+							if !seen[artifact.Name] {
+								newArtifacts = append(newArtifacts, *artifact)
+								seen[artifact.Name] = true
+							}
+						}
+
+						artifactArchMap2, err := GenerateArchMapForArtifacts(newArtifacts, &project, &repo)
+						if err != nil {
+							setInternalError(errorDetails, err)
+							return nil, err
+						}
+						c.log.Infof("artifactArchMap2: %v", artifactArchMap2)
+						archArtifacts = artifactArchMap2[arch]
+					}
+				}
+				if strings.HasSuffix(arch, "-debug") || arch == "src" {
+					streamDocument = nil
+				}
+			}
+
 			changes := &yumrepofspb.RepositoryChange{
 				Name:             fmt.Sprintf("%s-%s", repo.Name, arch),
 				AddedPackages:    []string{},
@@ -1104,9 +1185,15 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest
 			}
 			var modulesRoot []*modulemd.ModuleMd
 
-			idArch := fmt.Sprintf("%s-%s", repo.ID, arch)
+			idArch := fmt.Sprintf("%s-%s", repo.Name, arch)
+			idArchNoDebug := fmt.Sprintf("%s-%s", repo.Name, noDebugArch)
 			var currentRevision *models.RepositoryRevision
 			var groupsXml string
+			if cache.Repos[idArchNoDebug] != nil {
+				if cache.Repos[idArchNoDebug].Modulemd != nil {
+					modulesRoot = cache.Repos[idArchNoDebug].Modulemd
+				}
+			}
 			if cache.Repos[idArch] != nil {
 				if cache.Repos[idArch].PrimaryRoot != nil {
 					primaryRoot = *cache.Repos[idArch].PrimaryRoot
@@ -1117,17 +1204,27 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest
 				if cache.Repos[idArch].OtherRoot != nil {
 					otherRoot = *cache.Repos[idArch].OtherRoot
 				}
-				if cache.Repos[idArch].Modulemd != nil {
-					modulesRoot = cache.Repos[idArch].Modulemd
-				}
 				groupsXml = cache.Repos[idArch].GroupsXml
 			} else {
-				currentRevision, err = c.db.GetLatestActiveRepositoryRevision(repo.ID.String(), arch)
+				var noDebugRevision *models.RepositoryRevision
+
+				currentRevision, err = tx.GetLatestActiveRepositoryRevision(repo.ID.String(), arch)
 				if err != nil {
 					if err != sql.ErrNoRows {
 						return nil, fmt.Errorf("failed to get latest active repository revision: %v", err)
 					}
 				}
+				if strings.HasSuffix(arch, "-debug") {
+					noDebugRevision, err = tx.GetLatestActiveRepositoryRevision(repo.ID.String(), noDebugArch)
+					if err != nil {
+						if err != sql.ErrNoRows {
+							return nil, fmt.Errorf("failed to get latest active repository revision: %v", err)
+						}
+					}
+				} else {
+					noDebugRevision = currentRevision
+				}
+
 				if currentRevision != nil {
 					if currentRevision.PrimaryXml != "" {
 						var primaryXmlGz []byte
@@ -1171,11 +1268,14 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest
 							return nil, err
 						}
 					}
-					if currentRevision.ModulesYaml != "" {
+					groupsXml = currentRevision.GroupsXml
+				}
+				if noDebugRevision != nil {
+					if noDebugRevision.ModulesYaml != "" {
 						var modulesYamlGz []byte
 						var modulesYaml []byte
 						err := multiErrorCheck(
-							b64Decode(currentRevision.ModulesYaml, &modulesYamlGz),
+							b64Decode(noDebugRevision.ModulesYaml, &modulesYamlGz),
 							decompressWithGz(modulesYamlGz, &modulesYaml),
 						)
 						if err != nil {
@@ -1205,7 +1305,6 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest
 							modulesRoot = append(modulesRoot, &*&md)
 						}
 					}
-					groupsXml = currentRevision.GroupsXml
 				}
 			}
 
@@ -1219,6 +1318,7 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest
 					}
 				}
 			}
+			c.log.Infof("module artifacts: %v", moduleArtifacts)
 
 			for _, artifact := range archArtifacts {
 				// This shouldn't happen
@@ -1243,7 +1343,7 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest
 				}
 
 				shouldAdd := true
-				if arch != "src" {
+				if arch != "src" && moduleStream == nil {
 					// If repo has a list for inclusion, then the artifact has to pass that first
 					if len(repo.IncludeFilter) > 0 {
 						// If the artifact isn't forced, it should be in the include list or additional multilib list
@@ -1326,16 +1426,7 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest
 				var pkgId *string
 				var primaryIndex *int
 				for i, primaryPackage := range primaryRoot.Packages {
-					// We check module entries by artifact index from a previous revision
+					if moduleStream == nil {
-					if moduleStream != nil {
-						isStreamArtifact := moduleArtifacts[strings.TrimSuffix(filepath.Base(artifact.Name), ".rpm")]
-						if isStreamArtifact {
-							ix := i
-							primaryIndex = &ix
-							pkgId = &primaryPackage.Checksum.Value
-							break
-						}
-					} else {
 						// If not a module stream, search for a non-module entry
 						// Double check arch as well for multilib purposes
 						if primaryPackage.Name == name && !strings.Contains(primaryPackage.Version.Rel, ".module+") && primaryPackage.Arch == artifact.Arch {
@@ -1344,6 +1435,33 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest
 							pkgId = &primaryPackage.Checksum.Value
 							break
 						}
+					} else {
+						// If a module stream, search for a module entry
+						for _, streamArtifact := range artifacts {
+							var rpmName string
+							var rpmVersion string
+							var rpmRelease string
+							rpmBase := strings.TrimSuffix(filepath.Base(streamArtifact.Name), ".rpm")
+							if rpmutils.NVRUnusualRelease().MatchString(rpmBase) {
+								nvr := rpmutils.NVRUnusualRelease().FindStringSubmatch(rpmBase)
+								rpmName = nvr[1]
+								rpmVersion = nvr[2]
+								rpmRelease = nvr[3]
+							} else if rpmutils.NVR().MatchString(rpmBase) {
+								nvr := rpmutils.NVR().FindStringSubmatch(rpmBase)
+								rpmName = nvr[1]
+								rpmVersion = nvr[2]
+								rpmRelease = nvr[3]
+							}
+
+							if name == rpmName && primaryPackage.Name == name && primaryPackage.Version.Ver == rpmVersion && primaryPackage.Version.Rel == rpmRelease && primaryPackage.Arch == artifact.Arch {
+								c.log.Infof("Found module stream package: %s", rpmBase)
+								ix := i
+								primaryIndex = &ix
+								pkgId = &primaryPackage.Checksum.Value
+								break
+							}
+						}
 					}
 				}
 				if primaryIndex != nil {
@@ -1395,19 +1513,28 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest
 			var nFilelists []*yummeta.FilelistsPackage
 			var nOther []*yummeta.OtherPackage
 			var deleteIds []string
-			if !req.NoDeletePrevious {
+			c.log.Infof("modified packages: %v", changes.ModifiedPackages)
+			c.log.Infof("added packages: %v", changes.AddedPackages)
+			if !req.NoDeletePrevious || moduleStream != nil {
 				for _, pkg := range primaryRoot.Packages {
 					shouldAdd := true
 					for _, artifact := range currentActiveArtifacts {
-						if filepath.Base(artifact.Name) == filepath.Base(pkg.Location.Href) && !utils.StrContains(strings.TrimSuffix(filepath.Base(artifact.Name), ".rpm"), changes.ModifiedPackages) && !utils.StrContains(strings.TrimSuffix(filepath.Base(artifact.Name), ".rpm"), changes.AddedPackages) {
+						noRpmName := strings.TrimSuffix(filepath.Base(artifact.Name), ".rpm")
+						if filepath.Base(artifact.Name) == filepath.Base(pkg.Location.Href) && !utils.StrContains(noRpmName, changes.ModifiedPackages) && !utils.StrContains(noRpmName, changes.AddedPackages) && !utils.StrContains(noRpmName, skipDeleteArtifacts) {
 							shouldAdd = false
 						}
 					}
 					if !shouldAdd {
 						deleteIds = append(deleteIds, pkg.Checksum.Value)
+					}
+				}
+				for _, pkg := range primaryRoot.Packages {
+					if utils.StrContains(pkg.Checksum.Value, deleteIds) {
+						c.log.Infof("Deleting package %s-%s-%s from primary metadata", pkg.Name, pkg.Version.Ver, pkg.Version.Rel)
 						continue
 					}
-					nPackages = append(nPackages, &*pkg)
+
+					nPackages = append(nPackages, pkg)
 				}
 				for _, pkg := range filelistsRoot.Packages {
 					if utils.StrContains(pkg.PkgId, deleteIds) {
@@ -1432,31 +1559,22 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest
 			otherRoot.PackageCount = len(filelistsRoot.Packages)
 
 			// Module builds needs a few more steps
-			if moduleStream != nil {
+			if moduleStream != nil && streamDocument != nil {
-				streamDocument := moduleStream.ModuleStreamDocuments[arch]
+				// If a previous entry exists, we need to overwrite that
-				if streamDocument != nil {
+				var moduleIndex *int
-					newEntryNbc, err := modulemd.Parse(streamDocument.Streams[moduleStream.Stream])
+				for i, moduleMd := range modulesRoot {
-					if err != nil {
+					if moduleMd.Data.Name == moduleStream.Name && moduleMd.Data.Stream == moduleStream.Stream {
-						return nil, err
+						moduleIndex = &*&i
-					}
+						break
-					newEntry := newEntryNbc.V2
-
-					// If a previous entry exists, we need to overwrite that
-					var moduleIndex *int
-					for i, moduleMd := range modulesRoot {
-						if moduleMd.Data.Name == moduleStream.Name && moduleMd.Data.Stream == moduleStream.Stream {
-							moduleIndex = &*&i
-							break
-						}
-					}
-					if moduleIndex != nil {
-						changes.ModifiedModules = append(changes.ModifiedModules, fmt.Sprintf("%s:%s", moduleStream.Name, moduleStream.Stream))
-						modulesRoot[*moduleIndex] = newEntry
-					} else {
-						changes.AddedModules = append(changes.AddedModules, fmt.Sprintf("%s:%s", moduleStream.Name, moduleStream.Stream))
-						modulesRoot = append(modulesRoot, newEntry)
 					}
 				}
+				if moduleIndex != nil {
+					changes.ModifiedModules = append(changes.ModifiedModules, fmt.Sprintf("%s:%s", moduleStream.Name, moduleStream.Stream))
+					modulesRoot[*moduleIndex] = streamDocument
+				} else {
+					changes.AddedModules = append(changes.AddedModules, fmt.Sprintf("%s:%s", moduleStream.Name, moduleStream.Stream))
+					modulesRoot = append(modulesRoot, streamDocument)
+				}
 			}
 
 			nRepo := repo
@@ -1471,6 +1589,9 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest
 				ModuleDefaults: moduleDefaults,
 				GroupsXml:      groupsXml,
 			}
+			if strings.HasSuffix(arch, "-debug") || arch == "src" {
+				cache.Repos[idArch].Modulemd = nil
+			}
 
 			repoTask.Changes = append(repoTask.Changes, changes)
 		}

peridot/db/db.go

@@ -66,7 +66,7 @@ type Access interface {
 	NVRAExists(nvra string) (bool, error)
 	GetBuildByPackageNameAndVersionAndRelease(name string, version string, release string, projectId string) (*models.Build, error)
 	GetLatestBuildIdsByPackageName(name string, projectId string) ([]string, error)
-	GetLatestBuildsByPackageNameAndPackageVersionID(name string, packageVersionId string, projectId string) ([]string, error)
+	GetLatestBuildsByPackageNameAndBranchName(name string, branchName string, projectId string) ([]string, error)
 	GetActiveBuildIdsByTaskArtifactGlob(taskArtifactGlob string, projectId string) ([]string, error)
 	GetAllBuildIdsByPackageName(name string, projectId string) ([]string, error)
 

peridot/db/psql/build.go

@@ -428,7 +428,7 @@ func (a *Access) GetLatestBuildIdsByPackageName(name string, projectId string) (
 	return ret, nil
 }
 
-func (a *Access) GetLatestBuildsByPackageNameAndPackageVersionID(name string, packageVersionId string, projectId string) ([]string, error) {
+func (a *Access) GetLatestBuildsByPackageNameAndBranchName(name string, branchName string, projectId string) ([]string, error) {
 	var ret []string
 	err := a.query.Select(
 		&ret,
@@ -439,18 +439,19 @@ func (a *Access) GetLatestBuildsByPackageNameAndPackageVersionID(name string, pa
         inner join tasks t on t.id = b.task_id
         inner join packages p on p.id = b.package_id
         inner join project_package_versions ppv on ppv.package_version_id = b.package_version_id
+        inner join import_revisions ir on ir.package_version_id = b.package_version_id
         where
-            b.project_id = $1
+            b.project_id = $3
-            and p.name = $2
+            and p.name = $1
             and ppv.active_in_repo = true
             and ppv.project_id = b.project_id
-            and b.package_version_id = $3
+            and ir.scm_branch_name = $2
             and t.status = 3
         order by b.created_at asc
         `,
-		projectId,
 		name,
-		packageVersionId,
+		branchName,
+		projectId,
 	)
 	if err != nil {
 		return nil, err