Module mode now correctly replaces all artifacts

There was a bug in module mode where debuginfo+src would not get replaced correctly. That is now fixed. Additional changes to module artifact detection is also made after the build merge solution was implemented. Previously src.rpms could be collapsed because of multiple src.rpms in one build, leading to non-devel artifacts being placed in devel. That is also fixed with this.
This commit is contained in:
Mustafa Gezen 2022-11-14 11:06:16 +01:00
parent 4e9db547f6
commit e0a043afb3
Signed by untrusted user who does not match committer: mustafa
GPG Key ID: DCDF010D946438C1
5 changed files with 271 additions and 114 deletions

View File

@ -739,6 +739,12 @@ func (c *Controller) BuildWorkflow(ctx workflow.Context, req *peridotpb.SubmitBu
Arch: result.Arch, Arch: result.Arch,
Metadata: metadata, Metadata: metadata,
}) })
artifacts = append(artifacts, &peridotpb.TaskArtifact{
TaskId: result.Subtask.ID.String(),
Name: result.ObjectName,
HashSha256: result.HashSha256,
Arch: result.Arch,
})
} }
}) })
} }
@ -805,7 +811,7 @@ func (c *Controller) BuildWorkflow(ctx workflow.Context, req *peridotpb.SubmitBu
submitBuildTask = peridotpb.SubmitBuildTask{ submitBuildTask = peridotpb.SubmitBuildTask{
BuildId: buildID, BuildId: buildID,
BuildTaskId: taskID, BuildTaskId: task.ID.String(),
PackageName: pkg.Name, PackageName: pkg.Name,
ImportRevision: importRevision.ToProto(), ImportRevision: importRevision.ToProto(),
Artifacts: artifacts, Artifacts: artifacts,
@ -824,7 +830,6 @@ func (c *Controller) BuildWorkflow(ctx workflow.Context, req *peridotpb.SubmitBu
setInternalError(errorDetails, err) setInternalError(errorDetails, err)
return nil, err return nil, err
} }
submitBuildTask.Artifacts = nil
task.Status = peridotpb.TaskStatus_TASK_STATUS_RUNNING task.Status = peridotpb.TaskStatus_TASK_STATUS_RUNNING

View File

@ -442,14 +442,18 @@ func (c *Controller) BuildModuleWorkflow(ctx workflow.Context, req *peridotpb.Su
task.Status = peridotpb.TaskStatus_TASK_STATUS_FAILED task.Status = peridotpb.TaskStatus_TASK_STATUS_FAILED
var buildIDs []string
for _, options := range streamBuildOptions { for _, options := range streamBuildOptions {
buildIDs = append(buildIDs, options.BuildId)
}
yumrepoCtx := workflow.WithChildOptions(ctx, workflow.ChildWorkflowOptions{ yumrepoCtx := workflow.WithChildOptions(ctx, workflow.ChildWorkflowOptions{
TaskQueue: "yumrepofs", TaskQueue: "yumrepofs",
}) })
taskID := task.ID.String() taskID := task.ID.String()
updateRepoRequest := &UpdateRepoRequest{ updateRepoRequest := &UpdateRepoRequest{
ProjectID: req.ProjectId, ProjectID: req.ProjectId,
BuildIDs: []string{options.BuildId}, BuildIDs: buildIDs,
Delete: false, Delete: false,
TaskID: &taskID, TaskID: &taskID,
} }
@ -461,7 +465,6 @@ func (c *Controller) BuildModuleWorkflow(ctx workflow.Context, req *peridotpb.Su
} }
moduleBuildTask.RepoChanges.Changes = append(moduleBuildTask.RepoChanges.Changes, updateRepoTask.Changes...) moduleBuildTask.RepoChanges.Changes = append(moduleBuildTask.RepoChanges.Changes, updateRepoTask.Changes...)
}
task.Status = peridotpb.TaskStatus_TASK_STATUS_SUCCEEDED task.Status = peridotpb.TaskStatus_TASK_STATUS_SUCCEEDED
@ -637,13 +640,13 @@ func (c *Controller) BuildModuleStreamWorkflow(ctx workflow.Context, req *perido
// Create an index, so we don't have to re-unmarshal later on // Create an index, so we don't have to re-unmarshal later on
artifactPrimaryIndex := map[string]ArtifactIndex{} artifactPrimaryIndex := map[string]ArtifactIndex{}
// Pre-warm SRPM NEVRAs artifacts, err := c.db.GetArtifactsForBuild(streamBuildOptions.BuildId)
srpmNevras := map[string]string{}
for _, build := range buildTask.Builds {
artifacts, err := c.db.GetArtifactsForBuild(build.BuildId)
if err != nil { if err != nil {
return nil, err return nil, err
} }
// Pre-warm SRPM NEVRAs
srpmNevras := map[string]string{}
for _, artifact := range artifacts { for _, artifact := range artifacts {
if artifact.Arch != "src" { if artifact.Arch != "src" {
continue continue
@ -666,18 +669,26 @@ func (c *Controller) BuildModuleStreamWorkflow(ctx workflow.Context, req *perido
return nil, err return nil, err
} }
for _, build := range buildTask.Builds {
if build.PackageName == primary.Packages[0].Name {
srpmNevras[build.PackageName] = composetools.GenNevraPrimaryPkg(primary.Packages[0]) srpmNevras[build.PackageName] = composetools.GenNevraPrimaryPkg(primary.Packages[0])
} }
} }
}
// Group content licenses // Group content licenses
var licenses []string var licenses []string
for _, build := range buildTask.Builds { for _, build := range buildTask.Builds {
artifacts, err := c.db.GetArtifactsForBuild(build.BuildId) var buildTaskArtifactNames []string
if err != nil { for _, a := range build.Artifacts {
return nil, err buildTaskArtifactNames = append(buildTaskArtifactNames, a.Name)
} }
for _, artifact := range artifacts { for _, artifact := range artifacts {
if !utils.StrContains(artifact.Name, buildTaskArtifactNames) {
continue
}
rpmArtifactMetadata := &peridotpb.RpmArtifactMetadata{} rpmArtifactMetadata := &peridotpb.RpmArtifactMetadata{}
artifactMetadataAny := &anypb.Any{} artifactMetadataAny := &anypb.Any{}
err = protojson.Unmarshal(artifact.Metadata.JSONText, artifactMetadataAny) err = protojson.Unmarshal(artifact.Metadata.JSONText, artifactMetadataAny)
@ -791,12 +802,23 @@ func fillInRpmArtifactsForModuleMd(md *modulemd.ModuleMd, streamBuildOptions *Mo
newMd.Data.License.Content = licenses newMd.Data.License.Content = licenses
// Set buildrequires platform to the one used // Set buildrequires platform to the one used
didSetPlatform := false
platform := streamBuildOptions.Configuration.Platform
platformReq := fmt.Sprintf("el%d.%d.%d", platform.Major, platform.Minor, platform.Patch)
for _, dep := range newMd.Data.Dependencies { for _, dep := range newMd.Data.Dependencies {
if dep.BuildRequires["platform"] != nil { if dep.BuildRequires["platform"] != nil {
platform := streamBuildOptions.Configuration.Platform dep.BuildRequires["platform"] = []string{platformReq}
dep.BuildRequires["platform"] = []string{fmt.Sprintf("el%d.%d.%d", platform.Major, platform.Minor, platform.Patch)} didSetPlatform = true
break
} }
} }
if !didSetPlatform {
newMd.Data.Dependencies = append(newMd.Data.Dependencies, &modulemd.Dependencies{
BuildRequires: map[string][]string{
"platform": {platformReq},
},
})
}
// Set arch for components // Set arch for components
for _, component := range newMd.Data.Components.Rpms { for _, component := range newMd.Data.Components.Rpms {
@ -855,7 +877,15 @@ func fillInRpmArtifactsForModuleMd(md *modulemd.ModuleMd, streamBuildOptions *Mo
collected[nevra] = pkg collected[nevra] = pkg
} }
for nevra, rpmObj := range collected { // Sort collected by rpmObj.Name
var collectedNames []string
for name := range collected {
collectedNames = append(collectedNames, name)
}
sort.Strings(collectedNames)
for _, nevra := range collectedNames {
rpmObj := collected[nevra]
// Skip source RPMs for now as they're // Skip source RPMs for now as they're
// only added if the main RPM is included // only added if the main RPM is included
if rpmObj.Arch == "src" { if rpmObj.Arch == "src" {

View File

@ -951,7 +951,7 @@ func (c *Controller) UpdateRepoActivity(ctx context.Context, req *UpdateRepoRequ
// todo(mustafa): Convert to request struct // todo(mustafa): Convert to request struct
func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest, errorDetails *peridotpb.TaskErrorDetails, packageName string, buildId string, moduleStream *peridotpb.ModuleStream, gpgId *string, signArtifactsTasks *keykeeperpb.BatchSignArtifactsTask, cache *Cache) (*yumrepofspb.UpdateRepoTask, error) { func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest, errorDetails *peridotpb.TaskErrorDetails, packageName string, buildId string, moduleStream *peridotpb.ModuleStream, gpgId *string, signArtifactsTasks *keykeeperpb.BatchSignArtifactsTask, cache *Cache) (*yumrepofspb.UpdateRepoTask, error) {
build, err := c.db.GetBuild(req.ProjectID, buildId) build, err := tx.GetBuild(req.ProjectID, buildId)
if err != nil { if err != nil {
c.log.Errorf("error getting build: %v", err) c.log.Errorf("error getting build: %v", err)
return nil, err return nil, err
@ -970,33 +970,56 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest
} }
project := projects[0] project := projects[0]
artifacts, err := c.db.GetArtifactsForBuild(buildId) artifacts, err := tx.GetArtifactsForBuild(buildId)
if err != nil { if err != nil {
setInternalError(errorDetails, err) setInternalError(errorDetails, err)
return nil, fmt.Errorf("failed to get artifacts for build: %v", err) return nil, fmt.Errorf("failed to get artifacts for build: %v", err)
} }
var currentActiveArtifacts models.TaskArtifacts var currentActiveArtifacts models.TaskArtifacts
var skipDeleteArtifacts []string
if moduleStream == nil {
// Get currently active artifacts // Get currently active artifacts
latestBuilds, err := c.db.GetLatestBuildsByPackageNameAndPackageVersionID(build.PackageName, build.PackageVersionId, project.ID.String()) latestBuilds, err := tx.GetLatestBuildIdsByPackageName(build.PackageName, project.ID.String())
if err != nil { if err != nil {
setInternalError(errorDetails, err) setInternalError(errorDetails, err)
return nil, fmt.Errorf("failed to get latest build ids: %v", err) return nil, fmt.Errorf("failed to get latest build ids: %v", err)
} }
for _, latestBuild := range latestBuilds { for _, latestBuild := range latestBuilds {
buildArtifacts, err := c.db.GetArtifactsForBuild(latestBuild) buildArtifacts, err := tx.GetArtifactsForBuild(latestBuild)
if err != nil { if err != nil {
setInternalError(errorDetails, err) setInternalError(errorDetails, err)
return nil, fmt.Errorf("failed to get artifacts for build: %v", err) return nil, fmt.Errorf("failed to get artifacts for build: %v", err)
} }
currentActiveArtifacts = append(currentActiveArtifacts, buildArtifacts...) currentActiveArtifacts = append(currentActiveArtifacts, buildArtifacts...)
} }
} else {
// Get currently active artifacts
latestBuilds, err := tx.GetLatestBuildsByPackageNameAndBranchName(build.PackageName, moduleStream.ImportRevision.ScmBranchName, project.ID.String())
if err != nil {
setInternalError(errorDetails, err)
return nil, fmt.Errorf("failed to get latest build ids: %v", err)
}
for _, latestBuild := range latestBuilds {
buildArtifacts, err := tx.GetArtifactsForBuild(latestBuild)
if err != nil {
setInternalError(errorDetails, err)
return nil, fmt.Errorf("failed to get artifacts for build: %v", err)
}
currentActiveArtifacts = append(currentActiveArtifacts, buildArtifacts...)
}
}
// Get artifacts to skip deletion
for _, artifact := range artifacts {
skipDeleteArtifacts = append(skipDeleteArtifacts, strings.TrimSuffix(filepath.Base(artifact.Name), ".rpm"))
}
var repos models.Repositories var repos models.Repositories
if req.ForceRepoId != "" { if req.ForceRepoId != "" {
var err error var err error
repo, err := c.db.GetRepository(&req.ForceRepoId, nil, nil) repo, err := tx.GetRepository(&req.ForceRepoId, nil, nil)
if err != nil { if err != nil {
setInternalError(errorDetails, err) setInternalError(errorDetails, err)
return nil, fmt.Errorf("failed to get repo: %v", err) return nil, fmt.Errorf("failed to get repo: %v", err)
@ -1004,7 +1027,7 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest
repos = models.Repositories{*repo} repos = models.Repositories{*repo}
} else { } else {
var err error var err error
repos, err = c.db.FindRepositoriesForPackage(req.ProjectID, packageName, false) repos, err = tx.FindRepositoriesForPackage(req.ProjectID, packageName, false)
if err != nil { if err != nil {
setInternalError(errorDetails, err) setInternalError(errorDetails, err)
return nil, fmt.Errorf("failed to find repo: %v", err) return nil, fmt.Errorf("failed to find repo: %v", err)
@ -1018,6 +1041,7 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest
} }
for _, repo := range repos { for _, repo := range repos {
c.log.Infof("processing repo: %v", repo.Name)
artifactArchMap, err := GenerateArchMapForArtifacts(artifacts, &project, &repo) artifactArchMap, err := GenerateArchMapForArtifacts(artifacts, &project, &repo)
if err != nil { if err != nil {
setInternalError(errorDetails, err) setInternalError(errorDetails, err)
@ -1081,6 +1105,63 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest
} }
for arch, archArtifacts := range artifactArchMap { for arch, archArtifacts := range artifactArchMap {
c.log.Infof("processing arch %s", arch)
noDebugArch := strings.TrimSuffix(arch, "-debug")
var streamDocument *modulemd.ModuleMd
if moduleStream != nil {
var newArtifacts models.TaskArtifacts
streamDocuments := moduleStream.ModuleStreamDocuments[noDebugArch]
if streamDocuments != nil {
streamDocumentNbc, err := modulemd.Parse(streamDocuments.Streams[moduleStream.Stream])
if err != nil {
return nil, fmt.Errorf("failed to decode modulemd: %v", err)
}
streamDocument = streamDocumentNbc.V2
if arch != "src" {
for _, artifact := range archArtifacts {
for _, moduleArtifact := range streamDocument.Data.Artifacts.Rpms {
moduleArtifactNoEpoch := rpmutils.Epoch().ReplaceAllString(moduleArtifact, "")
if strings.TrimSuffix(filepath.Base(artifact.Name), ".rpm") == moduleArtifactNoEpoch {
newArtifacts = append(newArtifacts, *artifact)
}
}
}
artifactArchMap2, err := GenerateArchMapForArtifacts(newArtifacts, &project, &repo)
if err != nil {
setInternalError(errorDetails, err)
return nil, err
}
c.log.Infof("artifactArchMap2: %v", artifactArchMap2)
archArtifacts = artifactArchMap2[arch]
} else {
// Remove duplicates for src
seen := make(map[string]bool)
for _, artifact := range archArtifacts {
if !seen[artifact.Name] {
newArtifacts = append(newArtifacts, *artifact)
seen[artifact.Name] = true
}
}
artifactArchMap2, err := GenerateArchMapForArtifacts(newArtifacts, &project, &repo)
if err != nil {
setInternalError(errorDetails, err)
return nil, err
}
c.log.Infof("artifactArchMap2: %v", artifactArchMap2)
archArtifacts = artifactArchMap2[arch]
}
}
if strings.HasSuffix(arch, "-debug") || arch == "src" {
streamDocument = nil
}
}
changes := &yumrepofspb.RepositoryChange{ changes := &yumrepofspb.RepositoryChange{
Name: fmt.Sprintf("%s-%s", repo.Name, arch), Name: fmt.Sprintf("%s-%s", repo.Name, arch),
AddedPackages: []string{}, AddedPackages: []string{},
@ -1104,9 +1185,15 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest
} }
var modulesRoot []*modulemd.ModuleMd var modulesRoot []*modulemd.ModuleMd
idArch := fmt.Sprintf("%s-%s", repo.ID, arch) idArch := fmt.Sprintf("%s-%s", repo.Name, arch)
idArchNoDebug := fmt.Sprintf("%s-%s", repo.Name, noDebugArch)
var currentRevision *models.RepositoryRevision var currentRevision *models.RepositoryRevision
var groupsXml string var groupsXml string
if cache.Repos[idArchNoDebug] != nil {
if cache.Repos[idArchNoDebug].Modulemd != nil {
modulesRoot = cache.Repos[idArchNoDebug].Modulemd
}
}
if cache.Repos[idArch] != nil { if cache.Repos[idArch] != nil {
if cache.Repos[idArch].PrimaryRoot != nil { if cache.Repos[idArch].PrimaryRoot != nil {
primaryRoot = *cache.Repos[idArch].PrimaryRoot primaryRoot = *cache.Repos[idArch].PrimaryRoot
@ -1117,17 +1204,27 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest
if cache.Repos[idArch].OtherRoot != nil { if cache.Repos[idArch].OtherRoot != nil {
otherRoot = *cache.Repos[idArch].OtherRoot otherRoot = *cache.Repos[idArch].OtherRoot
} }
if cache.Repos[idArch].Modulemd != nil {
modulesRoot = cache.Repos[idArch].Modulemd
}
groupsXml = cache.Repos[idArch].GroupsXml groupsXml = cache.Repos[idArch].GroupsXml
} else { } else {
currentRevision, err = c.db.GetLatestActiveRepositoryRevision(repo.ID.String(), arch) var noDebugRevision *models.RepositoryRevision
currentRevision, err = tx.GetLatestActiveRepositoryRevision(repo.ID.String(), arch)
if err != nil { if err != nil {
if err != sql.ErrNoRows { if err != sql.ErrNoRows {
return nil, fmt.Errorf("failed to get latest active repository revision: %v", err) return nil, fmt.Errorf("failed to get latest active repository revision: %v", err)
} }
} }
if strings.HasSuffix(arch, "-debug") {
noDebugRevision, err = tx.GetLatestActiveRepositoryRevision(repo.ID.String(), noDebugArch)
if err != nil {
if err != sql.ErrNoRows {
return nil, fmt.Errorf("failed to get latest active repository revision: %v", err)
}
}
} else {
noDebugRevision = currentRevision
}
if currentRevision != nil { if currentRevision != nil {
if currentRevision.PrimaryXml != "" { if currentRevision.PrimaryXml != "" {
var primaryXmlGz []byte var primaryXmlGz []byte
@ -1171,11 +1268,14 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest
return nil, err return nil, err
} }
} }
if currentRevision.ModulesYaml != "" { groupsXml = currentRevision.GroupsXml
}
if noDebugRevision != nil {
if noDebugRevision.ModulesYaml != "" {
var modulesYamlGz []byte var modulesYamlGz []byte
var modulesYaml []byte var modulesYaml []byte
err := multiErrorCheck( err := multiErrorCheck(
b64Decode(currentRevision.ModulesYaml, &modulesYamlGz), b64Decode(noDebugRevision.ModulesYaml, &modulesYamlGz),
decompressWithGz(modulesYamlGz, &modulesYaml), decompressWithGz(modulesYamlGz, &modulesYaml),
) )
if err != nil { if err != nil {
@ -1205,7 +1305,6 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest
modulesRoot = append(modulesRoot, &*&md) modulesRoot = append(modulesRoot, &*&md)
} }
} }
groupsXml = currentRevision.GroupsXml
} }
} }
@ -1219,6 +1318,7 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest
} }
} }
} }
c.log.Infof("module artifacts: %v", moduleArtifacts)
for _, artifact := range archArtifacts { for _, artifact := range archArtifacts {
// This shouldn't happen // This shouldn't happen
@ -1243,7 +1343,7 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest
} }
shouldAdd := true shouldAdd := true
if arch != "src" { if arch != "src" && moduleStream == nil {
// If repo has a list for inclusion, then the artifact has to pass that first // If repo has a list for inclusion, then the artifact has to pass that first
if len(repo.IncludeFilter) > 0 { if len(repo.IncludeFilter) > 0 {
// If the artifact isn't forced, it should be in the include list or additional multilib list // If the artifact isn't forced, it should be in the include list or additional multilib list
@ -1326,16 +1426,7 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest
var pkgId *string var pkgId *string
var primaryIndex *int var primaryIndex *int
for i, primaryPackage := range primaryRoot.Packages { for i, primaryPackage := range primaryRoot.Packages {
// We check module entries by artifact index from a previous revision if moduleStream == nil {
if moduleStream != nil {
isStreamArtifact := moduleArtifacts[strings.TrimSuffix(filepath.Base(artifact.Name), ".rpm")]
if isStreamArtifact {
ix := i
primaryIndex = &ix
pkgId = &primaryPackage.Checksum.Value
break
}
} else {
// If not a module stream, search for a non-module entry // If not a module stream, search for a non-module entry
// Double check arch as well for multilib purposes // Double check arch as well for multilib purposes
if primaryPackage.Name == name && !strings.Contains(primaryPackage.Version.Rel, ".module+") && primaryPackage.Arch == artifact.Arch { if primaryPackage.Name == name && !strings.Contains(primaryPackage.Version.Rel, ".module+") && primaryPackage.Arch == artifact.Arch {
@ -1344,6 +1435,33 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest
pkgId = &primaryPackage.Checksum.Value pkgId = &primaryPackage.Checksum.Value
break break
} }
} else {
// If a module stream, search for a module entry
for _, streamArtifact := range artifacts {
var rpmName string
var rpmVersion string
var rpmRelease string
rpmBase := strings.TrimSuffix(filepath.Base(streamArtifact.Name), ".rpm")
if rpmutils.NVRUnusualRelease().MatchString(rpmBase) {
nvr := rpmutils.NVRUnusualRelease().FindStringSubmatch(rpmBase)
rpmName = nvr[1]
rpmVersion = nvr[2]
rpmRelease = nvr[3]
} else if rpmutils.NVR().MatchString(rpmBase) {
nvr := rpmutils.NVR().FindStringSubmatch(rpmBase)
rpmName = nvr[1]
rpmVersion = nvr[2]
rpmRelease = nvr[3]
}
if name == rpmName && primaryPackage.Name == name && primaryPackage.Version.Ver == rpmVersion && primaryPackage.Version.Rel == rpmRelease && primaryPackage.Arch == artifact.Arch {
c.log.Infof("Found module stream package: %s", rpmBase)
ix := i
primaryIndex = &ix
pkgId = &primaryPackage.Checksum.Value
break
}
}
} }
} }
if primaryIndex != nil { if primaryIndex != nil {
@ -1395,19 +1513,28 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest
var nFilelists []*yummeta.FilelistsPackage var nFilelists []*yummeta.FilelistsPackage
var nOther []*yummeta.OtherPackage var nOther []*yummeta.OtherPackage
var deleteIds []string var deleteIds []string
if !req.NoDeletePrevious { c.log.Infof("modified packages: %v", changes.ModifiedPackages)
c.log.Infof("added packages: %v", changes.AddedPackages)
if !req.NoDeletePrevious || moduleStream != nil {
for _, pkg := range primaryRoot.Packages { for _, pkg := range primaryRoot.Packages {
shouldAdd := true shouldAdd := true
for _, artifact := range currentActiveArtifacts { for _, artifact := range currentActiveArtifacts {
if filepath.Base(artifact.Name) == filepath.Base(pkg.Location.Href) && !utils.StrContains(strings.TrimSuffix(filepath.Base(artifact.Name), ".rpm"), changes.ModifiedPackages) && !utils.StrContains(strings.TrimSuffix(filepath.Base(artifact.Name), ".rpm"), changes.AddedPackages) { noRpmName := strings.TrimSuffix(filepath.Base(artifact.Name), ".rpm")
if filepath.Base(artifact.Name) == filepath.Base(pkg.Location.Href) && !utils.StrContains(noRpmName, changes.ModifiedPackages) && !utils.StrContains(noRpmName, changes.AddedPackages) && !utils.StrContains(noRpmName, skipDeleteArtifacts) {
shouldAdd = false shouldAdd = false
} }
} }
if !shouldAdd { if !shouldAdd {
deleteIds = append(deleteIds, pkg.Checksum.Value) deleteIds = append(deleteIds, pkg.Checksum.Value)
}
}
for _, pkg := range primaryRoot.Packages {
if utils.StrContains(pkg.Checksum.Value, deleteIds) {
c.log.Infof("Deleting package %s-%s-%s from primary metadata", pkg.Name, pkg.Version.Ver, pkg.Version.Rel)
continue continue
} }
nPackages = append(nPackages, &*pkg)
nPackages = append(nPackages, pkg)
} }
for _, pkg := range filelistsRoot.Packages { for _, pkg := range filelistsRoot.Packages {
if utils.StrContains(pkg.PkgId, deleteIds) { if utils.StrContains(pkg.PkgId, deleteIds) {
@ -1432,15 +1559,7 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest
otherRoot.PackageCount = len(filelistsRoot.Packages) otherRoot.PackageCount = len(filelistsRoot.Packages)
// Module builds needs a few more steps // Module builds needs a few more steps
if moduleStream != nil { if moduleStream != nil && streamDocument != nil {
streamDocument := moduleStream.ModuleStreamDocuments[arch]
if streamDocument != nil {
newEntryNbc, err := modulemd.Parse(streamDocument.Streams[moduleStream.Stream])
if err != nil {
return nil, err
}
newEntry := newEntryNbc.V2
// If a previous entry exists, we need to overwrite that // If a previous entry exists, we need to overwrite that
var moduleIndex *int var moduleIndex *int
for i, moduleMd := range modulesRoot { for i, moduleMd := range modulesRoot {
@ -1451,11 +1570,10 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest
} }
if moduleIndex != nil { if moduleIndex != nil {
changes.ModifiedModules = append(changes.ModifiedModules, fmt.Sprintf("%s:%s", moduleStream.Name, moduleStream.Stream)) changes.ModifiedModules = append(changes.ModifiedModules, fmt.Sprintf("%s:%s", moduleStream.Name, moduleStream.Stream))
modulesRoot[*moduleIndex] = newEntry modulesRoot[*moduleIndex] = streamDocument
} else { } else {
changes.AddedModules = append(changes.AddedModules, fmt.Sprintf("%s:%s", moduleStream.Name, moduleStream.Stream)) changes.AddedModules = append(changes.AddedModules, fmt.Sprintf("%s:%s", moduleStream.Name, moduleStream.Stream))
modulesRoot = append(modulesRoot, newEntry) modulesRoot = append(modulesRoot, streamDocument)
}
} }
} }
@ -1471,6 +1589,9 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest
ModuleDefaults: moduleDefaults, ModuleDefaults: moduleDefaults,
GroupsXml: groupsXml, GroupsXml: groupsXml,
} }
if strings.HasSuffix(arch, "-debug") || arch == "src" {
cache.Repos[idArch].Modulemd = nil
}
repoTask.Changes = append(repoTask.Changes, changes) repoTask.Changes = append(repoTask.Changes, changes)
} }

View File

@ -66,7 +66,7 @@ type Access interface {
NVRAExists(nvra string) (bool, error) NVRAExists(nvra string) (bool, error)
GetBuildByPackageNameAndVersionAndRelease(name string, version string, release string, projectId string) (*models.Build, error) GetBuildByPackageNameAndVersionAndRelease(name string, version string, release string, projectId string) (*models.Build, error)
GetLatestBuildIdsByPackageName(name string, projectId string) ([]string, error) GetLatestBuildIdsByPackageName(name string, projectId string) ([]string, error)
GetLatestBuildsByPackageNameAndPackageVersionID(name string, packageVersionId string, projectId string) ([]string, error) GetLatestBuildsByPackageNameAndBranchName(name string, branchName string, projectId string) ([]string, error)
GetActiveBuildIdsByTaskArtifactGlob(taskArtifactGlob string, projectId string) ([]string, error) GetActiveBuildIdsByTaskArtifactGlob(taskArtifactGlob string, projectId string) ([]string, error)
GetAllBuildIdsByPackageName(name string, projectId string) ([]string, error) GetAllBuildIdsByPackageName(name string, projectId string) ([]string, error)

View File

@ -428,7 +428,7 @@ func (a *Access) GetLatestBuildIdsByPackageName(name string, projectId string) (
return ret, nil return ret, nil
} }
func (a *Access) GetLatestBuildsByPackageNameAndPackageVersionID(name string, packageVersionId string, projectId string) ([]string, error) { func (a *Access) GetLatestBuildsByPackageNameAndBranchName(name string, branchName string, projectId string) ([]string, error) {
var ret []string var ret []string
err := a.query.Select( err := a.query.Select(
&ret, &ret,
@ -439,18 +439,19 @@ func (a *Access) GetLatestBuildsByPackageNameAndPackageVersionID(name string, pa
inner join tasks t on t.id = b.task_id inner join tasks t on t.id = b.task_id
inner join packages p on p.id = b.package_id inner join packages p on p.id = b.package_id
inner join project_package_versions ppv on ppv.package_version_id = b.package_version_id inner join project_package_versions ppv on ppv.package_version_id = b.package_version_id
inner join import_revisions ir on ir.package_version_id = b.package_version_id
where where
b.project_id = $1 b.project_id = $3
and p.name = $2 and p.name = $1
and ppv.active_in_repo = true and ppv.active_in_repo = true
and ppv.project_id = b.project_id and ppv.project_id = b.project_id
and b.package_version_id = $3 and ir.scm_branch_name = $2
and t.status = 3 and t.status = 3
order by b.created_at asc order by b.created_at asc
`, `,
projectId,
name, name,
packageVersionId, branchName,
projectId,
) )
if err != nil { if err != nil {
return nil, err return nil, err