mirror of
https://github.com/rocky-linux/peridot.git
synced 2024-12-21 02:08:29 +00:00
Module mode now correctly replaces all artifacts
There was a bug in module mode where debuginfo+src would not get replaced correctly. That is now fixed. Additional changes to module artifact detection is also made after the build merge solution was implemented. Previously src.rpms could be collapsed because of multiple src.rpms in one build, leading to non-devel artifacts being placed in devel. That is also fixed with this.
This commit is contained in:
parent
4e9db547f6
commit
e0a043afb3
@ -739,6 +739,12 @@ func (c *Controller) BuildWorkflow(ctx workflow.Context, req *peridotpb.SubmitBu
|
|||||||
Arch: result.Arch,
|
Arch: result.Arch,
|
||||||
Metadata: metadata,
|
Metadata: metadata,
|
||||||
})
|
})
|
||||||
|
artifacts = append(artifacts, &peridotpb.TaskArtifact{
|
||||||
|
TaskId: result.Subtask.ID.String(),
|
||||||
|
Name: result.ObjectName,
|
||||||
|
HashSha256: result.HashSha256,
|
||||||
|
Arch: result.Arch,
|
||||||
|
})
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@ -805,7 +811,7 @@ func (c *Controller) BuildWorkflow(ctx workflow.Context, req *peridotpb.SubmitBu
|
|||||||
|
|
||||||
submitBuildTask = peridotpb.SubmitBuildTask{
|
submitBuildTask = peridotpb.SubmitBuildTask{
|
||||||
BuildId: buildID,
|
BuildId: buildID,
|
||||||
BuildTaskId: taskID,
|
BuildTaskId: task.ID.String(),
|
||||||
PackageName: pkg.Name,
|
PackageName: pkg.Name,
|
||||||
ImportRevision: importRevision.ToProto(),
|
ImportRevision: importRevision.ToProto(),
|
||||||
Artifacts: artifacts,
|
Artifacts: artifacts,
|
||||||
@ -824,7 +830,6 @@ func (c *Controller) BuildWorkflow(ctx workflow.Context, req *peridotpb.SubmitBu
|
|||||||
setInternalError(errorDetails, err)
|
setInternalError(errorDetails, err)
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
submitBuildTask.Artifacts = nil
|
|
||||||
|
|
||||||
task.Status = peridotpb.TaskStatus_TASK_STATUS_RUNNING
|
task.Status = peridotpb.TaskStatus_TASK_STATUS_RUNNING
|
||||||
|
|
||||||
|
@ -442,14 +442,18 @@ func (c *Controller) BuildModuleWorkflow(ctx workflow.Context, req *peridotpb.Su
|
|||||||
|
|
||||||
task.Status = peridotpb.TaskStatus_TASK_STATUS_FAILED
|
task.Status = peridotpb.TaskStatus_TASK_STATUS_FAILED
|
||||||
|
|
||||||
|
var buildIDs []string
|
||||||
for _, options := range streamBuildOptions {
|
for _, options := range streamBuildOptions {
|
||||||
|
buildIDs = append(buildIDs, options.BuildId)
|
||||||
|
}
|
||||||
|
|
||||||
yumrepoCtx := workflow.WithChildOptions(ctx, workflow.ChildWorkflowOptions{
|
yumrepoCtx := workflow.WithChildOptions(ctx, workflow.ChildWorkflowOptions{
|
||||||
TaskQueue: "yumrepofs",
|
TaskQueue: "yumrepofs",
|
||||||
})
|
})
|
||||||
taskID := task.ID.String()
|
taskID := task.ID.String()
|
||||||
updateRepoRequest := &UpdateRepoRequest{
|
updateRepoRequest := &UpdateRepoRequest{
|
||||||
ProjectID: req.ProjectId,
|
ProjectID: req.ProjectId,
|
||||||
BuildIDs: []string{options.BuildId},
|
BuildIDs: buildIDs,
|
||||||
Delete: false,
|
Delete: false,
|
||||||
TaskID: &taskID,
|
TaskID: &taskID,
|
||||||
}
|
}
|
||||||
@ -461,7 +465,6 @@ func (c *Controller) BuildModuleWorkflow(ctx workflow.Context, req *peridotpb.Su
|
|||||||
}
|
}
|
||||||
|
|
||||||
moduleBuildTask.RepoChanges.Changes = append(moduleBuildTask.RepoChanges.Changes, updateRepoTask.Changes...)
|
moduleBuildTask.RepoChanges.Changes = append(moduleBuildTask.RepoChanges.Changes, updateRepoTask.Changes...)
|
||||||
}
|
|
||||||
|
|
||||||
task.Status = peridotpb.TaskStatus_TASK_STATUS_SUCCEEDED
|
task.Status = peridotpb.TaskStatus_TASK_STATUS_SUCCEEDED
|
||||||
|
|
||||||
@ -637,13 +640,13 @@ func (c *Controller) BuildModuleStreamWorkflow(ctx workflow.Context, req *perido
|
|||||||
// Create an index, so we don't have to re-unmarshal later on
|
// Create an index, so we don't have to re-unmarshal later on
|
||||||
artifactPrimaryIndex := map[string]ArtifactIndex{}
|
artifactPrimaryIndex := map[string]ArtifactIndex{}
|
||||||
|
|
||||||
// Pre-warm SRPM NEVRAs
|
artifacts, err := c.db.GetArtifactsForBuild(streamBuildOptions.BuildId)
|
||||||
srpmNevras := map[string]string{}
|
|
||||||
for _, build := range buildTask.Builds {
|
|
||||||
artifacts, err := c.db.GetArtifactsForBuild(build.BuildId)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Pre-warm SRPM NEVRAs
|
||||||
|
srpmNevras := map[string]string{}
|
||||||
for _, artifact := range artifacts {
|
for _, artifact := range artifacts {
|
||||||
if artifact.Arch != "src" {
|
if artifact.Arch != "src" {
|
||||||
continue
|
continue
|
||||||
@ -666,18 +669,26 @@ func (c *Controller) BuildModuleStreamWorkflow(ctx workflow.Context, req *perido
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
for _, build := range buildTask.Builds {
|
||||||
|
if build.PackageName == primary.Packages[0].Name {
|
||||||
srpmNevras[build.PackageName] = composetools.GenNevraPrimaryPkg(primary.Packages[0])
|
srpmNevras[build.PackageName] = composetools.GenNevraPrimaryPkg(primary.Packages[0])
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Group content licenses
|
// Group content licenses
|
||||||
var licenses []string
|
var licenses []string
|
||||||
for _, build := range buildTask.Builds {
|
for _, build := range buildTask.Builds {
|
||||||
artifacts, err := c.db.GetArtifactsForBuild(build.BuildId)
|
var buildTaskArtifactNames []string
|
||||||
if err != nil {
|
for _, a := range build.Artifacts {
|
||||||
return nil, err
|
buildTaskArtifactNames = append(buildTaskArtifactNames, a.Name)
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, artifact := range artifacts {
|
for _, artifact := range artifacts {
|
||||||
|
if !utils.StrContains(artifact.Name, buildTaskArtifactNames) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
rpmArtifactMetadata := &peridotpb.RpmArtifactMetadata{}
|
rpmArtifactMetadata := &peridotpb.RpmArtifactMetadata{}
|
||||||
artifactMetadataAny := &anypb.Any{}
|
artifactMetadataAny := &anypb.Any{}
|
||||||
err = protojson.Unmarshal(artifact.Metadata.JSONText, artifactMetadataAny)
|
err = protojson.Unmarshal(artifact.Metadata.JSONText, artifactMetadataAny)
|
||||||
@ -791,12 +802,23 @@ func fillInRpmArtifactsForModuleMd(md *modulemd.ModuleMd, streamBuildOptions *Mo
|
|||||||
newMd.Data.License.Content = licenses
|
newMd.Data.License.Content = licenses
|
||||||
|
|
||||||
// Set buildrequires platform to the one used
|
// Set buildrequires platform to the one used
|
||||||
|
didSetPlatform := false
|
||||||
|
platform := streamBuildOptions.Configuration.Platform
|
||||||
|
platformReq := fmt.Sprintf("el%d.%d.%d", platform.Major, platform.Minor, platform.Patch)
|
||||||
for _, dep := range newMd.Data.Dependencies {
|
for _, dep := range newMd.Data.Dependencies {
|
||||||
if dep.BuildRequires["platform"] != nil {
|
if dep.BuildRequires["platform"] != nil {
|
||||||
platform := streamBuildOptions.Configuration.Platform
|
dep.BuildRequires["platform"] = []string{platformReq}
|
||||||
dep.BuildRequires["platform"] = []string{fmt.Sprintf("el%d.%d.%d", platform.Major, platform.Minor, platform.Patch)}
|
didSetPlatform = true
|
||||||
|
break
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if !didSetPlatform {
|
||||||
|
newMd.Data.Dependencies = append(newMd.Data.Dependencies, &modulemd.Dependencies{
|
||||||
|
BuildRequires: map[string][]string{
|
||||||
|
"platform": {platformReq},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
// Set arch for components
|
// Set arch for components
|
||||||
for _, component := range newMd.Data.Components.Rpms {
|
for _, component := range newMd.Data.Components.Rpms {
|
||||||
@ -855,7 +877,15 @@ func fillInRpmArtifactsForModuleMd(md *modulemd.ModuleMd, streamBuildOptions *Mo
|
|||||||
collected[nevra] = pkg
|
collected[nevra] = pkg
|
||||||
}
|
}
|
||||||
|
|
||||||
for nevra, rpmObj := range collected {
|
// Sort collected by rpmObj.Name
|
||||||
|
var collectedNames []string
|
||||||
|
for name := range collected {
|
||||||
|
collectedNames = append(collectedNames, name)
|
||||||
|
}
|
||||||
|
sort.Strings(collectedNames)
|
||||||
|
|
||||||
|
for _, nevra := range collectedNames {
|
||||||
|
rpmObj := collected[nevra]
|
||||||
// Skip source RPMs for now as they're
|
// Skip source RPMs for now as they're
|
||||||
// only added if the main RPM is included
|
// only added if the main RPM is included
|
||||||
if rpmObj.Arch == "src" {
|
if rpmObj.Arch == "src" {
|
||||||
|
@ -951,7 +951,7 @@ func (c *Controller) UpdateRepoActivity(ctx context.Context, req *UpdateRepoRequ
|
|||||||
|
|
||||||
// todo(mustafa): Convert to request struct
|
// todo(mustafa): Convert to request struct
|
||||||
func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest, errorDetails *peridotpb.TaskErrorDetails, packageName string, buildId string, moduleStream *peridotpb.ModuleStream, gpgId *string, signArtifactsTasks *keykeeperpb.BatchSignArtifactsTask, cache *Cache) (*yumrepofspb.UpdateRepoTask, error) {
|
func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest, errorDetails *peridotpb.TaskErrorDetails, packageName string, buildId string, moduleStream *peridotpb.ModuleStream, gpgId *string, signArtifactsTasks *keykeeperpb.BatchSignArtifactsTask, cache *Cache) (*yumrepofspb.UpdateRepoTask, error) {
|
||||||
build, err := c.db.GetBuild(req.ProjectID, buildId)
|
build, err := tx.GetBuild(req.ProjectID, buildId)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
c.log.Errorf("error getting build: %v", err)
|
c.log.Errorf("error getting build: %v", err)
|
||||||
return nil, err
|
return nil, err
|
||||||
@ -970,33 +970,56 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest
|
|||||||
}
|
}
|
||||||
project := projects[0]
|
project := projects[0]
|
||||||
|
|
||||||
artifacts, err := c.db.GetArtifactsForBuild(buildId)
|
artifacts, err := tx.GetArtifactsForBuild(buildId)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
setInternalError(errorDetails, err)
|
setInternalError(errorDetails, err)
|
||||||
return nil, fmt.Errorf("failed to get artifacts for build: %v", err)
|
return nil, fmt.Errorf("failed to get artifacts for build: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
var currentActiveArtifacts models.TaskArtifacts
|
var currentActiveArtifacts models.TaskArtifacts
|
||||||
|
var skipDeleteArtifacts []string
|
||||||
|
if moduleStream == nil {
|
||||||
// Get currently active artifacts
|
// Get currently active artifacts
|
||||||
latestBuilds, err := c.db.GetLatestBuildsByPackageNameAndPackageVersionID(build.PackageName, build.PackageVersionId, project.ID.String())
|
latestBuilds, err := tx.GetLatestBuildIdsByPackageName(build.PackageName, project.ID.String())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
setInternalError(errorDetails, err)
|
setInternalError(errorDetails, err)
|
||||||
return nil, fmt.Errorf("failed to get latest build ids: %v", err)
|
return nil, fmt.Errorf("failed to get latest build ids: %v", err)
|
||||||
}
|
}
|
||||||
for _, latestBuild := range latestBuilds {
|
for _, latestBuild := range latestBuilds {
|
||||||
buildArtifacts, err := c.db.GetArtifactsForBuild(latestBuild)
|
buildArtifacts, err := tx.GetArtifactsForBuild(latestBuild)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
setInternalError(errorDetails, err)
|
setInternalError(errorDetails, err)
|
||||||
return nil, fmt.Errorf("failed to get artifacts for build: %v", err)
|
return nil, fmt.Errorf("failed to get artifacts for build: %v", err)
|
||||||
}
|
}
|
||||||
currentActiveArtifacts = append(currentActiveArtifacts, buildArtifacts...)
|
currentActiveArtifacts = append(currentActiveArtifacts, buildArtifacts...)
|
||||||
}
|
}
|
||||||
|
} else {
|
||||||
|
// Get currently active artifacts
|
||||||
|
latestBuilds, err := tx.GetLatestBuildsByPackageNameAndBranchName(build.PackageName, moduleStream.ImportRevision.ScmBranchName, project.ID.String())
|
||||||
|
if err != nil {
|
||||||
|
setInternalError(errorDetails, err)
|
||||||
|
return nil, fmt.Errorf("failed to get latest build ids: %v", err)
|
||||||
|
}
|
||||||
|
for _, latestBuild := range latestBuilds {
|
||||||
|
buildArtifacts, err := tx.GetArtifactsForBuild(latestBuild)
|
||||||
|
if err != nil {
|
||||||
|
setInternalError(errorDetails, err)
|
||||||
|
return nil, fmt.Errorf("failed to get artifacts for build: %v", err)
|
||||||
|
}
|
||||||
|
currentActiveArtifacts = append(currentActiveArtifacts, buildArtifacts...)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get artifacts to skip deletion
|
||||||
|
for _, artifact := range artifacts {
|
||||||
|
skipDeleteArtifacts = append(skipDeleteArtifacts, strings.TrimSuffix(filepath.Base(artifact.Name), ".rpm"))
|
||||||
|
}
|
||||||
|
|
||||||
var repos models.Repositories
|
var repos models.Repositories
|
||||||
|
|
||||||
if req.ForceRepoId != "" {
|
if req.ForceRepoId != "" {
|
||||||
var err error
|
var err error
|
||||||
repo, err := c.db.GetRepository(&req.ForceRepoId, nil, nil)
|
repo, err := tx.GetRepository(&req.ForceRepoId, nil, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
setInternalError(errorDetails, err)
|
setInternalError(errorDetails, err)
|
||||||
return nil, fmt.Errorf("failed to get repo: %v", err)
|
return nil, fmt.Errorf("failed to get repo: %v", err)
|
||||||
@ -1004,7 +1027,7 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest
|
|||||||
repos = models.Repositories{*repo}
|
repos = models.Repositories{*repo}
|
||||||
} else {
|
} else {
|
||||||
var err error
|
var err error
|
||||||
repos, err = c.db.FindRepositoriesForPackage(req.ProjectID, packageName, false)
|
repos, err = tx.FindRepositoriesForPackage(req.ProjectID, packageName, false)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
setInternalError(errorDetails, err)
|
setInternalError(errorDetails, err)
|
||||||
return nil, fmt.Errorf("failed to find repo: %v", err)
|
return nil, fmt.Errorf("failed to find repo: %v", err)
|
||||||
@ -1018,6 +1041,7 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest
|
|||||||
}
|
}
|
||||||
|
|
||||||
for _, repo := range repos {
|
for _, repo := range repos {
|
||||||
|
c.log.Infof("processing repo: %v", repo.Name)
|
||||||
artifactArchMap, err := GenerateArchMapForArtifacts(artifacts, &project, &repo)
|
artifactArchMap, err := GenerateArchMapForArtifacts(artifacts, &project, &repo)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
setInternalError(errorDetails, err)
|
setInternalError(errorDetails, err)
|
||||||
@ -1081,6 +1105,63 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest
|
|||||||
}
|
}
|
||||||
|
|
||||||
for arch, archArtifacts := range artifactArchMap {
|
for arch, archArtifacts := range artifactArchMap {
|
||||||
|
c.log.Infof("processing arch %s", arch)
|
||||||
|
|
||||||
|
noDebugArch := strings.TrimSuffix(arch, "-debug")
|
||||||
|
var streamDocument *modulemd.ModuleMd
|
||||||
|
|
||||||
|
if moduleStream != nil {
|
||||||
|
var newArtifacts models.TaskArtifacts
|
||||||
|
|
||||||
|
streamDocuments := moduleStream.ModuleStreamDocuments[noDebugArch]
|
||||||
|
if streamDocuments != nil {
|
||||||
|
streamDocumentNbc, err := modulemd.Parse(streamDocuments.Streams[moduleStream.Stream])
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to decode modulemd: %v", err)
|
||||||
|
}
|
||||||
|
streamDocument = streamDocumentNbc.V2
|
||||||
|
|
||||||
|
if arch != "src" {
|
||||||
|
for _, artifact := range archArtifacts {
|
||||||
|
for _, moduleArtifact := range streamDocument.Data.Artifacts.Rpms {
|
||||||
|
moduleArtifactNoEpoch := rpmutils.Epoch().ReplaceAllString(moduleArtifact, "")
|
||||||
|
if strings.TrimSuffix(filepath.Base(artifact.Name), ".rpm") == moduleArtifactNoEpoch {
|
||||||
|
newArtifacts = append(newArtifacts, *artifact)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
artifactArchMap2, err := GenerateArchMapForArtifacts(newArtifacts, &project, &repo)
|
||||||
|
if err != nil {
|
||||||
|
setInternalError(errorDetails, err)
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
c.log.Infof("artifactArchMap2: %v", artifactArchMap2)
|
||||||
|
archArtifacts = artifactArchMap2[arch]
|
||||||
|
} else {
|
||||||
|
// Remove duplicates for src
|
||||||
|
seen := make(map[string]bool)
|
||||||
|
for _, artifact := range archArtifacts {
|
||||||
|
if !seen[artifact.Name] {
|
||||||
|
newArtifacts = append(newArtifacts, *artifact)
|
||||||
|
seen[artifact.Name] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
artifactArchMap2, err := GenerateArchMapForArtifacts(newArtifacts, &project, &repo)
|
||||||
|
if err != nil {
|
||||||
|
setInternalError(errorDetails, err)
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
c.log.Infof("artifactArchMap2: %v", artifactArchMap2)
|
||||||
|
archArtifacts = artifactArchMap2[arch]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if strings.HasSuffix(arch, "-debug") || arch == "src" {
|
||||||
|
streamDocument = nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
changes := &yumrepofspb.RepositoryChange{
|
changes := &yumrepofspb.RepositoryChange{
|
||||||
Name: fmt.Sprintf("%s-%s", repo.Name, arch),
|
Name: fmt.Sprintf("%s-%s", repo.Name, arch),
|
||||||
AddedPackages: []string{},
|
AddedPackages: []string{},
|
||||||
@ -1104,9 +1185,15 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest
|
|||||||
}
|
}
|
||||||
var modulesRoot []*modulemd.ModuleMd
|
var modulesRoot []*modulemd.ModuleMd
|
||||||
|
|
||||||
idArch := fmt.Sprintf("%s-%s", repo.ID, arch)
|
idArch := fmt.Sprintf("%s-%s", repo.Name, arch)
|
||||||
|
idArchNoDebug := fmt.Sprintf("%s-%s", repo.Name, noDebugArch)
|
||||||
var currentRevision *models.RepositoryRevision
|
var currentRevision *models.RepositoryRevision
|
||||||
var groupsXml string
|
var groupsXml string
|
||||||
|
if cache.Repos[idArchNoDebug] != nil {
|
||||||
|
if cache.Repos[idArchNoDebug].Modulemd != nil {
|
||||||
|
modulesRoot = cache.Repos[idArchNoDebug].Modulemd
|
||||||
|
}
|
||||||
|
}
|
||||||
if cache.Repos[idArch] != nil {
|
if cache.Repos[idArch] != nil {
|
||||||
if cache.Repos[idArch].PrimaryRoot != nil {
|
if cache.Repos[idArch].PrimaryRoot != nil {
|
||||||
primaryRoot = *cache.Repos[idArch].PrimaryRoot
|
primaryRoot = *cache.Repos[idArch].PrimaryRoot
|
||||||
@ -1117,17 +1204,27 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest
|
|||||||
if cache.Repos[idArch].OtherRoot != nil {
|
if cache.Repos[idArch].OtherRoot != nil {
|
||||||
otherRoot = *cache.Repos[idArch].OtherRoot
|
otherRoot = *cache.Repos[idArch].OtherRoot
|
||||||
}
|
}
|
||||||
if cache.Repos[idArch].Modulemd != nil {
|
|
||||||
modulesRoot = cache.Repos[idArch].Modulemd
|
|
||||||
}
|
|
||||||
groupsXml = cache.Repos[idArch].GroupsXml
|
groupsXml = cache.Repos[idArch].GroupsXml
|
||||||
} else {
|
} else {
|
||||||
currentRevision, err = c.db.GetLatestActiveRepositoryRevision(repo.ID.String(), arch)
|
var noDebugRevision *models.RepositoryRevision
|
||||||
|
|
||||||
|
currentRevision, err = tx.GetLatestActiveRepositoryRevision(repo.ID.String(), arch)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if err != sql.ErrNoRows {
|
if err != sql.ErrNoRows {
|
||||||
return nil, fmt.Errorf("failed to get latest active repository revision: %v", err)
|
return nil, fmt.Errorf("failed to get latest active repository revision: %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if strings.HasSuffix(arch, "-debug") {
|
||||||
|
noDebugRevision, err = tx.GetLatestActiveRepositoryRevision(repo.ID.String(), noDebugArch)
|
||||||
|
if err != nil {
|
||||||
|
if err != sql.ErrNoRows {
|
||||||
|
return nil, fmt.Errorf("failed to get latest active repository revision: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
noDebugRevision = currentRevision
|
||||||
|
}
|
||||||
|
|
||||||
if currentRevision != nil {
|
if currentRevision != nil {
|
||||||
if currentRevision.PrimaryXml != "" {
|
if currentRevision.PrimaryXml != "" {
|
||||||
var primaryXmlGz []byte
|
var primaryXmlGz []byte
|
||||||
@ -1171,11 +1268,14 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if currentRevision.ModulesYaml != "" {
|
groupsXml = currentRevision.GroupsXml
|
||||||
|
}
|
||||||
|
if noDebugRevision != nil {
|
||||||
|
if noDebugRevision.ModulesYaml != "" {
|
||||||
var modulesYamlGz []byte
|
var modulesYamlGz []byte
|
||||||
var modulesYaml []byte
|
var modulesYaml []byte
|
||||||
err := multiErrorCheck(
|
err := multiErrorCheck(
|
||||||
b64Decode(currentRevision.ModulesYaml, &modulesYamlGz),
|
b64Decode(noDebugRevision.ModulesYaml, &modulesYamlGz),
|
||||||
decompressWithGz(modulesYamlGz, &modulesYaml),
|
decompressWithGz(modulesYamlGz, &modulesYaml),
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -1205,7 +1305,6 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest
|
|||||||
modulesRoot = append(modulesRoot, &*&md)
|
modulesRoot = append(modulesRoot, &*&md)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
groupsXml = currentRevision.GroupsXml
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1219,6 +1318,7 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
c.log.Infof("module artifacts: %v", moduleArtifacts)
|
||||||
|
|
||||||
for _, artifact := range archArtifacts {
|
for _, artifact := range archArtifacts {
|
||||||
// This shouldn't happen
|
// This shouldn't happen
|
||||||
@ -1243,7 +1343,7 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest
|
|||||||
}
|
}
|
||||||
|
|
||||||
shouldAdd := true
|
shouldAdd := true
|
||||||
if arch != "src" {
|
if arch != "src" && moduleStream == nil {
|
||||||
// If repo has a list for inclusion, then the artifact has to pass that first
|
// If repo has a list for inclusion, then the artifact has to pass that first
|
||||||
if len(repo.IncludeFilter) > 0 {
|
if len(repo.IncludeFilter) > 0 {
|
||||||
// If the artifact isn't forced, it should be in the include list or additional multilib list
|
// If the artifact isn't forced, it should be in the include list or additional multilib list
|
||||||
@ -1326,16 +1426,7 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest
|
|||||||
var pkgId *string
|
var pkgId *string
|
||||||
var primaryIndex *int
|
var primaryIndex *int
|
||||||
for i, primaryPackage := range primaryRoot.Packages {
|
for i, primaryPackage := range primaryRoot.Packages {
|
||||||
// We check module entries by artifact index from a previous revision
|
if moduleStream == nil {
|
||||||
if moduleStream != nil {
|
|
||||||
isStreamArtifact := moduleArtifacts[strings.TrimSuffix(filepath.Base(artifact.Name), ".rpm")]
|
|
||||||
if isStreamArtifact {
|
|
||||||
ix := i
|
|
||||||
primaryIndex = &ix
|
|
||||||
pkgId = &primaryPackage.Checksum.Value
|
|
||||||
break
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
// If not a module stream, search for a non-module entry
|
// If not a module stream, search for a non-module entry
|
||||||
// Double check arch as well for multilib purposes
|
// Double check arch as well for multilib purposes
|
||||||
if primaryPackage.Name == name && !strings.Contains(primaryPackage.Version.Rel, ".module+") && primaryPackage.Arch == artifact.Arch {
|
if primaryPackage.Name == name && !strings.Contains(primaryPackage.Version.Rel, ".module+") && primaryPackage.Arch == artifact.Arch {
|
||||||
@ -1344,6 +1435,33 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest
|
|||||||
pkgId = &primaryPackage.Checksum.Value
|
pkgId = &primaryPackage.Checksum.Value
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
|
} else {
|
||||||
|
// If a module stream, search for a module entry
|
||||||
|
for _, streamArtifact := range artifacts {
|
||||||
|
var rpmName string
|
||||||
|
var rpmVersion string
|
||||||
|
var rpmRelease string
|
||||||
|
rpmBase := strings.TrimSuffix(filepath.Base(streamArtifact.Name), ".rpm")
|
||||||
|
if rpmutils.NVRUnusualRelease().MatchString(rpmBase) {
|
||||||
|
nvr := rpmutils.NVRUnusualRelease().FindStringSubmatch(rpmBase)
|
||||||
|
rpmName = nvr[1]
|
||||||
|
rpmVersion = nvr[2]
|
||||||
|
rpmRelease = nvr[3]
|
||||||
|
} else if rpmutils.NVR().MatchString(rpmBase) {
|
||||||
|
nvr := rpmutils.NVR().FindStringSubmatch(rpmBase)
|
||||||
|
rpmName = nvr[1]
|
||||||
|
rpmVersion = nvr[2]
|
||||||
|
rpmRelease = nvr[3]
|
||||||
|
}
|
||||||
|
|
||||||
|
if name == rpmName && primaryPackage.Name == name && primaryPackage.Version.Ver == rpmVersion && primaryPackage.Version.Rel == rpmRelease && primaryPackage.Arch == artifact.Arch {
|
||||||
|
c.log.Infof("Found module stream package: %s", rpmBase)
|
||||||
|
ix := i
|
||||||
|
primaryIndex = &ix
|
||||||
|
pkgId = &primaryPackage.Checksum.Value
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if primaryIndex != nil {
|
if primaryIndex != nil {
|
||||||
@ -1395,19 +1513,28 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest
|
|||||||
var nFilelists []*yummeta.FilelistsPackage
|
var nFilelists []*yummeta.FilelistsPackage
|
||||||
var nOther []*yummeta.OtherPackage
|
var nOther []*yummeta.OtherPackage
|
||||||
var deleteIds []string
|
var deleteIds []string
|
||||||
if !req.NoDeletePrevious {
|
c.log.Infof("modified packages: %v", changes.ModifiedPackages)
|
||||||
|
c.log.Infof("added packages: %v", changes.AddedPackages)
|
||||||
|
if !req.NoDeletePrevious || moduleStream != nil {
|
||||||
for _, pkg := range primaryRoot.Packages {
|
for _, pkg := range primaryRoot.Packages {
|
||||||
shouldAdd := true
|
shouldAdd := true
|
||||||
for _, artifact := range currentActiveArtifacts {
|
for _, artifact := range currentActiveArtifacts {
|
||||||
if filepath.Base(artifact.Name) == filepath.Base(pkg.Location.Href) && !utils.StrContains(strings.TrimSuffix(filepath.Base(artifact.Name), ".rpm"), changes.ModifiedPackages) && !utils.StrContains(strings.TrimSuffix(filepath.Base(artifact.Name), ".rpm"), changes.AddedPackages) {
|
noRpmName := strings.TrimSuffix(filepath.Base(artifact.Name), ".rpm")
|
||||||
|
if filepath.Base(artifact.Name) == filepath.Base(pkg.Location.Href) && !utils.StrContains(noRpmName, changes.ModifiedPackages) && !utils.StrContains(noRpmName, changes.AddedPackages) && !utils.StrContains(noRpmName, skipDeleteArtifacts) {
|
||||||
shouldAdd = false
|
shouldAdd = false
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if !shouldAdd {
|
if !shouldAdd {
|
||||||
deleteIds = append(deleteIds, pkg.Checksum.Value)
|
deleteIds = append(deleteIds, pkg.Checksum.Value)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, pkg := range primaryRoot.Packages {
|
||||||
|
if utils.StrContains(pkg.Checksum.Value, deleteIds) {
|
||||||
|
c.log.Infof("Deleting package %s-%s-%s from primary metadata", pkg.Name, pkg.Version.Ver, pkg.Version.Rel)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
nPackages = append(nPackages, &*pkg)
|
|
||||||
|
nPackages = append(nPackages, pkg)
|
||||||
}
|
}
|
||||||
for _, pkg := range filelistsRoot.Packages {
|
for _, pkg := range filelistsRoot.Packages {
|
||||||
if utils.StrContains(pkg.PkgId, deleteIds) {
|
if utils.StrContains(pkg.PkgId, deleteIds) {
|
||||||
@ -1432,15 +1559,7 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest
|
|||||||
otherRoot.PackageCount = len(filelistsRoot.Packages)
|
otherRoot.PackageCount = len(filelistsRoot.Packages)
|
||||||
|
|
||||||
// Module builds needs a few more steps
|
// Module builds needs a few more steps
|
||||||
if moduleStream != nil {
|
if moduleStream != nil && streamDocument != nil {
|
||||||
streamDocument := moduleStream.ModuleStreamDocuments[arch]
|
|
||||||
if streamDocument != nil {
|
|
||||||
newEntryNbc, err := modulemd.Parse(streamDocument.Streams[moduleStream.Stream])
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
newEntry := newEntryNbc.V2
|
|
||||||
|
|
||||||
// If a previous entry exists, we need to overwrite that
|
// If a previous entry exists, we need to overwrite that
|
||||||
var moduleIndex *int
|
var moduleIndex *int
|
||||||
for i, moduleMd := range modulesRoot {
|
for i, moduleMd := range modulesRoot {
|
||||||
@ -1451,11 +1570,10 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest
|
|||||||
}
|
}
|
||||||
if moduleIndex != nil {
|
if moduleIndex != nil {
|
||||||
changes.ModifiedModules = append(changes.ModifiedModules, fmt.Sprintf("%s:%s", moduleStream.Name, moduleStream.Stream))
|
changes.ModifiedModules = append(changes.ModifiedModules, fmt.Sprintf("%s:%s", moduleStream.Name, moduleStream.Stream))
|
||||||
modulesRoot[*moduleIndex] = newEntry
|
modulesRoot[*moduleIndex] = streamDocument
|
||||||
} else {
|
} else {
|
||||||
changes.AddedModules = append(changes.AddedModules, fmt.Sprintf("%s:%s", moduleStream.Name, moduleStream.Stream))
|
changes.AddedModules = append(changes.AddedModules, fmt.Sprintf("%s:%s", moduleStream.Name, moduleStream.Stream))
|
||||||
modulesRoot = append(modulesRoot, newEntry)
|
modulesRoot = append(modulesRoot, streamDocument)
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1471,6 +1589,9 @@ func (c *Controller) makeRepoChanges(tx peridotdb.Access, req *UpdateRepoRequest
|
|||||||
ModuleDefaults: moduleDefaults,
|
ModuleDefaults: moduleDefaults,
|
||||||
GroupsXml: groupsXml,
|
GroupsXml: groupsXml,
|
||||||
}
|
}
|
||||||
|
if strings.HasSuffix(arch, "-debug") || arch == "src" {
|
||||||
|
cache.Repos[idArch].Modulemd = nil
|
||||||
|
}
|
||||||
|
|
||||||
repoTask.Changes = append(repoTask.Changes, changes)
|
repoTask.Changes = append(repoTask.Changes, changes)
|
||||||
}
|
}
|
||||||
|
@ -66,7 +66,7 @@ type Access interface {
|
|||||||
NVRAExists(nvra string) (bool, error)
|
NVRAExists(nvra string) (bool, error)
|
||||||
GetBuildByPackageNameAndVersionAndRelease(name string, version string, release string, projectId string) (*models.Build, error)
|
GetBuildByPackageNameAndVersionAndRelease(name string, version string, release string, projectId string) (*models.Build, error)
|
||||||
GetLatestBuildIdsByPackageName(name string, projectId string) ([]string, error)
|
GetLatestBuildIdsByPackageName(name string, projectId string) ([]string, error)
|
||||||
GetLatestBuildsByPackageNameAndPackageVersionID(name string, packageVersionId string, projectId string) ([]string, error)
|
GetLatestBuildsByPackageNameAndBranchName(name string, branchName string, projectId string) ([]string, error)
|
||||||
GetActiveBuildIdsByTaskArtifactGlob(taskArtifactGlob string, projectId string) ([]string, error)
|
GetActiveBuildIdsByTaskArtifactGlob(taskArtifactGlob string, projectId string) ([]string, error)
|
||||||
GetAllBuildIdsByPackageName(name string, projectId string) ([]string, error)
|
GetAllBuildIdsByPackageName(name string, projectId string) ([]string, error)
|
||||||
|
|
||||||
|
@ -428,7 +428,7 @@ func (a *Access) GetLatestBuildIdsByPackageName(name string, projectId string) (
|
|||||||
return ret, nil
|
return ret, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (a *Access) GetLatestBuildsByPackageNameAndPackageVersionID(name string, packageVersionId string, projectId string) ([]string, error) {
|
func (a *Access) GetLatestBuildsByPackageNameAndBranchName(name string, branchName string, projectId string) ([]string, error) {
|
||||||
var ret []string
|
var ret []string
|
||||||
err := a.query.Select(
|
err := a.query.Select(
|
||||||
&ret,
|
&ret,
|
||||||
@ -439,18 +439,19 @@ func (a *Access) GetLatestBuildsByPackageNameAndPackageVersionID(name string, pa
|
|||||||
inner join tasks t on t.id = b.task_id
|
inner join tasks t on t.id = b.task_id
|
||||||
inner join packages p on p.id = b.package_id
|
inner join packages p on p.id = b.package_id
|
||||||
inner join project_package_versions ppv on ppv.package_version_id = b.package_version_id
|
inner join project_package_versions ppv on ppv.package_version_id = b.package_version_id
|
||||||
|
inner join import_revisions ir on ir.package_version_id = b.package_version_id
|
||||||
where
|
where
|
||||||
b.project_id = $1
|
b.project_id = $3
|
||||||
and p.name = $2
|
and p.name = $1
|
||||||
and ppv.active_in_repo = true
|
and ppv.active_in_repo = true
|
||||||
and ppv.project_id = b.project_id
|
and ppv.project_id = b.project_id
|
||||||
and b.package_version_id = $3
|
and ir.scm_branch_name = $2
|
||||||
and t.status = 3
|
and t.status = 3
|
||||||
order by b.created_at asc
|
order by b.created_at asc
|
||||||
`,
|
`,
|
||||||
projectId,
|
|
||||||
name,
|
name,
|
||||||
packageVersionId,
|
branchName,
|
||||||
|
projectId,
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
|
Loading…
Reference in New Issue
Block a user