// Copyright (c) All respective contributors to the Peridot Project. All rights reserved. // Copyright (c) 2021-2022 Rocky Enterprise Software Foundation, Inc. All rights reserved. // Copyright (c) 2021-2022 Ctrl IQ, Inc. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are met: // // 1. Redistributions of source code must retain the above copyright notice, // this list of conditions and the following disclaimer. // // 2. Redistributions in binary form must reproduce the above copyright notice, // this list of conditions and the following disclaimer in the documentation // and/or other materials provided with the distribution. // // 3. Neither the name of the copyright holder nor the names of its contributors // may be used to endorse or promote products derived from this software without // specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE // ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE // LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR // CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF // SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS // INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN // CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) // ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE // POSSIBILITY OF SUCH DAMAGE. package legacy import ( "bytes" "compress/gzip" "crypto/sha256" "database/sql" "encoding/hex" "encoding/xml" "errors" "fmt" "github.com/go-git/go-billy/v5" "github.com/sirupsen/logrus" "io" "os" "path/filepath" "peridot.resf.org/publisher/updateinfo" "peridot.resf.org/secparse/db" secparsepb "peridot.resf.org/secparse/proto/v1" "peridot.resf.org/secparse/rpmutils" "strconv" "strings" "time" ) type Scanner struct { DB db.Access FS billy.Filesystem } type internalAdvisory struct { Pb *secparsepb.Advisory Db *db.Advisory } type rpm struct { Name string Src string Sha256 string Epoch string Advisory *internalAdvisory } func (s *Scanner) recursiveRPMScan(rootDir string) (map[string][]*rpm, error) { infos, err := s.FS.ReadDir(rootDir) if err != nil { return nil, err } ret := map[string][]*rpm{} for _, fi := range infos { if fi.IsDir() { nRpms, err := s.recursiveRPMScan(filepath.Join(rootDir, fi.Name())) if err != nil { // Ignore paths we can't access continue } for k, v := range nRpms { if ret[k] == nil { ret[k] = []*rpm{} } ret[k] = append(ret[k], v...) } } else { if strings.HasSuffix(fi.Name(), ".rpm") { k := filepath.Join(rootDir, "..") if ret[k] == nil { ret[k] = []*rpm{} } ret[k] = append(ret[k], &rpm{ Name: fi.Name(), }) } } } return ret, nil } func (s *Scanner) findRepoData(rootDir string) (string, error) { if rootDir == "." { return "", errors.New("could not find repodata") } repoDataPath := filepath.Join(rootDir, "repodata") stat, err := s.FS.Stat(repoDataPath) if err != nil { if os.IsNotExist(err) { return s.findRepoData(filepath.Join(rootDir, "..")) } else { return "", err } } if stat.IsDir() { return repoDataPath, nil } else { return s.findRepoData(filepath.Join(rootDir, "..")) } } func (s *Scanner) ScanAndPublish(from string, composeName string, productName string, productShort string, republish bool) error { _, err := s.FS.Stat(composeName) if err != nil { return err } rpms, err := s.recursiveRPMScan(composeName) if err != nil { return err } published := map[string][]*rpm{} beginTx, err := s.DB.Begin() if err != nil { logrus.Errorf("Could not initiate tx: %v", err) } tx := s.DB.UseTransaction(beginTx) rollback := false advisories, err := tx.GetAllAdvisories(false) if err != nil { return err } for _, advisory := range advisories { // Skip already published advisories if republish is disabled if advisory.PublishedAt.Valid && !republish { continue } advisoryPb := db.DTOAdvisoryToPB(advisory) touchedOnce := false for _, artifactWithSrpm := range advisory.BuildArtifacts { artifactSplit := strings.Split(artifactWithSrpm, ":::") artifact := artifactSplit[0] artifactSrc := rpmutils.Epoch().ReplaceAllString(artifactSplit[1], "") for repo, repoRpms := range rpms { if strings.HasSuffix(repo, "/Packages") { repo = strings.TrimSuffix(repo, "/Packages") } if published[repo] == nil { published[repo] = []*rpm{} } for _, repoRpm := range repoRpms { if repoRpm.Name == rpmutils.Epoch().ReplaceAllString(artifact, "") { hasher := sha256.New() f, err := s.FS.Open(filepath.Join(repo, "Packages", repoRpm.Name)) if err != nil { // If not found, then try sorted directory f, err = s.FS.Open(filepath.Join(repo, "Packages", strings.ToLower(string(repoRpm.Name[0])), repoRpm.Name)) if err != nil { logrus.Errorf("Could not open affected package: %v", err) rollback = true break } } _, err = io.Copy(hasher, f) _ = f.Close() if err != nil { logrus.Errorf("Could not hash affected package: %v", err) rollback = true break } logrus.Infof("Advisory %s affects %s", advisoryPb.Name, artifact) err = tx.AddAdvisoryRPM(advisory.ID, artifact) if err != nil { logrus.Errorf("Could not add advisory RPM: %v", err) rollback = true break } touchedOnce = true repoRpm.Sha256 = hex.EncodeToString(hasher.Sum(nil)) repoRpm.Epoch = strings.TrimSuffix(rpmutils.Epoch().FindStringSubmatch(artifact)[0], ":") repoRpm.Advisory = &internalAdvisory{ Pb: advisoryPb, Db: advisory, } repoRpm.Src = artifactSrc published[repo] = append(published[repo], repoRpm) } } } } if rollback { break } if !touchedOnce { continue } if !advisory.PublishedAt.Valid { advisory.PublishedAt = sql.NullTime{Valid: true, Time: time.Now()} _, err = tx.UpdateAdvisory(advisory) if err != nil { logrus.Errorf("Could not update advisory %s: %v", advisoryPb.Name, err) rollback = true break } } } publishedMappedByAdvisory := map[string]map[string][]*rpm{} advisoryByName := map[string]*internalAdvisory{} for repo, publishedRpms := range published { if publishedMappedByAdvisory[repo] == nil { publishedMappedByAdvisory[repo] = map[string][]*rpm{} } for _, publishedRpm := range publishedRpms { if publishedMappedByAdvisory[repo][publishedRpm.Advisory.Pb.Name] == nil { publishedMappedByAdvisory[repo][publishedRpm.Advisory.Pb.Name] = []*rpm{} } if advisoryByName[publishedRpm.Advisory.Pb.Name] == nil { advisoryByName[publishedRpm.Advisory.Pb.Name] = publishedRpm.Advisory } publishedMappedByAdvisory[repo][publishedRpm.Advisory.Pb.Name] = append(publishedMappedByAdvisory[repo][publishedRpm.Advisory.Pb.Name], publishedRpm) } } for repo, advisories := range publishedMappedByAdvisory { repoDataDir, err := s.findRepoData(repo) if err != nil { logrus.Error(err) rollback = true break } repoMdPath := filepath.Join(repoDataDir, "repomd.xml") f, err := s.FS.Open(repoMdPath) if err != nil { logrus.Errorf("Could not open repomd.xml: %v", err) rollback = true break } var repomd updateinfo.RepoMdRoot err = xml.NewDecoder(f).Decode(&repomd) if err != nil { logrus.Errorf("Could not decode repomd: %v", err) rollback = true break } _ = f.Close() var olderUpdateInfo string for _, e := range repomd.Data { if e.Type == "updateinfo" { olderUpdateInfo = e.Location.Href } } var updateInfo *updateinfo.UpdatesRoot if olderUpdateInfo == "" { updateInfo = &updateinfo.UpdatesRoot{ Updates: []*updateinfo.Update{}, } } else { if republish { updateInfo = &updateinfo.UpdatesRoot{ Updates: []*updateinfo.Update{}, } } else { olderF, err := s.FS.Open(filepath.Join(repoDataDir, "..", olderUpdateInfo)) if err != nil { logrus.Errorf("Could not open older updateinfo: %v", err) rollback = true break } var decoded bytes.Buffer r, err := gzip.NewReader(olderF) if err != nil { logrus.Errorf("Could not create new gzip reader: %v", err) rollback = true break } if _, err := io.Copy(&decoded, r); err != nil { logrus.Errorf("Could not copy gzip data: %v", err) rollback = true break } _ = r.Close() err = xml.NewDecoder(&decoded).Decode(&updateInfo) if err != nil { logrus.Errorf("Could not decode older updateinfo: %v", err) rollback = true break } _ = olderF.Close() if updateInfo.Updates == nil { updateInfo.Updates = []*updateinfo.Update{} } } } for advisoryName, publishedRpms := range advisories { advisory := advisoryByName[advisoryName] updateType := "enhancement" switch advisory.Pb.Type { case secparsepb.Advisory_BugFix: updateType = "bugfix" break case secparsepb.Advisory_Security: updateType = "security" break } severity := advisory.Pb.Severity.String() if advisory.Pb.Severity == secparsepb.Advisory_UnknownSeverity { severity = "None" } update := &updateinfo.Update{ From: from, Status: "final", Type: updateType, Version: "2", ID: advisory.Pb.Name, Title: advisory.Pb.Synopsis, Issued: &updateinfo.UpdateDate{ Date: advisory.Db.PublishedAt.Time.Format(updateinfo.TimeFormat), }, Updated: &updateinfo.UpdateDate{ Date: advisory.Db.RedHatIssuedAt.Time.Format(updateinfo.TimeFormat), }, Rights: "Copyright (C) 2021 Rocky Enterprise Software Foundation", Release: productName, PushCount: "1", Severity: severity, Summary: advisory.Pb.Topic, Description: fmt.Sprintf("For more information visit https://errata.rockylinux.org/%s", advisory.Pb.Name), References: &updateinfo.UpdateReferenceRoot{ References: []*updateinfo.UpdateReference{}, }, PkgList: &updateinfo.UpdateCollectionRoot{ Collections: []*updateinfo.UpdateCollection{ { Short: productShort, Name: productName, Packages: []*updateinfo.UpdatePackage{}, }, }, }, } for _, cve := range advisory.Pb.Cves { sourceCve := strings.Split(cve, ":::") sourceBy := sourceCve[0] sourceLink := sourceCve[1] id := sourceCve[2] referenceType := "erratum" if strings.HasPrefix(id, "CVE") { referenceType = "cve" } reference := &updateinfo.UpdateReference{ Href: sourceLink, ID: id, Type: referenceType, Title: fmt.Sprintf("Update information for %s is retrieved from %s", id, sourceBy), } update.References.References = append(update.References.References, reference) } for _, publishedRpm := range publishedRpms { nvr := rpmutils.NVR().FindStringSubmatch(publishedRpm.Name) update.PkgList.Collections[0].Packages = append(update.PkgList.Collections[0].Packages, &updateinfo.UpdatePackage{ Name: nvr[1], Version: nvr[2], Release: nvr[3], Epoch: publishedRpm.Epoch, Arch: nvr[4], Src: publishedRpm.Src, Filename: publishedRpm.Name, Sum: []*updateinfo.UpdatePackageSum{ { Type: "sha256", Value: publishedRpm.Sha256, }, }, }) } if rollback { break } updateInfo.Updates = append(updateInfo.Updates, update) } if rollback { break } xmlBytes, err := xml.MarshalIndent(updateInfo, "", " ") if err != nil { logrus.Errorf("Could not encode updateinfo xml: %v", err) rollback = true break } hasher := sha256.New() openSize := len(xmlBytes) _, err = hasher.Write(xmlBytes) if err != nil { logrus.Errorf("Could not hash updateinfo: %v", err) rollback = true break } openChecksum := hex.EncodeToString(hasher.Sum(nil)) hasher.Reset() var gzippedBuf bytes.Buffer w := gzip.NewWriter(&gzippedBuf) _, err = w.Write(xmlBytes) if err != nil { logrus.Errorf("Could not gzip encode: %v", err) rollback = true break } _ = w.Close() closedSize := len(gzippedBuf.Bytes()) _, err = hasher.Write(gzippedBuf.Bytes()) if err != nil { logrus.Errorf("Could not hash gzipped: %v", err) rollback = true break } closedChecksum := hex.EncodeToString(hasher.Sum(nil)) hasher.Reset() timestamp := strconv.FormatInt(time.Now().Unix(), 10) updateInfoPath := filepath.Join(repoDataDir, fmt.Sprintf("%s-updateinfo.xml.gz", closedChecksum)) updateInfoEntry := &updateinfo.RepoMdData{ Type: "updateinfo", Checksum: &updateinfo.RepoMdDataChecksum{ Type: "sha256", Value: closedChecksum, }, OpenChecksum: &updateinfo.RepoMdDataChecksum{ Type: "sha256", Value: openChecksum, }, Location: &updateinfo.RepoMdDataLocation{ Href: strings.ReplaceAll(updateInfoPath, repo+"/", ""), }, Timestamp: timestamp, Size: strconv.Itoa(closedSize), OpenSize: strconv.Itoa(openSize), } if olderUpdateInfo == "" { repomd.Data = append(repomd.Data, updateInfoEntry) } else { for i, e := range repomd.Data { if e.Type == "updateinfo" { repomd.Data[i] = updateInfoEntry } } } uif, err := s.FS.OpenFile(updateInfoPath, os.O_TRUNC|os.O_RDWR|os.O_CREATE, 0644) if err != nil { logrus.Errorf("Could not open updateinfo file %s: %v", updateInfoPath, err) rollback = true break } _, err = uif.Write(gzippedBuf.Bytes()) if err != nil { logrus.Errorf("Could not write gzipped updateinfo file: %v", err) rollback = true break } _ = uif.Close() if repomd.Rpm != "" && repomd.XmlnsRpm == "" { repomd.XmlnsRpm = repomd.Rpm repomd.Rpm = "" } updateF, err := s.FS.OpenFile(repoMdPath, os.O_TRUNC|os.O_RDWR|os.O_CREATE, 0644) if err != nil { logrus.Errorf("Could not open repomd file for update: %v", err) rollback = true break } _, _ = updateF.Write([]byte(xml.Header)) enc := xml.NewEncoder(updateF) enc.Indent("", " ") err = enc.Encode(&repomd) if err != nil { logrus.Errorf("Could not encode updated repomd file: %v", err) rollback = true break } _ = updateF.Close() if olderUpdateInfo != "" { _ = s.FS.Remove(filepath.Join(repo, olderUpdateInfo)) } } if rollback { err := beginTx.Rollback() if err != nil { logrus.Errorf("Could not rollback: %v", err) } return errors.New("rolled back") } err = beginTx.Commit() if err != nil { logrus.Errorf("Could not commit transaction: %v", err) } return nil }