2021-04-06 19:39:02 +00:00
// Copyright (c) 2021 The Srpmproc Authors
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
2021-08-19 10:09:53 +00:00
package srpmproc
2020-12-14 01:29:02 +00:00
import (
2022-09-28 03:32:50 +00:00
"bufio"
2020-12-14 03:51:03 +00:00
"encoding/hex"
"fmt"
2022-11-06 03:53:02 +00:00
"io"
"io/ioutil"
"log"
"os"
"os/exec"
"os/user"
"path/filepath"
"strings"
"time"
2021-09-03 21:07:02 +00:00
"github.com/go-git/go-billy/v5"
2022-04-21 04:30:33 +00:00
"github.com/go-git/go-billy/v5/osfs"
2022-09-28 03:32:50 +00:00
"github.com/go-git/go-git/v5/plumbing/format/gitignore"
2021-09-10 18:14:33 +00:00
"github.com/go-git/go-git/v5/plumbing/transport"
"github.com/go-git/go-git/v5/plumbing/transport/http"
2021-09-03 21:07:02 +00:00
"github.com/go-git/go-git/v5/plumbing/transport/ssh"
2021-09-08 15:00:56 +00:00
srpmprocpb "github.com/rocky-linux/srpmproc/pb"
2021-09-03 21:07:02 +00:00
"github.com/rocky-linux/srpmproc/pkg/blob"
"github.com/rocky-linux/srpmproc/pkg/blob/file"
"github.com/rocky-linux/srpmproc/pkg/blob/gcs"
"github.com/rocky-linux/srpmproc/pkg/blob/s3"
2021-09-08 15:00:56 +00:00
"github.com/rocky-linux/srpmproc/pkg/misc"
2021-09-03 21:07:02 +00:00
"github.com/rocky-linux/srpmproc/pkg/modes"
2021-09-08 15:00:56 +00:00
"github.com/rocky-linux/srpmproc/pkg/rpmutils"
2021-04-15 04:41:12 +00:00
"github.com/go-git/go-billy/v5/memfs"
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/config"
"github.com/go-git/go-git/v5/plumbing"
"github.com/go-git/go-git/v5/plumbing/object"
"github.com/go-git/go-git/v5/storage/memory"
2021-08-19 10:09:53 +00:00
"github.com/rocky-linux/srpmproc/pkg/data"
2020-12-14 01:29:02 +00:00
)
2021-09-03 21:07:02 +00:00
const (
RpmPrefixCentOS = "https://git.centos.org/rpms"
ModulePrefixCentOS = "https://git.centos.org/modules"
RpmPrefixRocky = "https://git.rockylinux.org/staging/rpms"
ModulePrefixRocky = "https://git.rockylinux.org/staging/modules"
UpstreamPrefixRocky = "https://git.rockylinux.org/staging"
)
type ProcessDataRequest struct {
// Required
Version int
StorageAddr string
Package string
// Optional
ModuleMode bool
TmpFsMode string
ModulePrefix string
RpmPrefix string
SshKeyLocation string
SshUser string
2021-09-10 18:14:33 +00:00
HttpUsername string
HttpPassword string
2021-09-03 21:07:02 +00:00
ManualCommits string
UpstreamPrefix string
GitCommitterName string
GitCommitterEmail string
ImportBranchPrefix string
BranchPrefix string
FsCreator data . FsCreatorFunc
NoDupMode bool
2022-04-21 04:30:33 +00:00
BranchSuffix string
StrictBranchMode bool
2021-09-03 21:07:02 +00:00
ModuleFallbackStream string
NoStorageUpload bool
NoStorageDownload bool
SingleTag string
2021-09-05 00:12:43 +00:00
CdnUrl string
2022-01-05 14:42:49 +00:00
LogWriter io . Writer
2022-05-05 02:25:07 +00:00
PackageVersion string
PackageRelease string
2022-09-28 03:32:50 +00:00
2023-02-23 04:23:47 +00:00
TaglessMode bool
Cdn string
}
type LookasidePath struct {
Distro string
Url string
2021-09-03 21:07:02 +00:00
}
2021-03-17 14:54:16 +00:00
func gitlabify ( str string ) string {
2021-04-11 22:08:42 +00:00
if str == "tree" {
return "treepkg"
}
2021-03-17 14:54:16 +00:00
return strings . Replace ( str , "+" , "plus" , - 1 )
}
2023-02-23 04:23:47 +00:00
// List of distros and their lookaside patterns
// If we find one of these passed as --cdn (ex: "--cdn fedora"), then we override, and assign this URL to be our --cdn-url
func StaticLookasides ( ) [ ] LookasidePath {
centos := LookasidePath { Distro : "centos" , Url : "https://git.centos.org/sources/{{.Name}}/{{.Branch}}/{{.Hash}}" }
centosStream := LookasidePath { Distro : "centos-stream" , Url : "https://sources.stream.centos.org/sources/rpms/{{.Name}}/{{.Filename}}/{{.Hashtype}}/{{.Hash}}/{{.Filename}}" }
rocky8 := LookasidePath { Distro : "rocky8" , Url : "https://rocky-linux-sources-staging.a1.rockylinux.org/{{.Hash}}" }
rocky := LookasidePath { Distro : "rocky" , Url : "https://sources.build.resf.org/{{.Hash}}" }
fedora := LookasidePath { Distro : "fedora" , Url : "https://src.fedoraproject.org/repo/pkgs/{{.Name}}/{{.Filename}}/{{.Hashtype}}/{{.Hash}}/{{.Filename}}" }
return [ ] LookasidePath { centos , centosStream , rocky8 , rocky , fedora }
}
2021-09-03 21:07:02 +00:00
func NewProcessData ( req * ProcessDataRequest ) ( * data . ProcessData , error ) {
2023-02-23 04:23:47 +00:00
// Build the logger to use for the data import
var writer io . Writer = os . Stdout
if req . LogWriter != nil {
writer = req . LogWriter
}
logger := log . New ( writer , "" , log . LstdFlags )
2021-09-03 21:07:02 +00:00
// Set defaults
if req . ModulePrefix == "" {
req . ModulePrefix = ModulePrefixCentOS
}
if req . RpmPrefix == "" {
req . RpmPrefix = RpmPrefixCentOS
}
if req . SshUser == "" {
req . SshUser = "git"
}
if req . UpstreamPrefix == "" {
req . UpstreamPrefix = UpstreamPrefixRocky
}
if req . GitCommitterName == "" {
req . GitCommitterName = "rockyautomation"
}
if req . GitCommitterEmail == "" {
req . GitCommitterEmail = "rockyautomation@rockylinux.org"
}
if req . ImportBranchPrefix == "" {
req . ImportBranchPrefix = "c"
}
if req . BranchPrefix == "" {
req . BranchPrefix = "r"
}
2023-02-23 04:23:47 +00:00
if req . CdnUrl == "" {
2021-09-05 00:12:43 +00:00
req . CdnUrl = "https://git.centos.org/sources"
}
2023-02-23 04:23:47 +00:00
// If a Cdn distro is defined, loop through StaticLookasides() array of structs,
// see if we have a match to --cdn (matching values are things like fedora, centos, rocky8, etc.)
// If we match, then we want to short-circuit the CdnUrl to the assigned distro's one
if req . Cdn != "" {
var foundDistro = false
for _ , distro := range StaticLookasides ( ) {
if distro . Distro == strings . ToLower ( req . Cdn ) {
foundDistro = true
req . CdnUrl = distro . Url
logger . Printf ( "Discovered --cdn distro: %s . Using override CDN URL Pattern: %s" , distro . Distro , req . CdnUrl )
break
}
}
if foundDistro == false {
return nil , fmt . Errorf ( "Error, distro name given as --cdn argument is not valid." )
}
2022-09-28 03:32:50 +00:00
}
2021-09-03 21:07:02 +00:00
// Validate required
if req . Package == "" {
return nil , fmt . Errorf ( "package cannot be empty" )
}
var importer data . ImportMode
var blobStorage blob . Storage
if strings . HasPrefix ( req . StorageAddr , "gs://" ) {
2021-09-10 20:31:59 +00:00
var err error
blobStorage , err = gcs . New ( strings . Replace ( req . StorageAddr , "gs://" , "" , 1 ) )
if err != nil {
return nil , err
}
2021-09-03 21:07:02 +00:00
} else if strings . HasPrefix ( req . StorageAddr , "s3://" ) {
blobStorage = s3 . New ( strings . Replace ( req . StorageAddr , "s3://" , "" , 1 ) )
} else if strings . HasPrefix ( req . StorageAddr , "file://" ) {
blobStorage = file . New ( strings . Replace ( req . StorageAddr , "file://" , "" , 1 ) )
} else {
2021-09-10 20:31:59 +00:00
return nil , fmt . Errorf ( "invalid blob storage" )
2021-09-03 21:07:02 +00:00
}
sourceRpmLocation := ""
if req . ModuleMode {
sourceRpmLocation = fmt . Sprintf ( "%s/%s" , req . ModulePrefix , req . Package )
} else {
sourceRpmLocation = fmt . Sprintf ( "%s/%s" , req . RpmPrefix , req . Package )
}
importer = & modes . GitMode { }
lastKeyLocation := req . SshKeyLocation
if lastKeyLocation == "" {
usr , err := user . Current ( )
if err != nil {
2021-09-10 20:31:59 +00:00
return nil , fmt . Errorf ( "could not get user: %v" , err )
2021-09-03 21:07:02 +00:00
}
lastKeyLocation = filepath . Join ( usr . HomeDir , ".ssh/id_rsa" )
}
2021-09-10 18:14:33 +00:00
var authenticator transport . AuthMethod
2021-09-03 21:07:02 +00:00
var err error
2021-09-10 18:14:33 +00:00
if req . HttpUsername != "" {
authenticator = & http . BasicAuth {
Username : req . HttpUsername ,
Password : req . HttpPassword ,
}
} else {
// create ssh key authenticator
authenticator , err = ssh . NewPublicKeysFromFile ( req . SshUser , lastKeyLocation , "" )
}
2021-09-03 21:07:02 +00:00
if err != nil {
2021-09-10 20:31:59 +00:00
return nil , fmt . Errorf ( "could not get git authenticator: %v" , err )
2021-09-03 21:07:02 +00:00
}
fsCreator := func ( branch string ) ( billy . Filesystem , error ) {
2022-04-21 05:39:10 +00:00
if req . TmpFsMode != "" {
2022-09-11 09:29:03 +00:00
return osfs . New ( "" ) , nil
2022-04-21 05:39:10 +00:00
}
return memfs . New ( ) , nil
2021-09-03 21:07:02 +00:00
}
2021-09-08 15:00:56 +00:00
reqFsCreator := fsCreator
2021-09-03 21:07:02 +00:00
if req . FsCreator != nil {
2021-09-08 15:00:56 +00:00
reqFsCreator = req . FsCreator
2021-09-03 21:07:02 +00:00
}
if req . TmpFsMode != "" {
2022-01-05 14:42:49 +00:00
logger . Printf ( "using tmpfs dir: %s" , req . TmpFsMode )
2021-09-03 21:07:02 +00:00
fsCreator = func ( branch string ) ( billy . Filesystem , error ) {
2021-09-08 15:00:56 +00:00
fs , err := reqFsCreator ( branch )
2021-09-03 21:07:02 +00:00
if err != nil {
return nil , err
}
tmpDir := filepath . Join ( req . TmpFsMode , branch )
2022-11-06 03:53:02 +00:00
err = fs . MkdirAll ( tmpDir , 0 o755 )
2021-09-03 21:07:02 +00:00
if err != nil {
2021-09-10 20:31:59 +00:00
return nil , fmt . Errorf ( "could not create tmpfs dir: %v" , err )
2021-09-03 21:07:02 +00:00
}
nFs , err := fs . Chroot ( tmpDir )
if err != nil {
return nil , err
}
return nFs , nil
}
2021-09-08 15:00:56 +00:00
} else {
fsCreator = reqFsCreator
2021-09-03 21:07:02 +00:00
}
var manualCs [ ] string
if strings . TrimSpace ( req . ManualCommits ) != "" {
manualCs = strings . Split ( req . ManualCommits , "," )
}
return & data . ProcessData {
Importer : importer ,
RpmLocation : sourceRpmLocation ,
UpstreamPrefix : req . UpstreamPrefix ,
Version : req . Version ,
BlobStorage : blobStorage ,
GitCommitterName : req . GitCommitterName ,
GitCommitterEmail : req . GitCommitterEmail ,
ModulePrefix : req . ModulePrefix ,
ImportBranchPrefix : req . ImportBranchPrefix ,
BranchPrefix : req . BranchPrefix ,
SingleTag : req . SingleTag ,
Authenticator : authenticator ,
NoDupMode : req . NoDupMode ,
ModuleMode : req . ModuleMode ,
TmpFsMode : req . TmpFsMode ,
NoStorageDownload : req . NoStorageDownload ,
NoStorageUpload : req . NoStorageUpload ,
ManualCommits : manualCs ,
ModuleFallbackStream : req . ModuleFallbackStream ,
2022-04-21 04:30:33 +00:00
BranchSuffix : req . BranchSuffix ,
StrictBranchMode : req . StrictBranchMode ,
2021-09-03 21:07:02 +00:00
FsCreator : fsCreator ,
2021-09-08 15:00:56 +00:00
CdnUrl : req . CdnUrl ,
2022-01-05 14:42:49 +00:00
Log : logger ,
2022-05-05 02:25:07 +00:00
PackageVersion : req . PackageVersion ,
PackageRelease : req . PackageRelease ,
2022-09-28 03:32:50 +00:00
TaglessMode : req . TaglessMode ,
2023-02-23 04:23:47 +00:00
Cdn : req . Cdn ,
2021-09-03 21:07:02 +00:00
} , nil
}
2020-12-14 01:29:02 +00:00
// ProcessRPM checks the RPM specs and discards any remote files
// This functions also sorts files into directories
// .spec files goes into -> SPECS
// metadata files goes to root
// source files goes into -> SOURCES
// all files that are remote goes into .gitignore
2021-02-24 07:27:51 +00:00
// all ignored files' hash goes into .{Name}.metadata
2021-09-08 15:00:56 +00:00
func ProcessRPM ( pd * data . ProcessData ) ( * srpmprocpb . ProcessResponse , error ) {
2022-09-28 03:32:50 +00:00
// if we are using "tagless mode", then we need to jump to a completely different import process:
// Version info needs to be derived from rpmbuild + spec file, not tags
if pd . TaglessMode {
result , err := processRPMTagless ( pd )
return result , err
}
2021-09-03 21:07:02 +00:00
md , err := pd . Importer . RetrieveSource ( pd )
if err != nil {
2021-09-04 21:33:08 +00:00
return nil , err
2021-04-27 17:14:01 +00:00
}
2021-02-24 07:27:51 +00:00
md . BlobCache = map [ string ] [ ] byte { }
2020-12-14 03:51:03 +00:00
2021-02-14 20:44:44 +00:00
remotePrefix := "rpms"
2020-12-22 16:28:04 +00:00
if pd . ModuleMode {
remotePrefix = "modules"
}
2021-02-20 04:12:16 +00:00
latestHashForBranch := map [ string ] string { }
2021-09-08 15:00:56 +00:00
versionForBranch := map [ string ] * srpmprocpb . VersionRelease { }
2021-02-19 15:22:36 +00:00
// already uploaded blobs are skipped
var alreadyUploadedBlobs [ ] string
2020-12-22 05:13:27 +00:00
// if no-dup-mode is enabled then skip already imported versions
var tagIgnoreList [ ] string
if pd . NoDupMode {
repo , err := git . Init ( memory . NewStorage ( ) , memfs . New ( ) )
if err != nil {
2021-09-04 21:33:08 +00:00
return nil , fmt . Errorf ( "could not init git repo: %v" , err )
2020-12-22 05:13:27 +00:00
}
2021-09-03 21:07:02 +00:00
remoteUrl := fmt . Sprintf ( "%s/%s/%s.git" , pd . UpstreamPrefix , remotePrefix , gitlabify ( md . Name ) )
2020-12-22 05:13:27 +00:00
refspec := config . RefSpec ( "+refs/heads/*:refs/remotes/origin/*" )
remote , err := repo . CreateRemote ( & config . RemoteConfig {
Name : "origin" ,
URLs : [ ] string { remoteUrl } ,
Fetch : [ ] config . RefSpec { refspec } ,
} )
if err != nil {
2021-09-04 21:33:08 +00:00
return nil , fmt . Errorf ( "could not create remote: %v" , err )
2020-12-22 05:13:27 +00:00
}
list , err := remote . List ( & git . ListOptions {
Auth : pd . Authenticator ,
} )
2023-02-23 04:23:47 +00:00
2020-12-22 05:13:27 +00:00
if err != nil {
2020-12-22 16:28:04 +00:00
log . Println ( "ignoring no-dup-mode" )
2020-12-22 05:19:53 +00:00
} else {
for _ , ref := range list {
if ! strings . HasPrefix ( string ( ref . Name ( ) ) , "refs/tags/imports" ) {
continue
}
tagIgnoreList = append ( tagIgnoreList , string ( ref . Name ( ) ) )
2020-12-22 05:13:27 +00:00
}
}
}
2021-02-24 07:27:51 +00:00
sourceRepo := * md . Repo
sourceWorktree := * md . Worktree
2021-04-06 19:35:46 +00:00
commitPin := map [ string ] string { }
2021-02-24 07:27:51 +00:00
if pd . SingleTag != "" {
md . Branches = [ ] string { fmt . Sprintf ( "refs/tags/%s" , pd . SingleTag ) }
2021-04-06 19:35:46 +00:00
} else if len ( pd . ManualCommits ) > 0 {
md . Branches = [ ] string { }
for _ , commit := range pd . ManualCommits {
branchCommit := strings . Split ( commit , ":" )
if len ( branchCommit ) != 2 {
2021-09-10 20:31:59 +00:00
return nil , fmt . Errorf ( "invalid manual commit list" )
2021-04-06 19:35:46 +00:00
}
2021-09-03 21:07:02 +00:00
head := fmt . Sprintf ( "refs/tags/imports/%s/%s-%s" , branchCommit [ 0 ] , md . Name , branchCommit [ 1 ] )
2021-04-06 19:35:46 +00:00
md . Branches = append ( md . Branches , head )
commitPin [ head ] = branchCommit [ 1 ]
}
2021-02-24 07:27:51 +00:00
}
2020-12-14 03:51:03 +00:00
2023-02-23 04:23:47 +00:00
// If we have no valid branches to consider, then we'll automatically switch to attempt a tagless import:
if len ( md . Branches ) <= 0 {
log . Println ( "No valid tags (refs/tags/imports/*) found in repository! Switching to perform a tagless import." )
pd . TaglessMode = true
result , err := processRPMTagless ( pd )
return result , err
}
2021-02-24 07:27:51 +00:00
for _ , branch := range md . Branches {
md . Repo = & sourceRepo
md . Worktree = & sourceWorktree
md . TagBranch = branch
for _ , source := range md . SourcesToIgnore {
source . Expired = true
2021-01-01 15:55:21 +00:00
}
2020-12-20 09:36:13 +00:00
2020-12-22 16:28:04 +00:00
var matchString string
2022-04-21 04:30:33 +00:00
if ! misc . GetTagImportRegex ( pd ) . MatchString ( md . TagBranch ) {
2020-12-22 16:28:04 +00:00
if pd . ModuleMode {
2021-02-15 04:20:42 +00:00
prefix := fmt . Sprintf ( "refs/heads/%s%d" , pd . ImportBranchPrefix , pd . Version )
2021-02-24 07:27:51 +00:00
if strings . HasPrefix ( md . TagBranch , prefix ) {
replace := strings . Replace ( md . TagBranch , "refs/heads/" , "" , 1 )
2020-12-22 16:28:04 +00:00
matchString = fmt . Sprintf ( "refs/tags/imports/%s/%s" , replace , filepath . Base ( pd . RpmLocation ) )
2022-01-05 14:42:49 +00:00
pd . Log . Printf ( "using match string: %s" , matchString )
2020-12-22 16:28:04 +00:00
}
}
2022-04-21 04:30:33 +00:00
if ! misc . GetTagImportRegex ( pd ) . MatchString ( matchString ) {
2021-02-15 04:20:42 +00:00
continue
2020-12-22 16:28:04 +00:00
}
} else {
2021-02-24 07:27:51 +00:00
matchString = md . TagBranch
2020-12-16 10:46:33 +00:00
}
2020-12-14 03:51:03 +00:00
2022-04-21 04:30:33 +00:00
match := misc . GetTagImportRegex ( pd ) . FindStringSubmatch ( matchString )
2022-09-28 03:32:50 +00:00
2021-02-24 07:27:51 +00:00
md . PushBranch = pd . BranchPrefix + strings . TrimPrefix ( match [ 2 ] , pd . ImportBranchPrefix )
2022-09-28 03:32:50 +00:00
2021-02-21 13:51:30 +00:00
newTag := "imports/" + pd . BranchPrefix + strings . TrimPrefix ( match [ 1 ] , "imports/" + pd . ImportBranchPrefix )
2021-04-26 02:00:00 +00:00
newTag = strings . Replace ( newTag , "%" , "_" , - 1 )
2020-12-22 05:13:27 +00:00
2021-09-03 21:07:02 +00:00
createdFs , err := pd . FsCreator ( md . PushBranch )
if err != nil {
2021-09-04 21:33:08 +00:00
return nil , err
2021-09-03 21:07:02 +00:00
}
2021-03-18 17:44:02 +00:00
// create new Repo for final dist
2021-09-03 21:07:02 +00:00
repo , err := git . Init ( memory . NewStorage ( ) , createdFs )
2021-03-18 17:44:02 +00:00
if err != nil {
2021-09-04 21:33:08 +00:00
return nil , fmt . Errorf ( "could not create new dist Repo: %v" , err )
2021-03-18 17:44:02 +00:00
}
w , err := repo . Worktree ( )
if err != nil {
2021-09-04 21:33:08 +00:00
return nil , fmt . Errorf ( "could not get dist Worktree: %v" , err )
2021-03-18 17:44:02 +00:00
}
2020-12-22 05:13:27 +00:00
shouldContinue := true
for _ , ignoredTag := range tagIgnoreList {
if ignoredTag == "refs/tags/" + newTag {
2022-01-05 14:42:49 +00:00
pd . Log . Printf ( "skipping %s" , ignoredTag )
2020-12-22 05:13:27 +00:00
shouldContinue = false
}
}
if ! shouldContinue {
continue
}
2020-12-14 03:51:03 +00:00
2020-12-20 09:36:13 +00:00
// create a new remote
2021-09-03 21:07:02 +00:00
remoteUrl := fmt . Sprintf ( "%s/%s/%s.git" , pd . UpstreamPrefix , remotePrefix , gitlabify ( md . Name ) )
2022-01-05 14:42:49 +00:00
pd . Log . Printf ( "using remote: %s" , remoteUrl )
2021-02-24 07:27:51 +00:00
refspec := config . RefSpec ( fmt . Sprintf ( "+refs/heads/%s:refs/remotes/origin/%s" , md . PushBranch , md . PushBranch ) )
2022-01-05 14:42:49 +00:00
pd . Log . Printf ( "using refspec: %s" , refspec )
2020-12-20 09:36:13 +00:00
_ , err = repo . CreateRemote ( & config . RemoteConfig {
Name : "origin" ,
URLs : [ ] string { remoteUrl } ,
Fetch : [ ] config . RefSpec { refspec } ,
2020-12-14 03:51:03 +00:00
} )
if err != nil {
2021-09-04 21:33:08 +00:00
return nil , fmt . Errorf ( "could not create remote: %v" , err )
2020-12-14 03:51:03 +00:00
}
2020-12-20 09:36:13 +00:00
err = repo . Fetch ( & git . FetchOptions {
RemoteName : "origin" ,
RefSpecs : [ ] config . RefSpec { refspec } ,
Auth : pd . Authenticator ,
} )
2020-12-14 03:51:03 +00:00
2021-02-24 07:27:51 +00:00
refName := plumbing . NewBranchReferenceName ( md . PushBranch )
2022-01-05 14:42:49 +00:00
pd . Log . Printf ( "set reference to ref: %s" , refName )
2020-12-16 10:46:33 +00:00
2021-04-06 19:35:46 +00:00
var hash plumbing . Hash
if commitPin [ md . PushBranch ] != "" {
hash = plumbing . NewHash ( commitPin [ md . PushBranch ] )
}
2020-12-14 03:51:03 +00:00
if err != nil {
2020-12-20 09:36:13 +00:00
h := plumbing . NewSymbolicReference ( plumbing . HEAD , refName )
if err := repo . Storer . CheckAndSetReference ( h , nil ) ; err != nil {
2021-09-04 21:33:08 +00:00
return nil , fmt . Errorf ( "could not set reference: %v" , err )
2020-12-20 09:36:13 +00:00
}
} else {
err = w . Checkout ( & git . CheckoutOptions {
2021-02-24 07:27:51 +00:00
Branch : plumbing . NewRemoteReferenceName ( "origin" , md . PushBranch ) ,
2021-04-06 19:35:46 +00:00
Hash : hash ,
2020-12-20 09:36:13 +00:00
Force : true ,
} )
if err != nil {
2021-09-04 21:33:08 +00:00
return nil , fmt . Errorf ( "could not checkout: %v" , err )
2020-12-20 09:36:13 +00:00
}
2020-12-14 03:51:03 +00:00
}
2021-09-03 21:07:02 +00:00
err = pd . Importer . WriteSource ( pd , md )
if err != nil {
2021-09-04 21:33:08 +00:00
return nil , err
2021-09-03 21:07:02 +00:00
}
2020-12-14 03:51:03 +00:00
2021-09-07 11:30:50 +00:00
err = data . CopyFromFs ( md . Worktree . Filesystem , w . Filesystem , "." )
if err != nil {
return nil , err
}
2021-02-24 07:27:51 +00:00
md . Repo = repo
md . Worktree = w
2020-12-14 03:51:03 +00:00
2020-12-22 16:28:04 +00:00
if pd . ModuleMode {
2021-09-03 21:07:02 +00:00
err := patchModuleYaml ( pd , md )
if err != nil {
2021-09-04 21:33:08 +00:00
return nil , err
2021-09-03 21:07:02 +00:00
}
2020-12-22 16:28:04 +00:00
} else {
2021-09-03 21:07:02 +00:00
err := executePatchesRpm ( pd , md )
if err != nil {
2021-09-04 21:33:08 +00:00
return nil , err
2021-09-03 21:07:02 +00:00
}
2020-12-22 16:28:04 +00:00
}
2020-12-14 03:51:03 +00:00
2021-02-24 07:27:51 +00:00
// get ignored files hash and add to .{Name}.metadata
2022-05-09 17:57:43 +00:00
metadataFile := ""
ls , err := md . Worktree . Filesystem . ReadDir ( "." )
if err != nil {
return nil , fmt . Errorf ( "could not read directory: %v" , err )
}
for _ , f := range ls {
if strings . HasSuffix ( f . Name ( ) , ".metadata" ) {
if metadataFile != "" {
return nil , fmt . Errorf ( "multiple metadata files found" )
}
metadataFile = f . Name ( )
}
}
if metadataFile == "" {
metadataFile = fmt . Sprintf ( ".%s.metadata" , md . Name )
}
2020-12-20 09:36:13 +00:00
metadata , err := w . Filesystem . Create ( metadataFile )
2020-12-14 03:51:03 +00:00
if err != nil {
2021-09-04 21:33:08 +00:00
return nil , fmt . Errorf ( "could not create metadata file: %v" , err )
2020-12-14 03:51:03 +00:00
}
2021-02-24 07:27:51 +00:00
for _ , source := range md . SourcesToIgnore {
sourcePath := source . Name
2021-02-14 17:14:21 +00:00
_ , err := w . Filesystem . Stat ( sourcePath )
2021-02-24 07:27:51 +00:00
if source . Expired || err != nil {
2021-01-01 15:55:21 +00:00
continue
}
2020-12-20 09:36:13 +00:00
sourceFile , err := w . Filesystem . Open ( sourcePath )
if err != nil {
2021-09-04 21:33:08 +00:00
return nil , fmt . Errorf ( "could not open ignored source file %s: %v" , sourcePath , err )
2020-12-20 09:36:13 +00:00
}
sourceFileBts , err := ioutil . ReadAll ( sourceFile )
if err != nil {
2021-09-04 21:33:08 +00:00
return nil , fmt . Errorf ( "could not read the whole of ignored source file: %v" , err )
2020-12-20 09:36:13 +00:00
}
2020-12-14 03:51:03 +00:00
2021-02-24 07:27:51 +00:00
source . HashFunction . Reset ( )
_ , err = source . HashFunction . Write ( sourceFileBts )
2020-12-14 03:51:03 +00:00
if err != nil {
2021-09-04 21:33:08 +00:00
return nil , fmt . Errorf ( "could not write bytes to hash function: %v" , err )
2020-12-14 03:51:03 +00:00
}
2021-02-24 07:27:51 +00:00
checksum := hex . EncodeToString ( source . HashFunction . Sum ( nil ) )
2020-12-20 17:38:25 +00:00
checksumLine := fmt . Sprintf ( "%s %s\n" , checksum , sourcePath )
2020-12-20 09:36:13 +00:00
_ , err = metadata . Write ( [ ] byte ( checksumLine ) )
if err != nil {
2021-09-04 21:33:08 +00:00
return nil , fmt . Errorf ( "could not write to metadata file: %v" , err )
2020-12-14 03:51:03 +00:00
}
2020-12-20 17:38:25 +00:00
2021-04-08 02:54:44 +00:00
if data . StrContains ( alreadyUploadedBlobs , checksum ) {
2020-12-23 13:38:46 +00:00
continue
}
2021-09-10 20:31:59 +00:00
exists , err := pd . BlobStorage . Exists ( checksum )
if err != nil {
return nil , err
}
if ! exists && ! pd . NoStorageUpload {
err := pd . BlobStorage . Write ( checksum , sourceFileBts )
if err != nil {
return nil , err
}
2022-01-05 14:42:49 +00:00
pd . Log . Printf ( "wrote %s to blob storage" , checksum )
2021-02-19 15:22:36 +00:00
}
2021-03-18 17:56:47 +00:00
alreadyUploadedBlobs = append ( alreadyUploadedBlobs , checksum )
2020-12-14 03:51:03 +00:00
}
2020-12-20 09:36:13 +00:00
_ , err = w . Add ( metadataFile )
2020-12-14 03:51:03 +00:00
if err != nil {
2021-09-04 21:33:08 +00:00
return nil , fmt . Errorf ( "could not add metadata file: %v" , err )
2020-12-14 03:51:03 +00:00
}
2020-12-20 09:36:13 +00:00
lastFilesToAdd := [ ] string { ".gitignore" , "SPECS" }
for _ , f := range lastFilesToAdd {
2020-12-22 16:28:04 +00:00
_ , err := w . Filesystem . Stat ( f )
if err == nil {
_ , err := w . Add ( f )
if err != nil {
2021-09-04 21:33:08 +00:00
return nil , fmt . Errorf ( "could not add %s: %v" , f , err )
2020-12-22 16:28:04 +00:00
}
2020-12-20 09:36:13 +00:00
}
2020-12-15 08:56:09 +00:00
}
2021-09-08 15:00:56 +00:00
nvrMatch := rpmutils . Nvr . FindStringSubmatch ( match [ 3 ] )
if len ( nvrMatch ) >= 4 {
versionForBranch [ md . PushBranch ] = & srpmprocpb . VersionRelease {
Version : nvrMatch [ 2 ] ,
Release : nvrMatch [ 3 ] ,
}
}
2021-03-18 17:56:47 +00:00
if pd . TmpFsMode != "" {
continue
}
2021-09-03 21:07:02 +00:00
err = pd . Importer . PostProcess ( md )
if err != nil {
2021-09-04 21:33:08 +00:00
return nil , err
2021-09-03 21:07:02 +00:00
}
2020-12-20 09:36:13 +00:00
// show status
status , _ := w . Status ( )
2022-01-05 14:42:49 +00:00
pd . Log . Printf ( "successfully processed:\n%s" , status )
2020-12-20 09:36:13 +00:00
2021-02-20 04:12:16 +00:00
statusLines := strings . Split ( status . String ( ) , "\n" )
for _ , line := range statusLines {
trimmed := strings . TrimSpace ( line )
if strings . HasPrefix ( trimmed , "D" ) {
path := strings . TrimPrefix ( trimmed , "D " )
_ , err := w . Remove ( path )
if err != nil {
2021-09-04 21:33:08 +00:00
return nil , fmt . Errorf ( "could not delete extra file %s: %v" , path , err )
2021-02-20 04:12:16 +00:00
}
}
}
2020-12-20 09:36:13 +00:00
var hashes [ ] plumbing . Hash
var pushRefspecs [ ] config . RefSpec
head , err := repo . Head ( )
if err != nil {
hashes = nil
pushRefspecs = append ( pushRefspecs , "*:*" )
} else {
2022-01-05 14:42:49 +00:00
pd . Log . Printf ( "tip %s" , head . String ( ) )
2020-12-20 09:36:13 +00:00
hashes = append ( hashes , head . Hash ( ) )
2021-02-24 07:27:51 +00:00
refOrigin := "refs/heads/" + md . PushBranch
2020-12-20 09:36:13 +00:00
pushRefspecs = append ( pushRefspecs , config . RefSpec ( fmt . Sprintf ( "HEAD:%s" , refOrigin ) ) )
}
2021-02-24 07:27:51 +00:00
// we are now finished with the tree and are going to push it to the src Repo
2020-12-20 09:36:13 +00:00
// create import commit
commit , err := w . Commit ( "import " + pd . Importer . ImportName ( pd , md ) , & git . CommitOptions {
Author : & object . Signature {
Name : pd . GitCommitterName ,
Email : pd . GitCommitterEmail ,
When : time . Now ( ) ,
} ,
Parents : hashes ,
} )
2020-12-14 03:51:03 +00:00
if err != nil {
2021-09-04 21:33:08 +00:00
return nil , fmt . Errorf ( "could not commit object: %v" , err )
2020-12-14 03:51:03 +00:00
}
2020-12-20 09:36:13 +00:00
obj , err := repo . CommitObject ( commit )
2020-12-14 03:51:03 +00:00
if err != nil {
2021-09-04 21:33:08 +00:00
return nil , fmt . Errorf ( "could not get commit object: %v" , err )
2020-12-14 03:51:03 +00:00
}
2022-01-05 14:42:49 +00:00
pd . Log . Printf ( "committed:\n%s" , obj . String ( ) )
2020-12-14 03:51:03 +00:00
2020-12-20 09:36:13 +00:00
_ , err = repo . CreateTag ( newTag , commit , & git . CreateTagOptions {
Tagger : & object . Signature {
Name : pd . GitCommitterName ,
Email : pd . GitCommitterEmail ,
When : time . Now ( ) ,
} ,
2021-02-24 07:27:51 +00:00
Message : "import " + md . TagBranch + " from " + pd . RpmLocation ,
2020-12-20 09:36:13 +00:00
SignKey : nil ,
} )
if err != nil {
2021-09-04 21:33:08 +00:00
return nil , fmt . Errorf ( "could not create tag: %v" , err )
2020-12-20 09:36:13 +00:00
}
2020-12-14 03:51:03 +00:00
2021-04-13 04:36:44 +00:00
pushRefspecs = append ( pushRefspecs , config . RefSpec ( "HEAD:" + plumbing . NewTagReferenceName ( newTag ) ) )
2020-12-14 03:51:03 +00:00
2020-12-20 09:36:13 +00:00
err = repo . Push ( & git . PushOptions {
RemoteName : "origin" ,
Auth : pd . Authenticator ,
RefSpecs : pushRefspecs ,
Force : true ,
} )
if err != nil {
2021-09-04 21:33:08 +00:00
return nil , fmt . Errorf ( "could not push to remote: %v" , err )
2020-12-20 09:36:13 +00:00
}
2021-02-19 15:22:36 +00:00
hashString := obj . Hash . String ( )
2021-02-24 07:27:51 +00:00
latestHashForBranch [ md . PushBranch ] = hashString
2021-02-19 15:22:36 +00:00
}
2021-09-08 15:00:56 +00:00
return & srpmprocpb . ProcessResponse {
BranchCommits : latestHashForBranch ,
BranchVersions : versionForBranch ,
} , nil
2020-12-14 01:29:02 +00:00
}
2022-09-28 03:32:50 +00:00
// Process for when we want to import a tagless repo (like from CentOS Stream)
//
func processRPMTagless ( pd * data . ProcessData ) ( * srpmprocpb . ProcessResponse , error ) {
pd . Log . Println ( "Tagless mode detected, attempting import of latest commit" )
// In tagless mode, we *automatically* set StrictBranchMode to true
// Only the exact <PREFIX><VERSION><SUFFIX> branch should be pulled from the source repo
pd . StrictBranchMode = true
// our return values: a mapping of branches -> commits (1:1) that we're bringing in,
// and a mapping of branches to: version = X, release = Y
latestHashForBranch := map [ string ] string { }
versionForBranch := map [ string ] * srpmprocpb . VersionRelease { }
md , err := pd . Importer . RetrieveSource ( pd )
if err != nil {
pd . Log . Println ( "Error detected in RetrieveSource!" )
return nil , err
}
md . BlobCache = map [ string ] [ ] byte { }
// TODO: add tagless module support
remotePrefix := "rpms"
if pd . ModuleMode {
remotePrefix = "modules"
}
// Set up our remote URL for pushing our repo to
var tagIgnoreList [ ] string
if pd . NoDupMode {
repo , err := git . Init ( memory . NewStorage ( ) , memfs . New ( ) )
if err != nil {
return nil , fmt . Errorf ( "could not init git repo: %v" , err )
}
remoteUrl := fmt . Sprintf ( "%s/%s/%s.git" , pd . UpstreamPrefix , remotePrefix , gitlabify ( md . Name ) )
refspec := config . RefSpec ( "+refs/heads/*:refs/remotes/origin/*" )
remote , err := repo . CreateRemote ( & config . RemoteConfig {
Name : "origin" ,
URLs : [ ] string { remoteUrl } ,
Fetch : [ ] config . RefSpec { refspec } ,
} )
if err != nil {
return nil , fmt . Errorf ( "could not create remote: %v" , err )
}
list , err := remote . List ( & git . ListOptions {
Auth : pd . Authenticator ,
} )
if err != nil {
log . Println ( "ignoring no-dup-mode" )
} else {
for _ , ref := range list {
if ! strings . HasPrefix ( string ( ref . Name ( ) ) , "refs/tags/imports" ) {
continue
}
tagIgnoreList = append ( tagIgnoreList , string ( ref . Name ( ) ) )
}
}
}
sourceRepo := * md . Repo
sourceWorktree := * md . Worktree
localPath := ""
for _ , branch := range md . Branches {
md . Repo = & sourceRepo
md . Worktree = & sourceWorktree
md . TagBranch = branch
for _ , source := range md . SourcesToIgnore {
source . Expired = true
}
// Create a temporary place to check out our tag/branch : /tmp/srpmproctmp_<PKG_NAME><RANDOMSTRING>/
localPath , _ = os . MkdirTemp ( "/tmp" , fmt . Sprintf ( "srpmproctmp_%s" , md . Name ) )
if err := os . RemoveAll ( localPath ) ; err != nil {
return nil , fmt . Errorf ( "Could not remove previous temporary directory: %s" , localPath )
}
2022-11-06 03:53:02 +00:00
if err := os . Mkdir ( localPath , 0 o755 ) ; err != nil {
2022-09-28 03:32:50 +00:00
return nil , fmt . Errorf ( "Could not create temporary directory: %s" , localPath )
}
// Clone repo into the temporary path, but only the tag we're interested in:
// (TODO: will probably need to assign this a variable or use the md struct gitrepo object to perform a successful tag+push later)
_ , _ = git . PlainClone ( localPath , false , & git . CloneOptions {
URL : pd . RpmLocation ,
SingleBranch : true ,
ReferenceName : plumbing . ReferenceName ( branch ) ,
} )
// Now that we're cloned into localPath, we need to "covert" the import into the old format
// We want sources to become .PKGNAME.metadata, we want SOURCES and SPECS folders, etc.
repoFixed , _ := convertLocalRepo ( md . Name , localPath )
if ! repoFixed {
return nil , fmt . Errorf ( "Error converting repository into SOURCES + SPECS + .package.metadata format" )
}
// call extra function to determine the proper way to convert the tagless branch name.
// c9s becomes r9s (in the usual case), or in the modular case, stream-httpd-2.4-rhel-9.1.0 becomes r9s-stream-httpd-2.4_r9.1.0
md . PushBranch = taglessBranchName ( branch , pd )
rpmVersion := ""
// get name-version-release of tagless repo, only if we're not a module repo:
if ! pd . ModuleMode {
nvrString := getVersionFromSpec ( md . Name , localPath , pd . Version )
if nvrString == "" {
return nil , fmt . Errorf ( "Error using rpm or rpmbuild to build SRPM and determine version info! (tagless mode)" )
}
// Set version and release fields we extracted (name|version|release are separated by pipes)
pd . PackageVersion = strings . Split ( nvrString , "|" ) [ 1 ]
pd . PackageRelease = strings . Split ( nvrString , "|" ) [ 2 ]
// Set full rpm version: name-version-release (for tagging properly)
rpmVersion = fmt . Sprintf ( "%s-%s-%s" , md . Name , pd . PackageVersion , pd . PackageRelease )
pd . Log . Println ( "Successfully determined version of tagless checkout: " , rpmVersion )
} else {
// In case of module mode, we just set rpmVersion to the current date - that's what our tag will end up being
rpmVersion = time . Now ( ) . Format ( "2006-01-02" )
}
// Make an initial repo we will use to push to our target
pushRepo , err := git . PlainInit ( localPath + "_gitpush" , false )
if err != nil {
return nil , fmt . Errorf ( "could not create new dist Repo: %v" , err )
}
w , err := pushRepo . Worktree ( )
if err != nil {
return nil , fmt . Errorf ( "could not get dist Worktree: %v" , err )
}
// Create a remote "origin" in our empty git, make the upstream equal to the branch we want to modify
pushUrl := fmt . Sprintf ( "%s/%s/%s.git" , pd . UpstreamPrefix , remotePrefix , gitlabify ( md . Name ) )
refspec := config . RefSpec ( fmt . Sprintf ( "+refs/heads/%s:refs/remotes/origin/%s" , md . PushBranch , md . PushBranch ) )
// Make our remote repo the target one - the one we want to push our update to
pushRepoRemote , err := pushRepo . CreateRemote ( & config . RemoteConfig {
Name : "origin" ,
URLs : [ ] string { pushUrl } ,
Fetch : [ ] config . RefSpec { refspec } ,
} )
if err != nil {
return nil , fmt . Errorf ( "could not create remote: %v" , err )
}
// fetch our branch data (md.PushBranch) into this new repo
err = pushRepo . Fetch ( & git . FetchOptions {
RemoteName : "origin" ,
RefSpecs : [ ] config . RefSpec { refspec } ,
Auth : pd . Authenticator ,
} )
refName := plumbing . NewBranchReferenceName ( md . PushBranch )
var hash plumbing . Hash
h := plumbing . NewSymbolicReference ( plumbing . HEAD , refName )
if err := pushRepo . Storer . CheckAndSetReference ( h , nil ) ; err != nil {
return nil , fmt . Errorf ( "Could not set symbolic reference: %v" , err )
}
err = w . Checkout ( & git . CheckoutOptions {
Branch : plumbing . NewRemoteReferenceName ( "origin" , md . PushBranch ) ,
Hash : hash ,
Force : true ,
} )
os . Rename ( fmt . Sprintf ( "%s/SPECS" , localPath ) , fmt . Sprintf ( "%s_gitpush/SPECS" , localPath ) )
os . Rename ( fmt . Sprintf ( "%s/SOURCES" , localPath ) , fmt . Sprintf ( "%s_gitpush/SOURCES" , localPath ) )
os . Rename ( fmt . Sprintf ( "%s/.gitignore" , localPath ) , fmt . Sprintf ( "%s_gitpush/.gitignore" , localPath ) )
os . Rename ( fmt . Sprintf ( "%s/.%s.metadata" , localPath , md . Name ) , fmt . Sprintf ( "%s_gitpush/.%s.metadata" , localPath , md . Name ) )
md . Repo = pushRepo
md . Worktree = w
// Download lookaside sources (tarballs) into the push git repo:
err = pd . Importer . WriteSource ( pd , md )
if err != nil {
return nil , err
}
// Call function to upload source to target lookaside and
// ensure the sources are added to .gitignore
err = processLookasideSources ( pd , md , localPath + "_gitpush" )
if err != nil {
return nil , err
}
// Apply patch(es) if needed:
if pd . ModuleMode {
err := patchModuleYaml ( pd , md )
if err != nil {
return nil , err
}
} else {
err := executePatchesRpm ( pd , md )
if err != nil {
return nil , err
}
}
err = w . AddWithOptions ( & git . AddOptions { All : true } )
if err != nil {
return nil , fmt . Errorf ( "Error adding SOURCES/ , SPECS/ or .metadata file to commit list." )
}
status , err := w . Status ( )
pd . Log . Printf ( "successfully processed:\n%s" , status )
// assign tag for our new remote we're about to push (derived from the SRPM version)
newTag := "refs/tags/imports/" + md . PushBranch + "/" + rpmVersion
newTag = strings . Replace ( newTag , "%" , "_" , - 1 )
// pushRefspecs is a list of all the references we want to push (tags + heads)
// It's an array of colon-separated strings which map local references to their remote counterparts
var pushRefspecs [ ] config . RefSpec
// We need to find out if the remote repo already has this branch
// If it doesn't, we want to add *:* to our references for commit. This will allow us to push the new branch
// If it does, we can simply push HEAD:refs/heads/<BRANCH>
newRepo := true
refList , _ := pushRepoRemote . List ( & git . ListOptions { Auth : pd . Authenticator } )
for _ , ref := range refList {
if strings . HasSuffix ( ref . Name ( ) . String ( ) , fmt . Sprintf ( "heads/%s" , md . PushBranch ) ) {
newRepo = false
break
}
}
if newRepo {
pushRefspecs = append ( pushRefspecs , config . RefSpec ( "*:*" ) )
pd . Log . Printf ( "New remote repo detected, creating new remote branch" )
}
// Identify specific references we want to push
// Should be refs/heads/<target_branch>, and a tag called imports/<target_branch>/<rpm_nvr>
pushRefspecs = append ( pushRefspecs , config . RefSpec ( fmt . Sprintf ( "HEAD:refs/heads/%s" , md . PushBranch ) ) )
pushRefspecs = append ( pushRefspecs , config . RefSpec ( fmt . Sprintf ( "HEAD:%s" , newTag ) ) )
// Actually do the commit (locally)
commit , err := w . Commit ( "import from tagless source " + pd . Importer . ImportName ( pd , md ) , & git . CommitOptions {
Author : & object . Signature {
Name : pd . GitCommitterName ,
Email : pd . GitCommitterEmail ,
When : time . Now ( ) ,
} ,
} )
if err != nil {
return nil , fmt . Errorf ( "could not commit object: %v" , err )
}
obj , err := pushRepo . CommitObject ( commit )
if err != nil {
return nil , fmt . Errorf ( "could not get commit object: %v" , err )
}
pd . Log . Printf ( "Committed local repo tagless mode transform:\n%s" , obj . String ( ) )
// After commit, we will now tag our local repo on disk:
_ , err = pushRepo . CreateTag ( newTag , commit , & git . CreateTagOptions {
Tagger : & object . Signature {
Name : pd . GitCommitterName ,
Email : pd . GitCommitterEmail ,
When : time . Now ( ) ,
} ,
Message : "import " + md . TagBranch + " from " + pd . RpmLocation + "(import from tagless source)" ,
SignKey : nil ,
} )
if err != nil {
return nil , fmt . Errorf ( "could not create tag: %v" , err )
}
pd . Log . Printf ( "Pushing these references to the remote: %+v \n" , pushRefspecs )
// Do the actual push to the remote target repository
err = pushRepo . Push ( & git . PushOptions {
RemoteName : "origin" ,
Auth : pd . Authenticator ,
RefSpecs : pushRefspecs ,
Force : true ,
} )
if err != nil {
return nil , fmt . Errorf ( "could not push to remote: %v" , err )
}
if err := os . RemoveAll ( localPath ) ; err != nil {
log . Printf ( "Error cleaning up temporary git checkout directory %s . Non-fatal, continuing anyway...\n" , localPath )
}
if err := os . RemoveAll ( fmt . Sprintf ( "%s_gitpush" , localPath ) ) ; err != nil {
log . Printf ( "Error cleaning up temporary git checkout directory %s . Non-fatal, continuing anyway...\n" , fmt . Sprintf ( "%s_gitpush" , localPath ) )
}
// append our processed branch to the return structures:
latestHashForBranch [ md . PushBranch ] = obj . Hash . String ( )
versionForBranch [ md . PushBranch ] = & srpmprocpb . VersionRelease {
Version : pd . PackageVersion ,
Release : pd . PackageRelease ,
}
}
// return struct with all our branch:commit and branch:version+release mappings
return & srpmprocpb . ProcessResponse {
BranchCommits : latestHashForBranch ,
BranchVersions : versionForBranch ,
} , nil
}
// Given a local repo on disk, ensure it's in the "traditional" format. This means:
// - metadata file is named .pkgname.metadata
// - metadata file has the old "<SHASUM> SOURCES/<filename>" format
// - SPECS/ and SOURCES/ exist and are populated correctly
func convertLocalRepo ( pkgName string , localRepo string ) ( bool , error ) {
// Make sure we have a SPECS and SOURCES folder made:
2022-11-06 03:53:02 +00:00
if err := os . MkdirAll ( fmt . Sprintf ( "%s/SOURCES" , localRepo ) , 0 o755 ) ; err != nil {
2022-09-28 03:32:50 +00:00
return false , fmt . Errorf ( "Could not create SOURCES directory in: %s" , localRepo )
}
2022-11-06 03:53:02 +00:00
if err := os . MkdirAll ( fmt . Sprintf ( "%s/SPECS" , localRepo ) , 0 o755 ) ; err != nil {
2022-09-28 03:32:50 +00:00
return false , fmt . Errorf ( "Could not create SPECS directory in: %s" , localRepo )
}
// Loop through each file/folder and operate accordingly:
files , err := ioutil . ReadDir ( localRepo )
if err != nil {
return false , err
}
2022-11-06 03:53:02 +00:00
for _ , f := range files {
2022-09-28 03:32:50 +00:00
// We don't want to process SOURCES, SPECS, or any of our .git folders
2022-11-06 03:53:02 +00:00
if f . Name ( ) == "SOURCES" || f . Name ( ) == "SPECS" || strings . HasPrefix ( f . Name ( ) , ".git" ) || f . Name ( ) == "." + pkgName + ".metadata" {
2022-09-28 03:32:50 +00:00
continue
}
// If we have a metadata "sources" file, we need to read it and convert to the old .<pkgname>.metadata format
2022-11-06 03:53:02 +00:00
if f . Name ( ) == "sources" {
2022-09-28 03:32:50 +00:00
convertStatus := convertMetaData ( pkgName , localRepo )
if convertStatus != true {
return false , fmt . Errorf ( "Error converting sources metadata file to .metadata format" )
}
continue
}
// Any file that ends in a ".spec" should be put into SPECS/
2022-11-06 03:53:02 +00:00
if strings . HasSuffix ( f . Name ( ) , ".spec" ) {
err := os . Rename ( fmt . Sprintf ( "%s/%s" , localRepo , f . Name ( ) ) , fmt . Sprintf ( "%s/SPECS/%s" , localRepo , f . Name ( ) ) )
2022-09-28 03:32:50 +00:00
if err != nil {
return false , fmt . Errorf ( "Error moving .spec file to SPECS/" )
}
}
// if a file isn't skipped in one of the above checks, then it must be a file that belongs in SOURCES/
2022-11-06 03:53:02 +00:00
os . Rename ( fmt . Sprintf ( "%s/%s" , localRepo , f . Name ( ) ) , fmt . Sprintf ( "%s/SOURCES/%s" , localRepo , f . Name ( ) ) )
2022-09-28 03:32:50 +00:00
}
return true , nil
}
// Given a local "sources" metadata file (new CentOS Stream format), convert it into the older
// classic CentOS style: "<HASH> SOURCES/<FILENAME>"
func convertMetaData ( pkgName string , localRepo string ) bool {
lookAside , err := os . Open ( fmt . Sprintf ( "%s/sources" , localRepo ) )
if err != nil {
return false
}
// Split file into lines and start processing:
scanner := bufio . NewScanner ( lookAside )
scanner . Split ( bufio . ScanLines )
// convertedLA is our array of new "converted" lookaside lines
var convertedLA [ ] string
// loop through each line, and:
// - split by whitespace
// - check each line begins with "SHA" or "MD" - validate
// - take the
// Then check
for scanner . Scan ( ) {
tmpLine := strings . Fields ( scanner . Text ( ) )
// make sure line starts with a "SHA" or "MD" before processing - otherwise it might not be a valid format lookaside line!
if ! ( strings . HasPrefix ( tmpLine [ 0 ] , "SHA" ) || strings . HasPrefix ( tmpLine [ 0 ] , "MD" ) ) {
continue
}
// Strip out "( )" characters from file name and prepend SOURCES/ to it
tmpLine [ 1 ] = strings . ReplaceAll ( tmpLine [ 1 ] , "(" , "" )
tmpLine [ 1 ] = strings . ReplaceAll ( tmpLine [ 1 ] , ")" , "" )
tmpLine [ 1 ] = fmt . Sprintf ( "SOURCES/%s" , tmpLine [ 1 ] )
convertedLA = append ( convertedLA , fmt . Sprintf ( "%s %s" , tmpLine [ 3 ] , tmpLine [ 1 ] ) )
}
lookAside . Close ( )
// open .<NAME>.metadata file for writing our old-format lines
2022-11-06 03:53:02 +00:00
lookAside , err = os . OpenFile ( fmt . Sprintf ( "%s/.%s.metadata" , localRepo , pkgName ) , os . O_APPEND | os . O_CREATE | os . O_WRONLY , 0 o644 )
2022-09-28 03:32:50 +00:00
if err != nil {
fmt . Errorf ( "Error opening new .metadata file for writing." )
return false
}
writer := bufio . NewWriter ( lookAside )
for _ , convertedLine := range convertedLA {
_ , _ = writer . WriteString ( convertedLine + "\n" )
}
writer . Flush ( )
lookAside . Close ( )
// Remove old "sources" metadata file - we don't need it now that conversion is complete
os . Remove ( fmt . Sprintf ( "%s/sources" , localRepo ) )
return true
}
// Given a local checked out folder and package name, including SPECS/ , SOURCES/ , and .package.metadata, this will:
// - create a "dummy" SRPM (using dummy sources files we use to populate tarballs from lookaside)
// - extract RPM version info from that SRPM, and return it
// If we are in tagless mode, we need to get a package version somehow!
func getVersionFromSpec ( pkgName string , localRepo string , majorVersion int ) string {
// Make sure we have "rpm" and "rpmbuild" and "cp" available in our PATH. Otherwise, this won't work:
_ , err := exec . LookPath ( "rpm" )
if err != nil {
return ""
}
_ , err = exec . LookPath ( "rpmbuild" )
if err != nil {
return ""
}
_ , err = exec . LookPath ( "cp" )
if err != nil {
return ""
}
// create separate temp folder space to do our RPM work - we don't want to accidentally contaminate the main Git area:
rpmBuildPath := fmt . Sprintf ( "%s_rpm" , localRepo )
2022-11-06 03:53:02 +00:00
os . Mkdir ( rpmBuildPath , 0 o755 )
2022-09-28 03:32:50 +00:00
// Copy SOURCES/ and SPECS/ into the temp rpmbuild directory recursively
// Yes, we could create or import an elaborate Go-native way to do this, but damnit this is easier:
cmdArgs := strings . Fields ( fmt . Sprintf ( "cp -rp %s/SOURCES %s/SPECS %s/" , localRepo , localRepo , rpmBuildPath ) )
if err := exec . Command ( cmdArgs [ 0 ] , cmdArgs [ 1 : ] ... ) . Run ( ) ; err != nil {
log . Println ( err )
return ""
}
// Loop through .<package>.metadata and get the file names we need to make our SRPM:
lookAside , err := os . Open ( fmt . Sprintf ( "%s/.%s.metadata" , localRepo , pkgName ) )
if err != nil {
log . Println ( err )
return ""
}
// Split file into lines and start processing:
scanner := bufio . NewScanner ( lookAside )
scanner . Split ( bufio . ScanLines )
// loop through each line, and:
// - isolate the SOURCES/filename entry
// - write out a dummy file of the same name to rpmBuildPath/SOURCES
for scanner . Scan ( ) {
// lookaside source is always the 2nd part of the line (after the long SHA sum)
srcFile := strings . Fields ( scanner . Text ( ) ) [ 1 ]
// write a dummy file of the same name into the rpmbuild SOURCES/ directory:
2022-11-06 03:53:02 +00:00
dummyFile , err := os . OpenFile ( fmt . Sprintf ( "%s/%s" , rpmBuildPath , srcFile ) , os . O_APPEND | os . O_CREATE | os . O_WRONLY , 0 o644 )
2022-09-28 03:32:50 +00:00
if err != nil {
return ""
}
writer := bufio . NewWriter ( dummyFile )
_ , _ = writer . WriteString ( "This is a dummy lookaside file generated by srpmproc. It is only needed to get a working SRPM and extract version information. Please disregard\n" )
writer . Flush ( )
dummyFile . Close ( )
}
lookAside . Close ( )
// Now, call rpmbuild to produce the dummy src file:
// Example: rpmbuild --define "_topdir /tmp/srpmproctmp_httpd1988142783_rpm" -bs /tmp/srpmproctmp_httpd1988142783_rpm/SPECS/*.spec
cmd := exec . Command ( "rpmbuild" , fmt . Sprintf ( ` --define=_topdir %s ` , rpmBuildPath ) , fmt . Sprintf ( ` --define=dist .el%d ` , majorVersion ) , "-bs" , fmt . Sprintf ( "%s/SPECS/%s.spec" , rpmBuildPath , pkgName ) )
if err := cmd . Run ( ) ; err != nil {
log . Println ( err )
return ""
}
// Read the first file from the SRPMS/ folder in rpmBuildPath. It should be the SRPM that rpmbuild produced above
// (there should only be one file - we check that it ends in ".rpm" just to be sure!)
lsTmp , err := ioutil . ReadDir ( fmt . Sprintf ( "%s/SRPMS/" , rpmBuildPath ) )
if err != nil {
log . Println ( err )
return ""
}
srpmFile := lsTmp [ 0 ] . Name ( )
if ! strings . HasSuffix ( srpmFile , ".rpm" ) {
log . Println ( "Error, file found in dummy SRPMS directory did not have an .rpm extension! Perhaps rpmbuild didn't produce a proper source RPM?" )
return ""
}
// Call the rpm binary to extract the version-release info out of it, and tack on ".el<VERSION>" at the end:
cmd = exec . Command ( "rpm" , "-qp" , "--qf" , ` % { NAME}|% { VERSION}|% { RELEASE}\n ` , fmt . Sprintf ( "%s/SRPMS/%s" , rpmBuildPath , srpmFile ) )
nvrTmp , err := cmd . CombinedOutput ( )
if err != nil {
log . Println ( "Error running rpm command to extract temporary SRPM name-version-release identifiers." )
log . Println ( "rpmbuild output: " , string ( nvrTmp ) )
log . Println ( "rpmbuild command: " , cmd . String ( ) )
return ""
}
// Pull first line of the rpm command's output to get the name-version-release number (there should only be 1 line)
nvr := string ( nvrTmp )
nvr = strings . Fields ( nvr ) [ 0 ]
// Clean up: delete the temporary directory
if err := os . RemoveAll ( rpmBuildPath ) ; err != nil {
log . Printf ( "Error cleaning up temporary RPM directory %s . Non-fatal, continuing anyway...\n" , rpmBuildPath )
}
// return name-version-release string we derived:
log . Printf ( "Derived NVR %s from tagless repo via temporary SRPM build\n" , nvr )
return nvr
}
// We need to loop through the lookaside blob files ("SourcesToIgnore"),
// and upload them to our target storage (usually an S3 bucket, but could be a local folder)
//
// We also need to add the source paths to .gitignore in the git repo, so we don't accidentally commit + push them
func processLookasideSources ( pd * data . ProcessData , md * data . ModeData , localDir string ) error {
w := md . Worktree
metadata , err := w . Filesystem . Create ( fmt . Sprintf ( ".%s.metadata" , md . Name ) )
if err != nil {
return fmt . Errorf ( "could not create metadata file: %v" , err )
}
// Keep track of files we've already uploaded - don't want duplicates!
var alreadyUploadedBlobs [ ] string
2022-11-06 03:53:02 +00:00
gitIgnore , err := os . OpenFile ( fmt . Sprintf ( "%s/.gitignore" , localDir ) , os . O_APPEND | os . O_CREATE | os . O_WRONLY , 0 o644 )
2022-09-28 03:32:50 +00:00
if err != nil {
return err
}
for _ , source := range md . SourcesToIgnore {
sourcePath := source . Name
_ , err := w . Filesystem . Stat ( sourcePath )
if source . Expired || err != nil {
continue
}
sourceFile , err := w . Filesystem . Open ( sourcePath )
if err != nil {
return fmt . Errorf ( "could not open ignored source file %s: %v" , sourcePath , err )
}
sourceFileBts , err := ioutil . ReadAll ( sourceFile )
if err != nil {
return fmt . Errorf ( "could not read the whole of ignored source file: %v" , err )
}
source . HashFunction . Reset ( )
_ , err = source . HashFunction . Write ( sourceFileBts )
if err != nil {
return fmt . Errorf ( "could not write bytes to hash function: %v" , err )
}
checksum := hex . EncodeToString ( source . HashFunction . Sum ( nil ) )
checksumLine := fmt . Sprintf ( "%s %s\n" , checksum , sourcePath )
_ , err = metadata . Write ( [ ] byte ( checksumLine ) )
if err != nil {
return fmt . Errorf ( "could not write to metadata file: %v" , err )
}
if data . StrContains ( alreadyUploadedBlobs , checksum ) {
continue
}
exists , err := pd . BlobStorage . Exists ( checksum )
if err != nil {
return err
}
if ! exists && ! pd . NoStorageUpload {
err := pd . BlobStorage . Write ( checksum , sourceFileBts )
if err != nil {
return err
}
pd . Log . Printf ( "wrote %s to blob storage" , checksum )
}
alreadyUploadedBlobs = append ( alreadyUploadedBlobs , checksum )
// Add this SOURCES/ lookaside file to be excluded
w . Excludes = append ( w . Excludes , gitignore . ParsePattern ( sourcePath , nil ) )
// Append the SOURCES/<file> path to .gitignore:
_ , err = gitIgnore . Write ( [ ] byte ( fmt . Sprintf ( "%s\n" , sourcePath ) ) )
if err != nil {
return err
}
}
err = gitIgnore . Close ( )
if err != nil {
return err
}
return nil
}
// Given an input branch name to import from, like "refs/heads/c9s", produce the tagless branch name we want to commit to, like "r9s"
// Modular translation of CentOS stream branches i is also done - branch stream-maven-3.8-rhel-9.1.0 ----> r9s-stream-maven-3.8_9.1.0
func taglessBranchName ( fullBranch string , pd * data . ProcessData ) string {
// Split the full branch name "refs/heads/blah" to only get the short name - last entry
tmpBranch := strings . Split ( fullBranch , "/" )
branch := tmpBranch [ len ( tmpBranch ) - 1 ]
// Simple case: if our branch is not a modular stream branch, just return the normal <prefix><version><suffix> pattern
if ! strings . HasPrefix ( branch , "stream-" ) {
return fmt . Sprintf ( "%s%d%s" , pd . BranchPrefix , pd . Version , pd . BranchSuffix )
}
// index where the "-rhel-" starts near the end of the string
rhelSpot := strings . LastIndex ( branch , "-rhel-" )
// module name will be everything from the start until that "-rhel-" string (like "stream-httpd-2.4")
moduleString := branch [ 0 : rhelSpot ]
// major minor version is everything after the "-rhel-" string
2022-11-06 03:53:02 +00:00
majorMinor := branch [ rhelSpot + 6 : ]
2022-09-28 03:32:50 +00:00
// return translated modular branch:
return fmt . Sprintf ( "%s%d%s-%s_%s" , pd . BranchPrefix , pd . Version , pd . BranchSuffix , moduleString , majorMinor )
}