Merge pull request #42 from mstg/misc-prow

Further Prow adjustments
2022-10-30 06:27:23 +00:00 · 2022-10-30 06:27:23 +00:00 · ab6861caf0
parent 3b19040cfd 5e5bfc032f
commit ab6861caf0
14 changed files with 155 additions and 19 deletions
--- a/.bazelrc
+++ b/.bazelrc
@ -16,9 +16,9 @@ build:toplevel --config=inmemory
 build:toplevel --remote_download_outputs=toplevel

 build:remote --config=toplevel
-build:remote --remote_cache=grpc://buildcache.default.svc.cluster.local:9092
-build:remote --experimental_remote_downloader=grpc://buildcache.default.svc.cluster.local:9092
-build:remote --noremote_upload_local_results
+build:remote --remote_cache=grpc://buildcache.default.svc.cluster.local:9092 --experimental_remote_downloader=grpc://buildcache.default.svc.cluster.local:9092
+query:remote --remote_cache=grpc://buildcache.default.svc.cluster.local:9092 --experimental_remote_downloader=grpc://buildcache.default.svc.cluster.local:9092
+fetch:remote --remote_cache=grpc://buildcache.default.svc.cluster.local:9092 --experimental_remote_downloader=grpc://buildcache.default.svc.cluster.local:9092
 build:remote --remote_timeout=3600
 build:remote --bes_results_url=https://bz.build.resf.org/invocation/
 build:remote --bes_backend=grpc://buildbuddy-grpc.default.svc.cluster.local:1985
--- a/config/jobs/postsubmits.yaml
+++ b/config/jobs/postsubmits.yaml
@ -0,0 +1,20 @@
+postsubmits:
+  rocky-linux/peridot:
+    - name: post-peridot-unit-test
+      always_run: true
+      decorate: true
+      branches:
+        - main
+      spec:
+        tolerations:
+          - key: peridot.rockylinux.org/workflow-tolerates-arch
+            operator: Equal
+            value: amd64
+            effect: NoSchedule
+        nodeSelector:
+          peridot.rockylinux.org/workflow-tolerates-arch: amd64
+        containers:
+          - image: quay.io/peridot/ci:v0.3.16
+            command:
+              - hack/unit_tests.sh
+
--- a/config/jobs/presubmits.yaml
+++ b/config/jobs/presubmits.yaml
@ -20,6 +20,13 @@ presubmits:
      always_run: true
      decorate: true
      spec:
+        tolerations:
+          - key: peridot.rockylinux.org/workflow-tolerates-arch
+            operator: Equal
+            value: amd64
+            effect: NoSchedule
+        nodeSelector:
+          peridot.rockylinux.org/workflow-tolerates-arch: amd64
        containers:
          - image: quay.io/peridot/ci:v0.3.16
            command:
@ -31,7 +38,25 @@ presubmits:
      always_run: true
      decorate: true
      spec:
+        tolerations:
+          - key: peridot.rockylinux.org/workflow-tolerates-arch
+            operator: Equal
+            value: amd64
+            effect: NoSchedule
+        nodeSelector:
+          peridot.rockylinux.org/workflow-tolerates-arch: amd64
        containers:
          - image: quay.io/peridot/ci:v0.3.16
            command:
              - hack/non_manual_build.sh
+
+    - name: pull-peridot-gofmt-check
+      branches:
+        - main
+      always_run: true
+      decorate: true
+      spec:
+        containers:
+          - image: quay.io/peridot/ci:v0.3.16
+            command:
+              - hack/gofmt_check.sh
--- a/go.mod
+++ b/go.mod
@ -26,6 +26,8 @@ require (
 	github.com/gocolly/colly/v2 v2.1.0
 	github.com/gogo/status v1.1.0
 	github.com/google/uuid v1.3.0
+	github.com/gorilla/feeds v1.1.1 // indirect
+	github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect
 	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.6.0
 	github.com/imdario/mergo v0.3.11 // indirect
@ -33,6 +35,7 @@ require (
 	github.com/lib/pq v1.10.2
 	github.com/ory/hydra-client-go v1.10.6
 	github.com/pelletier/go-toml v1.8.1 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
 	github.com/prometheus/client_golang v1.13.0
 	github.com/rocky-linux/srpmproc v0.3.16
 	github.com/sirupsen/logrus v1.8.1
@ -57,13 +60,12 @@ require (
 	k8s.io/apimachinery v0.22.1
 	k8s.io/client-go v0.22.1
 	openapi.peridot.resf.org/peridotopenapi v0.0.0-00010101000000-000000000000
+	peridot.resf.org/apollo/pb v0.0.0-00010101000000-000000000000 // indirect
 	peridot.resf.org/common v0.0.0-00010101000000-000000000000
 	peridot.resf.org/obsidian/pb v0.0.0-00010101000000-000000000000
 	peridot.resf.org/peridot/keykeeper/pb v0.0.0-00010101000000-000000000000
 	peridot.resf.org/peridot/pb v0.0.0-00010101000000-000000000000
 	peridot.resf.org/peridot/yumrepofs/pb v0.0.0-00010101000000-000000000000
-	peridot.resf.org/secparse/admin/proto v0.0.0-00010101000000-000000000000
-	peridot.resf.org/secparse/proto v0.0.0-00010101000000-000000000000
 )

 // Manual replace
@ -74,6 +76,7 @@ replace (

 // sync-replace-start
 replace (
+	peridot.resf.org/apollo/pb => ./bazel-bin/apollo/proto/v1/apollopb_go_proto_/peridot.resf.org/apollo/pb
 	bazel.build/protobuf => ./bazel-bin/build/bazel/protobuf/bazelbuild_go_proto_/bazel.build/protobuf
 	bazel.build/remote/execution/v2 => ./bazel-bin/build/bazel/remote/execution/v2/remoteexecution_go_proto_/bazel.build/remote/execution/v2
 	bazel.build/semver => ./bazel-bin/build/bazel/semver/semver_go_proto_/bazel.build/semver
@ -82,8 +85,6 @@ replace (
 	peridot.resf.org/peridot/keykeeper/pb => ./bazel-bin/peridot/proto/v1/keykeeper/keykeeperpb_go_proto_/peridot.resf.org/peridot/keykeeper/pb
 	peridot.resf.org/peridot/yumrepofs/pb => ./bazel-bin/peridot/proto/v1/yumrepofs/yumrepofspb_go_proto_/peridot.resf.org/peridot/yumrepofs/pb
 	peridot.resf.org/common => ./bazel-bin/proto/commonpb_go_proto_/peridot.resf.org/common
-	peridot.resf.org/secparse/admin/proto => ./bazel-bin/secparse/admin/proto/v1/secparseadminpb_go_proto_/peridot.resf.org/secparse/admin/proto
-	peridot.resf.org/secparse/proto => ./bazel-bin/secparse/proto/v1/secparsepb_go_proto_/peridot.resf.org/secparse/proto
 	github.com/envoyproxy/protoc-gen-validate/validate => ./bazel-bin/vendor/github.com/envoyproxy/protoc-gen-validate/validate/go_default_library_/github.com/envoyproxy/protoc-gen-validate/validate
 )
 // sync-replace-end
--- a/go.sum
+++ b/go.sum
@ -435,6 +435,8 @@ github.com/googleapis/gnostic v0.5.5 h1:9fHAtK0uDfpveeqqo1hkEZJcFvYXAiCN3UutL8F9
 github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
+github.com/gorilla/feeds v1.1.1 h1:HwKXxqzcRNg9to+BbvJog4+f3s/xzvtZXICcQGutYfY=
+github.com/gorilla/feeds v1.1.1/go.mod h1:Nk0jZrvPFZX1OBe5NPiddPw7CfwF6Q9eqzaBbaightA=
 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
--- a/hack/bazel_setup.sh
+++ b/hack/bazel_setup.sh
@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+export SILO_KEY=rocky86-peridot-prow-1
+export REMOTE_DEF=cache-silo-key=$SILO_KEY
+
+CI=${CI:-}
+QUERY_FLAGS=""
+FLAGS="--show_progress_rate_limit=5 --color=yes --ui_actions_shown=30 --terminal_columns=140 --show_timestamps --verbose_failures --announce_rc --experimental_repository_cache_hardlinks --disk_cache= --sandbox_tmpfs_path=/tmp --experimental_guard_against_concurrent_changes"
+
+if [[ -n ${CI} ]]; then
+  FLAGS="--config=ci $FLAGS"
+  QUERY_FLAGS="--config=ci"
+fi
+
+BAZEL_B="bazel $SOPTIONS build $FLAGS --remote_default_exec_properties=$REMOTE_DEF"
+BAZEL_R="bazel $SOPTIONS run $FLAGS --remote_default_exec_properties=$REMOTE_DEF"
+BAZEL_T="bazel $SOPTIONS test $FLAGS --remote_default_exec_properties=$REMOTE_DEF --test_arg=-test.v --flaky_test_attempts=3 --build_tests_only --test_output=errors"
+BAZEL_QR="bazel $SOPTIONS query $QUERY_FLAGS --keep_going --noshow_progress"
--- a/hack/gofmt_check.sh
+++ b/hack/gofmt_check.sh
@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+set -o errexit
+
+source hack/bazel_setup.sh
+
+$BAZEL_B @go_sdk//...
+
+GO_BIN="$(bazel info output_base)/external/go_sdk/bin/go"
+
+if [[ -n $($GO_BIN fmt -n $(go list ./... | grep -v /vendor/) | sed 's/ -w//') ]]; then
+  echo "Go files must be formatted with gofmt. Please run:"
+  echo "  go fmt $(go list ./... | grep -v /vendor/)"
+  exit 1
+fi
+
+
--- a/hack/non_manual_build.sh
+++ b/hack/non_manual_build.sh
@ -1,6 +1,7 @@
 #!/usr/bin/env bash

 set -o errexit
-set -x

-bazel build --config=ci $(bazel query "//... except attr(tags, 'manual', //...) except //vendor/...")
+source hack/bazel_setup.sh
+
+$BAZEL_B $($BAZEL_QR "//... except attr(tags, 'manual', //...) except //vendor/...")
--- a/hack/unit_tests.sh
+++ b/hack/unit_tests.sh
@ -1,6 +1,7 @@
 #!/usr/bin/env bash

 set -o errexit
-set -x

-bazel test --config=ci //...
+source hack/bazel_setup.sh
+
+$BAZEL_T //...
--- a/infrastructure/bazel-remote/003-deployment.yaml
+++ b/infrastructure/bazel-remote/003-deployment.yaml
@ -34,6 +34,7 @@ spec:
            - --s3.endpoint=s3.us-east-2.amazonaws.com
            - --max_size=98
            - --dir=/buildcache
+            - --experimental_remote_asset_api
          ports:
            - name: http
              containerPort: 8080
--- a/infrastructure/buildbuddy/005-virtualservice.yaml
+++ b/infrastructure/buildbuddy/005-virtualservice.yaml
@ -13,9 +13,6 @@ spec:
    - bz.build.resf.org
  http:
    - name: buildbuddy
-      match:
-        - uri:
-            prefix: "/invocation"
      route:
        - destination:
            host: buildbuddy.default.svc.cluster.local
--- a/infrastructure/prow/deployments.yaml
+++ b/infrastructure/prow/deployments.yaml
@ -30,6 +30,7 @@ spec:
          args:
            - --dry-run=false
            - --config-path=/etc/config/config.yaml
+            - --job-config-path=/etc/job-config
            - --github-endpoint=http://ghproxy
            - --github-endpoint=https://api.github.com
            - --github-app-id=$(GITHUB_APP_ID)
@ -53,6 +54,9 @@ spec:
            - name: config
              mountPath: /etc/config
              readOnly: true
+            - name: job-config
+              mountPath: /etc/job-config
+              readOnly: true
            - name: plugins
              mountPath: /etc/plugins
              readOnly: true
@ -79,6 +83,9 @@ spec:
        - name: config
          configMap:
            name: config
+        - name: job-config
+          configMap:
+            name: job-config
        - name: plugins
          configMap:
            name: plugins
@ -106,15 +113,22 @@ spec:
          image: gcr.io/k8s-prow/sinker:v20221028-a8625c1f93
          args:
            - --config-path=/etc/config/config.yaml
+            - --job-config-path=/etc/job-config
            - --dry-run=false
          volumeMounts:
            - name: config
              mountPath: /etc/config
              readOnly: true
+            - name: job-config
+              mountPath: /etc/job-config
+              readOnly: true
      volumes:
        - name: config
          configMap:
            name: config
+        - name: job-config
+          configMap:
+            name: job-config
 ---
 apiVersion: apps/v1
 kind: Deployment
@ -145,6 +159,7 @@ spec:
          image: gcr.io/k8s-prow/deck:v20221028-a8625c1f93
          args:
            - --config-path=/etc/config/config.yaml
+            - --job-config-path=/etc/job-config
            - --plugin-config=/etc/plugins/plugins.yaml
            - --tide-url=http://tide/
            - --hook-url=http://hook:8888/plugin-help
@ -168,6 +183,9 @@ spec:
            - name: config
              mountPath: /etc/config
              readOnly: true
+            - name: job-config
+              mountPath: /etc/job-config
+              readOnly: true
            - name: github-token
              mountPath: /etc/github
              readOnly: true
@ -194,6 +212,9 @@ spec:
        - name: config
          configMap:
            name: config
+        - name: job-config
+          configMap:
+            name: job-config
        - name: github-token
          secret:
            secretName: github-token
@ -231,14 +252,21 @@ spec:
          args:
            - --dry-run=false
            - --config-path=/etc/config/config.yaml
+            - --job-config-path=/etc/job-config
          volumeMounts:
            - name: config
              mountPath: /etc/config
              readOnly: true
+            - name: job-config
+              mountPath: /etc/job-config
+              readOnly: true
      volumes:
        - name: config
          configMap:
            name: config
+        - name: job-config
+          configMap:
+            name: job-config
 ---
 apiVersion: apps/v1
 kind: Deployment
@ -266,6 +294,7 @@ spec:
          args:
            - --dry-run=false
            - --config-path=/etc/config/config.yaml
+            - --job-config-path=/etc/job-config
            - --github-endpoint=http://ghproxy
            - --github-endpoint=https://api.github.com
            - --github-graphql-endpoint=http://ghproxy/graphql
@ -290,6 +319,9 @@ spec:
            - name: config
              mountPath: /etc/config
              readOnly: true
+            - name: job-config
+              mountPath: /etc/job-config
+              readOnly: true
            - name: s3-credentials
              mountPath: /etc/s3-credentials
              readOnly: true
@ -300,6 +332,9 @@ spec:
        - name: config
          configMap:
            name: config
+        - name: job-config
+          configMap:
+            name: job-config
        - name: s3-credentials
          secret:
            secretName: s3-credentials
@ -331,6 +366,7 @@ spec:
            - --continue-on-error=true
            - --plugin-config=/etc/plugins/plugins.yaml
            - --config-path=/etc/config/config.yaml
+            - --job-config-path=/etc/job-config
            - --github-endpoint=http://ghproxy
            - --github-endpoint=https://api.github.com
            - --s3-credentials-file=/etc/s3-credentials/service-account.json
@ -350,6 +386,9 @@ spec:
            - name: config
              mountPath: /etc/config
              readOnly: true
+            - name: job-config
+              mountPath: /etc/job-config
+              readOnly: true
            - name: plugins
              mountPath: /etc/plugins
              readOnly: true
@ -363,6 +402,9 @@ spec:
        - name: config
          configMap:
            name: config
+        - name: job-config
+          configMap:
+            name: job-config
        - name: plugins
          configMap:
            name: plugins
@ -430,6 +472,7 @@ spec:
          args:
            - --dry-run=false
            - --config-path=/etc/config/config.yaml
+            - --job-config-path=/etc/job-config
            - --github-endpoint=http://ghproxy
            - --github-endpoint=https://api.github.com
            - --enable-controller=plank
@ -449,6 +492,9 @@ spec:
            - name: config
              mountPath: /etc/config
              readOnly: true
+            - name: job-config
+              mountPath: /etc/job-config
+              readOnly: true
      volumes:
        - name: github-token
          secret:
@ -456,6 +502,9 @@ spec:
        - name: config
          configMap:
            name: config
+        - name: job-config
+          configMap:
+            name: job-config
 ---
 apiVersion: apps/v1
 kind: Deployment
@ -482,6 +531,7 @@ spec:
          args:
            - --blob-storage-workers=10
            - --config-path=/etc/config/config.yaml
+            - --job-config-path=/etc/job-config
            - --s3-credentials-file=/etc/s3-credentials/service-account.json
            - --github-endpoint=http://ghproxy
            - --github-endpoint=https://api.github.com
@ -499,6 +549,9 @@ spec:
            - name: config
              mountPath: /etc/config
              readOnly: true
+            - name: job-config
+              mountPath: /etc/job-config
+              readOnly: true
            - name: github-token
              mountPath: /etc/github
              readOnly: true
@ -509,6 +562,9 @@ spec:
        - name: config
          configMap:
            name: config
+        - name: job-config
+          configMap:
+            name: job-config
        - name: github-token
          secret:
            secretName: github-token
--- a/peridot/cmd/v1/peridotbuilder/main.go
+++ b/peridot/cmd/v1/peridotbuilder/main.go
@ -31,6 +31,8 @@
 package main

 import (
+	"log"
+
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
@ -39,7 +41,6 @@ import (
 	"google.golang.org/grpc/credentials/insecure"
 	"google.golang.org/protobuf/encoding/protojson"
 	"google.golang.org/protobuf/types/known/anypb"
-	"log"
 	builderv1 "peridot.resf.org/peridot/builder/v1"
 	peridotcommon "peridot.resf.org/peridot/common"
 	serverconnector "peridot.resf.org/peridot/db/connector"
--- a/vendor/modules.txt
+++ b/vendor/modules.txt
@ -403,8 +403,6 @@ github.com/matttproud/golang_protobuf_extensions/pbutil
 github.com/mitchellh/go-homedir
 # github.com/mitchellh/mapstructure v1.4.1
 github.com/mitchellh/mapstructure
-# github.com/mmcdole/gofeed v1.1.3
-## explicit
 # github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd
 github.com/modern-go/concurrent
 # github.com/modern-go/reflect2 v1.0.2
@ -447,8 +445,6 @@ github.com/prometheus/common/model
 github.com/prometheus/procfs
 github.com/prometheus/procfs/internal/fs
 github.com/prometheus/procfs/internal/util
-# github.com/quay/goval-parser v0.8.7
-## explicit
 # github.com/rivo/uniseg v0.2.0
 github.com/rivo/uniseg
 # github.com/robfig/cron v1.2.0