From b303e3ffbd55f652670330b0ff7f50bd7d8ed766 Mon Sep 17 00:00:00 2001 From: Mustafa Gezen Date: Sun, 30 Oct 2022 05:03:28 +0100 Subject: [PATCH 01/12] Add job-config mount and argument to Prow deployments --- infrastructure/prow/deployments.yaml | 56 ++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/infrastructure/prow/deployments.yaml b/infrastructure/prow/deployments.yaml index ab3c49b..166bf8b 100644 --- a/infrastructure/prow/deployments.yaml +++ b/infrastructure/prow/deployments.yaml @@ -30,6 +30,7 @@ spec: args: - --dry-run=false - --config-path=/etc/config/config.yaml + - --job-config-path=/etc/job-config - --github-endpoint=http://ghproxy - --github-endpoint=https://api.github.com - --github-app-id=$(GITHUB_APP_ID) @@ -53,6 +54,9 @@ spec: - name: config mountPath: /etc/config readOnly: true + - name: job-config + mountPath: /etc/job-config + readOnly: true - name: plugins mountPath: /etc/plugins readOnly: true @@ -79,6 +83,9 @@ spec: - name: config configMap: name: config + - name: job-config + configMap: + name: job-config - name: plugins configMap: name: plugins @@ -106,15 +113,22 @@ spec: image: gcr.io/k8s-prow/sinker:v20221028-a8625c1f93 args: - --config-path=/etc/config/config.yaml + - --job-config-path=/etc/job-config - --dry-run=false volumeMounts: - name: config mountPath: /etc/config readOnly: true + - name: job-config + mountPath: /etc/job-config + readOnly: true volumes: - name: config configMap: name: config + - name: job-config + configMap: + name: job-config --- apiVersion: apps/v1 kind: Deployment @@ -145,6 +159,7 @@ spec: image: gcr.io/k8s-prow/deck:v20221028-a8625c1f93 args: - --config-path=/etc/config/config.yaml + - --job-config-path=/etc/job-config - --plugin-config=/etc/plugins/plugins.yaml - --tide-url=http://tide/ - --hook-url=http://hook:8888/plugin-help @@ -168,6 +183,9 @@ spec: - name: config mountPath: /etc/config readOnly: true + - name: job-config + mountPath: /etc/job-config + readOnly: true - name: github-token mountPath: /etc/github readOnly: true @@ -194,6 +212,9 @@ spec: - name: config configMap: name: config + - name: job-config + configMap: + name: job-config - name: github-token secret: secretName: github-token @@ -231,14 +252,21 @@ spec: args: - --dry-run=false - --config-path=/etc/config/config.yaml + - --job-config-path=/etc/job-config volumeMounts: - name: config mountPath: /etc/config readOnly: true + - name: job-config + mountPath: /etc/job-config + readOnly: true volumes: - name: config configMap: name: config + - name: job-config + configMap: + name: job-config --- apiVersion: apps/v1 kind: Deployment @@ -266,6 +294,7 @@ spec: args: - --dry-run=false - --config-path=/etc/config/config.yaml + - --job-config-path=/etc/job-config - --github-endpoint=http://ghproxy - --github-endpoint=https://api.github.com - --github-graphql-endpoint=http://ghproxy/graphql @@ -290,6 +319,9 @@ spec: - name: config mountPath: /etc/config readOnly: true + - name: job-config + mountPath: /etc/job-config + readOnly: true - name: s3-credentials mountPath: /etc/s3-credentials readOnly: true @@ -300,6 +332,9 @@ spec: - name: config configMap: name: config + - name: job-config + configMap: + name: job-config - name: s3-credentials secret: secretName: s3-credentials @@ -331,6 +366,7 @@ spec: - --continue-on-error=true - --plugin-config=/etc/plugins/plugins.yaml - --config-path=/etc/config/config.yaml + - --job-config-path=/etc/job-config - --github-endpoint=http://ghproxy - --github-endpoint=https://api.github.com - --s3-credentials-file=/etc/s3-credentials/service-account.json @@ -350,6 +386,9 @@ spec: - name: config mountPath: /etc/config readOnly: true + - name: job-config + mountPath: /etc/job-config + readOnly: true - name: plugins mountPath: /etc/plugins readOnly: true @@ -363,6 +402,9 @@ spec: - name: config configMap: name: config + - name: job-config + configMap: + name: job-config - name: plugins configMap: name: plugins @@ -430,6 +472,7 @@ spec: args: - --dry-run=false - --config-path=/etc/config/config.yaml + - --job-config-path=/etc/job-config - --github-endpoint=http://ghproxy - --github-endpoint=https://api.github.com - --enable-controller=plank @@ -449,6 +492,9 @@ spec: - name: config mountPath: /etc/config readOnly: true + - name: job-config + mountPath: /etc/job-config + readOnly: true volumes: - name: github-token secret: @@ -456,6 +502,9 @@ spec: - name: config configMap: name: config + - name: job-config + configMap: + name: job-config --- apiVersion: apps/v1 kind: Deployment @@ -482,6 +531,7 @@ spec: args: - --blob-storage-workers=10 - --config-path=/etc/config/config.yaml + - --job-config-path=/etc/job-config - --s3-credentials-file=/etc/s3-credentials/service-account.json - --github-endpoint=http://ghproxy - --github-endpoint=https://api.github.com @@ -499,6 +549,9 @@ spec: - name: config mountPath: /etc/config readOnly: true + - name: job-config + mountPath: /etc/job-config + readOnly: true - name: github-token mountPath: /etc/github readOnly: true @@ -509,6 +562,9 @@ spec: - name: config configMap: name: config + - name: job-config + configMap: + name: job-config - name: github-token secret: secretName: github-token From 65f67b7fd60f88cef7b08d0fd99d2676b2689967 Mon Sep 17 00:00:00 2001 From: Mustafa Gezen Date: Sun, 30 Oct 2022 05:16:07 +0100 Subject: [PATCH 02/12] Enable experimental remote asset API --- .bazelrc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.bazelrc b/.bazelrc index c8ad09c..c9e535d 100644 --- a/.bazelrc +++ b/.bazelrc @@ -17,7 +17,7 @@ build:toplevel --remote_download_outputs=toplevel build:remote --config=toplevel build:remote --remote_cache=grpc://buildcache.default.svc.cluster.local:9092 -build:remote --experimental_remote_downloader=grpc://buildcache.default.svc.cluster.local:9092 +build:remote --experimental_remote_asset_api --experimental_remote_downloader=grpc://buildcache.default.svc.cluster.local:9092 build:remote --noremote_upload_local_results build:remote --remote_timeout=3600 build:remote --bes_results_url=https://bz.build.resf.org/invocation/ From 35be9b6184552a02469b97dfec339112bffc651c Mon Sep 17 00:00:00 2001 From: Mustafa Gezen Date: Sun, 30 Oct 2022 05:19:32 +0100 Subject: [PATCH 03/12] It's bazel-remote not bazelrc (experimental_remote_api) --- .bazelrc | 2 +- infrastructure/bazel-remote/003-deployment.yaml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.bazelrc b/.bazelrc index c9e535d..c8ad09c 100644 --- a/.bazelrc +++ b/.bazelrc @@ -17,7 +17,7 @@ build:toplevel --remote_download_outputs=toplevel build:remote --config=toplevel build:remote --remote_cache=grpc://buildcache.default.svc.cluster.local:9092 -build:remote --experimental_remote_asset_api --experimental_remote_downloader=grpc://buildcache.default.svc.cluster.local:9092 +build:remote --experimental_remote_downloader=grpc://buildcache.default.svc.cluster.local:9092 build:remote --noremote_upload_local_results build:remote --remote_timeout=3600 build:remote --bes_results_url=https://bz.build.resf.org/invocation/ diff --git a/infrastructure/bazel-remote/003-deployment.yaml b/infrastructure/bazel-remote/003-deployment.yaml index 727d46f..df875b8 100644 --- a/infrastructure/bazel-remote/003-deployment.yaml +++ b/infrastructure/bazel-remote/003-deployment.yaml @@ -34,6 +34,7 @@ spec: - --s3.endpoint=s3.us-east-2.amazonaws.com - --max_size=98 - --dir=/buildcache + - --experimental_remote_asset_api ports: - name: http containerPort: 8080 From 48788cfd4c39472b7720518c508069276228cdc5 Mon Sep 17 00:00:00 2001 From: Mustafa Gezen Date: Sun, 30 Oct 2022 06:02:42 +0100 Subject: [PATCH 04/12] Update govendor --- go.mod | 9 +++++---- go.sum | 2 ++ peridot/cmd/v1/peridotbuilder/main.go | 3 ++- vendor/modules.txt | 4 ---- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 40bb9f1..6296fdd 100644 --- a/go.mod +++ b/go.mod @@ -26,6 +26,8 @@ require ( github.com/gocolly/colly/v2 v2.1.0 github.com/gogo/status v1.1.0 github.com/google/uuid v1.3.0 + github.com/gorilla/feeds v1.1.1 // indirect + github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.6.0 github.com/imdario/mergo v0.3.11 // indirect @@ -33,6 +35,7 @@ require ( github.com/lib/pq v1.10.2 github.com/ory/hydra-client-go v1.10.6 github.com/pelletier/go-toml v1.8.1 // indirect + github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.13.0 github.com/rocky-linux/srpmproc v0.3.16 github.com/sirupsen/logrus v1.8.1 @@ -57,13 +60,12 @@ require ( k8s.io/apimachinery v0.22.1 k8s.io/client-go v0.22.1 openapi.peridot.resf.org/peridotopenapi v0.0.0-00010101000000-000000000000 + peridot.resf.org/apollo/pb v0.0.0-00010101000000-000000000000 // indirect peridot.resf.org/common v0.0.0-00010101000000-000000000000 peridot.resf.org/obsidian/pb v0.0.0-00010101000000-000000000000 peridot.resf.org/peridot/keykeeper/pb v0.0.0-00010101000000-000000000000 peridot.resf.org/peridot/pb v0.0.0-00010101000000-000000000000 peridot.resf.org/peridot/yumrepofs/pb v0.0.0-00010101000000-000000000000 - peridot.resf.org/secparse/admin/proto v0.0.0-00010101000000-000000000000 - peridot.resf.org/secparse/proto v0.0.0-00010101000000-000000000000 ) // Manual replace @@ -74,6 +76,7 @@ replace ( // sync-replace-start replace ( + peridot.resf.org/apollo/pb => ./bazel-bin/apollo/proto/v1/apollopb_go_proto_/peridot.resf.org/apollo/pb bazel.build/protobuf => ./bazel-bin/build/bazel/protobuf/bazelbuild_go_proto_/bazel.build/protobuf bazel.build/remote/execution/v2 => ./bazel-bin/build/bazel/remote/execution/v2/remoteexecution_go_proto_/bazel.build/remote/execution/v2 bazel.build/semver => ./bazel-bin/build/bazel/semver/semver_go_proto_/bazel.build/semver @@ -82,8 +85,6 @@ replace ( peridot.resf.org/peridot/keykeeper/pb => ./bazel-bin/peridot/proto/v1/keykeeper/keykeeperpb_go_proto_/peridot.resf.org/peridot/keykeeper/pb peridot.resf.org/peridot/yumrepofs/pb => ./bazel-bin/peridot/proto/v1/yumrepofs/yumrepofspb_go_proto_/peridot.resf.org/peridot/yumrepofs/pb peridot.resf.org/common => ./bazel-bin/proto/commonpb_go_proto_/peridot.resf.org/common - peridot.resf.org/secparse/admin/proto => ./bazel-bin/secparse/admin/proto/v1/secparseadminpb_go_proto_/peridot.resf.org/secparse/admin/proto - peridot.resf.org/secparse/proto => ./bazel-bin/secparse/proto/v1/secparsepb_go_proto_/peridot.resf.org/secparse/proto github.com/envoyproxy/protoc-gen-validate/validate => ./bazel-bin/vendor/github.com/envoyproxy/protoc-gen-validate/validate/go_default_library_/github.com/envoyproxy/protoc-gen-validate/validate ) // sync-replace-end diff --git a/go.sum b/go.sum index 1d0f898..db475bf 100644 --- a/go.sum +++ b/go.sum @@ -435,6 +435,8 @@ github.com/googleapis/gnostic v0.5.5 h1:9fHAtK0uDfpveeqqo1hkEZJcFvYXAiCN3UutL8F9 github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/feeds v1.1.1 h1:HwKXxqzcRNg9to+BbvJog4+f3s/xzvtZXICcQGutYfY= +github.com/gorilla/feeds v1.1.1/go.mod h1:Nk0jZrvPFZX1OBe5NPiddPw7CfwF6Q9eqzaBbaightA= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= diff --git a/peridot/cmd/v1/peridotbuilder/main.go b/peridot/cmd/v1/peridotbuilder/main.go index fc3c899..ffe1b9b 100644 --- a/peridot/cmd/v1/peridotbuilder/main.go +++ b/peridot/cmd/v1/peridotbuilder/main.go @@ -31,6 +31,8 @@ package main import ( + "log" + "github.com/sirupsen/logrus" "github.com/spf13/cobra" "github.com/spf13/viper" @@ -39,7 +41,6 @@ import ( "google.golang.org/grpc/credentials/insecure" "google.golang.org/protobuf/encoding/protojson" "google.golang.org/protobuf/types/known/anypb" - "log" builderv1 "peridot.resf.org/peridot/builder/v1" peridotcommon "peridot.resf.org/peridot/common" serverconnector "peridot.resf.org/peridot/db/connector" diff --git a/vendor/modules.txt b/vendor/modules.txt index 0c46774..eb84519 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -403,8 +403,6 @@ github.com/matttproud/golang_protobuf_extensions/pbutil github.com/mitchellh/go-homedir # github.com/mitchellh/mapstructure v1.4.1 github.com/mitchellh/mapstructure -# github.com/mmcdole/gofeed v1.1.3 -## explicit # github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd github.com/modern-go/concurrent # github.com/modern-go/reflect2 v1.0.2 @@ -447,8 +445,6 @@ github.com/prometheus/common/model github.com/prometheus/procfs github.com/prometheus/procfs/internal/fs github.com/prometheus/procfs/internal/util -# github.com/quay/goval-parser v0.8.7 -## explicit # github.com/rivo/uniseg v0.2.0 github.com/rivo/uniseg # github.com/robfig/cron v1.2.0 From b05450b3b38431f54cec7cf9ddf24e63ac496e63 Mon Sep 17 00:00:00 2001 From: Mustafa Gezen Date: Sun, 30 Oct 2022 06:02:57 +0100 Subject: [PATCH 05/12] Support full buildbuddy paths --- infrastructure/buildbuddy/005-virtualservice.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/infrastructure/buildbuddy/005-virtualservice.yaml b/infrastructure/buildbuddy/005-virtualservice.yaml index 58dcee5..5090318 100644 --- a/infrastructure/buildbuddy/005-virtualservice.yaml +++ b/infrastructure/buildbuddy/005-virtualservice.yaml @@ -13,9 +13,6 @@ spec: - bz.build.resf.org http: - name: buildbuddy - match: - - uri: - prefix: "/invocation" route: - destination: host: buildbuddy.default.svc.cluster.local From 842af93afa5540a9c75743127872fa1775a3e7b0 Mon Sep 17 00:00:00 2001 From: Mustafa Gezen Date: Sun, 30 Oct 2022 06:03:06 +0100 Subject: [PATCH 06/12] Add gofmt checks to presubmit --- config/jobs/presubmits.yaml | 11 +++++++++++ hack/gofmt_check.sh | 15 +++++++++++++++ 2 files changed, 26 insertions(+) create mode 100755 hack/gofmt_check.sh diff --git a/config/jobs/presubmits.yaml b/config/jobs/presubmits.yaml index da95518..aae4ae4 100644 --- a/config/jobs/presubmits.yaml +++ b/config/jobs/presubmits.yaml @@ -35,3 +35,14 @@ presubmits: - image: quay.io/peridot/ci:v0.3.16 command: - hack/non_manual_build.sh + + - name: pull-peridot-gofmt-check + branches: + - main + always_run: true + decorate: true + spec: + containers: + - image: quay.io/peridot/ci:v0.3.16 + command: + - hack/gofmt_check.sh diff --git a/hack/gofmt_check.sh b/hack/gofmt_check.sh new file mode 100755 index 0000000..545b3a9 --- /dev/null +++ b/hack/gofmt_check.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +set -o errexit + +bazel build --config=ci @go_sdk//... + +GO_BIN="$(bazel info output_base)/external/go_sdk/bin/go" + +if [[ -n $($GO_BIN fmt -n $(go list ./... | grep -v /vendor/) | sed 's/ -w//') ]]; then + echo "Go files must be formatted with gofmt. Please run:" + echo " go fmt $(go list ./... | grep -v /vendor/)" + exit 1 +fi + + From bb7993168370e5dd28cd5904b2100583038335be Mon Sep 17 00:00:00 2001 From: Mustafa Gezen Date: Sun, 30 Oct 2022 06:18:12 +0100 Subject: [PATCH 07/12] Standard bazel commands + remote_def --- hack/bazel_setup.sh | 16 ++++++++++++++++ hack/gofmt_check.sh | 4 +++- hack/non_manual_build.sh | 4 +++- hack/unit_tests.sh | 4 +++- 4 files changed, 25 insertions(+), 3 deletions(-) create mode 100644 hack/bazel_setup.sh diff --git a/hack/bazel_setup.sh b/hack/bazel_setup.sh new file mode 100644 index 0000000..2b0f6b9 --- /dev/null +++ b/hack/bazel_setup.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +export SILO_KEY=rocky86-peridot-prow-1 +export REMOTE_DEF=cache-silo-key=$SILO_KEY + +CI=${CI:-} +FLAGS="--show_progress_rate_limit=5 --color=yes --ui_actions_shown=30 --terminal_columns=140 --show_timestamps --verbose_failures --announce_rc --experimental_repository_cache_hardlinks --disk_cache= --sandbox_tmpfs_path=/tmp --experimental_guard_against_concurrent_changes" + +if [[ -n ${CI} ]]; then + FLAGS="--config=ci $FLAGS" +fi + +BAZEL_B="bazel $SOPTIONS build $FLAGS --remote_default_exec_properties=$REMOTE_DEF" +BAZEL_R="bazel $SOPTIONS run $FLAGS --remote_default_exec_properties=$REMOTE_DEF" +BAZEL_T="bazel $SOPTIONS test $FLAGS --remote_default_exec_properties=$REMOTE_DEF --test_arg=-test.v --flaky_test_attempts=3 --build_tests_only --test_output=errors" +BAZEL_QR="bazel $SOPTIONS query --keep_going --noshow_progress" diff --git a/hack/gofmt_check.sh b/hack/gofmt_check.sh index 545b3a9..8b51df6 100755 --- a/hack/gofmt_check.sh +++ b/hack/gofmt_check.sh @@ -2,7 +2,9 @@ set -o errexit -bazel build --config=ci @go_sdk//... +source hack/bazel_setup.sh + +$BAZEL_B @go_sdk//... GO_BIN="$(bazel info output_base)/external/go_sdk/bin/go" diff --git a/hack/non_manual_build.sh b/hack/non_manual_build.sh index 5856a0a..00abbf7 100755 --- a/hack/non_manual_build.sh +++ b/hack/non_manual_build.sh @@ -3,4 +3,6 @@ set -o errexit set -x -bazel build --config=ci $(bazel query "//... except attr(tags, 'manual', //...) except //vendor/...") +source hack/bazel_setup.sh + +$BAZEL_B $(BAZEL_QR query "//... except attr(tags, 'manual', //...) except //vendor/...") diff --git a/hack/unit_tests.sh b/hack/unit_tests.sh index 268d372..3f1c06b 100755 --- a/hack/unit_tests.sh +++ b/hack/unit_tests.sh @@ -3,4 +3,6 @@ set -o errexit set -x -bazel test --config=ci //... +source hack/bazel_setup.sh + +$BAZEL_T //... From 343a8f9ce4c08b1222c89c21fc21e03f8f1f2a5d Mon Sep 17 00:00:00 2001 From: Mustafa Gezen Date: Sun, 30 Oct 2022 06:20:56 +0100 Subject: [PATCH 08/12] Disable set -x and fix BAZEL_QR --- hack/non_manual_build.sh | 3 +-- hack/unit_tests.sh | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/hack/non_manual_build.sh b/hack/non_manual_build.sh index 00abbf7..4a0b270 100755 --- a/hack/non_manual_build.sh +++ b/hack/non_manual_build.sh @@ -1,8 +1,7 @@ #!/usr/bin/env bash set -o errexit -set -x source hack/bazel_setup.sh -$BAZEL_B $(BAZEL_QR query "//... except attr(tags, 'manual', //...) except //vendor/...") +$BAZEL_B $($BAZEL_QR query "//... except attr(tags, 'manual', //...) except //vendor/...") diff --git a/hack/unit_tests.sh b/hack/unit_tests.sh index 3f1c06b..b09be5b 100755 --- a/hack/unit_tests.sh +++ b/hack/unit_tests.sh @@ -1,7 +1,6 @@ #!/usr/bin/env bash set -o errexit -set -x source hack/bazel_setup.sh From be2d10790928630a231d6231701032a7c53fc031 Mon Sep 17 00:00:00 2001 From: Mustafa Gezen Date: Sun, 30 Oct 2022 06:22:13 +0100 Subject: [PATCH 09/12] Fix non_manual_build query --- hack/non_manual_build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hack/non_manual_build.sh b/hack/non_manual_build.sh index 4a0b270..69826c2 100755 --- a/hack/non_manual_build.sh +++ b/hack/non_manual_build.sh @@ -4,4 +4,4 @@ set -o errexit source hack/bazel_setup.sh -$BAZEL_B $($BAZEL_QR query "//... except attr(tags, 'manual', //...) except //vendor/...") +$BAZEL_B $($BAZEL_QR "//... except attr(tags, 'manual', //...) except //vendor/...") From 9f0644e272f2c9a35fed73527a47b2e7eb77dc1f Mon Sep 17 00:00:00 2001 From: Mustafa Gezen Date: Sun, 30 Oct 2022 06:47:57 +0100 Subject: [PATCH 10/12] Run bazel builds/tests on better nodes and add postsubmit test --- config/jobs/postsubmits.yaml | 20 ++++++++++++++++++++ config/jobs/presubmits.yaml | 14 ++++++++++++++ 2 files changed, 34 insertions(+) create mode 100644 config/jobs/postsubmits.yaml diff --git a/config/jobs/postsubmits.yaml b/config/jobs/postsubmits.yaml new file mode 100644 index 0000000..59e9174 --- /dev/null +++ b/config/jobs/postsubmits.yaml @@ -0,0 +1,20 @@ +postsubmits: + rocky-linux/peridot: + - name: post-peridot-unit-test + always_run: true + decorate: true + branches: + - main + spec: + tolerations: + - key: peridot.rockylinux.org/workflow-tolerates-arch + operator: Equal + value: amd64 + effect: NoSchedule + nodeSelector: + peridot.rockylinux.org/workflow-tolerates-arch: amd64 + containers: + - image: quay.io/peridot/ci:v0.3.16 + command: + - hack/unit_tests.sh + diff --git a/config/jobs/presubmits.yaml b/config/jobs/presubmits.yaml index aae4ae4..0f59c7b 100644 --- a/config/jobs/presubmits.yaml +++ b/config/jobs/presubmits.yaml @@ -20,6 +20,13 @@ presubmits: always_run: true decorate: true spec: + tolerations: + - key: peridot.rockylinux.org/workflow-tolerates-arch + operator: Equal + value: amd64 + effect: NoSchedule + nodeSelector: + peridot.rockylinux.org/workflow-tolerates-arch: amd64 containers: - image: quay.io/peridot/ci:v0.3.16 command: @@ -31,6 +38,13 @@ presubmits: always_run: true decorate: true spec: + tolerations: + - key: peridot.rockylinux.org/workflow-tolerates-arch + operator: Equal + value: amd64 + effect: NoSchedule + nodeSelector: + peridot.rockylinux.org/workflow-tolerates-arch: amd64 containers: - image: quay.io/peridot/ci:v0.3.16 command: From d6d5f1c191553b0a8a2a9309f921ebdb400cc98e Mon Sep 17 00:00:00 2001 From: Mustafa Gezen Date: Sun, 30 Oct 2022 06:55:36 +0100 Subject: [PATCH 11/12] Use remote downloader during fetch --- .bazelrc | 5 +++-- hack/bazel_setup.sh | 4 +++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.bazelrc b/.bazelrc index c8ad09c..a794dd5 100644 --- a/.bazelrc +++ b/.bazelrc @@ -16,8 +16,9 @@ build:toplevel --config=inmemory build:toplevel --remote_download_outputs=toplevel build:remote --config=toplevel -build:remote --remote_cache=grpc://buildcache.default.svc.cluster.local:9092 -build:remote --experimental_remote_downloader=grpc://buildcache.default.svc.cluster.local:9092 +build:remote --remote_cache=grpc://buildcache.default.svc.cluster.local:9092 --experimental_remote_downloader=grpc://buildcache.default.svc.cluster.local:9092 +query:remote --remote_cache=grpc://buildcache.default.svc.cluster.local:9092 --experimental_remote_downloader=grpc://buildcache.default.svc.cluster.local:9092 +fetch:remote --remote_cache=grpc://buildcache.default.svc.cluster.local:9092 --experimental_remote_downloader=grpc://buildcache.default.svc.cluster.local:9092 build:remote --noremote_upload_local_results build:remote --remote_timeout=3600 build:remote --bes_results_url=https://bz.build.resf.org/invocation/ diff --git a/hack/bazel_setup.sh b/hack/bazel_setup.sh index 2b0f6b9..35787f5 100644 --- a/hack/bazel_setup.sh +++ b/hack/bazel_setup.sh @@ -4,13 +4,15 @@ export SILO_KEY=rocky86-peridot-prow-1 export REMOTE_DEF=cache-silo-key=$SILO_KEY CI=${CI:-} +QUERY_FLAGS="" FLAGS="--show_progress_rate_limit=5 --color=yes --ui_actions_shown=30 --terminal_columns=140 --show_timestamps --verbose_failures --announce_rc --experimental_repository_cache_hardlinks --disk_cache= --sandbox_tmpfs_path=/tmp --experimental_guard_against_concurrent_changes" if [[ -n ${CI} ]]; then FLAGS="--config=ci $FLAGS" + QUERY_FLAGS="--config=ci" fi BAZEL_B="bazel $SOPTIONS build $FLAGS --remote_default_exec_properties=$REMOTE_DEF" BAZEL_R="bazel $SOPTIONS run $FLAGS --remote_default_exec_properties=$REMOTE_DEF" BAZEL_T="bazel $SOPTIONS test $FLAGS --remote_default_exec_properties=$REMOTE_DEF --test_arg=-test.v --flaky_test_attempts=3 --build_tests_only --test_output=errors" -BAZEL_QR="bazel $SOPTIONS query --keep_going --noshow_progress" +BAZEL_QR="bazel $SOPTIONS query $QUERY_FLAGS --keep_going --noshow_progress" From 5e5bfc032fdeb2bf16ad350c5309c4f5be391ab6 Mon Sep 17 00:00:00 2001 From: Mustafa Gezen Date: Sun, 30 Oct 2022 07:04:22 +0100 Subject: [PATCH 12/12] Upload local actions to cache --- .bazelrc | 1 - 1 file changed, 1 deletion(-) diff --git a/.bazelrc b/.bazelrc index a794dd5..c65be71 100644 --- a/.bazelrc +++ b/.bazelrc @@ -19,7 +19,6 @@ build:remote --config=toplevel build:remote --remote_cache=grpc://buildcache.default.svc.cluster.local:9092 --experimental_remote_downloader=grpc://buildcache.default.svc.cluster.local:9092 query:remote --remote_cache=grpc://buildcache.default.svc.cluster.local:9092 --experimental_remote_downloader=grpc://buildcache.default.svc.cluster.local:9092 fetch:remote --remote_cache=grpc://buildcache.default.svc.cluster.local:9092 --experimental_remote_downloader=grpc://buildcache.default.svc.cluster.local:9092 -build:remote --noremote_upload_local_results build:remote --remote_timeout=3600 build:remote --bes_results_url=https://bz.build.resf.org/invocation/ build:remote --bes_backend=grpc://buildbuddy-grpc.default.svc.cluster.local:1985