From 8a1cabe9944ad240563e120e0a8494a8b4fbd0d2 Mon Sep 17 00:00:00 2001 From: Mustafa Gezen Date: Tue, 1 Nov 2022 01:14:38 +0100 Subject: [PATCH] Postsubmit: Deploy impacted targets --- config/jobs/postsubmits.yaml | 23 +++++++++++++++++++++-- config/jobs/presubmits.yaml | 8 ++++---- hack/bazel_setup.sh | 7 +++++++ hack/deploy_impacted.sh | 33 +++++++++++++++++++++++++++++++++ hack/frontend_build.sh | 18 ++---------------- hack/get_impacted_targets.sh | 23 +++++++++++++++++++++++ rules_resf/defs.bzl | 6 ++++-- 7 files changed, 94 insertions(+), 24 deletions(-) create mode 100755 hack/deploy_impacted.sh create mode 100755 hack/get_impacted_targets.sh diff --git a/config/jobs/postsubmits.yaml b/config/jobs/postsubmits.yaml index bc48d601..dcee8fe8 100644 --- a/config/jobs/postsubmits.yaml +++ b/config/jobs/postsubmits.yaml @@ -14,7 +14,7 @@ postsubmits: nodeSelector: peridot.rockylinux.org/workflow-tolerates-arch: amd64 containers: - - image: quay.io/peridot/ci:v0.3.16 + - image: quay.io/peridot/ci:v0.3.17 command: - hack/unit_tests.sh @@ -32,6 +32,25 @@ postsubmits: nodeSelector: peridot.rockylinux.org/workflow-tolerates-arch: amd64 containers: - - image: quay.io/peridot/ci:v0.3.16 + - image: quay.io/peridot/ci:v0.3.17 command: - hack/build_all_frontends.sh + + - name: post-peridot-deploy-impacted + always_run: true + decorate: true + branches: + - main + spec: + serviceAccountName: peridot-deploy + tolerations: + - key: peridot.rockylinux.org/workflow-tolerates-arch + operator: Equal + value: amd64 + effect: NoSchedule + nodeSelector: + peridot.rockylinux.org/workflow-tolerates-arch: amd64 + containers: + - image: quay.io/peridot/ci:v0.3.17 + command: + - hack/deploy_impacted.sh diff --git a/config/jobs/presubmits.yaml b/config/jobs/presubmits.yaml index 9bff83ce..b4e6372c 100644 --- a/config/jobs/presubmits.yaml +++ b/config/jobs/presubmits.yaml @@ -28,7 +28,7 @@ presubmits: nodeSelector: peridot.rockylinux.org/workflow-tolerates-arch: amd64 containers: - - image: quay.io/peridot/ci:v0.3.16 + - image: quay.io/peridot/ci:v0.3.17 command: - hack/unit_tests.sh @@ -46,7 +46,7 @@ presubmits: nodeSelector: peridot.rockylinux.org/workflow-tolerates-arch: amd64 containers: - - image: quay.io/peridot/ci:v0.3.16 + - image: quay.io/peridot/ci:v0.3.17 command: - hack/non_manual_build.sh @@ -64,7 +64,7 @@ presubmits: nodeSelector: peridot.rockylinux.org/workflow-tolerates-arch: amd64 containers: - - image: quay.io/peridot/ci:v0.3.16 + - image: quay.io/peridot/ci:v0.3.17 command: - hack/frontend_build.sh @@ -75,6 +75,6 @@ presubmits: decorate: true spec: containers: - - image: quay.io/peridot/ci:v0.3.16 + - image: quay.io/peridot/ci:v0.3.17 command: - hack/gofmt_check.sh diff --git a/hack/bazel_setup.sh b/hack/bazel_setup.sh index 35787f51..c220facf 100644 --- a/hack/bazel_setup.sh +++ b/hack/bazel_setup.sh @@ -16,3 +16,10 @@ BAZEL_B="bazel $SOPTIONS build $FLAGS --remote_default_exec_properties=$REMOTE_D BAZEL_R="bazel $SOPTIONS run $FLAGS --remote_default_exec_properties=$REMOTE_DEF" BAZEL_T="bazel $SOPTIONS test $FLAGS --remote_default_exec_properties=$REMOTE_DEF --test_arg=-test.v --flaky_test_attempts=3 --build_tests_only --test_output=errors" BAZEL_QR="bazel $SOPTIONS query $QUERY_FLAGS --keep_going --noshow_progress" + +return_if_impacted_targets_empty() { + if [[ -z "$(cat impacted_targets)" ]]; then + echo "No impacted targets found" + exit 0 + fi +} diff --git a/hack/deploy_impacted.sh b/hack/deploy_impacted.sh new file mode 100755 index 00000000..68c4dec5 --- /dev/null +++ b/hack/deploy_impacted.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +set -o errexit + +source .envrc.prod.resf +source hack/bazel_setup.sh + +hack/get_impacted_targets.sh + +return_if_impacted_targets_empty + +aws eks --region us-east-2 update-kubeconfig --name peridot-T8WbrA + +AWS_JWT="$(aws ecr get-login-password --region us-east-2)" +B64_AWS_AUTH="$(echo -n "AWS:$AWS_JWT")" +mkdir -p ~/.docker +echo '{"auths":{"893168113496.dkr.ecr.us-east-2.amazonaws.com":{"auth":"'"$B64_AWS_AUTH"'"}}}' > ~/.docker/config.json + +starting_query="attr(tags, 'push_apply'," + +for t in `cat impacted_targets`; do + starting_query="$starting_query $t union" +done + +starting_query=${starting_query%" union"} +starting_query="$starting_query)" + +TARGETS=$($BAZEL_QR "$starting_query") +for target in $TARGETS; do + $BAZEL_R "$target" +done + + diff --git a/hack/frontend_build.sh b/hack/frontend_build.sh index ab87a3c3..e85503f8 100755 --- a/hack/frontend_build.sh +++ b/hack/frontend_build.sh @@ -4,23 +4,9 @@ set -o errexit source hack/bazel_setup.sh -bazel_bin="$(which bazel)" -workspace_dir="$(pwd)" +hack/get_impacted_targets.sh -$BAZEL_B //:bazel-diff - -# Generate starting hashes -git checkout "$PULL_BASE_SHA" --quiet -bazel-bin/bazel-diff generate-hashes -w "$workspace_dir" -b "$bazel_bin" starting_hashes_json - -# Generate ending hashes -git checkout "$PULL_PULL_SHA" --quiet -bazel-bin/bazel-diff generate-hashes -w "$workspace_dir" -b "$bazel_bin" ending_hashes_json - -# Get impacted targets -bazel-bin/bazel-diff get-impacted-targets -sh starting_hashes_json -fh ending_hashes_json -o impacted_targets +return_if_impacted_targets_empty # Build impacted targets hack/build_impacted_frontend.sh - - diff --git a/hack/get_impacted_targets.sh b/hack/get_impacted_targets.sh new file mode 100755 index 00000000..bece2850 --- /dev/null +++ b/hack/get_impacted_targets.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +set -o errexit + +source hack/bazel_setup.sh + +bazel_bin="$(which bazel)" +workspace_dir="$(pwd)" + +$BAZEL_B //:bazel-diff + +# Generate starting hashes +git checkout "$PULL_BASE_SHA" --quiet +bazel-bin/bazel-diff generate-hashes -w "$workspace_dir" -b "$bazel_bin" starting_hashes_json + +# Generate ending hashes +git checkout "$PULL_PULL_SHA" --quiet +bazel-bin/bazel-diff generate-hashes -w "$workspace_dir" -b "$bazel_bin" ending_hashes_json + +# Get impacted targets +bazel-bin/bazel-diff get-impacted-targets -sh starting_hashes_json -fh ending_hashes_json -o impacted_targets + + diff --git a/rules_resf/defs.bzl b/rules_resf/defs.bzl index 81610f21..b90c9285 100644 --- a/rules_resf/defs.bzl +++ b/rules_resf/defs.bzl @@ -37,6 +37,7 @@ def tag_default_update(defaults, append): return tdict def gen_from_jsonnet(name, src, outs, tags, force_normal_tags, helm_mode, **kwargs): + def_tag = "peridot_k8s" ext_str_nested = "{STABLE_OCI_REGISTRY_NO_NESTED_SUPPORT_IN_2022_SHAME_ON_YOU_AWS}" if force_normal_tags: ext_str_nested = "false" @@ -56,13 +57,14 @@ def gen_from_jsonnet(name, src, outs, tags, force_normal_tags, helm_mode, **kwar } if helm_mode: ext_strs["helm_mode"] = "true" + def_tag = "peridot_helm" jsonnet_to_json( name = name, src = src, outs = outs, tags = tags + [ "manual", - "peridot_k8s", + def_tag, ], ext_strs = select({ "//platforms:arm64": dict(ext_strs, arch = "arm64"), @@ -123,7 +125,7 @@ def peridot_k8s(name, src, tags = [], outs = [], static = False, prod_only = Fal ":%s.push" % name, ":%s.apply" % name, ], - tags = ["manual"], + tags = ["manual", "push_apply"], ) def resf_frontend(name, tags = [], **kwargs):