peridot/ci/kubernetes.jsonnet

650 lines
21 KiB
Plaintext
Raw Normal View History

2022-07-07 20:11:50 +00:00
local stage = std.extVar('stage');
local tag = std.extVar('tag');
local ociRegistry = std.extVar('oci_registry');
local ociRegistryRepo = std.extVar('oci_registry_repo');
local ociRegistryDocker = std.extVar('oci_registry_docker');
local ociNoNestedSupport = std.extVar('oci_registry_no_nested_support_in_2022_shame_on_you_aws') == 'true';
local site = std.extVar('site');
local arch = std.extVar('arch');
local localEnvironment = std.extVar('local_environment') == '1';
local stageNoDash = std.strReplace(stage, '-', '');
local imagePullPolicy = if stageNoDash == 'dev' then 'Always' else 'IfNotPresent';
local utils = import 'ci/utils.jsonnet';
local defaultEnvs = [
{
name: 'BYC_ENV',
value: stageNoDash,
},
{
name: 'BYC_NS',
valueFrom: true,
field: 'metadata.namespace',
},
{
name: 'BYC_SERVICE_ACCOUNT',
valueFrom: true,
field: 'spec.serviceAccountName',
},
{
name: 'AWS_REGION',
value: 'us-east-2',
},
{
name: 'LOCALSTACK_ENDPOINT',
value: if utils.local_image then 'http://localstack.default.svc.cluster.local:4566' else '',
}
];
local define_env(envsOrig) = std.filter(function(x) x != null, [
if field != null then {
name: field.name,
value: if std.objectHas(field, 'value') then field.value,
valueFrom: if std.objectHas(field, 'valueFrom') && field.valueFrom == true then {
secretKeyRef: if std.objectHas(field, 'secret') then {
name: field.secret.name,
key: field.secret.key,
},
fieldRef: if std.objectHas(field, 'field') then {
fieldPath: field.field,
},
},
}
for field in (envsOrig + defaultEnvs)
]);
local define_volumes(volumes) = [
{
name: vm.name,
persistentVolumeClaim: if std.objectHas(vm, 'pvc') then {
claimName: vm.name,
},
emptyDir: if std.objectHas(vm, 'emptyDir') then {},
secret: if std.objectHas(vm, 'secret') then vm.secret,
configMap: if std.objectHas(vm, 'configMap') then vm.configMap,
2022-10-30 01:59:25 +00:00
hostPath: if std.objectHas(vm, 'hostPath') then vm.hostPath,
2022-07-07 20:11:50 +00:00
}
for vm in volumes
];
local define_volume_mounts(volumes) = [
{
name: vm.name,
mountPath: vm.path,
}
for vm in volumes
];
local define_init_containers(initc_) = std.filter(function(x) x != null, [
if initc != null && std.objectHas(initc, 'name') then {
name: initc.name,
image: initc.image,
imagePullPolicy: imagePullPolicy,
command: if std.objectHas(initc, 'command') then initc.command,
args: if std.objectHas(initc, 'args') then initc.args,
env: define_env(if std.objectHas(initc, 'env') then initc.env else []),
volumeMounts: if std.objectHas(initc, 'volumes') && initc.volumes != null then define_volume_mounts(initc.volumes),
}
for initc in initc_
]);
local default_labels = {
env: stageNoDash
};
local fix_metadata(metadata) = metadata {
namespace: metadata.namespace,
};
local prod() = stage == '-prod';
local dev() = stage == '-dev';
{
// For reference
metadata: {
name: 'empty',
namespace: 'namespace',
annotations: {},
},
// Namespace
define_namespace(name, metadata={})::
{
apiVersion: 'v1',
kind: 'Namespace',
metadata: metadata {
name: name,
},
},
// Deployment
define_deployment(metadataOrig, deporig)::
local _ = std.assertEqual(true, std.objectHas(deporig, 'image'));
local _ = std.assertEqual(true, std.objectHas(deporig, 'tag'));
local metadata = fix_metadata(metadataOrig);
local deployment = deporig {
annotations: if !std.objectHas(deporig, 'annotations') then {} else deporig.annotations,
labels: if !std.objectHas(deporig, 'labels') then default_labels else deporig.labels + default_labels,
volumes: if !std.objectHas(deporig, 'volumes') then [] else deporig.volumes,
imagePulLSecrets: if !std.objectHas(deporig, 'imagePullSecrets') then deporig.imagePullSecrets else deporig.imagePullSecrets,
env: if !std.objectHas(deporig, 'env') then [] else deporig.env,
ports: if !std.objectHas(deporig, 'ports') then [{ containerPort: 80, protocol: 'TCP' }] else deporig.ports,
initContainers: if !std.objectHas(deporig, 'initContainers') then [] else deporig.initContainers,
2022-10-30 01:59:25 +00:00
limits: if std.objectHas(deporig, 'limits') then deporig.limits,
2022-07-07 20:11:50 +00:00
requests: if !std.objectHas(deporig, 'requests') || deporig.requests == null then { cpu: '0.001', memory: '128M' } else deporig.requests,
};
{
apiVersion: 'apps/v1',
kind: 'Deployment',
metadata: metadata {
name: metadata.name + '-deployment',
},
spec: {
revisionHistoryLimit: 15,
selector: {
matchLabels: {
app: metadata.name,
env: stageNoDash
},
},
replicas: deployment.replicas,
strategy: {
type: 'RollingUpdate',
rollingUpdate: {
maxSurge: '300%',
maxUnavailable: '0%',
},
},
template: {
metadata: {
annotations: deployment.annotations,
labels: deployment.labels {
app: metadata.name,
env: stageNoDash,
version: deployment.tag,
},
},
spec: {
automountServiceAccountToken: true,
serviceAccountName: if std.objectHas(deployment, 'serviceAccount') then deployment.serviceAccount,
initContainers: if std.objectHas(deployment, 'initContainers') && deployment.initContainers != null then define_init_containers(deployment.initContainers),
securityContext: {
fsGroup: 1000,
},
containers: [
{
image: deployment.image + (if ociNoNestedSupport then '-' else ':') + deployment.tag,
imagePullPolicy: imagePullPolicy,
name: metadata.name,
command: if std.objectHas(deployment, 'command') then deployment.command else null,
args: if std.objectHas(deployment, 'args') then deployment.args else null,
ports: deployment.ports,
env: define_env(deployment.env),
volumeMounts: if std.objectHas(deployment, 'volumes') && deployment.volumes != null then define_volume_mounts(deployment.volumes),
securityContext: {
runAsGroup: if std.objectHas(deployment, 'fsGroup') then deployment.fsGroup else null,
runAsUser: if std.objectHas(deployment, 'fsUser') then deployment.fsUser else null,
},
resources: {
limits: deployment.limits,
requests: deployment.requests,
},
readinessProbe: if std.objectHas(deployment, 'health') && deployment.health != null then {
httpGet: if !std.objectHas(deployment.health, 'grpc') || !deployment.health.grpc then {
path: if std.objectHas(deployment.health, 'path') then deployment.health.path else '/_/healthz',
port: deployment.health.port,
httpHeaders: [
{
name: 'byc-internal-req',
value: 'yes',
},
],
},
exec: if std.objectHas(deployment.health, 'grpc') && deployment.health.grpc then {
command: ["grpc_health_probe", "-connect-timeout=4s", "-v", "-addr=localhost:"+deployment.health.port],
},
initialDelaySeconds: if std.objectHas(deployment.health, 'initialDelaySeconds') then deployment.health.initialDelaySeconds else 1,
periodSeconds: if std.objectHas(deployment.health, 'periodSeconds') then deployment.health.periodSeconds else 3,
timeoutSeconds: if std.objectHas(deployment.health, 'timeoutSeconds') then deployment.health.timeoutSeconds else 5,
successThreshold: if std.objectHas(deployment.health, 'successThreshold') then deployment.health.successThreshold else 1,
failureThreshold: if std.objectHas(deployment.health, 'failureTreshold') then deployment.health.failureTreshold else 30,
} else if std.objectHas(deployment, 'health_tcp') && deployment.health_tcp != null then {
tcpSocket: {
port: deployment.health_tcp.port,
},
initialDelaySeconds: if std.objectHas(deployment.health, 'initialDelaySeconds') then deployment.health.initialDelaySeconds else 5,
periodSeconds: if std.objectHas(deployment.health, 'periodSeconds') then deployment.health.periodSeconds else 5,
},
},
],
affinity: if !std.objectHas(deployment, 'no_anti_affinity') || !deployment.no_anti_affinity then {
podAntiAffinity: {
preferredDuringSchedulingIgnoredDuringExecution: [
{
weight: 99,
podAffinityTerm: {
labelSelector: {
matchExpressions: [
{
key: 'app',
operator: 'In',
values: [
metadata.name,
],
},
],
},
topologyKey: 'kubernetes.io/hostname',
},
},
{
weight: 100,
podAffinityTerm: {
labelSelector: {
matchExpressions: [
{
key: 'app',
operator: 'In',
values: [
metadata.name,
],
},
],
},
topologyKey: 'failure-domain.beta.kubernetes.io/zone',
},
},
],
},
},
restartPolicy: 'Always',
imagePullSecrets: if std.objectHas(deployment, 'imagePullSecrets') && deployment.imagePullSecrets != null then [
{
name: secret,
}
for secret in deployment.imagePullSecrets
],
volumes: if std.objectHas(deployment, 'volumes') && deployment.volumes != null then define_volumes(deployment.volumes),
},
},
},
},
// Ingress
2022-10-30 01:59:25 +00:00
define_ingress(metadataOrig, host, srvName=null, path='/', port=80)::
2022-07-07 20:11:50 +00:00
local metadata = fix_metadata(metadataOrig);
{
2022-10-30 01:59:25 +00:00
apiVersion: 'networking.k8s.io/v1',
2022-07-07 20:11:50 +00:00
kind: 'Ingress',
metadata: metadata {
name: metadata.name + '-ingress',
},
spec: {
rules: [{
host: host,
http: {
paths: [
{
path: path,
2022-10-30 01:59:25 +00:00
pathType: 'Prefix',
2022-07-07 20:11:50 +00:00
backend: {
2022-10-30 01:59:25 +00:00
service: {
name: if srvName != null then srvName else metadata.name + '-service',
port: {
number: port,
}
}
2022-07-07 20:11:50 +00:00
},
},
],
},
}],
2022-10-30 01:59:25 +00:00
} + ({
tls: [{
hosts: [
host,
],
secretName: metadata.name + '-tls',
}],
}),
2022-07-07 20:11:50 +00:00
},
// Service
define_service(metadataOrig, externalPort=80, internalPort=80, protocol='TCP', portName='http', selector='', env='canary')::
local metadata = fix_metadata(metadataOrig);
{
apiVersion: 'v1',
kind: 'Service',
metadata: metadata {
name: metadata.name + '-service',
},
spec: {
type: 'ClusterIP',
ports: [{
name: portName,
port: externalPort,
protocol: protocol,
targetPort: internalPort,
2022-10-30 01:59:25 +00:00
}] + (if portName == 'http' && externalPort != 80 then [{
name: portName + "-80",
port: 80,
protocol: protocol,
targetPort: internalPort,
}] else []),
2022-07-07 20:11:50 +00:00
selector: {
app: if selector != '' then selector else metadata.name,
env: env,
},
},
},
// Virtual Service
define_virtual_service(metadataOrig, spec)::
local metadata = fix_metadata(metadataOrig);
{
apiVersion: 'networking.istio.io/v1alpha3',
kind: 'VirtualService',
metadata: metadata {
name: metadata.name + '-vs',
},
spec: spec,
},
// Destination rule
define_destination_rule(metadataOrig, spec)::
local metadata = fix_metadata(metadataOrig);
{
apiVersion: 'networking.istio.io/v1alpha3',
kind: 'DestinationRule',
metadata: metadata {
name: metadata.name + '-dsr',
},
spec: spec,
},
// Service entry
define_service_entry(metadataOrig, hosts, ports, resolution, location)::
local metadata = fix_metadata(metadataOrig);
{
apiVersion: 'networking.istio.io/v1alpha3',
kind: 'ServiceEntry',
metadata: metadata {
name: metadata.name + '-se',
},
spec: {
hosts: hosts,
ports: ports,
resolution: resolution,
location: location,
},
},
// Job
define_job(metadataOrig, joborig)::
local metadata = fix_metadata(metadataOrig);
local job = joborig {
env: if !std.objectHas(joborig, 'env') then [] else joborig.env,
labels: if !std.objectHas(joborig, 'labels') then {} else joborig.labels,
annotations: if !std.objectHas(joborig, 'annotations') then {} else joborig.annotations,
initContainers: if !std.objectHas(joborig, 'initContainers') then [] else joborig.initContainers,
volumes: if !std.objectHas(joborig, 'volumes') then [] else joborig.volumes,
args: if !std.objectHas(joborig, 'args') then [] else joborig.args,
};
local name = metadata.name + '-job';
{
apiVersion: 'batch/v1',
kind: 'Job',
metadata: metadata {
name: name,
},
spec: {
ttlSecondsAfterFinished: 120,
template: {
metadata: {
labels: job.labels,
annotations: job.annotations,
},
spec: {
automountServiceAccountToken: true,
serviceAccountName: if std.objectHas(job, 'serviceAccount') then job.serviceAccount,
imagePullSecrets: if std.objectHas(job, 'imagePullSecrets') && job.imagePullSecrets != null then [
{
name: secret,
}
for secret in job.imagePullSecrets
],
initContainers: define_init_containers(job.initContainers),
containers: [{
name: name,
image: job.image + (if ociNoNestedSupport then '-' else ':') + job.tag,
command: if std.objectHas(job, 'command') then job.command else null,
args: job.args,
env: define_env(job.env),
volumeMounts: if std.objectHas(job, 'volumes') && job.volumes != null then define_volume_mounts(job.volumes),
}],
restartPolicy: 'Never',
volumes: if std.objectHas(job, 'volumes') && job.volumes != null then define_volumes(job.volumes),
},
},
},
},
// ServiceAccount
define_service_account(metadataOrig)::
local metadata = fix_metadata(metadataOrig);
{
apiVersion: 'v1',
kind: 'ServiceAccount',
metadata: metadata {
name: metadata.name + '-serviceaccount',
},
},
// Role
define_role(metadataOrig, rules)::
local metadata = fix_metadata(metadataOrig);
{
apiVersion: 'rbac.authorization.k8s.io/v1',
kind: 'Role',
metadata: metadata {
name: metadata.name + '-role',
},
rules: rules,
},
define_role_v2(metadataOrig, name, rules)::
$.define_role(metadataOrig { name: '%s-%s' % [metadataOrig.name, name] }, rules),
// ClusterRole
define_cluster_role(metadataOrig, rules)::
local metadata = fix_metadata(metadataOrig);
{
apiVersion: 'rbac.authorization.k8s.io/v1',
kind: 'ClusterRole',
metadata: metadata {
name: metadata.name + '-clusterrole',
},
rules: rules,
},
// RoleBinding
define_role_binding(metadataOrig, roleName, subjects)::
local metadata = fix_metadata(metadataOrig);
{
apiVersion: 'rbac.authorization.k8s.io/v1',
kind: 'RoleBinding',
metadata: metadata {
name: metadata.name + '-rolebinding',
},
roleRef: {
apiGroup: 'rbac.authorization.k8s.io',
kind: 'Role',
name: roleName,
},
subjects: subjects,
},
bind_to_role_sa(role, serviceAccount)::
$.define_role_binding(role.metadata, role.metadata.name, [{
kind: 'ServiceAccount',
name: serviceAccount,
namespace: role.metadata.namespace,
}]),
// ClusterRoleBinding
define_cluster_role_binding(metadataOrig, roleName, subjects)::
local metadata = fix_metadata(metadataOrig);
{
apiVersion: 'rbac.authorization.k8s.io/v1',
kind: 'ClusterRoleBinding',
metadata: metadata {
name: metadata.name + '-clusterrolebinding',
},
roleRef: {
apiGroup: 'rbac.authorization.k8s.io',
kind: 'ClusterRole',
name: roleName,
},
subjects: subjects,
},
// PersistentVolumeClaim
define_persistent_volume_claim(metadataOrig, storage, access_mode='ReadWriteOnce')::
local metadata = fix_metadata(metadataOrig);
{
apiVersion: 'v1',
kind: 'PersistentVolumeClaim',
metadata: metadata {
name: metadata.name + '-pvc',
},
spec: {
accessModes: [access_mode],
resources: {
requests: {
storage: storage,
},
},
},
},
// ConfigMap
define_config_map(metadataOrig, data)::
local metadata = fix_metadata(metadataOrig);
{
apiVersion: 'v1',
kind: 'ConfigMap',
metadata: metadata {
name: metadata.name + '-cm',
},
data: data,
},
request_cdb_certs_volumes()::
[
{
name: 'client-certs',
path: '/cockroach-certs',
emptyDir: true,
},
],
request_cdb_certs(user)::
{
name: 'init-certs',
image: ociRegistryDocker + '/cockroachdb/cockroach-k8s-request-cert',
tag: '0.4',
annotations: {
'sidecar.istio.io/inject': 'false',
},
command: [
'/bin/ash',
],
args: [
'-ecx',
'/request-cert -namespace=${POD_NAMESPACE} -certs-dir=/cockroach-certs -type=client -user=' + user + ' -symlink-ca-from=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt && ' +
'chown -R 1000:1000 /cockroach-certs',
],
volumes: $.request_cdb_certs_volumes(),
env: [{
name: 'POD_NAMESPACE',
valueFrom: true,
field: 'metadata.namespace',
}],
},
cdb_sa_roles(metadataOrig)::
local metadata = fix_metadata(metadataOrig);
{
apiVersion: 'v1',
kind: 'List',
items: [
$.define_service_account(metadataOrig),
$.define_role(metadataOrig, [{
apiGroups: [''],
resources: ['secrets'],
verbs: ['create', 'get'],
}]),
$.define_role_binding(metadataOrig, metadata.name + '-role', [{
kind: 'ServiceAccount',
name: metadata.name + '-serviceaccount',
namespace: metadata.namespace,
}]),
$.define_cluster_role(metadataOrig, [{
apiGroups: ['certificates.k8s.io'],
resources: ['certificatesigningrequests'],
verbs: ['create', 'get', 'watch'],
}]),
$.define_cluster_role_binding(metadataOrig, metadata.name + '-clusterrole', [{
kind: 'ServiceAccount',
name: metadata.name + '-serviceaccount',
namespace: metadata.namespace,
}]),
],
},
chown_vm(name, path, id, volumes)::
{
name: 'chown-vm-' + name,
image: 'alpine:3.9.3',
command: [
'chown',
'-R',
'%d:%d' % [id, id],
path,
],
volumes: volumes,
},
istio_labels()::
{
'istio-injection': 'enabled',
},
tag(name, extra=false)::
'%s/%s%s%s%s' % [
2022-10-30 01:59:25 +00:00
std.strReplace(ociRegistry, 'host.docker.internal.local', 'registry'),
2022-07-07 20:11:50 +00:00
ociRegistryRepo,
if ociNoNestedSupport then ':' else '/',
name,
if !extra then (if (arch != 'amd64' && !localEnvironment) then '_'+arch else '') else '',
],
tagVersion(name, version)::
'%s%s%s' % [$.tag(name, true), (if ociNoNestedSupport then '-' else ':'), version],
fix_metadata: fix_metadata,
prod: prod,
dev: dev,
version: tag,
}