From dd543e311bb38e1c3b3a2a53ada60f75877b044b Mon Sep 17 00:00:00 2001 From: Neil Hanlon Date: Mon, 11 Mar 2024 15:22:34 -0400 Subject: [PATCH] WIP: various updates to enable development --- WORKSPACE | 10 ++++++++++ ci/resfdeploy.jsonnet | 11 +++++------ ci/utils.jsonnet | 19 +++++++++++++++++++ common/frontend_server/index.mjs | 2 +- hydra/deploy/common.jsonnet | 2 +- hydra/pkg/hydra/autosignup.mjs | 4 ++-- .../istio-dev/istio-base-gateway.yaml | 2 ++ obsidian/ui/server/index.mjs | 2 +- rules_resf/internal/container/container.bzl | 2 +- wrksp/python_download.bzl | 2 +- 10 files changed, 43 insertions(+), 13 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index 7d775e1b..ceba71f0 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -24,6 +24,16 @@ load("//wrksp:python_deps.bzl", "python_deps") python_deps() # --end python-- +http_archive( + name = "rules_pkg", + urls = [ + "https://github.com/bazelbuild/rules_pkg/releases/download/0.10.1/rules_pkg-0.10.1.tar.gz", + ], + sha256 = "d250924a2ecc5176808fc4c25d5cf5e9e79e6346d79d5ab1c493e289e722d1d0", +) +load("@rules_pkg//:deps.bzl", "rules_pkg_dependencies") +rules_pkg_dependencies() + http_archive( name = "com_google_protobuf", sha256 = "d19643d265b978383352b3143f04c0641eea75a75235c111cc01a1350173180e", diff --git a/ci/resfdeploy.jsonnet b/ci/resfdeploy.jsonnet index e799c3d8..321b32de 100644 --- a/ci/resfdeploy.jsonnet +++ b/ci/resfdeploy.jsonnet @@ -63,7 +63,7 @@ local manifestYamlStream = function (value, indent_array_in_object=false, c_docu protocol: 'TCP', }]); local services = if std.objectHas(info, 'services') then info.services else - [{ name: '%s-%s-%s' % [metadata.name, port.name, env], port: port.containerPort, portName: port.name, expose: if std.objectHas(port, 'expose') then port.expose else false } for env in envs for port in ports]; + [{ name: '%s-%s-%s' % [metadata.name, port.name, env], port: port.containerPort, expose: if std.objectHas(port, 'expose') then port.expose else false } for env in envs for port in ports]; local file_yaml_prefix = if helm_mode then 'helm-' else ''; local nssa = '%s001-ns-sa.yaml' % file_yaml_prefix; @@ -109,7 +109,7 @@ local manifestYamlStream = function (value, indent_array_in_object=false, c_docu image: image, tag: tag, }; - local istio_mode = if helm_mode then false else if utils.local_image then false else true; + local istio_mode = true; #if helm_mode then false else if utils.local_image then false else true; { [nssa]: (if helm_mode then '{{ if not .Values.serviceAccountName }}\n' else '') + manifestYamlStream([ @@ -248,7 +248,7 @@ local manifestYamlStream = function (value, indent_array_in_object=false, c_docu 'prometheus.io/port': '7332', }), volumes: (if std.objectHas(info, 'volumes') then info.volumes(metadata) else []), - ports: std.map(function(x) x { expose: null, external: null }, ports), + ports: [utils.filterObjectFields(port, ['expose']) for port in ports], health: if std.objectHas(info, 'health') then info.health, env: env + (if dbname != '' && info.backend then ([dbPassEnv]) else []) + [ { @@ -258,7 +258,7 @@ local manifestYamlStream = function (value, indent_array_in_object=false, c_docu ] + [ if std.objectHas(srv, 'expose') && srv.expose then (if helm_mode then { name: '%s_PUBLIC_URL' % [std.asciiUpper(std.strReplace(std.strReplace(srv.name, stage, ''), '-', '_'))], - value: 'https://{{ .Values.%s.ingressHost }}!!' % [srv.portName], + value: 'https://{{ .Values.%s.ingressHost }}!!' % [srv.name], } else { name: '%s_PUBLIC_URL' % [std.asciiUpper(std.strReplace(std.strReplace(srv.name, stage, ''), '-', '_'))], value: 'https://%s' % mappings.get(srv.name, user), @@ -283,7 +283,6 @@ local manifestYamlStream = function (value, indent_array_in_object=false, c_docu }, srv.port, srv.port, - portName=srv.portName, selector=metadata.name, env=mappings.get_env_from_svc(srv.name) ) for srv in services]) + @@ -298,7 +297,7 @@ local manifestYamlStream = function (value, indent_array_in_object=false, c_docu 'konghq.com/protocols': (if helm_mode then '{{ .Values.kongProtocols | default !"%ss!" }}' else '%ss') % std.strReplace(std.strReplace(std.strReplace(srv.name, metadata.name, ''), stage, ''), '-', ''), } }, - host=if helm_mode then '{{ .Values.%s.ingressHost }}' % srv.portName else mappings.get(srv.name, user), + host=if helm_mode then '{{ .Values.%s.ingressHost }}' % srv.name else mappings.get(srv.name, user), port=srv.port, srvName=srv.name + '-service', ) else null for srv in services]) + diff --git a/ci/utils.jsonnet b/ci/utils.jsonnet index b93d7315..e3b14733 100644 --- a/ci/utils.jsonnet +++ b/ci/utils.jsonnet @@ -18,4 +18,23 @@ local stage_no_dash = std.strReplace(stage, '-', ''); stage: stage, user: user, stage_no_dash: stage_no_dash, + + // Function to filter an object by excluding specified fields. + // Parameters: + // - inputObject: The object to be filtered. + // - fieldsToIgnore: List of fields to be ignored from the input object. + filterObjectFields(inputObject, fieldsToIgnore):: + // Iterating over the fields in the input object and creating a new object + // without the fields specified in `fieldsToIgnore`. + std.foldl(function(filteredObject, currentField) + // If current field is in `fieldsToIgnore`, return the filtered object as is. + // Otherwise, add the current field to the filtered object. + ( + if std.member(fieldsToIgnore, currentField) then + filteredObject + else + filteredObject + { [currentField]: inputObject[currentField] } + ), + // Starting with an empty object and iterating over each field in the input object. + std.objectFields(inputObject), {}), } diff --git a/common/frontend_server/index.mjs b/common/frontend_server/index.mjs index 7ce7c941..e2d9262c 100644 --- a/common/frontend_server/index.mjs +++ b/common/frontend_server/index.mjs @@ -91,7 +91,7 @@ export default async function (opts) { opts.issuerBaseURL.endsWith('.localhost/')) && process.env['RESF_ENV'] ) { - const kong = 'kong-proxy.kong.svc.cluster.local'; + const kong = 'istio-ingressgateway.istio-system.svc.cluster.local'; const urlObject = new URL(opts.issuerBaseURL); console.warn(`Forcing ${urlObject.hostname} to resolve to ${kong}`); const lookup = async () => { diff --git a/hydra/deploy/common.jsonnet b/hydra/deploy/common.jsonnet index 5f678316..651a103f 100644 --- a/hydra/deploy/common.jsonnet +++ b/hydra/deploy/common.jsonnet @@ -5,7 +5,7 @@ local utils = import 'ci/utils.jsonnet'; local tag = std.extVar('tag'); local DSN = db.dsn('hydra'); -local authn = if kubernetes.prod() then 'https://id.build.resf.org' else 'http://obsidian.pdot.localhost:16000'; +local authn = if kubernetes.prod() then 'https://id.build.resf.org' else 'https://id-dev.internal.pdev.resf.localhost'; { image: 'oryd/hydra', diff --git a/hydra/pkg/hydra/autosignup.mjs b/hydra/pkg/hydra/autosignup.mjs index b2e1eaaf..ae4bf2c2 100644 --- a/hydra/pkg/hydra/autosignup.mjs +++ b/hydra/pkg/hydra/autosignup.mjs @@ -49,7 +49,7 @@ export function hydraPublicUrl() { if (process.env['HYDRA_PUBLIC_URL']) { return process.env['HYDRA_PUBLIC_URL']; } - return 'https://hdr-dev.internal.rdev.ciq.localhost'; + return 'https://hdr-dev.internal.pdev.resf.localhost'; } const svc = svcNameHttp('hydra-public'); return endpointHttp(svc, NS('hydra-public'), ':4444'); @@ -59,7 +59,7 @@ export function hydraPublicUrl() { function hydraAdminUrl() { return envOverridable('hydra_admin', 'http', () => { if (!process.env['RESF_ENV']) { - return 'https://hdr-admin-dev.internal.rdev.ciq.localhost'; + return 'https://hdr-admin-dev.internal.pdev.resf.localhost'; } const svc = svcNameHttp('hydra-admin'); return endpointHttp(svc, NS('hydra-admin'), ':4445'); diff --git a/infrastructure/istio-dev/istio-base-gateway.yaml b/infrastructure/istio-dev/istio-base-gateway.yaml index d8efecfe..b13a3213 100644 --- a/infrastructure/istio-dev/istio-base-gateway.yaml +++ b/infrastructure/istio-dev/istio-base-gateway.yaml @@ -13,6 +13,7 @@ spec: protocol: HTTP hosts: - "*.pdev.resf.localhost" + - "*.pdev.resf.local" tls: httpsRedirect: true - port: @@ -21,6 +22,7 @@ spec: protocol: HTTPS hosts: - "*.pdev.resf.localhost" + - "*.pdev.resf.local" tls: mode: SIMPLE credentialName: default-cert diff --git a/obsidian/ui/server/index.mjs b/obsidian/ui/server/index.mjs index f88212f8..2f032433 100644 --- a/obsidian/ui/server/index.mjs +++ b/obsidian/ui/server/index.mjs @@ -40,7 +40,7 @@ import { } from '../../../common/frontend_server/upstream.mjs'; export default async function run(webpackConfig) { - const devFrontendUrl = 'http://obsidian.pdot.localhost:16000'; + const devFrontendUrl = 'https://id-dev.internal.pdev.resf.localhost'; const envPublicUrl = process.env['OBSIDIAN_FRONTEND_HTTP_PUBLIC_URL']; const frontendUrl = process.env['RESF_NS'] ? envPublicUrl : devFrontendUrl; diff --git a/rules_resf/internal/container/container.bzl b/rules_resf/internal/container/container.bzl index adc2ed08..18e37162 100644 --- a/rules_resf/internal/container/container.bzl +++ b/rules_resf/internal/container/container.bzl @@ -1,4 +1,4 @@ -load("@bazel_tools//tools/build_defs/pkg:pkg.bzl", "pkg_tar") +load("@rules_pkg//:pkg.bzl", "pkg_tar") load("@io_bazel_rules_docker//container:container.bzl", "container_image", "container_layer", "container_push") load("@io_bazel_rules_docker//nodejs:image.bzl", "nodejs_image") diff --git a/wrksp/python_download.bzl b/wrksp/python_download.bzl index 97766463..651aacea 100644 --- a/wrksp/python_download.bzl +++ b/wrksp/python_download.bzl @@ -3,6 +3,6 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") def python_download(): http_archive( name = "rules_python", - url = "https://github.com/bazelbuild/rules_python/releases/download/0.2.0/rules_python-0.2.0.tar.gz", + url = "https://github.com/bazelbuild/rules_python/releases/download/0.31.0/rules_python-0.2.0.tar.gz", sha256 = "778197e26c5fbeb07ac2a2c5ae405b30f6cb7ad1f5510ea6fdac03bded96cc6f", )