Add kernelmanager

This commit is contained in:
Mustafa Gezen 2023-10-04 02:55:49 +02:00
parent 91fc789dcb
commit 60e664b8ae
Signed by: mustafa
GPG Key ID: DCDF010D946438C1
137 changed files with 55194 additions and 378 deletions

View File

@ -24,6 +24,11 @@
# We want to allow certain POST methods to set body to something other than "*".
# Useful for non-JSON payloads.
- "core::0136::http-body"
# I don't really understand this requirement but looks like Google has a special
# use case when it comes to user provided IDs.
- "core::0133::request-id-field"
# We don't require update mask support
- "core::0134::request-mask-required"
- included_paths:
- "third_party/**/*.proto"
- "vendor/**/*.proto"

View File

@ -6,3 +6,4 @@ bazel-out
bazel-testlogs
bazel-peridot
node_modules
vendor/go.resf.org

View File

@ -44,216 +44,202 @@ const (
EnvVarStoragePathStyle EnvVar = "STORAGE_PATH_STYLE"
)
var defaultCliFlagsDatabaseOnly = []cli.Flag{
&cli.StringFlag{
Name: "database-url",
Aliases: []string{"d"},
Usage: "database url",
EnvVars: []string{string(EnvVarDatabaseURL)},
Value: "postgres://postgres:postgres@localhost:5432/postgres?sslmode=disable",
},
func WithDatabaseFlags(appName string) []cli.Flag {
if appName == "" {
appName = "postgres"
}
return []cli.Flag{
&cli.StringFlag{
Name: "database-url",
Aliases: []string{"d"},
Usage: "database url",
EnvVars: []string{string(EnvVarDatabaseURL)},
Value: "postgres://postgres:postgres@localhost:5432/" + appName + "?sslmode=disable",
},
}
}
var defaultCliFlagsTemporal = append(defaultCliFlagsDatabaseOnly, []cli.Flag{
&cli.StringFlag{
Name: "temporal-namespace",
Aliases: []string{"n"},
Usage: "temporal namespace",
EnvVars: []string{string(EnvVarTemporalNamespace)},
Value: "default",
},
&cli.StringFlag{
Name: "temporal-address",
Aliases: []string{"a"},
Usage: "temporal address",
EnvVars: []string{string(EnvVarTemporalAddress)},
Value: "localhost:7233",
},
&cli.StringFlag{
Name: "temporal-task-queue",
Aliases: []string{"q"},
Usage: "temporal task queue",
EnvVars: []string{string(EnvVarTemporalTaskQueue)},
},
}...)
func WithTemporalFlags(defaultNamespace string, defaultTaskQueue string) []cli.Flag {
if defaultNamespace == "" {
defaultNamespace = "default"
}
var defaultCliFlagsNoAuth = append(defaultCliFlagsDatabaseOnly, []cli.Flag{
&cli.IntFlag{
Name: "grpc-port",
Usage: "gRPC port",
EnvVars: []string{string(EnvVarGRPCPort)},
Value: 8080,
},
&cli.IntFlag{
Name: "gateway-port",
Usage: "gRPC gateway port",
EnvVars: []string{string(EnvVarGatewayPort)},
Value: 8081,
},
}...)
var defaultCliFlagsNoAuthTemporal = append(defaultCliFlagsTemporal, []cli.Flag{
&cli.IntFlag{
Name: "grpc-port",
Usage: "gRPC port",
EnvVars: []string{string(EnvVarGRPCPort)},
Value: 8080,
},
&cli.IntFlag{
Name: "gateway-port",
Usage: "gRPC gateway port",
EnvVars: []string{string(EnvVarGatewayPort)},
Value: 8081,
},
}...)
var defaultCliFlags = append(defaultCliFlagsNoAuth, []cli.Flag{
&cli.StringFlag{
Name: "oidc-issuer",
Usage: "OIDC issuer",
EnvVars: []string{string(EnvVarFrontendOIDCIssuer)},
Value: "https://accounts.rockylinux.org/auth/realms/rocky",
},
&cli.StringFlag{
Name: "required-oidc-group",
Usage: "OIDC group that is required to access the frontend",
EnvVars: []string{string(EnvVarFrontendRequiredOIDCGroup)},
},
}...)
var defaultCliFlagsTemporalClient = append(defaultCliFlagsNoAuthTemporal, []cli.Flag{
&cli.StringFlag{
Name: "oidc-issuer",
Usage: "OIDC issuer",
EnvVars: []string{string(EnvVarFrontendOIDCIssuer)},
Value: "https://accounts.rockylinux.org/auth/realms/rocky",
},
&cli.StringFlag{
Name: "required-oidc-group",
Usage: "OIDC group that is required to access the frontend",
EnvVars: []string{string(EnvVarFrontendRequiredOIDCGroup)},
},
}...)
var defaultFrontendNoAuthCliFlags = []cli.Flag{
&cli.IntFlag{
Name: "port",
Usage: "frontend port",
EnvVars: []string{string(EnvVarFrontendPort)},
Value: 9111,
},
return []cli.Flag{
&cli.StringFlag{
Name: "temporal-namespace",
Aliases: []string{"n"},
Usage: "temporal namespace",
EnvVars: []string{string(EnvVarTemporalNamespace)},
Value: defaultNamespace,
},
&cli.StringFlag{
Name: "temporal-address",
Aliases: []string{"a"},
Usage: "temporal address",
EnvVars: []string{string(EnvVarTemporalAddress)},
Value: "localhost:7233",
},
&cli.StringFlag{
Name: "temporal-task-queue",
Aliases: []string{"q"},
Usage: "temporal task queue",
EnvVars: []string{string(EnvVarTemporalTaskQueue)},
Value: defaultTaskQueue,
},
}
}
var defaultFrontendCliFlags = append(defaultFrontendNoAuthCliFlags, []cli.Flag{
&cli.StringFlag{
Name: "oidc-issuer",
Usage: "OIDC issuer",
EnvVars: []string{string(EnvVarFrontendOIDCIssuer)},
Value: "https://accounts.rockylinux.org/auth/realms/rocky",
},
&cli.StringFlag{
Name: "oidc-client-id",
Usage: "OIDC client ID",
EnvVars: []string{string(EnvVarFrontendOIDCClientID)},
},
&cli.StringFlag{
Name: "oidc-client-secret",
Usage: "OIDC client secret",
EnvVars: []string{string(EnvVarFrontendOIDCClientSecret)},
},
&cli.StringFlag{
Name: "oidc-userinfo-override",
Usage: "OIDC userinfo override",
EnvVars: []string{string(EnvVarFrontendOIDCUserInfoOverride)},
},
&cli.StringFlag{
Name: "required-oidc-group",
Usage: "OIDC group that is required to access the frontend",
EnvVars: []string{string(EnvVarFrontendRequiredOIDCGroup)},
},
&cli.StringFlag{
Name: "self",
Usage: "Endpoint pointing to the frontend",
EnvVars: []string{string(EnvVarFrontendSelf)},
},
}...)
func WithGrpcFlags(defaultPort int) []cli.Flag {
if defaultPort == 0 {
defaultPort = 8080
}
var storageFlags = []cli.Flag{
&cli.StringFlag{
Name: "storage-endpoint",
Usage: "storage endpoint",
EnvVars: []string{string(EnvVarStorageEndpoint)},
Value: "",
},
&cli.StringFlag{
Name: "storage-connection-string",
Usage: "storage connection string",
EnvVars: []string{string(EnvVarStorageConnectionString)},
},
&cli.StringFlag{
Name: "storage-region",
Usage: "storage region",
EnvVars: []string{string(EnvVarStorageRegion)},
// RESF default region
Value: "us-east-2",
},
&cli.BoolFlag{
Name: "storage-secure",
Usage: "storage secure",
EnvVars: []string{string(EnvVarStorageSecure)},
Value: true,
},
&cli.BoolFlag{
Name: "storage-path-style",
Usage: "storage path style",
EnvVars: []string{string(EnvVarStoragePathStyle)},
Value: false,
},
return []cli.Flag{
&cli.IntFlag{
Name: "grpc-port",
Usage: "gRPC port",
EnvVars: []string{string(EnvVarGRPCPort)},
Value: defaultPort,
},
}
}
// WithDefaultCliFlags adds the default cli flags to the app.
func WithDefaultCliFlags(flags ...cli.Flag) []cli.Flag {
return append(defaultCliFlags, flags...)
func WithGatewayFlags(defaultPort int) []cli.Flag {
if defaultPort == 0 {
defaultPort = 8081
}
return []cli.Flag{
&cli.IntFlag{
Name: "gateway-port",
Usage: "gRPC gateway port",
EnvVars: []string{string(EnvVarGatewayPort)},
Value: defaultPort,
},
}
}
// WithDefaultCliFlagsTemporalClient adds the default cli flags to the app.
func WithDefaultCliFlagsTemporalClient(flags ...cli.Flag) []cli.Flag {
return append(defaultCliFlagsTemporalClient, flags...)
func WithOidcFlags(defaultOidcIssuer string, defaultGroup string) []cli.Flag {
if defaultOidcIssuer == "" {
defaultOidcIssuer = "https://accounts.rockylinux.org/auth/realms/rocky"
}
return []cli.Flag{
&cli.StringFlag{
Name: "oidc-issuer",
Usage: "OIDC issuer",
EnvVars: []string{string(EnvVarFrontendOIDCIssuer)},
Value: defaultOidcIssuer,
},
&cli.StringFlag{
Name: "required-oidc-group",
Usage: "OIDC group that is required to access the frontend",
EnvVars: []string{string(EnvVarFrontendRequiredOIDCGroup)},
Value: defaultGroup,
},
}
}
// WithDefaultCliFlagsNoAuth adds the default cli flags to the app.
func WithDefaultCliFlagsNoAuth(flags ...cli.Flag) []cli.Flag {
return append(defaultCliFlagsNoAuth, flags...)
func WithFrontendFlags(defaultPort int) []cli.Flag {
if defaultPort == 0 {
defaultPort = 9111
}
return []cli.Flag{
&cli.IntFlag{
Name: "port",
Usage: "frontend port",
EnvVars: []string{string(EnvVarFrontendPort)},
Value: defaultPort,
},
}
}
// WithDefaultCliFlagsNoAuthTemporal adds the default cli flags to the app.
func WithDefaultCliFlagsNoAuthTemporal(flags ...cli.Flag) []cli.Flag {
return append(defaultCliFlagsNoAuthTemporal, flags...)
}
func WithFrontendAuthFlags(defaultOidcIssuer string) []cli.Flag {
if defaultOidcIssuer == "" {
defaultOidcIssuer = "https://accounts.rockylinux.org/auth/realms/rocky"
}
// WithDefaultCliFlagsTemporal adds the default cli flags to the app.
func WithDefaultCliFlagsTemporal(flags ...cli.Flag) []cli.Flag {
return append(defaultCliFlagsTemporal, flags...)
}
// WithDefaultCliFlagsDatabaseOnly adds the default cli flags to the app.
func WithDefaultCliFlagsDatabaseOnly(flags ...cli.Flag) []cli.Flag {
return append(defaultCliFlagsDatabaseOnly, flags...)
}
// WithDefaultFrontendNoAuthCliFlags adds the default frontend cli flags to the app.
func WithDefaultFrontendNoAuthCliFlags(flags ...cli.Flag) []cli.Flag {
return append(defaultFrontendNoAuthCliFlags, flags...)
}
// WithDefaultFrontendCliFlags adds the default frontend cli flags to the app.
func WithDefaultFrontendCliFlags(flags ...cli.Flag) []cli.Flag {
return append(defaultFrontendCliFlags, flags...)
return []cli.Flag{
&cli.StringFlag{
Name: "oidc-issuer",
Usage: "OIDC issuer",
EnvVars: []string{string(EnvVarFrontendOIDCIssuer)},
Value: defaultOidcIssuer,
},
&cli.StringFlag{
Name: "oidc-client-id",
Usage: "OIDC client ID",
EnvVars: []string{string(EnvVarFrontendOIDCClientID)},
},
&cli.StringFlag{
Name: "oidc-client-secret",
Usage: "OIDC client secret",
EnvVars: []string{string(EnvVarFrontendOIDCClientSecret)},
},
&cli.StringFlag{
Name: "oidc-userinfo-override",
Usage: "OIDC userinfo override",
EnvVars: []string{string(EnvVarFrontendOIDCUserInfoOverride)},
},
&cli.StringFlag{
Name: "required-oidc-group",
Usage: "OIDC group that is required to access the frontend",
EnvVars: []string{string(EnvVarFrontendRequiredOIDCGroup)},
},
&cli.StringFlag{
Name: "self",
Usage: "Endpoint pointing to the frontend",
EnvVars: []string{string(EnvVarFrontendSelf)},
},
}
}
// WithStorageFlags adds the storage flags to the app.
func WithStorageFlags(flags ...cli.Flag) []cli.Flag {
return append(storageFlags, flags...)
func WithStorageFlags() []cli.Flag {
return []cli.Flag{
&cli.StringFlag{
Name: "storage-endpoint",
Usage: "storage endpoint",
EnvVars: []string{string(EnvVarStorageEndpoint)},
Value: "",
},
&cli.StringFlag{
Name: "storage-connection-string",
Usage: "storage connection string",
EnvVars: []string{string(EnvVarStorageConnectionString)},
},
&cli.StringFlag{
Name: "storage-region",
Usage: "storage region",
EnvVars: []string{string(EnvVarStorageRegion)},
// RESF default region
Value: "us-east-2",
},
&cli.BoolFlag{
Name: "storage-secure",
Usage: "storage secure",
EnvVars: []string{string(EnvVarStorageSecure)},
Value: true,
},
&cli.BoolFlag{
Name: "storage-path-style",
Usage: "storage path style",
EnvVars: []string{string(EnvVarStoragePathStyle)},
Value: false,
},
}
}
func WithFlags(flags ...[]cli.Flag) []cli.Flag {
var result []cli.Flag
for _, f := range flags {
result = append(result, f...)
}
return result
}
// FlagsToGRPCServerOptions converts the cli flags to gRPC server options.
@ -311,19 +297,6 @@ func GetTemporalClientFromFlags(ctx *cli.Context, opts client.Options) (client.C
)
}
// ChangeDefaultForEnvVar changes the default value of a flag based on an environment variable.
func ChangeDefaultForEnvVar(envVar EnvVar, newDefault string) {
// Check if the environment variable is set.
if _, ok := os.LookupEnv(string(envVar)); ok {
return
}
// Change the default value.
if err := os.Setenv(string(envVar), newDefault); err != nil {
LogFatalf("failed to set environment variable %s: %v", envVar, err)
}
}
// RareUseChangeDefault changes the default value of an arbitrary environment variable.
func RareUseChangeDefault(envVar string, newDefault string) {
// Check if the environment variable is set.
@ -336,8 +309,3 @@ func RareUseChangeDefault(envVar string, newDefault string) {
LogFatalf("failed to set environment variable %s: %v", envVar, err)
}
}
// ChangeDefaultDatabaseURL changes the default value of the database url based on an environment variable.
func ChangeDefaultDatabaseURL(appName string) {
ChangeDefaultForEnvVar(EnvVarDatabaseURL, "postgres://postgres:postgres@localhost:5432/"+appName+"?sslmode=disable")
}

View File

@ -20,7 +20,7 @@ go_library(
"caching.go",
"forge.go",
],
importpath = "go.resf.org/peridot/tools/mothership/worker_server/forge",
importpath = "go.resf.org/peridot/base/go/forge",
visibility = ["//visibility:public"],
deps = ["//vendor/github.com/go-git/go-git/v5/plumbing/transport"],
)

View File

@ -34,4 +34,5 @@ type Forge interface {
GetRemote(repo string) string
GetCommitViewerURL(repo string, commit string) string
EnsureRepositoryExists(auth *Authenticator, repo string) error
WithNamespace(namespace string) Forge
}

View File

@ -17,10 +17,10 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
go_library(
name = "github",
srcs = ["github.go"],
importpath = "go.resf.org/peridot/tools/mothership/worker_server/forge/github",
importpath = "go.resf.org/peridot/base/go/forge/github",
visibility = ["//visibility:public"],
deps = [
"//tools/mothership/worker_server/forge",
"//base/go/forge",
"//vendor/github.com/go-git/go-git/v5/plumbing/transport/http",
"//vendor/github.com/golang-jwt/jwt/v5:jwt",
],

View File

@ -22,7 +22,7 @@ import (
transport_http "github.com/go-git/go-git/v5/plumbing/transport/http"
"github.com/golang-jwt/jwt/v5"
_ "github.com/golang-jwt/jwt/v5"
"go.resf.org/peridot/tools/mothership/worker_server/forge"
"go.resf.org/peridot/base/go/forge"
"net/http"
"path/filepath"
"strconv"
@ -259,3 +259,9 @@ func (f *Forge) EnsureRepositoryExists(auth *forge.Authenticator, repo string) e
return nil
}
func (f *Forge) WithNamespace(namespace string) forge.Forge {
newF := *f
newF.organization = namespace
return &newF
}

12
base/go/forge/gitlab/BUILD vendored Normal file
View File

@ -0,0 +1,12 @@
load("@io_bazel_rules_go//go:def.bzl", "go_library")
go_library(
name = "gitlab",
srcs = ["gitlab.go"],
importpath = "go.resf.org/peridot/base/go/forge/gitlab",
visibility = ["//visibility:public"],
deps = [
"//base/go/forge",
"//vendor/github.com/go-git/go-git/v5/plumbing/transport/http",
],
)

View File

@ -0,0 +1,163 @@
package gitlab
import (
"bytes"
"encoding/json"
"fmt"
transport_http "github.com/go-git/go-git/v5/plumbing/transport/http"
"go.resf.org/peridot/base/go/forge"
"io"
"net/http"
"net/url"
"time"
)
type Forge struct {
host string
group string
username string
password string
authorName string
authorEmail string
shouldMakeRepoPublic bool
}
func New(host string, group string, username string, password string, authorName string, authorEmail string, shouldMakeRepoPublic bool) *Forge {
return &Forge{
host: host,
group: group,
username: username,
password: password,
authorName: authorName,
authorEmail: authorEmail,
shouldMakeRepoPublic: shouldMakeRepoPublic,
}
}
func (f *Forge) GetAuthenticator() (*forge.Authenticator, error) {
transporter := &transport_http.BasicAuth{
Username: f.username,
Password: f.password,
}
// We're assuming never expiring tokens for now
// Set it to 100 years from now
expires := time.Now().AddDate(100, 0, 0)
return &forge.Authenticator{
AuthMethod: transporter,
AuthorName: f.authorName,
AuthorEmail: f.authorEmail,
Expires: expires,
}, nil
}
func (f *Forge) GetRemote(repo string) string {
return fmt.Sprintf("https://%s/%s/%s", f.host, f.group, repo)
}
func (f *Forge) GetCommitViewerURL(repo string, commit string) string {
return fmt.Sprintf(
"https://%s/%s/%s/-/commit/%s",
f.host,
f.group,
repo,
commit,
)
}
func (f *Forge) EnsureRepositoryExists(auth *forge.Authenticator, repo string) error {
// Cast AuthMethod to BasicAuth
basicAuth := auth.AuthMethod.(*transport_http.BasicAuth)
token := basicAuth.Password
client := &http.Client{
Timeout: time.Second * 10,
}
// Check if the repo exists
urlEncodedPath := url.PathEscape(fmt.Sprintf("%s/%s", f.group, repo))
endpoint := fmt.Sprintf("https://%s/api/v4/projects/%s", f.host, urlEncodedPath)
req, err := http.NewRequest("GET", endpoint, nil)
if err != nil {
return err
}
req.Header.Add("Authorization", "Bearer "+token)
resp, err := client.Do(req)
if err != nil {
return err
}
if resp.StatusCode == 200 {
// Repo exists, we're done
return nil
}
// Repo doesn't exist, create it
// First get namespace id
endpoint = fmt.Sprintf("https://%s/api/v4/namespaces/%s", f.host, url.PathEscape(f.group))
req, err = http.NewRequest("GET", endpoint, nil)
if err != nil {
return err
}
req.Header.Add("Authorization", "Bearer "+token)
resp, err = client.Do(req)
if err != nil {
return err
}
if resp.StatusCode != 200 {
return fmt.Errorf("namespace %s does not exist", f.group)
}
mapBody := map[string]any{}
err = json.NewDecoder(resp.Body).Decode(&mapBody)
if err != nil {
return err
}
namespaceId := mapBody["id"].(float64)
mapBody = map[string]any{
"name": repo,
"namespace_id": namespaceId,
}
if f.shouldMakeRepoPublic {
mapBody["visibility"] = "public"
} else {
mapBody["visibility"] = "private"
}
endpoint = fmt.Sprintf("https://%s/api/v4/projects", f.host)
body, err := json.Marshal(mapBody)
if err != nil {
return err
}
req, err = http.NewRequest("POST", endpoint, bytes.NewReader(body))
if err != nil {
return err
}
req.Header.Add("Content-Type", "application/json")
req.Header.Add("Authorization", "Bearer "+token)
resp, err = client.Do(req)
if err != nil {
return err
}
if resp.StatusCode != 201 && resp.StatusCode != 200 {
body, _ := io.ReadAll(resp.Body)
return fmt.Errorf("failed to create repo %s: %s", repo, string(body))
}
return nil
}
func (f *Forge) WithNamespace(namespace string) forge.Forge {
newF := *f
newF.group = namespace
return &newF
}

View File

@ -98,7 +98,7 @@ var frontendHtmlTemplate = `
name="viewport"
content="width=device-width, initial-scale=1, viewport-fit=cover"
/>
<title>{{.Title}}</title>
<title>{{-Title-}}</title>
<link rel="icon" type="image/png" href="/_ga/favicon.png" />
@ -127,6 +127,7 @@ var frontendHtmlTemplate = `
}
</script>
{{end}}
{{-Beta-}}
{{if .Prefix}}
<script>window.__peridot_prefix__ = '{{.Prefix}}'.replace('\\', '');</script>
{{end}}
@ -144,7 +145,7 @@ var frontendUnauthenticated = `
name="viewport"
content="width=device-width, initial-scale=1, viewport-fit=cover"
/>
<title>{{.Title}} - Unauthenticated</title>
<title>{{-Title-}} - Ouch</title>
<link rel="icon" type="image/png" href="/_ga/favicon.png" />
@ -222,45 +223,50 @@ func (info *FrontendInfo) frontendAuthHandler(provider OidcProvider, h http.Hand
}
}
ctx := r.Context()
// get auth cookie
authCookie, err := r.Cookie(frontendAuthCookieKey)
if err != nil {
// redirect to login
http.Redirect(w, r, info.Self+"/auth/oidc/login", http.StatusFound)
return
}
// verify the token
userInfo, err := provider.UserInfo(r.Context(), oauth2.StaticTokenSource(&oauth2.Token{
AccessToken: authCookie.Value,
TokenType: "Bearer",
}))
if err != nil {
// redirect to login
http.Redirect(w, r, info.Self+"/auth/oidc/login", http.StatusFound)
return
}
// Check if the user is in the group
var claims oidcClaims
err = userInfo.Claims(&claims)
if err != nil {
// redirect to login
http.Redirect(w, r, info.Self+"/auth/oidc/login", http.StatusFound)
return
}
groups := claims.Groups
if info.OIDCGroup != "" {
if !Contains(groups, info.OIDCGroup) {
// show unauthenticated page
info.renderUnauthorized(w, fmt.Sprintf("User is not in group %s", info.OIDCGroup))
// only redirect if not allowed unauthenticated
if !info.AllowUnauthenticated {
// redirect to login
http.Redirect(w, r, info.Self+"/auth/oidc/login", http.StatusFound)
return
}
} else {
// verify the token
userInfo, err := provider.UserInfo(r.Context(), oauth2.StaticTokenSource(&oauth2.Token{
AccessToken: authCookie.Value,
TokenType: "Bearer",
}))
if err != nil {
// redirect to login
http.Redirect(w, r, info.Self+"/auth/oidc/login", http.StatusFound)
return
}
}
// Add the user to the context
ctx := context.WithValue(r.Context(), "user", userInfo)
// Check if the user is in the group
var claims oidcClaims
err = userInfo.Claims(&claims)
if err != nil {
// redirect to login
http.Redirect(w, r, info.Self+"/auth/oidc/login", http.StatusFound)
return
}
groups := claims.Groups
if info.OIDCGroup != "" {
if !Contains(groups, info.OIDCGroup) {
// show unauthenticated page
info.renderUnauthorized(w, fmt.Sprintf("User is not in group %s", info.OIDCGroup))
return
}
}
// Add the user to the context
ctx = context.WithValue(ctx, "user", userInfo)
}
h.ServeHTTP(w, r.WithContext(ctx))
})
@ -290,8 +296,8 @@ func FrontendServer(info *FrontendInfo, embedfs *embed.FS) error {
if info.Title == "" {
info.Title = "Peridot"
}
newTemplate = strings.ReplaceAll(newTemplate, "{{.Title}}", info.Title)
newUnauthenticatedTemplate = strings.ReplaceAll(newUnauthenticatedTemplate, "{{.Title}}", info.Title)
newTemplate = strings.ReplaceAll(newTemplate, "{{-Title-}}", info.Title)
newUnauthenticatedTemplate = strings.ReplaceAll(newUnauthenticatedTemplate, "{{-Title-}}", info.Title)
info.unauthenticatedTemplate = newUnauthenticatedTemplate
@ -354,7 +360,15 @@ func FrontendServer(info *FrontendInfo, embedfs *embed.FS) error {
http.HandleFunc(prefix+"/", func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/html")
tmpl, err := template.New("index.html").Parse(newTemplate)
// Set beta script (if beta basically set window.__beta__ = true)
srvTemplate := newTemplate
betaScript := ""
if r.Header.Get("x-peridot-beta") == "true" {
betaScript = "<script>window.__beta__ = true;</script>"
}
srvTemplate = strings.ReplaceAll(srvTemplate, "{{-Beta-}}", betaScript)
tmpl, err := template.New("index.html").Parse(srvTemplate)
if err != nil {
info.renderUnauthorized(w, fmt.Sprintf("Failed to parse template: %v", err))
return
@ -523,8 +537,8 @@ func FrontendServer(info *FrontendInfo, embedfs *embed.FS) error {
}
var handler http.Handler = nil
// if auth is enabled as well as AllowUnauthenticated is false, then wrap the handler with the auth handler
if !info.NoAuth && !info.AllowUnauthenticated {
// if auth is enabled as well, then wrap the handler with the auth handler
if !info.NoAuth {
handler = info.frontendAuthHandler(provider, http.DefaultServeMux)
} else {
handler = http.DefaultServeMux

8
base/go/kv/BUILD vendored Normal file
View File

@ -0,0 +1,8 @@
load("@io_bazel_rules_go//go:def.bzl", "go_library")
go_library(
name = "kv",
srcs = ["kv.go"],
importpath = "go.resf.org/peridot/base/go/kv",
visibility = ["//visibility:public"],
)

17
base/go/kv/dynamodb/BUILD vendored Normal file
View File

@ -0,0 +1,17 @@
load("@io_bazel_rules_go//go:def.bzl", "go_library")
go_library(
name = "dynamodb",
srcs = ["dynamodb.go"],
importpath = "go.resf.org/peridot/base/go/kv/dynamodb",
visibility = ["//visibility:public"],
deps = [
"//base/go/awsutils",
"//base/go/kv",
"//base/proto:pb",
"//vendor/github.com/aws/aws-sdk-go/aws",
"//vendor/github.com/aws/aws-sdk-go/aws/session",
"//vendor/github.com/aws/aws-sdk-go/service/dynamodb",
"@org_golang_google_protobuf//proto",
],
)

View File

@ -0,0 +1,328 @@
package dynamodb
import (
"context"
"crypto/rand"
"errors"
"fmt"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/dynamodb"
"go.resf.org/peridot/base/go/awsutils"
"go.resf.org/peridot/base/go/kv"
basepb "go.resf.org/peridot/base/go/pb"
"google.golang.org/protobuf/proto"
"strings"
"time"
)
type DynamoDB struct {
db *dynamodb.DynamoDB
tableName string
}
// New creates a new DynamoDB storage backend.
func New(endpoint string, tableName string) (*DynamoDB, error) {
awsCfg := &aws.Config{}
awsutils.FillOutConfig(awsCfg)
if endpoint != "" {
awsCfg.Endpoint = aws.String(endpoint)
}
sess, err := session.NewSession(awsCfg)
if err != nil {
return nil, err
}
svc := dynamodb.New(sess)
// Create the table if it doesn't exist.
// First check if the table exists.
_, err = svc.DescribeTable(&dynamodb.DescribeTableInput{
TableName: aws.String(tableName),
})
if err != nil {
_, err = svc.CreateTable(&dynamodb.CreateTableInput{
TableName: aws.String(tableName),
AttributeDefinitions: []*dynamodb.AttributeDefinition{
{
AttributeName: aws.String("Key"),
AttributeType: aws.String("S"),
},
{
AttributeName: aws.String("Path"),
AttributeType: aws.String("S"),
},
},
KeySchema: []*dynamodb.KeySchemaElement{
{
AttributeName: aws.String("Key"),
KeyType: aws.String("HASH"),
},
{
AttributeName: aws.String("Path"),
KeyType: aws.String("RANGE"),
},
},
ProvisionedThroughput: &dynamodb.ProvisionedThroughput{
ReadCapacityUnits: aws.Int64(5),
WriteCapacityUnits: aws.Int64(5),
},
})
if err != nil {
return nil, err
}
}
return &DynamoDB{
db: svc,
tableName: tableName,
}, nil
}
func (d *DynamoDB) Get(ctx context.Context, key string) (*kv.Pair, error) {
trimmed := strings.TrimPrefix(key, "/")
parts := strings.Split(trimmed, "/")
if len(parts) < 2 {
return nil, kv.ErrNoNamespace
}
ns := parts[0]
result, err := d.db.GetItem(&dynamodb.GetItemInput{
TableName: aws.String(d.tableName),
Key: map[string]*dynamodb.AttributeValue{
"Key": {
S: aws.String(ns),
},
"Path": {
S: aws.String(strings.Join(parts[1:], "/")),
},
},
})
if err != nil {
return nil, err
}
if result.Item == nil {
return nil, kv.ErrNotFound
}
return &kv.Pair{
Key: *result.Item["Key"].S,
Value: result.Item["Value"].B,
}, nil
}
func (d *DynamoDB) Set(ctx context.Context, key string, value []byte) error {
trimmed := strings.TrimPrefix(key, "/")
parts := strings.Split(trimmed, "/")
if len(parts) < 2 {
return kv.ErrNoNamespace
}
ns := parts[0]
_, err := d.db.PutItem(&dynamodb.PutItemInput{
TableName: aws.String(d.tableName),
Item: map[string]*dynamodb.AttributeValue{
"Key": {
S: aws.String(ns),
},
"Path": {
S: aws.String(strings.Join(parts[1:], "/")),
},
"Value": {
B: value,
},
},
})
return err
}
func (d *DynamoDB) Delete(ctx context.Context, key string) error {
trimmed := strings.TrimPrefix(key, "/")
parts := strings.Split(trimmed, "/")
if len(parts) < 2 {
return kv.ErrNoNamespace
}
ns := parts[0]
_, err := d.db.DeleteItem(&dynamodb.DeleteItemInput{
TableName: aws.String(d.tableName),
Key: map[string]*dynamodb.AttributeValue{
"Key": {
S: aws.String(ns),
},
"Path": {
S: aws.String(strings.Join(parts[1:], "/")),
},
},
})
return err
}
func (d *DynamoDB) RangePrefix(ctx context.Context, prefix string, pageSize int32, pageToken string) (*kv.Query, error) {
if pageSize <= 0 {
pageSize = 20
}
if pageSize > 100 {
pageSize = 100
}
// Check if there is a page token.
var fromKey string
var fromPath string
if pageToken != "" {
// Get the page token from the database.
res, err := d.Get(ctx, fmt.Sprintf("/_internals/page_tokens/%s", pageToken))
if err != nil {
if errors.Is(err, kv.ErrNotFound) {
return nil, kv.ErrPageTokenNotFound
}
return nil, err
}
// Parse the page token.
pt := &basepb.DynamoDbPageToken{}
err = proto.Unmarshal(res.Value, pt)
if err != nil {
return nil, err
}
fromKey = pt.LastKey
fromPath = pt.LastPath
}
trimmed := strings.TrimPrefix(prefix, "/")
parts := strings.Split(trimmed, "/")
if len(parts) < 2 {
return nil, kv.ErrNoNamespace
}
ns := parts[0]
queryInput := &dynamodb.QueryInput{
TableName: aws.String(d.tableName),
KeyConditions: map[string]*dynamodb.Condition{
"Key": {
ComparisonOperator: aws.String("EQ"),
AttributeValueList: []*dynamodb.AttributeValue{
{
S: aws.String(ns),
},
},
},
"Path": {
ComparisonOperator: aws.String("BEGINS_WITH"),
AttributeValueList: []*dynamodb.AttributeValue{
{
S: aws.String(strings.Join(parts[1:], "/")),
},
},
},
},
Limit: aws.Int64(int64(pageSize + 1)),
}
if fromKey != "" && fromPath != "" {
queryInput.ExclusiveStartKey = map[string]*dynamodb.AttributeValue{
"Key": {
S: aws.String(fromKey),
},
"Path": {
S: aws.String(fromPath),
},
}
}
result, err := d.db.Query(queryInput)
if err != nil {
return nil, err
}
pairs := make([]*kv.Pair, 0, len(result.Items))
for _, item := range result.Items {
// Since we fetch pageSize+1, stop if we have enough.
if len(pairs) >= int(pageSize) {
break
}
pairs = append(pairs, &kv.Pair{
Key: *item["Key"].S,
Value: item["Value"].B,
})
}
var nextToken string
var lastEvalKey string
var lastEvalPath string
// We always fetch pageSize+1, so if we have pageSize+1 results, we need to
// create a page token. We can't continue from result.LastEvaluatedKey since
// that will skip over the last result. So we should continue from the last
// visible result.
if len(result.Items) > int(pageSize) {
lastEvalKey = *result.Items[len(pairs)-1]["Key"].S
lastEvalPath = *result.Items[len(pairs)-1]["Path"].S
} else if result.LastEvaluatedKey != nil {
lastEvalKey = *result.LastEvaluatedKey["Key"].S
lastEvalPath = *result.LastEvaluatedKey["Path"].S
}
if lastEvalKey != "" && lastEvalPath != "" {
// Add page token to the database.
ptKey, err := generatePageToken()
if err != nil {
return nil, err
}
ptBytes, err := proto.Marshal(&basepb.DynamoDbPageToken{
LastKey: lastEvalKey,
LastPath: lastEvalPath,
})
if err != nil {
return nil, err
}
// Create the page token in the database, and it should expire in 2 days.
_, err = d.db.PutItem(&dynamodb.PutItemInput{
TableName: aws.String(d.tableName),
Item: map[string]*dynamodb.AttributeValue{
"Key": {
S: aws.String("_internals"),
},
"Path": {
S: aws.String(fmt.Sprintf("page_tokens/%s", ptKey)),
},
"Value": {
B: ptBytes,
},
"ExpiresAt": {
N: aws.String(fmt.Sprintf("%d", time.Now().Add(48*time.Hour).Unix())),
},
},
})
if err != nil {
return nil, err
}
nextToken = ptKey
}
return &kv.Query{
Prefix: prefix,
Pairs: pairs,
NextToken: nextToken,
}, nil
}
func generatePageToken() (string, error) {
possible := "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
token := make([]byte, 48)
_, err := rand.Read(token)
if err != nil {
return "", err
}
for i := 0; i < len(token); i++ {
token[i] = possible[token[i]%byte(len(possible))]
}
return "v1." + string(token), nil
}

33
base/go/kv/kv.go Normal file
View File

@ -0,0 +1,33 @@
package kv
import (
"context"
"errors"
)
var (
ErrNotFound = errors.New("key not found")
ErrPageTokenNotFound = errors.New("page token not found")
ErrNoNamespace = errors.New("no namespace")
)
type Pair struct {
Key string
Value []byte
}
type Query struct {
Prefix string
Pairs []*Pair
NextToken string
}
type KV interface {
// Get returns the contents of a file from the storage backend.
// Key must have a namespace prefix.
// Example: /kernels/entries/123, where kernels is the namespace and the rest is the range key.
Get(ctx context.Context, key string) (*Pair, error)
Set(ctx context.Context, key string, value []byte) error
Delete(ctx context.Context, key string) error
RangePrefix(ctx context.Context, prefix string, pageSize int32, pageToken string) (*Query, error)
}

23
base/proto/BUILD vendored Normal file
View File

@ -0,0 +1,23 @@
load("@rules_proto//proto:defs.bzl", "proto_library")
load("@io_bazel_rules_go//go:def.bzl", "go_library")
load("@io_bazel_rules_go//proto:def.bzl", "go_proto_library")
proto_library(
name = "basepb_proto",
srcs = ["kv_page_token.proto"],
visibility = ["//visibility:public"],
)
go_proto_library(
name = "basepb_go_proto",
importpath = "go.resf.org/peridot/base/go/pb",
proto = ":basepb_proto",
visibility = ["//visibility:public"],
)
go_library(
name = "pb",
embed = [":basepb_go_proto"],
importpath = "go.resf.org/peridot/base/go/pb",
visibility = ["//visibility:public"],
)

View File

@ -0,0 +1,17 @@
syntax = "proto3";
package base.go;
option java_multiple_files = true;
option java_outer_classname = "KvPageTokenProto";
option java_package = "org.resf.base.go";
option go_package = "go.resf.org/peridot/base/go/pb;basepb";
// DynamoDbPageToken is the page token used for DynamoDB.
message DynamoDbPageToken {
// key is the last evaluated key.
string last_key = 1;
// path is the last evaluated path.
string last_path = 2;
}

1
base/ts/global.d.ts vendored
View File

@ -23,5 +23,6 @@ declare global {
interface Window {
__peridot_prefix__: string;
__peridot_user__: PeridotUser;
__beta__: boolean;
}
}

View File

@ -49,6 +49,9 @@ export interface ResourceTableProps<T> {
// Default filter to start with
defaultFilter?: string;
// Whether to hide the filter input
hideFilter?: boolean;
// load is usually the OpenAPI SDK function that loads the resource.
load(pageSize: number, pageToken?: string, filter?: string): Promise<any>;
@ -102,6 +105,7 @@ export function ResourceTable<T extends StandardResource>(
const [rows, setRows] = React.useState<T[] | undefined>(undefined);
const [loading, setLoading] = React.useState<boolean>(false);
const [filter, setFilter] = React.useState<string | undefined>(initFilter);
const [filterValue, setFilterValue] = React.useState<string | undefined>(initFilter);
const updateSearch = (replace = false) => {
const search = new URLSearchParams(location.search);
@ -219,7 +223,7 @@ export function ResourceTable<T extends StandardResource>(
// Load the resource using useEffect
React.useEffect(() => {
fetchResource().then();
}, [pageToken, rowsPerPage]);
}, [filter, pageToken, rowsPerPage]);
// For filter, we're going to wait for the user to stop typing for 500ms
// before we actually fetch the resource.
@ -231,9 +235,9 @@ export function ResourceTable<T extends StandardResource>(
clearTimeout(timeout.current);
}
timeout.current = setTimeout(() => {
fetchResource().then();
setFilter(filterValue);
}, 500);
}, [filter]);
}, [filterValue]);
// Create table header
const header = props.fields.map((field) => {
@ -262,20 +266,21 @@ export function ResourceTable<T extends StandardResource>(
// Create a search box for filter input
// This can be disabled if the request does not support filtering
const searchBox = (
const searchBox = !props.hideFilter && (
<Box sx={{ display: 'flex', alignItems: 'center', mb: 2, width: '100%' }}>
<TextField
sx={{ mr: 2, flexGrow: 1 }}
label="Filter"
variant="outlined"
size="small"
value={filter}
onChange={(event: React.ChangeEvent<HTMLInputElement>) => setFilter(event.target.value)}
value={filterValue}
onChange={(event: React.ChangeEvent<HTMLInputElement>) => setFilterValue(event.target.value)}
/>
<Button
variant="contained"
onClick={() => {
setFilter('');
setFilterValue('');
setPageToken(undefined);
}}
>

View File

@ -638,6 +638,7 @@ def go_dependencies():
sum = "h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=",
version = "v1.0.1",
)
go_repository(
name = "com_github_elazarl_goproxy",
importpath = "github.com/elazarl/goproxy",
@ -1885,6 +1886,7 @@ def go_dependencies():
sum = "h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI=",
version = "v2.0.1+incompatible",
)
go_repository(
name = "com_github_pjbgf_sha1cd",
importpath = "github.com/pjbgf/sha1cd",

View File

@ -130,11 +130,12 @@ func spawnIbazel(target string) error {
func run(ctx *cli.Context) error {
// add default frontend flags if needed
if ctx.Bool("dev-frontend-flags") {
base.ChangeDefaultForEnvVar(base.EnvVarFrontendSelf, "http://localhost:9111")
base.ChangeDefaultForEnvVar(base.EnvVarFrontendOIDCIssuer, "http://127.0.0.1:5556/dex")
base.ChangeDefaultForEnvVar(base.EnvVarFrontendOIDCClientID, "local")
base.ChangeDefaultForEnvVar(base.EnvVarFrontendOIDCClientSecret, "local")
_ = os.Setenv(string(base.EnvVarFrontendSelf), "http://localhost:9111")
_ = os.Setenv(string(base.EnvVarFrontendOIDCIssuer), "http://127.0.0.1:5556/dex")
_ = os.Setenv(string(base.EnvVarFrontendOIDCClientID), "local")
_ = os.Setenv(string(base.EnvVarFrontendOIDCClientSecret), "local")
}
_ = os.Setenv("AWS_ENDPOINT", "http://localhost:4566")
// get current wd and make the ibazel path relative to it