Add kernelmanager

.api-linter.yaml

@@ -24,6 +24,11 @@
     # We want to allow certain POST methods to set body to something other than "*".
     # Useful for non-JSON payloads.
     - "core::0136::http-body"
+    # I don't really understand this requirement but looks like Google has a special
+    # use case when it comes to user provided IDs.
+    - "core::0133::request-id-field"
+    # We don't require update mask support
+    - "core::0134::request-mask-required"
 - included_paths:
     - "third_party/**/*.proto"
     - "vendor/**/*.proto"

.bazelignore

@@ -6,3 +6,4 @@ bazel-out
 bazel-testlogs
 bazel-peridot
 node_modules
+vendor/go.resf.org

base/go/flags.go

@@ -44,216 +44,202 @@ const (
 	EnvVarStoragePathStyle             EnvVar = "STORAGE_PATH_STYLE"
 )
 
-var defaultCliFlagsDatabaseOnly = []cli.Flag{
-	&cli.StringFlag{
-		Name:    "database-url",
-		Aliases: []string{"d"},
-		Usage:   "database url",
-		EnvVars: []string{string(EnvVarDatabaseURL)},
-		Value:   "postgres://postgres:postgres@localhost:5432/postgres?sslmode=disable",
-	},
+func WithDatabaseFlags(appName string) []cli.Flag {
+	if appName == "" {
+		appName = "postgres"
+	}
+
+	return []cli.Flag{
+		&cli.StringFlag{
+			Name:    "database-url",
+			Aliases: []string{"d"},
+			Usage:   "database url",
+			EnvVars: []string{string(EnvVarDatabaseURL)},
+			Value:   "postgres://postgres:postgres@localhost:5432/" + appName + "?sslmode=disable",
+		},
+	}
 }
 
-var defaultCliFlagsTemporal = append(defaultCliFlagsDatabaseOnly, []cli.Flag{
-	&cli.StringFlag{
-		Name:    "temporal-namespace",
-		Aliases: []string{"n"},
-		Usage:   "temporal namespace",
-		EnvVars: []string{string(EnvVarTemporalNamespace)},
-		Value:   "default",
-	},
-	&cli.StringFlag{
-		Name:    "temporal-address",
-		Aliases: []string{"a"},
-		Usage:   "temporal address",
-		EnvVars: []string{string(EnvVarTemporalAddress)},
-		Value:   "localhost:7233",
-	},
-	&cli.StringFlag{
-		Name:    "temporal-task-queue",
-		Aliases: []string{"q"},
-		Usage:   "temporal task queue",
-		EnvVars: []string{string(EnvVarTemporalTaskQueue)},
-	},
-}...)
+func WithTemporalFlags(defaultNamespace string, defaultTaskQueue string) []cli.Flag {
+	if defaultNamespace == "" {
+		defaultNamespace = "default"
+	}
 
-var defaultCliFlagsNoAuth = append(defaultCliFlagsDatabaseOnly, []cli.Flag{
-	&cli.IntFlag{
-		Name:    "grpc-port",
-		Usage:   "gRPC port",
-		EnvVars: []string{string(EnvVarGRPCPort)},
-		Value:   8080,
-	},
-	&cli.IntFlag{
-		Name:    "gateway-port",
-		Usage:   "gRPC gateway port",
-		EnvVars: []string{string(EnvVarGatewayPort)},
-		Value:   8081,
-	},
-}...)
-
-var defaultCliFlagsNoAuthTemporal = append(defaultCliFlagsTemporal, []cli.Flag{
-	&cli.IntFlag{
-		Name:    "grpc-port",
-		Usage:   "gRPC port",
-		EnvVars: []string{string(EnvVarGRPCPort)},
-		Value:   8080,
-	},
-	&cli.IntFlag{
-		Name:    "gateway-port",
-		Usage:   "gRPC gateway port",
-		EnvVars: []string{string(EnvVarGatewayPort)},
-		Value:   8081,
-	},
-}...)
-
-var defaultCliFlags = append(defaultCliFlagsNoAuth, []cli.Flag{
-	&cli.StringFlag{
-		Name:    "oidc-issuer",
-		Usage:   "OIDC issuer",
-		EnvVars: []string{string(EnvVarFrontendOIDCIssuer)},
-		Value:   "https://accounts.rockylinux.org/auth/realms/rocky",
-	},
-	&cli.StringFlag{
-		Name:    "required-oidc-group",
-		Usage:   "OIDC group that is required to access the frontend",
-		EnvVars: []string{string(EnvVarFrontendRequiredOIDCGroup)},
-	},
-}...)
-
-var defaultCliFlagsTemporalClient = append(defaultCliFlagsNoAuthTemporal, []cli.Flag{
-	&cli.StringFlag{
-		Name:    "oidc-issuer",
-		Usage:   "OIDC issuer",
-		EnvVars: []string{string(EnvVarFrontendOIDCIssuer)},
-		Value:   "https://accounts.rockylinux.org/auth/realms/rocky",
-	},
-	&cli.StringFlag{
-		Name:    "required-oidc-group",
-		Usage:   "OIDC group that is required to access the frontend",
-		EnvVars: []string{string(EnvVarFrontendRequiredOIDCGroup)},
-	},
-}...)
-
-var defaultFrontendNoAuthCliFlags = []cli.Flag{
-	&cli.IntFlag{
-		Name:    "port",
-		Usage:   "frontend port",
-		EnvVars: []string{string(EnvVarFrontendPort)},
-		Value:   9111,
-	},
+	return []cli.Flag{
+		&cli.StringFlag{
+			Name:    "temporal-namespace",
+			Aliases: []string{"n"},
+			Usage:   "temporal namespace",
+			EnvVars: []string{string(EnvVarTemporalNamespace)},
+			Value:   defaultNamespace,
+		},
+		&cli.StringFlag{
+			Name:    "temporal-address",
+			Aliases: []string{"a"},
+			Usage:   "temporal address",
+			EnvVars: []string{string(EnvVarTemporalAddress)},
+			Value:   "localhost:7233",
+		},
+		&cli.StringFlag{
+			Name:    "temporal-task-queue",
+			Aliases: []string{"q"},
+			Usage:   "temporal task queue",
+			EnvVars: []string{string(EnvVarTemporalTaskQueue)},
+			Value:   defaultTaskQueue,
+		},
+	}
 }
 
-var defaultFrontendCliFlags = append(defaultFrontendNoAuthCliFlags, []cli.Flag{
-	&cli.StringFlag{
-		Name:    "oidc-issuer",
-		Usage:   "OIDC issuer",
-		EnvVars: []string{string(EnvVarFrontendOIDCIssuer)},
-		Value:   "https://accounts.rockylinux.org/auth/realms/rocky",
-	},
-	&cli.StringFlag{
-		Name:    "oidc-client-id",
-		Usage:   "OIDC client ID",
-		EnvVars: []string{string(EnvVarFrontendOIDCClientID)},
-	},
-	&cli.StringFlag{
-		Name:    "oidc-client-secret",
-		Usage:   "OIDC client secret",
-		EnvVars: []string{string(EnvVarFrontendOIDCClientSecret)},
-	},
-	&cli.StringFlag{
-		Name:    "oidc-userinfo-override",
-		Usage:   "OIDC userinfo override",
-		EnvVars: []string{string(EnvVarFrontendOIDCUserInfoOverride)},
-	},
-	&cli.StringFlag{
-		Name:    "required-oidc-group",
-		Usage:   "OIDC group that is required to access the frontend",
-		EnvVars: []string{string(EnvVarFrontendRequiredOIDCGroup)},
-	},
-	&cli.StringFlag{
-		Name:    "self",
-		Usage:   "Endpoint pointing to the frontend",
-		EnvVars: []string{string(EnvVarFrontendSelf)},
-	},
-}...)
+func WithGrpcFlags(defaultPort int) []cli.Flag {
+	if defaultPort == 0 {
+		defaultPort = 8080
+	}
 
-var storageFlags = []cli.Flag{
-	&cli.StringFlag{
-		Name:    "storage-endpoint",
-		Usage:   "storage endpoint",
-		EnvVars: []string{string(EnvVarStorageEndpoint)},
-		Value:   "",
-	},
-	&cli.StringFlag{
-		Name:    "storage-connection-string",
-		Usage:   "storage connection string",
-		EnvVars: []string{string(EnvVarStorageConnectionString)},
-	},
-	&cli.StringFlag{
-		Name:    "storage-region",
-		Usage:   "storage region",
-		EnvVars: []string{string(EnvVarStorageRegion)},
-		// RESF default region
-		Value: "us-east-2",
-	},
-	&cli.BoolFlag{
-		Name:    "storage-secure",
-		Usage:   "storage secure",
-		EnvVars: []string{string(EnvVarStorageSecure)},
-		Value:   true,
-	},
-	&cli.BoolFlag{
-		Name:    "storage-path-style",
-		Usage:   "storage path style",
-		EnvVars: []string{string(EnvVarStoragePathStyle)},
-		Value:   false,
-	},
+	return []cli.Flag{
+		&cli.IntFlag{
+			Name:    "grpc-port",
+			Usage:   "gRPC port",
+			EnvVars: []string{string(EnvVarGRPCPort)},
+			Value:   defaultPort,
+		},
+	}
 }
 
-// WithDefaultCliFlags adds the default cli flags to the app.
-func WithDefaultCliFlags(flags ...cli.Flag) []cli.Flag {
-	return append(defaultCliFlags, flags...)
+func WithGatewayFlags(defaultPort int) []cli.Flag {
+	if defaultPort == 0 {
+		defaultPort = 8081
+	}
+
+	return []cli.Flag{
+		&cli.IntFlag{
+			Name:    "gateway-port",
+			Usage:   "gRPC gateway port",
+			EnvVars: []string{string(EnvVarGatewayPort)},
+			Value:   defaultPort,
+		},
+	}
 }
 
-// WithDefaultCliFlagsTemporalClient adds the default cli flags to the app.
-func WithDefaultCliFlagsTemporalClient(flags ...cli.Flag) []cli.Flag {
-	return append(defaultCliFlagsTemporalClient, flags...)
+func WithOidcFlags(defaultOidcIssuer string, defaultGroup string) []cli.Flag {
+	if defaultOidcIssuer == "" {
+		defaultOidcIssuer = "https://accounts.rockylinux.org/auth/realms/rocky"
+	}
+
+	return []cli.Flag{
+		&cli.StringFlag{
+			Name:    "oidc-issuer",
+			Usage:   "OIDC issuer",
+			EnvVars: []string{string(EnvVarFrontendOIDCIssuer)},
+			Value:   defaultOidcIssuer,
+		},
+		&cli.StringFlag{
+			Name:    "required-oidc-group",
+			Usage:   "OIDC group that is required to access the frontend",
+			EnvVars: []string{string(EnvVarFrontendRequiredOIDCGroup)},
+			Value:   defaultGroup,
+		},
+	}
 }
 
-// WithDefaultCliFlagsNoAuth adds the default cli flags to the app.
-func WithDefaultCliFlagsNoAuth(flags ...cli.Flag) []cli.Flag {
-	return append(defaultCliFlagsNoAuth, flags...)
+func WithFrontendFlags(defaultPort int) []cli.Flag {
+	if defaultPort == 0 {
+		defaultPort = 9111
+	}
+
+	return []cli.Flag{
+		&cli.IntFlag{
+			Name:    "port",
+			Usage:   "frontend port",
+			EnvVars: []string{string(EnvVarFrontendPort)},
+			Value:   defaultPort,
+		},
+	}
 }
 
-// WithDefaultCliFlagsNoAuthTemporal adds the default cli flags to the app.
-func WithDefaultCliFlagsNoAuthTemporal(flags ...cli.Flag) []cli.Flag {
-	return append(defaultCliFlagsNoAuthTemporal, flags...)
-}
+func WithFrontendAuthFlags(defaultOidcIssuer string) []cli.Flag {
+	if defaultOidcIssuer == "" {
+		defaultOidcIssuer = "https://accounts.rockylinux.org/auth/realms/rocky"
+	}
 
-// WithDefaultCliFlagsTemporal adds the default cli flags to the app.
-func WithDefaultCliFlagsTemporal(flags ...cli.Flag) []cli.Flag {
-	return append(defaultCliFlagsTemporal, flags...)
-}
-
-// WithDefaultCliFlagsDatabaseOnly adds the default cli flags to the app.
-func WithDefaultCliFlagsDatabaseOnly(flags ...cli.Flag) []cli.Flag {
-	return append(defaultCliFlagsDatabaseOnly, flags...)
-}
-
-// WithDefaultFrontendNoAuthCliFlags adds the default frontend cli flags to the app.
-func WithDefaultFrontendNoAuthCliFlags(flags ...cli.Flag) []cli.Flag {
-	return append(defaultFrontendNoAuthCliFlags, flags...)
-}
-
-// WithDefaultFrontendCliFlags adds the default frontend cli flags to the app.
-func WithDefaultFrontendCliFlags(flags ...cli.Flag) []cli.Flag {
-	return append(defaultFrontendCliFlags, flags...)
+	return []cli.Flag{
+		&cli.StringFlag{
+			Name:    "oidc-issuer",
+			Usage:   "OIDC issuer",
+			EnvVars: []string{string(EnvVarFrontendOIDCIssuer)},
+			Value:   defaultOidcIssuer,
+		},
+		&cli.StringFlag{
+			Name:    "oidc-client-id",
+			Usage:   "OIDC client ID",
+			EnvVars: []string{string(EnvVarFrontendOIDCClientID)},
+		},
+		&cli.StringFlag{
+			Name:    "oidc-client-secret",
+			Usage:   "OIDC client secret",
+			EnvVars: []string{string(EnvVarFrontendOIDCClientSecret)},
+		},
+		&cli.StringFlag{
+			Name:    "oidc-userinfo-override",
+			Usage:   "OIDC userinfo override",
+			EnvVars: []string{string(EnvVarFrontendOIDCUserInfoOverride)},
+		},
+		&cli.StringFlag{
+			Name:    "required-oidc-group",
+			Usage:   "OIDC group that is required to access the frontend",
+			EnvVars: []string{string(EnvVarFrontendRequiredOIDCGroup)},
+		},
+		&cli.StringFlag{
+			Name:    "self",
+			Usage:   "Endpoint pointing to the frontend",
+			EnvVars: []string{string(EnvVarFrontendSelf)},
+		},
+	}
 }
 
 // WithStorageFlags adds the storage flags to the app.
-func WithStorageFlags(flags ...cli.Flag) []cli.Flag {
-	return append(storageFlags, flags...)
+func WithStorageFlags() []cli.Flag {
+	return []cli.Flag{
+		&cli.StringFlag{
+			Name:    "storage-endpoint",
+			Usage:   "storage endpoint",
+			EnvVars: []string{string(EnvVarStorageEndpoint)},
+			Value:   "",
+		},
+		&cli.StringFlag{
+			Name:    "storage-connection-string",
+			Usage:   "storage connection string",
+			EnvVars: []string{string(EnvVarStorageConnectionString)},
+		},
+		&cli.StringFlag{
+			Name:    "storage-region",
+			Usage:   "storage region",
+			EnvVars: []string{string(EnvVarStorageRegion)},
+			// RESF default region
+			Value: "us-east-2",
+		},
+		&cli.BoolFlag{
+			Name:    "storage-secure",
+			Usage:   "storage secure",
+			EnvVars: []string{string(EnvVarStorageSecure)},
+			Value:   true,
+		},
+		&cli.BoolFlag{
+			Name:    "storage-path-style",
+			Usage:   "storage path style",
+			EnvVars: []string{string(EnvVarStoragePathStyle)},
+			Value:   false,
+		},
+	}
+}
+
+func WithFlags(flags ...[]cli.Flag) []cli.Flag {
+	var result []cli.Flag
+
+	for _, f := range flags {
+		result = append(result, f...)
+	}
+
+	return result
 }
 
 // FlagsToGRPCServerOptions converts the cli flags to gRPC server options.
@@ -311,19 +297,6 @@ func GetTemporalClientFromFlags(ctx *cli.Context, opts client.Options) (client.C
 	)
 }
 
-// ChangeDefaultForEnvVar changes the default value of a flag based on an environment variable.
-func ChangeDefaultForEnvVar(envVar EnvVar, newDefault string) {
-	// Check if the environment variable is set.
-	if _, ok := os.LookupEnv(string(envVar)); ok {
-		return
-	}
-
-	// Change the default value.
-	if err := os.Setenv(string(envVar), newDefault); err != nil {
-		LogFatalf("failed to set environment variable %s: %v", envVar, err)
-	}
-}
-
 // RareUseChangeDefault changes the default value of an arbitrary environment variable.
 func RareUseChangeDefault(envVar string, newDefault string) {
 	// Check if the environment variable is set.
@@ -336,8 +309,3 @@ func RareUseChangeDefault(envVar string, newDefault string) {
 		LogFatalf("failed to set environment variable %s: %v", envVar, err)
 	}
 }
-
-// ChangeDefaultDatabaseURL changes the default value of the database url based on an environment variable.
-func ChangeDefaultDatabaseURL(appName string) {
-	ChangeDefaultForEnvVar(EnvVarDatabaseURL, "postgres://postgres:postgres@localhost:5432/"+appName+"?sslmode=disable")
-}

base/go/forge/BUILD

@@ -20,7 +20,7 @@ go_library(
         "caching.go",
         "forge.go",
     ],
-    importpath = "go.resf.org/peridot/tools/mothership/worker_server/forge",
+    importpath = "go.resf.org/peridot/base/go/forge",
     visibility = ["//visibility:public"],
     deps = ["//vendor/github.com/go-git/go-git/v5/plumbing/transport"],
 )

base/go/forge/forge.go

@@ -34,4 +34,5 @@ type Forge interface {
 	GetRemote(repo string) string
 	GetCommitViewerURL(repo string, commit string) string
 	EnsureRepositoryExists(auth *Authenticator, repo string) error
+	WithNamespace(namespace string) Forge
 }

base/go/forge/github/BUILD

@@ -17,10 +17,10 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
 go_library(
     name = "github",
     srcs = ["github.go"],
-    importpath = "go.resf.org/peridot/tools/mothership/worker_server/forge/github",
+    importpath = "go.resf.org/peridot/base/go/forge/github",
     visibility = ["//visibility:public"],
     deps = [
-        "//tools/mothership/worker_server/forge",
+        "//base/go/forge",
         "//vendor/github.com/go-git/go-git/v5/plumbing/transport/http",
         "//vendor/github.com/golang-jwt/jwt/v5:jwt",
     ],

base/go/forge/github/github.go

@@ -22,7 +22,7 @@ import (
 	transport_http "github.com/go-git/go-git/v5/plumbing/transport/http"
 	"github.com/golang-jwt/jwt/v5"
 	_ "github.com/golang-jwt/jwt/v5"
-	"go.resf.org/peridot/tools/mothership/worker_server/forge"
+	"go.resf.org/peridot/base/go/forge"
 	"net/http"
 	"path/filepath"
 	"strconv"
@@ -259,3 +259,9 @@ func (f *Forge) EnsureRepositoryExists(auth *forge.Authenticator, repo string) e
 
 	return nil
 }
+
+func (f *Forge) WithNamespace(namespace string) forge.Forge {
+	newF := *f
+	newF.organization = namespace
+	return &newF
+}

base/go/forge/gitlab/BUILD

@@ -0,0 +1,12 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "gitlab",
+    srcs = ["gitlab.go"],
+    importpath = "go.resf.org/peridot/base/go/forge/gitlab",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//base/go/forge",
+        "//vendor/github.com/go-git/go-git/v5/plumbing/transport/http",
+    ],
+)

base/go/forge/gitlab/gitlab.go

@@ -0,0 +1,163 @@
+package gitlab
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	transport_http "github.com/go-git/go-git/v5/plumbing/transport/http"
+	"go.resf.org/peridot/base/go/forge"
+	"io"
+	"net/http"
+	"net/url"
+	"time"
+)
+
+type Forge struct {
+	host                 string
+	group                string
+	username             string
+	password             string
+	authorName           string
+	authorEmail          string
+	shouldMakeRepoPublic bool
+}
+
+func New(host string, group string, username string, password string, authorName string, authorEmail string, shouldMakeRepoPublic bool) *Forge {
+	return &Forge{
+		host:                 host,
+		group:                group,
+		username:             username,
+		password:             password,
+		authorName:           authorName,
+		authorEmail:          authorEmail,
+		shouldMakeRepoPublic: shouldMakeRepoPublic,
+	}
+}
+
+func (f *Forge) GetAuthenticator() (*forge.Authenticator, error) {
+	transporter := &transport_http.BasicAuth{
+		Username: f.username,
+		Password: f.password,
+	}
+
+	// We're assuming never expiring tokens for now
+	// Set it to 100 years from now
+	expires := time.Now().AddDate(100, 0, 0)
+
+	return &forge.Authenticator{
+		AuthMethod:  transporter,
+		AuthorName:  f.authorName,
+		AuthorEmail: f.authorEmail,
+		Expires:     expires,
+	}, nil
+}
+
+func (f *Forge) GetRemote(repo string) string {
+	return fmt.Sprintf("https://%s/%s/%s", f.host, f.group, repo)
+}
+
+func (f *Forge) GetCommitViewerURL(repo string, commit string) string {
+	return fmt.Sprintf(
+		"https://%s/%s/%s/-/commit/%s",
+		f.host,
+		f.group,
+		repo,
+		commit,
+	)
+}
+
+func (f *Forge) EnsureRepositoryExists(auth *forge.Authenticator, repo string) error {
+	// Cast AuthMethod to BasicAuth
+	basicAuth := auth.AuthMethod.(*transport_http.BasicAuth)
+	token := basicAuth.Password
+
+	client := &http.Client{
+		Timeout: time.Second * 10,
+	}
+
+	// Check if the repo exists
+	urlEncodedPath := url.PathEscape(fmt.Sprintf("%s/%s", f.group, repo))
+	endpoint := fmt.Sprintf("https://%s/api/v4/projects/%s", f.host, urlEncodedPath)
+	req, err := http.NewRequest("GET", endpoint, nil)
+	if err != nil {
+		return err
+	}
+	req.Header.Add("Authorization", "Bearer "+token)
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return err
+	}
+
+	if resp.StatusCode == 200 {
+		// Repo exists, we're done
+		return nil
+	}
+
+	// Repo doesn't exist, create it
+	// First get namespace id
+	endpoint = fmt.Sprintf("https://%s/api/v4/namespaces/%s", f.host, url.PathEscape(f.group))
+	req, err = http.NewRequest("GET", endpoint, nil)
+	if err != nil {
+		return err
+	}
+	req.Header.Add("Authorization", "Bearer "+token)
+
+	resp, err = client.Do(req)
+	if err != nil {
+		return err
+	}
+
+	if resp.StatusCode != 200 {
+		return fmt.Errorf("namespace %s does not exist", f.group)
+	}
+
+	mapBody := map[string]any{}
+	err = json.NewDecoder(resp.Body).Decode(&mapBody)
+	if err != nil {
+		return err
+	}
+
+	namespaceId := mapBody["id"].(float64)
+
+	mapBody = map[string]any{
+		"name":         repo,
+		"namespace_id": namespaceId,
+	}
+	if f.shouldMakeRepoPublic {
+		mapBody["visibility"] = "public"
+	} else {
+		mapBody["visibility"] = "private"
+	}
+
+	endpoint = fmt.Sprintf("https://%s/api/v4/projects", f.host)
+	body, err := json.Marshal(mapBody)
+	if err != nil {
+		return err
+	}
+
+	req, err = http.NewRequest("POST", endpoint, bytes.NewReader(body))
+	if err != nil {
+		return err
+	}
+	req.Header.Add("Content-Type", "application/json")
+	req.Header.Add("Authorization", "Bearer "+token)
+
+	resp, err = client.Do(req)
+	if err != nil {
+		return err
+	}
+
+	if resp.StatusCode != 201 && resp.StatusCode != 200 {
+		body, _ := io.ReadAll(resp.Body)
+		return fmt.Errorf("failed to create repo %s: %s", repo, string(body))
+	}
+
+	return nil
+}
+
+func (f *Forge) WithNamespace(namespace string) forge.Forge {
+	newF := *f
+	newF.group = namespace
+	return &newF
+}

base/go/frontend_server.go

@@ -98,7 +98,7 @@ var frontendHtmlTemplate = `
       name="viewport"
       content="width=device-width, initial-scale=1, viewport-fit=cover"
     />
-    <title>{{.Title}}</title>
+    <title>{{-Title-}}</title>
 
     <link rel="icon" type="image/png" href="/_ga/favicon.png" />
 
@@ -127,6 +127,7 @@ var frontendHtmlTemplate = `
         }
     </script>
     {{end}}
+    {{-Beta-}}
     {{if .Prefix}}
     <script>window.__peridot_prefix__ = '{{.Prefix}}'.replace('\\', '');</script>
     {{end}}
@@ -144,7 +145,7 @@ var frontendUnauthenticated = `
       name="viewport"
       content="width=device-width, initial-scale=1, viewport-fit=cover"
     />
-    <title>{{.Title}} - Unauthenticated</title>
+    <title>{{-Title-}} - Ouch</title>
 
     <link rel="icon" type="image/png" href="/_ga/favicon.png" />
 
@@ -222,45 +223,50 @@ func (info *FrontendInfo) frontendAuthHandler(provider OidcProvider, h http.Hand
 			}
 		}
 
+		ctx := r.Context()
+
 		// get auth cookie
 		authCookie, err := r.Cookie(frontendAuthCookieKey)
 		if err != nil {
-			// redirect to login
-			http.Redirect(w, r, info.Self+"/auth/oidc/login", http.StatusFound)
-			return
-		}
-
-		// verify the token
-		userInfo, err := provider.UserInfo(r.Context(), oauth2.StaticTokenSource(&oauth2.Token{
-			AccessToken: authCookie.Value,
-			TokenType:   "Bearer",
-		}))
-		if err != nil {
-			// redirect to login
-			http.Redirect(w, r, info.Self+"/auth/oidc/login", http.StatusFound)
-			return
-		}
-
-		// Check if the user is in the group
-		var claims oidcClaims
-		err = userInfo.Claims(&claims)
-		if err != nil {
-			// redirect to login
-			http.Redirect(w, r, info.Self+"/auth/oidc/login", http.StatusFound)
-			return
-		}
-
-		groups := claims.Groups
-		if info.OIDCGroup != "" {
-			if !Contains(groups, info.OIDCGroup) {
-				// show unauthenticated page
-				info.renderUnauthorized(w, fmt.Sprintf("User is not in group %s", info.OIDCGroup))
+			// only redirect if not allowed unauthenticated
+			if !info.AllowUnauthenticated {
+				// redirect to login
+				http.Redirect(w, r, info.Self+"/auth/oidc/login", http.StatusFound)
+				return
+			}
+		} else {
+			// verify the token
+			userInfo, err := provider.UserInfo(r.Context(), oauth2.StaticTokenSource(&oauth2.Token{
+				AccessToken: authCookie.Value,
+				TokenType:   "Bearer",
+			}))
+			if err != nil {
+				// redirect to login
+				http.Redirect(w, r, info.Self+"/auth/oidc/login", http.StatusFound)
 				return
 			}
-		}
 
-		// Add the user to the context
-		ctx := context.WithValue(r.Context(), "user", userInfo)
+			// Check if the user is in the group
+			var claims oidcClaims
+			err = userInfo.Claims(&claims)
+			if err != nil {
+				// redirect to login
+				http.Redirect(w, r, info.Self+"/auth/oidc/login", http.StatusFound)
+				return
+			}
+
+			groups := claims.Groups
+			if info.OIDCGroup != "" {
+				if !Contains(groups, info.OIDCGroup) {
+					// show unauthenticated page
+					info.renderUnauthorized(w, fmt.Sprintf("User is not in group %s", info.OIDCGroup))
+					return
+				}
+			}
+
+			// Add the user to the context
+			ctx = context.WithValue(ctx, "user", userInfo)
+		}
 
 		h.ServeHTTP(w, r.WithContext(ctx))
 	})
@@ -290,8 +296,8 @@ func FrontendServer(info *FrontendInfo, embedfs *embed.FS) error {
 	if info.Title == "" {
 		info.Title = "Peridot"
 	}
-	newTemplate = strings.ReplaceAll(newTemplate, "{{.Title}}", info.Title)
-	newUnauthenticatedTemplate = strings.ReplaceAll(newUnauthenticatedTemplate, "{{.Title}}", info.Title)
+	newTemplate = strings.ReplaceAll(newTemplate, "{{-Title-}}", info.Title)
+	newUnauthenticatedTemplate = strings.ReplaceAll(newUnauthenticatedTemplate, "{{-Title-}}", info.Title)
 
 	info.unauthenticatedTemplate = newUnauthenticatedTemplate
 
@@ -354,7 +360,15 @@ func FrontendServer(info *FrontendInfo, embedfs *embed.FS) error {
 	http.HandleFunc(prefix+"/", func(w http.ResponseWriter, r *http.Request) {
 		w.Header().Set("Content-Type", "text/html")
 
-		tmpl, err := template.New("index.html").Parse(newTemplate)
+		// Set beta script (if beta basically set window.__beta__ = true)
+		srvTemplate := newTemplate
+		betaScript := ""
+		if r.Header.Get("x-peridot-beta") == "true" {
+			betaScript = "<script>window.__beta__ = true;</script>"
+		}
+		srvTemplate = strings.ReplaceAll(srvTemplate, "{{-Beta-}}", betaScript)
+
+		tmpl, err := template.New("index.html").Parse(srvTemplate)
 		if err != nil {
 			info.renderUnauthorized(w, fmt.Sprintf("Failed to parse template: %v", err))
 			return
@@ -523,8 +537,8 @@ func FrontendServer(info *FrontendInfo, embedfs *embed.FS) error {
 	}
 
 	var handler http.Handler = nil
-	// if auth is enabled as well as AllowUnauthenticated is false, then wrap the handler with the auth handler
-	if !info.NoAuth && !info.AllowUnauthenticated {
+	// if auth is enabled as well, then wrap the handler with the auth handler
+	if !info.NoAuth {
 		handler = info.frontendAuthHandler(provider, http.DefaultServeMux)
 	} else {
 		handler = http.DefaultServeMux

base/go/kv/BUILD

@@ -0,0 +1,8 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "kv",
+    srcs = ["kv.go"],
+    importpath = "go.resf.org/peridot/base/go/kv",
+    visibility = ["//visibility:public"],
+)

base/go/kv/dynamodb/BUILD

@@ -0,0 +1,17 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "dynamodb",
+    srcs = ["dynamodb.go"],
+    importpath = "go.resf.org/peridot/base/go/kv/dynamodb",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//base/go/awsutils",
+        "//base/go/kv",
+        "//base/proto:pb",
+        "//vendor/github.com/aws/aws-sdk-go/aws",
+        "//vendor/github.com/aws/aws-sdk-go/aws/session",
+        "//vendor/github.com/aws/aws-sdk-go/service/dynamodb",
+        "@org_golang_google_protobuf//proto",
+    ],
+)

base/go/kv/dynamodb/dynamodb.go

@@ -0,0 +1,328 @@
+package dynamodb
+
+import (
+	"context"
+	"crypto/rand"
+	"errors"
+	"fmt"
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/dynamodb"
+	"go.resf.org/peridot/base/go/awsutils"
+	"go.resf.org/peridot/base/go/kv"
+	basepb "go.resf.org/peridot/base/go/pb"
+	"google.golang.org/protobuf/proto"
+	"strings"
+	"time"
+)
+
+type DynamoDB struct {
+	db        *dynamodb.DynamoDB
+	tableName string
+}
+
+// New creates a new DynamoDB storage backend.
+func New(endpoint string, tableName string) (*DynamoDB, error) {
+	awsCfg := &aws.Config{}
+	awsutils.FillOutConfig(awsCfg)
+
+	if endpoint != "" {
+		awsCfg.Endpoint = aws.String(endpoint)
+	}
+
+	sess, err := session.NewSession(awsCfg)
+	if err != nil {
+		return nil, err
+	}
+
+	svc := dynamodb.New(sess)
+
+	// Create the table if it doesn't exist.
+	// First check if the table exists.
+	_, err = svc.DescribeTable(&dynamodb.DescribeTableInput{
+		TableName: aws.String(tableName),
+	})
+	if err != nil {
+		_, err = svc.CreateTable(&dynamodb.CreateTableInput{
+			TableName: aws.String(tableName),
+			AttributeDefinitions: []*dynamodb.AttributeDefinition{
+				{
+					AttributeName: aws.String("Key"),
+					AttributeType: aws.String("S"),
+				},
+				{
+					AttributeName: aws.String("Path"),
+					AttributeType: aws.String("S"),
+				},
+			},
+			KeySchema: []*dynamodb.KeySchemaElement{
+				{
+					AttributeName: aws.String("Key"),
+					KeyType:       aws.String("HASH"),
+				},
+				{
+					AttributeName: aws.String("Path"),
+					KeyType:       aws.String("RANGE"),
+				},
+			},
+			ProvisionedThroughput: &dynamodb.ProvisionedThroughput{
+				ReadCapacityUnits:  aws.Int64(5),
+				WriteCapacityUnits: aws.Int64(5),
+			},
+		})
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return &DynamoDB{
+		db:        svc,
+		tableName: tableName,
+	}, nil
+}
+
+func (d *DynamoDB) Get(ctx context.Context, key string) (*kv.Pair, error) {
+	trimmed := strings.TrimPrefix(key, "/")
+	parts := strings.Split(trimmed, "/")
+	if len(parts) < 2 {
+		return nil, kv.ErrNoNamespace
+	}
+	ns := parts[0]
+
+	result, err := d.db.GetItem(&dynamodb.GetItemInput{
+		TableName: aws.String(d.tableName),
+		Key: map[string]*dynamodb.AttributeValue{
+			"Key": {
+				S: aws.String(ns),
+			},
+			"Path": {
+				S: aws.String(strings.Join(parts[1:], "/")),
+			},
+		},
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	if result.Item == nil {
+		return nil, kv.ErrNotFound
+	}
+
+	return &kv.Pair{
+		Key:   *result.Item["Key"].S,
+		Value: result.Item["Value"].B,
+	}, nil
+}
+
+func (d *DynamoDB) Set(ctx context.Context, key string, value []byte) error {
+	trimmed := strings.TrimPrefix(key, "/")
+	parts := strings.Split(trimmed, "/")
+	if len(parts) < 2 {
+		return kv.ErrNoNamespace
+	}
+	ns := parts[0]
+
+	_, err := d.db.PutItem(&dynamodb.PutItemInput{
+		TableName: aws.String(d.tableName),
+		Item: map[string]*dynamodb.AttributeValue{
+			"Key": {
+				S: aws.String(ns),
+			},
+			"Path": {
+				S: aws.String(strings.Join(parts[1:], "/")),
+			},
+			"Value": {
+				B: value,
+			},
+		},
+	})
+	return err
+}
+
+func (d *DynamoDB) Delete(ctx context.Context, key string) error {
+	trimmed := strings.TrimPrefix(key, "/")
+	parts := strings.Split(trimmed, "/")
+	if len(parts) < 2 {
+		return kv.ErrNoNamespace
+	}
+	ns := parts[0]
+
+	_, err := d.db.DeleteItem(&dynamodb.DeleteItemInput{
+		TableName: aws.String(d.tableName),
+		Key: map[string]*dynamodb.AttributeValue{
+			"Key": {
+				S: aws.String(ns),
+			},
+			"Path": {
+				S: aws.String(strings.Join(parts[1:], "/")),
+			},
+		},
+	})
+	return err
+}
+
+func (d *DynamoDB) RangePrefix(ctx context.Context, prefix string, pageSize int32, pageToken string) (*kv.Query, error) {
+	if pageSize <= 0 {
+		pageSize = 20
+	}
+	if pageSize > 100 {
+		pageSize = 100
+	}
+
+	// Check if there is a page token.
+	var fromKey string
+	var fromPath string
+	if pageToken != "" {
+		// Get the page token from the database.
+		res, err := d.Get(ctx, fmt.Sprintf("/_internals/page_tokens/%s", pageToken))
+		if err != nil {
+			if errors.Is(err, kv.ErrNotFound) {
+				return nil, kv.ErrPageTokenNotFound
+			}
+			return nil, err
+		}
+
+		// Parse the page token.
+		pt := &basepb.DynamoDbPageToken{}
+		err = proto.Unmarshal(res.Value, pt)
+		if err != nil {
+			return nil, err
+		}
+
+		fromKey = pt.LastKey
+		fromPath = pt.LastPath
+	}
+
+	trimmed := strings.TrimPrefix(prefix, "/")
+	parts := strings.Split(trimmed, "/")
+	if len(parts) < 2 {
+		return nil, kv.ErrNoNamespace
+	}
+	ns := parts[0]
+
+	queryInput := &dynamodb.QueryInput{
+		TableName: aws.String(d.tableName),
+		KeyConditions: map[string]*dynamodb.Condition{
+			"Key": {
+				ComparisonOperator: aws.String("EQ"),
+				AttributeValueList: []*dynamodb.AttributeValue{
+					{
+						S: aws.String(ns),
+					},
+				},
+			},
+			"Path": {
+				ComparisonOperator: aws.String("BEGINS_WITH"),
+				AttributeValueList: []*dynamodb.AttributeValue{
+					{
+						S: aws.String(strings.Join(parts[1:], "/")),
+					},
+				},
+			},
+		},
+		Limit: aws.Int64(int64(pageSize + 1)),
+	}
+	if fromKey != "" && fromPath != "" {
+		queryInput.ExclusiveStartKey = map[string]*dynamodb.AttributeValue{
+			"Key": {
+				S: aws.String(fromKey),
+			},
+			"Path": {
+				S: aws.String(fromPath),
+			},
+		}
+	}
+	result, err := d.db.Query(queryInput)
+	if err != nil {
+		return nil, err
+	}
+
+	pairs := make([]*kv.Pair, 0, len(result.Items))
+	for _, item := range result.Items {
+		// Since we fetch pageSize+1, stop if we have enough.
+		if len(pairs) >= int(pageSize) {
+			break
+		}
+		pairs = append(pairs, &kv.Pair{
+			Key:   *item["Key"].S,
+			Value: item["Value"].B,
+		})
+	}
+
+	var nextToken string
+	var lastEvalKey string
+	var lastEvalPath string
+
+	// We always fetch pageSize+1, so if we have pageSize+1 results, we need to
+	// create a page token. We can't continue from result.LastEvaluatedKey since
+	// that will skip over the last result. So we should continue from the last
+	// visible result.
+	if len(result.Items) > int(pageSize) {
+		lastEvalKey = *result.Items[len(pairs)-1]["Key"].S
+		lastEvalPath = *result.Items[len(pairs)-1]["Path"].S
+	} else if result.LastEvaluatedKey != nil {
+		lastEvalKey = *result.LastEvaluatedKey["Key"].S
+		lastEvalPath = *result.LastEvaluatedKey["Path"].S
+	}
+
+	if lastEvalKey != "" && lastEvalPath != "" {
+		// Add page token to the database.
+		ptKey, err := generatePageToken()
+		if err != nil {
+			return nil, err
+		}
+
+		ptBytes, err := proto.Marshal(&basepb.DynamoDbPageToken{
+			LastKey:  lastEvalKey,
+			LastPath: lastEvalPath,
+		})
+		if err != nil {
+			return nil, err
+		}
+
+		// Create the page token in the database, and it should expire in 2 days.
+		_, err = d.db.PutItem(&dynamodb.PutItemInput{
+			TableName: aws.String(d.tableName),
+			Item: map[string]*dynamodb.AttributeValue{
+				"Key": {
+					S: aws.String("_internals"),
+				},
+				"Path": {
+					S: aws.String(fmt.Sprintf("page_tokens/%s", ptKey)),
+				},
+				"Value": {
+					B: ptBytes,
+				},
+				"ExpiresAt": {
+					N: aws.String(fmt.Sprintf("%d", time.Now().Add(48*time.Hour).Unix())),
+				},
+			},
+		})
+		if err != nil {
+			return nil, err
+		}
+
+		nextToken = ptKey
+	}
+
+	return &kv.Query{
+		Prefix:    prefix,
+		Pairs:     pairs,
+		NextToken: nextToken,
+	}, nil
+}
+
+func generatePageToken() (string, error) {
+	possible := "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+	token := make([]byte, 48)
+	_, err := rand.Read(token)
+	if err != nil {
+		return "", err
+	}
+
+	for i := 0; i < len(token); i++ {
+		token[i] = possible[token[i]%byte(len(possible))]
+	}
+
+	return "v1." + string(token), nil
+}

base/go/kv/kv.go

@@ -0,0 +1,33 @@
+package kv
+
+import (
+	"context"
+	"errors"
+)
+
+var (
+	ErrNotFound          = errors.New("key not found")
+	ErrPageTokenNotFound = errors.New("page token not found")
+	ErrNoNamespace       = errors.New("no namespace")
+)
+
+type Pair struct {
+	Key   string
+	Value []byte
+}
+
+type Query struct {
+	Prefix    string
+	Pairs     []*Pair
+	NextToken string
+}
+
+type KV interface {
+	// Get returns the contents of a file from the storage backend.
+	// Key must have a namespace prefix.
+	// Example: /kernels/entries/123, where kernels is the namespace and the rest is the range key.
+	Get(ctx context.Context, key string) (*Pair, error)
+	Set(ctx context.Context, key string, value []byte) error
+	Delete(ctx context.Context, key string) error
+	RangePrefix(ctx context.Context, prefix string, pageSize int32, pageToken string) (*Query, error)
+}

base/proto/BUILD

@@ -0,0 +1,23 @@
+load("@rules_proto//proto:defs.bzl", "proto_library")
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+load("@io_bazel_rules_go//proto:def.bzl", "go_proto_library")
+
+proto_library(
+    name = "basepb_proto",
+    srcs = ["kv_page_token.proto"],
+    visibility = ["//visibility:public"],
+)
+
+go_proto_library(
+    name = "basepb_go_proto",
+    importpath = "go.resf.org/peridot/base/go/pb",
+    proto = ":basepb_proto",
+    visibility = ["//visibility:public"],
+)
+
+go_library(
+    name = "pb",
+    embed = [":basepb_go_proto"],
+    importpath = "go.resf.org/peridot/base/go/pb",
+    visibility = ["//visibility:public"],
+)

base/proto/kv_page_token.proto

@@ -0,0 +1,17 @@
+syntax = "proto3";
+
+package base.go;
+
+option java_multiple_files = true;
+option java_outer_classname = "KvPageTokenProto";
+option java_package = "org.resf.base.go";
+option go_package = "go.resf.org/peridot/base/go/pb;basepb";
+
+// DynamoDbPageToken is the page token used for DynamoDB.
+message DynamoDbPageToken {
+  // key is the last evaluated key.
+  string last_key = 1;
+
+  // path is the last evaluated path.
+  string last_path = 2;
+}

base/ts/global.d.ts

@@ -23,5 +23,6 @@ declare global {
   interface Window {
     __peridot_prefix__: string;
     __peridot_user__: PeridotUser;
+    __beta__: boolean;
   }
 }

base/ts/mui/ResourceTable.tsx

@@ -49,6 +49,9 @@ export interface ResourceTableProps<T> {
   // Default filter to start with
   defaultFilter?: string;
 
+  // Whether to hide the filter input
+  hideFilter?: boolean;
+
   // load is usually the OpenAPI SDK function that loads the resource.
   load(pageSize: number, pageToken?: string, filter?: string): Promise<any>;
 
@@ -102,6 +105,7 @@ export function ResourceTable<T extends StandardResource>(
   const [rows, setRows] = React.useState<T[] | undefined>(undefined);
   const [loading, setLoading] = React.useState<boolean>(false);
   const [filter, setFilter] = React.useState<string | undefined>(initFilter);
+  const [filterValue, setFilterValue] = React.useState<string | undefined>(initFilter);
 
   const updateSearch = (replace = false) => {
     const search = new URLSearchParams(location.search);
@@ -219,7 +223,7 @@ export function ResourceTable<T extends StandardResource>(
   // Load the resource using useEffect
   React.useEffect(() => {
     fetchResource().then();
-  }, [pageToken, rowsPerPage]);
+  }, [filter, pageToken, rowsPerPage]);
 
   // For filter, we're going to wait for the user to stop typing for 500ms
   // before we actually fetch the resource.
@@ -231,9 +235,9 @@ export function ResourceTable<T extends StandardResource>(
       clearTimeout(timeout.current);
     }
     timeout.current = setTimeout(() => {
-      fetchResource().then();
+      setFilter(filterValue);
     }, 500);
-  }, [filter]);
+  }, [filterValue]);
 
   // Create table header
   const header = props.fields.map((field) => {
@@ -262,20 +266,21 @@ export function ResourceTable<T extends StandardResource>(
 
   // Create a search box for filter input
   // This can be disabled if the request does not support filtering
-  const searchBox = (
+  const searchBox = !props.hideFilter && (
     <Box sx={{ display: 'flex', alignItems: 'center', mb: 2, width: '100%' }}>
       <TextField
         sx={{ mr: 2, flexGrow: 1 }}
         label="Filter"
         variant="outlined"
         size="small"
-        value={filter}
-        onChange={(event: React.ChangeEvent<HTMLInputElement>) => setFilter(event.target.value)}
+        value={filterValue}
+        onChange={(event: React.ChangeEvent<HTMLInputElement>) => setFilterValue(event.target.value)}
       />
       <Button
         variant="contained"
         onClick={() => {
           setFilter('');
+          setFilterValue('');
           setPageToken(undefined);
         }}
       >

deps.bzl

@@ -638,6 +638,7 @@ def go_dependencies():
         sum = "h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=",
         version = "v1.0.1",
     )
+
     go_repository(
         name = "com_github_elazarl_goproxy",
         importpath = "github.com/elazarl/goproxy",
@@ -1885,6 +1886,7 @@ def go_dependencies():
         sum = "h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI=",
         version = "v2.0.1+incompatible",
     )
+
     go_repository(
         name = "com_github_pjbgf_sha1cd",
         importpath = "github.com/pjbgf/sha1cd",

devtools/taskrunner2/main.go

@@ -130,11 +130,12 @@ func spawnIbazel(target string) error {
 func run(ctx *cli.Context) error {
 	// add default frontend flags if needed
 	if ctx.Bool("dev-frontend-flags") {
-