Add Batch resource

tools/mothership/proto/v1/BUILD

@@ -20,7 +20,9 @@ load("//tools/build_rules/oapi_gen:defs.bzl", "oapi_gen_ts")
 proto_library(
     name = "mothershippb_proto",
     srcs = [
+        "batch.proto",
         "entry.proto",
+        "process_rpm.proto",
         "srpm_archiver.proto",
     ],
     visibility = ["//visibility:public"],
@@ -28,6 +30,7 @@ proto_library(
         "@com_google_protobuf//:timestamp_proto",
         "@com_google_protobuf//:wrappers_proto",
         "@go_googleapis//google/api:annotations_proto",
+        "@go_googleapis//google/longrunning:longrunning_proto",
         "@googleapis//google/api:annotations_proto",
     ],
 )
@@ -49,6 +52,7 @@ go_proto_library(
     visibility = ["//visibility:public"],
     deps = [
         "@go_googleapis//google/api:annotations_go_proto",
+        "@go_googleapis//google/longrunning:longrunning_go_proto",
         "@org_golang_google_genproto//googleapis/api/annotations",
     ],
 )

tools/mothership/proto/v1/batch.proto

@@ -0,0 +1,49 @@
+// Copyright 2023 Peridot Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto3";
+
+package peridot.tools.mothership.v1;
+
+import "google/api/field_behavior.proto";
+import "google/protobuf/timestamp.proto";
+import "google/protobuf/wrappers.proto";
+
+option java_multiple_files = true;
+option java_outer_classname = "BatchProto";
+option java_package = "org.resf.peridot.tools.mothership.v1";
+option go_package = "go.resf.org/peridot/tools/mothership/pb;mothershippb";
+
+// Batch is a collection of Entries that were imported closely
+// together. It usually indicates an update batch from a single
+// source.
+message Batch {
+  // Output only. Unique ID of the batch.
+  string name = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
+
+  // Custom ID of the batch. Optional
+  string batch_id = 2;
+
+  // Output only. Timestamp when the batch was created.
+  google.protobuf.Timestamp create_time = 3 [(google.api.field_behavior) = OUTPUT_ONLY];
+
+  // Output only. Timestamp when the batch was last updated.
+  google.protobuf.Timestamp update_time = 4 [(google.api.field_behavior) = OUTPUT_ONLY];
+
+  // Output only. Timestamp when the batch was sealed.
+  google.protobuf.Timestamp seal_time = 5 [(google.api.field_behavior) = OUTPUT_ONLY];
+
+  // Output only. Bugtracker URI of the batch.
+  google.protobuf.StringValue bugtracker_uri = 6 [(google.api.field_behavior) = OUTPUT_ONLY];
+}

tools/mothership/proto/v1/srpm_archiver.proto

@@ -19,7 +19,10 @@ package peridot.tools.mothership.v1;
 import "google/api/annotations.proto";
 import "google/api/client.proto";
 import "google/api/field_behavior.proto";
+import "google/longrunning/operations.proto";
+import "tools/mothership/proto/v1/batch.proto";
 import "tools/mothership/proto/v1/entry.proto";
+import "tools/mothership/proto/v1/process_rpm.proto";
 
 option java_multiple_files = true;
 option java_outer_classname = "SrpmArchiverProto";
@@ -34,6 +37,31 @@ option go_package = "go.resf.org/peridot/tools/mothership/pb;mothershippb";
 service SrpmArchiver {
   option (google.api.default_host) = "mship.resf.org";
 
+  // Returns a batch
+  rpc GetBatch(GetBatchRequest) returns (Batch) {
+    option (google.api.http) = {
+      get: "/v1/{name=batches/*}"
+    };
+    option (google.api.method_signature) = "name";
+  }
+
+  // Returns a list of batches that match the filter criteria.
+  rpc ListBatches(ListBatchesRequest) returns (ListBatchesResponse) {
+    option (google.api.http) = {
+      get: "/v1/batches"
+    };
+  }
+
+  // Creates a batch.
+  // Only worker credentials can create a batch.
+  rpc CreateBatch(CreateBatchRequest) returns (Batch) {
+    option (google.api.http) = {
+      post: "/v1/batches"
+      body: "batch"
+    };
+    option (google.api.method_signature) = "batch";
+  }
+
   // Returns an entry
   rpc GetEntry(GetEntryRequest) returns (Entry) {
     option (google.api.http) = {
@@ -48,6 +76,97 @@ service SrpmArchiver {
       get: "/v1/entries"
     };
   }
+
+  // Submits an SRPM to be archived.
+  // A worker can call this method to submit an SRPM to be archived.
+  // The call can occur even before uploading the SRPM to the object storage
+  // that way it can be ensured that a certain hash is "leased" by the worker.
+  // Other workers will still keep the hash in their backlog until the SRPM is
+  // verified processed.
+  // Until they can query an entry with `sha256_sum=X` matching the hash of the
+  // SRPM, it will not be deleted from the backlog.
+  // If after 2 hours the SRPM is not processed, the worker can assume that
+  // the SRPM is lost and can be re-uploaded. It that case, the entry will be
+  // re-assigned to the worker.
+  rpc SubmitEntry(SubmitEntryRequest) returns (google.longrunning.Operation) {
+    option (google.api.http) = {
+      post: "/v1/entries:submitEntry"
+      body: "*"
+    };
+    option (google.longrunning.operation_info) = {
+      response_type: "ProcessRPMResponse"
+      metadata_type: "ProcessRPMMetadata"
+    };
+  }
+
+  // WorkerUploadObject is used by workers to upload objects to the
+  // object storage service.
+  // Returns 409 (Conflict) if the SRPM already exists. This should indicate to the
+  // worker that they should not continue processing the SRPM.
+  rpc WorkerUploadObject(stream WorkerUploadObjectRequest) returns (WorkerUploadObjectResponse) {
+    option (google.api.http) = {
+      post: "/v1/srpms:workerUploadObject"
+      body: "chunk"
+    };
+  }
+}
+
+// Request message for GetBatch method.
+message GetBatchRequest {
+  // The name of the batch to retrieve.
+  // For example: "batches/1234".
+  string name = 1 [(google.api.field_behavior) = REQUIRED];
+}
+
+// Request message for ListBatches method.
+message ListBatchesRequest {
+  // The maximum number of batches to return.
+  // The service may return fewer than this value.
+  // If unspecified, at most 50 batches will be returned.
+  // The maximum value is 1000; values above 1000 will return an error
+  int32 page_size = 1;
+
+  // A page token, received from a previous `ListBatches` call.
+  // Provide this to retrieve the subsequent page.
+  //
+  // When paginating, all other parameters provided to `ListBatches` must
+  // match the call that provided the page token.
+  string page_token = 2;
+
+  // The filter string, following the syntax described in
+  // https://google.aip.dev/160.
+  // Supports all fields of the `Batch` resource.
+  // Examples:
+  //   - By name: `name="batches/1234"`
+  string filter = 3;
+
+  // The order to sort the results by. For example: `name desc`.
+  // Supports all fields of the `Batch` resource.
+  // Needs a suffix of either `asc` or `desc`.
+  // Example: `name asc`, `created_at desc`.
+  string order_by = 4;
+}
+
+// Response message for ListBatches method.
+message ListBatchesResponse {
+  // The list of batches.
+  repeated Batch batches = 1;
+
+  // A token to retrieve next page of results.
+  // Pass this value in the
+  // [ListBatchesRequest.page_token][peridot.tools.mothership.v1.ListBatchesRequest.page_token]
+  // field in the subsequent call to `ListBatches` method to retrieve the next
+  // page of results.
+  string next_page_token = 2;
+}
+
+// Request message for CreateBatch method.
+message CreateBatchRequest {
+  // The batch to create.
+  Batch batch = 1 [(google.api.field_behavior) = REQUIRED];
+
+  // Custom ID for the batch. Optional
+  string batch_id = 2;
 }
 
 // Request message for GetEntry method.
@@ -62,7 +181,7 @@ message ListEntriesRequest {
   // The maximum number of entries to return.
   // The service may return fewer than this value.
   // If unspecified, at most 50 entries will be returned.
-  // The maximum value is 1000; values above 1000 will be coerced to 1000.
+  // The maximum value is 1000; values above 1000 will return an error
   int32 page_size = 1;
 
   // A page token, received from a previous `ListEntries` call.
@@ -98,3 +217,26 @@ message ListEntriesResponse {
   // page of results.
   string next_page_token = 2;
 }
+
+// Request message for SubmitEntry method.
+message SubmitEntryRequest {
+  // Process request for RPM.
+  // This request is sent to the worker to process the RPM.
+  ProcessRPMRequest process_rpm_request = 1 [(google.api.field_behavior) = REQUIRED];
+
+  // The name of the batch to submit the entry to.
+  // For example: "batches/1234".
+  string batch = 2;
+}
+
+// Request message for WorkerUploadObject method.
+message WorkerUploadObjectRequest {
+  // The object to upload.
+  bytes chunk = 1 [(google.api.field_behavior) = REQUIRED];
+}
+
+// Response message for WorkerUploadObject method.
+message WorkerUploadObjectResponse {
+  // The object URI.
+  string uri = 1 [(google.api.field_behavior) = REQUIRED];
+}