diff --git a/model/interactive/src/main/proto/org/apache/beam/model/interactive/v1/beam_interactive_api.proto b/model/interactive/src/main/proto/org/apache/beam/model/interactive/v1/beam_interactive_api.proto index 6351b2472579..faae80261402 100644 --- a/model/interactive/src/main/proto/org/apache/beam/model/interactive/v1/beam_interactive_api.proto +++ b/model/interactive/src/main/proto/org/apache/beam/model/interactive/v1/beam_interactive_api.proto @@ -50,11 +50,15 @@ message TestStreamFileRecord { org.apache.beam.model.pipeline.v1.TestStreamPayload.Event recorded_event = 1; } +// A gRPC service that serves cached TestStream events to a TestStream source. +// Used by Interactive Beam to replay recorded pipeline elements from a streaming cache. service TestStreamService { // A TestStream will request for events using this RPC. rpc Events(EventsRequest) returns (stream org.apache.beam.model.pipeline.v1.TestStreamPayload.Event) {} } +// A request for TestStream events from a streaming cache. +// The client specifies which PCollection output tags to read events for. message EventsRequest { // The set of PCollections to read from. These are the PTransform outputs // local names. These are a subset of the TestStream's outputs. This allows diff --git a/model/job-management/src/main/proto/org/apache/beam/model/job_management/v1/beam_artifact_api.proto b/model/job-management/src/main/proto/org/apache/beam/model/job_management/v1/beam_artifact_api.proto index 35cd4f0c73d1..a64352c9289e 100644 --- a/model/job-management/src/main/proto/org/apache/beam/model/job_management/v1/beam_artifact_api.proto +++ b/model/job-management/src/main/proto/org/apache/beam/model/job_management/v1/beam_artifact_api.proto @@ -78,18 +78,23 @@ message ResolveArtifactsResponse { // A request to get an artifact. message GetArtifactRequest { + // (Required) The artifact to retrieve, typically a resolved artifact from ResolveArtifacts. org.apache.beam.model.pipeline.v1.ArtifactInformation artifact = 1; } // Part of a response to getting an artifact. message GetArtifactResponse { + // A chunk of the artifact's binary content. bytes data = 1; } // Wraps an ArtifactRetrievalService request for use in ReverseArtifactRetrievalService. +// The server sends these requests to the client during the reverse staging flow. message ArtifactRequestWrapper { oneof request { + // Request to resolve a set of artifacts into concrete artifact references. ResolveArtifactsRequest resolve_artifact = 1000; + // Request to retrieve the bytes of a specific artifact. GetArtifactRequest get_artifact = 1001; } } @@ -151,16 +156,22 @@ message ArtifactMetadata { // A collection of artifacts. message Manifest { + // (Required) The list of artifacts staged for a job. repeated ArtifactMetadata artifact = 1; } // A manifest with location information. message ProxyManifest { + // (Required) The manifest of staged artifacts. Manifest manifest = 1; + // A location (name + URI) at which a staged artifact can be retrieved. message Location { - string name = 1; - string uri = 2; + // (Required) The name of the artifact, matching the name in the Manifest. + string name = 1; + // (Required) The URI at which the artifact content can be retrieved. + string uri = 2; } + // (Required) The list of artifact locations for retrieval. repeated Location location = 2; } @@ -173,6 +184,7 @@ message GetManifestRequest { // A response containing a job manifest. message GetManifestResponse { + // (Required) The manifest of staged artifacts for the job. Manifest manifest = 1; } @@ -187,9 +199,11 @@ message LegacyGetArtifactRequest { // Part of an artifact. message ArtifactChunk { + // A chunk of the artifact's binary content. bytes data = 1; } +// Metadata for an artifact being staged, sent as the first message in a PutArtifact stream. message PutArtifactMetadata { // (Required) A token for artifact staging session. This token can be obtained // from PrepareJob request in JobService @@ -211,6 +225,7 @@ message PutArtifactRequest { } } +// A response to staging an artifact. An empty response indicating the artifact was staged successfully. message PutArtifactResponse { } diff --git a/model/job-management/src/main/proto/org/apache/beam/model/job_management/v1/beam_expansion_api.proto b/model/job-management/src/main/proto/org/apache/beam/model/job_management/v1/beam_expansion_api.proto index a4736f8b4938..0e3b1adbffd6 100644 --- a/model/job-management/src/main/proto/org/apache/beam/model/job_management/v1/beam_expansion_api.proto +++ b/model/job-management/src/main/proto/org/apache/beam/model/job_management/v1/beam_expansion_api.proto @@ -33,6 +33,8 @@ import "google/protobuf/struct.proto"; import "org/apache/beam/model/pipeline/v1/beam_runner_api.proto"; import "org/apache/beam/model/pipeline/v1/schema.proto"; +// A request to expand a single PTransform in a remote SDK. The expansion service +// resolves the transform into its constituent subtransforms and outputs. message ExpansionRequest { // Set of components needed to interpret the transform, or which // may be useful for its expansion. This includes the input @@ -64,6 +66,8 @@ message ExpansionRequest { google.protobuf.Struct pipeline_options = 6; } +// The result of expanding a PTransform, containing the expanded transform and +// all newly created components. message ExpansionResponse { // Set of components needed to execute the expanded transform, // including the (original) inputs, outputs, and subtransforms. @@ -82,9 +86,13 @@ message ExpansionResponse { string error = 10; } +// A request to discover all SchemaTransformProviders registered with the expansion service. +// This is used by cross-language pipeline construction to enumerate available transforms. message DiscoverSchemaTransformRequest { } +// Configuration metadata for a SchemaTransform, describing its inputs, outputs, +// and configuration schema. Used to enable cross-language transform discovery. message SchemaTransformConfig { // Config schema of the SchemaTransform org.apache.beam.model.pipeline.v1.Schema config_schema = 1; @@ -102,6 +110,7 @@ message SchemaTransformConfig { string description = 4; } +// A response containing all discovered SchemaTransform configurations. message DiscoverSchemaTransformResponse { // A mapping from SchemaTransform ID to schema transform config of discovered // SchemaTransforms diff --git a/model/job-management/src/main/proto/org/apache/beam/model/job_management/v1/beam_job_api.proto b/model/job-management/src/main/proto/org/apache/beam/model/job_management/v1/beam_job_api.proto index 8e196d5bedd3..353eaec57994 100644 --- a/model/job-management/src/main/proto/org/apache/beam/model/job_management/v1/beam_job_api.proto +++ b/model/job-management/src/main/proto/org/apache/beam/model/job_management/v1/beam_job_api.proto @@ -112,6 +112,7 @@ message RunJobRequest { } +// The response from submitting a job for execution. message RunJobResponse { string job_id = 1; // (required) The ID for the executing job } @@ -144,6 +145,7 @@ message DrainJobResponse { } // A subset of info provided by ProvisionApi.ProvisionInfo +// Represents metadata about a submitted job. message JobInfo { string job_id = 1; // (required) string job_name = 2; // (required) @@ -155,6 +157,7 @@ message JobInfo { // Throws error GRPC_STATUS_UNAVAILABLE if server is down message GetJobsRequest { } +// The response from requesting a list of all invoked jobs. message GetJobsResponse { repeated JobInfo job_info = 1; // (required) } @@ -168,6 +171,7 @@ message GetJobStateRequest { } +// An event representing a job state transition. Emitted by GetState and GetStateStream. message JobStateEvent { JobState.Enum state = 1; // (required) google.protobuf.Timestamp timestamp = 2; // (required) @@ -182,6 +186,7 @@ message GetJobPipelineRequest { } +// The response from requesting a job's pipeline representation. message GetJobPipelineResponse { org.apache.beam.model.pipeline.v1.Pipeline pipeline = 1; // (required) } @@ -195,25 +200,41 @@ message JobMessagesRequest { string job_id = 1; // (required) } +// A single log message or diagnostic event from a running job. message JobMessage { + // (Required) A unique identifier for this message. string message_id = 1; + // (Required) The time at which the message was emitted, as a string representation. string time = 2; + // (Required) The importance level of this message. MessageImportance importance = 3; + // (Required) The text content of the message. string message_text = 4; + // Importance levels for job messages, ordered from least to most severe. enum MessageImportance { + // The importance was not specified. MESSAGE_IMPORTANCE_UNSPECIFIED = 0; + // Debug-level messages, typically very verbose. JOB_MESSAGE_DEBUG = 1; + // Detailed informational messages. JOB_MESSAGE_DETAILED = 2; + // Basic informational messages. JOB_MESSAGE_BASIC = 3; + // Warning messages indicating potential issues. JOB_MESSAGE_WARNING = 4; + // Error messages indicating failures. JOB_MESSAGE_ERROR = 5; } } +// A streaming response from GetMessageStream, containing either a job message +// or a job state change event. message JobMessagesResponse { oneof response { + // A log message or diagnostic event from the job. JobMessage message_response = 1; + // A job state transition event. JobStateEvent state_response = 2; } } @@ -270,17 +291,22 @@ message JobState { } +// A request to fetch metrics for a given job. message GetJobMetricsRequest { string job_id = 1; // (required) } +// A response containing metrics for a given job. message GetJobMetricsResponse { + // (Required) The metrics results containing both attempted and committed values. MetricResults metrics = 1; } // All metrics for a given job. Runners may support one or the other or both. message MetricResults { + // Metrics reflecting the result of attempted (non-committed) computations. repeated org.apache.beam.model.pipeline.v1.MonitoringInfo attempted = 1; + // Metrics reflecting the result of committed computations. repeated org.apache.beam.model.pipeline.v1.MonitoringInfo committed = 2; } @@ -296,12 +322,17 @@ message DescribePipelineOptionsRequest { // Types mirror those of JSON, since that's how pipeline options are serialized. message PipelineOptionType { enum Enum { + // A string value. STRING = 0; + // A boolean (true/false) value. BOOLEAN = 1; // whole numbers, see https://json-schema.org/understanding-json-schema/reference/numeric.html INTEGER = 2; + // A floating-point number. NUMBER = 3; + // An array of values. ARRAY = 4; + // A nested object. OBJECT = 5; }; } @@ -324,6 +355,7 @@ message PipelineOptionDescriptor { string group = 5; } +// The response from describing pipeline options, containing a list of option descriptors. message DescribePipelineOptionsResponse { // List of pipeline option descriptors. repeated PipelineOptionDescriptor options = 1;