feat: add featurestore, index, metadata, monitoring, pipeline, and te…

…nsorboard services to aiplatform v1beta1. (#118)

PiperOrigin-RevId: 371996826
Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com>
Co-authored-by: Jeff Ching <chingor@google.com>

linkinator.config.json

@@ -3,10 +3,7 @@
   "skip": [
     "https://codecov.io/gh/googleapis/",
     "www.googleapis.com",
-    "img.shields.io",
-    "https://github.com/googleapis/nodejs-ai-platform/blob/master/samples/create-batch-prediction-job-text-classification.js",
-    "https://github.com/googleapis/nodejs-ai-platform/blob/master/samples/create-batch-prediction-job-text-entity-extraction.js",
-    "https://github.com/googleapis/nodejs-ai-platform/blob/master/samples/create-batch-prediction-job-text-sentiment-analysis.js"
+    "img.shields.io"
   ],
   "silent": true,
   "concurrency": 10

protos/google/cloud/aiplatform/v1beta1/artifact.proto

@@ -0,0 +1,107 @@
+// Copyright 2021 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto3";
+
+package google.cloud.aiplatform.v1beta1;
+
+import "google/api/field_behavior.proto";
+import "google/api/resource.proto";
+import "google/cloud/aiplatform/v1beta1/value.proto";
+import "google/protobuf/struct.proto";
+import "google/protobuf/timestamp.proto";
+import "google/api/annotations.proto";
+
+option go_package = "google.golang.org/genproto/googleapis/cloud/aiplatform/v1beta1;aiplatform";
+option java_multiple_files = true;
+option java_outer_classname = "ArtifactProto";
+option java_package = "com.google.cloud.aiplatform.v1beta1";
+
+// Instance of a general artifact.
+message Artifact {
+  option (google.api.resource) = {
+    type: "aiplatform.googleapis.com/Artifact"
+    pattern: "projects/{project}/locations/{location}/metadataStores/{metadata_store}/artifacts/{artifact}"
+  };
+
+  // Describes the state of the Artifact.
+  enum State {
+    // Unspecified state for the Artifact.
+    STATE_UNSPECIFIED = 0;
+
+    // A state used by systems like Managed Pipelines to indicate that the
+    // underlying data item represented by this Artifact is being created.
+    PENDING = 1;
+
+    // A state indicating that the Artifact should exist, unless something
+    // external to the system deletes it.
+    LIVE = 2;
+  }
+
+  // Output only. The resource name of the Artifact.
+  string name = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
+
+  // User provided display name of the Artifact.
+  // May be up to 128 Unicode characters.
+  string display_name = 2;
+
+  // The uniform resource identifier of the artifact file.
+  // May be empty if there is no actual artifact file.
+  string uri = 6;
+
+  // An eTag used to perform consistent read-modify-write updates. If not set, a
+  // blind "overwrite" update happens.
+  string etag = 9;
+
+  // The labels with user-defined metadata to organize your Artifacts.
+  //
+  // Label keys and values can be no longer than 64 characters
+  // (Unicode codepoints), can only contain lowercase letters, numeric
+  // characters, underscores and dashes. International characters are allowed.
+  // No more than 64 user labels can be associated with one Artifact (System
+  // labels are excluded).
+  map<string, string> labels = 10;
+
+  // Output only. Timestamp when this Artifact was created.
+  google.protobuf.Timestamp create_time = 11 [(google.api.field_behavior) = OUTPUT_ONLY];
+
+  // Output only. Timestamp when this Artifact was last updated.
+  google.protobuf.Timestamp update_time = 12 [(google.api.field_behavior) = OUTPUT_ONLY];
+
+  // The state of this Artifact. This is a property of the Artifact, and does
+  // not imply or capture any ongoing process. This property is managed by
+  // clients (such as AI Platform Pipelines), and the system does not prescribe
+  // or check the validity of state transitions.
+  State state = 13;
+
+  // The title of the schema describing the metadata.
+  //
+  // Schema title and version is expected to be registered in earlier Create
+  // Schema calls. And both are used together as unique identifiers to identify
+  // schemas within the local metadata store.
+  string schema_title = 14;
+
+  // The version of the schema in schema_name to use.
+  //
+  // Schema title and version is expected to be registered in earlier Create
+  // Schema calls. And both are used together as unique identifiers to identify
+  // schemas within the local metadata store.
+  string schema_version = 15;
+
+  // Properties of the Artifact.
+  google.protobuf.Struct metadata = 16;
+
+  // Description of the Artifact
+  string description = 17;
+}

protos/google/cloud/aiplatform/v1beta1/batch_prediction_job.proto

@@ -25,6 +25,7 @@ import "google/cloud/aiplatform/v1beta1/io.proto";
 import "google/cloud/aiplatform/v1beta1/job_state.proto";
 import "google/cloud/aiplatform/v1beta1/machine_resources.proto";
 import "google/cloud/aiplatform/v1beta1/manual_batch_tuning_parameters.proto";
+import "google/cloud/aiplatform/v1beta1/model_monitoring.proto";
 import "google/protobuf/struct.proto";
 import "google/protobuf/timestamp.proto";
 import "google/rpc/status.proto";
@@ -97,8 +98,8 @@ message BatchPredictionJob {
       // containing only `code` and `message` fields.
       GcsDestination gcs_destination = 2;
 
-      // The BigQuery project location where the output is to be written to.
-      // In the given project a new dataset is created with name
+      // The BigQuery project or dataset location where the output is to be
+      // written to. If project is provided, a new dataset is created with name
       // `prediction_<model-display-name>_<job-create-time>`
       // where <model-display-name> is made
       // BigQuery-dataset-name compatible (for example, most special characters

protos/google/cloud/aiplatform/v1beta1/context.proto

@@ -0,0 +1,92 @@
+// Copyright 2021 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto3";
+
+package google.cloud.aiplatform.v1beta1;
+
+import "google/api/field_behavior.proto";
+import "google/api/resource.proto";
+import "google/cloud/aiplatform/v1beta1/value.proto";
+import "google/protobuf/struct.proto";
+import "google/protobuf/timestamp.proto";
+import "google/api/annotations.proto";
+
+option go_package = "google.golang.org/genproto/googleapis/cloud/aiplatform/v1beta1;aiplatform";
+option java_multiple_files = true;
+option java_outer_classname = "ContextProto";
+option java_package = "com.google.cloud.aiplatform.v1beta1";
+
+// Instance of a general context.
+message Context {
+  option (google.api.resource) = {
+    type: "aiplatform.googleapis.com/Context"
+    pattern: "projects/{project}/locations/{location}/metadataStores/{metadata_store}/contexts/{context}"
+  };
+
+  // Output only. The resource name of the Context.
+  string name = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
+
+  // User provided display name of the Context.
+  // May be up to 128 Unicode characters.
+  string display_name = 2;
+
+  // An eTag used to perform consistent read-modify-write updates. If not set, a
+  // blind "overwrite" update happens.
+  string etag = 8;
+
+  // The labels with user-defined metadata to organize your Contexts.
+  //
+  // Label keys and values can be no longer than 64 characters
+  // (Unicode codepoints), can only contain lowercase letters, numeric
+  // characters, underscores and dashes. International characters are allowed.
+  // No more than 64 user labels can be associated with one Context (System
+  // labels are excluded).
+  map<string, string> labels = 9;
+
+  // Output only. Timestamp when this Context was created.
+  google.protobuf.Timestamp create_time = 10 [(google.api.field_behavior) = OUTPUT_ONLY];
+
+  // Output only. Timestamp when this Context was last updated.
+  google.protobuf.Timestamp update_time = 11 [(google.api.field_behavior) = OUTPUT_ONLY];
+
+  // Output only. A list of resource names of Contexts that are parents of this Context.
+  // A Context may have at most 10 parent_contexts.
+  repeated string parent_contexts = 12 [
+    (google.api.field_behavior) = OUTPUT_ONLY,
+    (google.api.resource_reference) = {
+      type: "aiplatform.googleapis.com/Context"
+    }
+  ];
+
+  // The title of the schema describing the metadata.
+  //
+  // Schema title and version is expected to be registered in earlier Create
+  // Schema calls. And both are used together as unique identifiers to identify
+  // schemas within the local metadata store.
+  string schema_title = 13;
+
+  // The version of the schema in schema_name to use.
+  //
+  // Schema title and version is expected to be registered in earlier Create
+  // Schema calls. And both are used together as unique identifiers to identify
+  // schemas within the local metadata store.
+  string schema_version = 14;
+
+  // Properties of the Context.
+  google.protobuf.Struct metadata = 15;
+
+  // Description of the Context
+  string description = 16;
+}

protos/google/cloud/aiplatform/v1beta1/custom_job.proto

@@ -116,7 +116,9 @@ message CustomJobSpec {
   //
   // Private services access must already be configured for the network. If left
   // unspecified, the job is not peered with any network.
-  string network = 5;
+  string network = 5 [(google.api.resource_reference) = {
+                        type: "compute.googleapis.com/Network"
+                      }];
 
   // The Cloud Storage location to store the output of this CustomJob or
   // HyperparameterTuningJob. For HyperparameterTuningJob,
@@ -140,6 +142,17 @@ message CustomJobSpec {
   //   * AIP_CHECKPOINT_DIR = `<base_output_directory>/<trial_id>/checkpoints/`
   //   * AIP_TENSORBOARD_LOG_DIR = `<base_output_directory>/<trial_id>/logs/`
   GcsDestination base_output_directory = 6;
+
+  // Optional. The name of an AI Platform [Tensorboard][google.cloud.aiplatform.v1beta1.Tensorboard] resource to which this CustomJob
+  // will upload Tensorboard logs.
+  // Format:
+  // `projects/{project}/locations/{location}/tensorboards/{tensorboard}`
+  string tensorboard = 7 [
+    (google.api.field_behavior) = OPTIONAL,
+    (google.api.resource_reference) = {
+      type: "aiplatform.googleapis.com/Tensorboard"
+    }
+  ];
 }
 
 // Represents the spec of a worker pool in a job.

protos/google/cloud/aiplatform/v1beta1/deployed_index_ref.proto

@@ -0,0 +1,40 @@
+// Copyright 2021 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto3";
+
+package google.cloud.aiplatform.v1beta1;
+
+import "google/api/field_behavior.proto";
+import "google/api/resource.proto";
+import "google/api/annotations.proto";
+
+option go_package = "google.golang.org/genproto/googleapis/cloud/aiplatform/v1beta1;aiplatform";
+option java_multiple_files = true;
+option java_outer_classname = "DeployedIndexRefProto";
+option java_package = "com.google.cloud.aiplatform.v1beta1";
+
+// Points to a DeployedIndex.
+message DeployedIndexRef {
+  // Immutable. A resource name of the IndexEndpoint.
+  string index_endpoint = 1 [
+    (google.api.field_behavior) = IMMUTABLE,
+    (google.api.resource_reference) = {
+      type: "aiplatform.googleapis.com/IndexEndpoint"
+    }
+  ];
+
+  // Immutable. The ID of the DeployedIndex in the above IndexEndpoint.
+  string deployed_index_id = 2 [(google.api.field_behavior) = IMMUTABLE];
+}

protos/google/cloud/aiplatform/v1beta1/endpoint.proto

@@ -21,6 +21,7 @@ import "google/api/resource.proto";
 import "google/cloud/aiplatform/v1beta1/encryption_spec.proto";
 import "google/cloud/aiplatform/v1beta1/explanation.proto";
 import "google/cloud/aiplatform/v1beta1/machine_resources.proto";
+import "google/cloud/aiplatform/v1beta1/model_deployment_monitoring_job.proto";
 import "google/protobuf/timestamp.proto";
 import "google/api/annotations.proto";
 

protos/google/cloud/aiplatform/v1beta1/entity_type.proto

@@ -0,0 +1,84 @@
+// Copyright 2021 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto3";
+
+package google.cloud.aiplatform.v1beta1;
+
+import "google/api/field_behavior.proto";
+import "google/api/resource.proto";
+import "google/cloud/aiplatform/v1beta1/feature.proto";
+import "google/cloud/aiplatform/v1beta1/featurestore_monitoring.proto";
+import "google/protobuf/timestamp.proto";
+import "google/api/annotations.proto";
+
+option go_package = "google.golang.org/genproto/googleapis/cloud/aiplatform/v1beta1;aiplatform";
+option java_multiple_files = true;
+option java_outer_classname = "EntityTypeProto";
+option java_package = "com.google.cloud.aiplatform.v1beta1";
+
+// An entity type is a type of object in a system that needs to be modeled and
+// have stored information about. For example, driver is an entity type, and
+// driver0 is an instance of an entity type driver.
+message EntityType {
+  option (google.api.resource) = {
+    type: "aiplatform.googleapis.com/EntityType"
+    pattern: "projects/{project}/locations/{location}/featurestores/{featurestore}/entityTypes/{entity_type}"
+  };
+
+  // Immutable. Name of the EntityType.
+  // Format:
+  // `projects/{project}/locations/{location}/featurestores/{featurestore}/entityTypes/{entity_type}`
+  //
+  // The last part entity_type is assigned by the client. The entity_type can be
+  // up to 64 characters long and can consist only of ASCII Latin letters A-Z
+  // and a-z and underscore(_), and ASCII digits 0-9 starting with a letter. The
+  // value will be unique given a featurestore.
+  string name = 1 [(google.api.field_behavior) = IMMUTABLE];
+
+  // Optional. Description of the EntityType.
+  string description = 2 [(google.api.field_behavior) = OPTIONAL];
+
+  // Output only. Timestamp when this EntityType was created.
+  google.protobuf.Timestamp create_time = 3 [(google.api.field_behavior) = OUTPUT_ONLY];
+
+  // Output only. Timestamp when this EntityType was most recently updated.
+  google.protobuf.Timestamp update_time = 4 [(google.api.field_behavior) = OUTPUT_ONLY];
+
+  // Optional. The labels with user-defined metadata to organize your EntityTypes.
+  //
+  // Label keys and values can be no longer than 64 characters
+  // (Unicode codepoints), can only contain lowercase letters, numeric
+  // characters, underscores and dashes. International characters are allowed.
+  //
+  // See https://goo.gl/xmQnxf for more information on and examples of labels.
+  // No more than 64 user labels can be associated with one EntityType (System
+  // labels are excluded)."
+  // System reserved label keys are prefixed with "aiplatform.googleapis.com/"
+  // and are immutable.
+  map<string, string> labels = 6 [(google.api.field_behavior) = OPTIONAL];
+
+  // Optional. Used to perform a consistent read-modify-write updates. If not set, a blind
+  // "overwrite" update happens.
+  string etag = 7 [(google.api.field_behavior) = OPTIONAL];
+
+  // Optional. The default monitoring configuration for all Features under this
+  // EntityType.
+  //
+  // If this is populated with
+  // [FeaturestoreMonitoringConfig.monitoring_interval] specified, snapshot
+  // analysis monitoring is enabled. Otherwise, snapshot analysis monitoring is
+  // disabled.
+  FeaturestoreMonitoringConfig monitoring_config = 8 [(google.api.field_behavior) = OPTIONAL];
+}