Skip to content

Commit

Permalink
feat: Add support for notebook tasks (#100)
Browse files Browse the repository at this point in the history
* feat: Add support for notebook tasks

PiperOrigin-RevId: 477935449

Source-Link: googleapis/googleapis@9e3a1dd

Source-Link: googleapis/googleapis-gen@a5443ab
Copy-Tag: eyJwIjoiLmdpdGh1Yi8uT3dsQm90LnlhbWwiLCJoIjoiYTU0NDNhYjQ4ZTMwYmRkNjA4YWU3YWUxZDZjZjJmYWJjMTdiNmRjYSJ9

* 🦉 Updates from OwlBot post-processor

See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md

Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com>
  • Loading branch information
gcf-owl-bot[bot] and gcf-owl-bot[bot] authored Sep 30, 2022
1 parent c8b0069 commit 467f271
Show file tree
Hide file tree
Showing 2 changed files with 79 additions and 1 deletion.
Original file line number Diff line number Diff line change
Expand Up @@ -328,7 +328,8 @@ class SessionEvent(proto.Message):
The log message.
user_id (str):
The information about the user that created
the session.
the session. It will be the email address of the
user.
session_id (str):
Unique identifier for the session.
type_ (google.cloud.dataplex_v1.types.SessionEvent.EventType):
Expand All @@ -337,6 +338,13 @@ class SessionEvent(proto.Message):
The execution details of the query.
This field is a member of `oneof`_ ``detail``.
event_succeeded (bool):
The status of the event.
warm_pool_enabled (bool):
If the session is a warm pooled session.
unassigned_duration (google.protobuf.duration_pb2.Duration):
The idle duration of a warm pooled session
before it is assigned to user.
"""

class EventType(proto.Enum):
Expand All @@ -345,6 +353,7 @@ class EventType(proto.Enum):
START = 1
STOP = 2
QUERY = 3
CREATE = 4

class QueryDetail(proto.Message):
r"""Execution details of the query.
Expand Down Expand Up @@ -420,6 +429,19 @@ class Engine(proto.Enum):
oneof="detail",
message=QueryDetail,
)
event_succeeded = proto.Field(
proto.BOOL,
number=6,
)
warm_pool_enabled = proto.Field(
proto.BOOL,
number=7,
)
unassigned_duration = proto.Field(
proto.MESSAGE,
number=8,
message=duration_pb2.Duration,
)


__all__ = tuple(sorted(__protobuf__.manifest))
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,11 @@
class Task(proto.Message):
r"""A task represents a user-visible job.
This message has `oneof`_ fields (mutually exclusive fields).
For each oneof, at most one member field can be set at the same time.
Setting any member of the oneof automatically clears all other
members.
.. _oneof: https://proto-plus-python.readthedocs.io/en/stable/fields.html#oneofs-mutually-exclusive-fields
Attributes:
Expand Down Expand Up @@ -70,6 +75,11 @@ class Task(proto.Message):
spark (google.cloud.dataplex_v1.types.Task.SparkTaskConfig):
Config related to running custom Spark tasks.
This field is a member of `oneof`_ ``config``.
notebook (google.cloud.dataplex_v1.types.Task.NotebookTaskConfig):
Config related to running scheduled
Notebooks.
This field is a member of `oneof`_ ``config``.
"""

Expand Down Expand Up @@ -445,6 +455,46 @@ class SparkTaskConfig(proto.Message):
message="Task.InfrastructureSpec",
)

class NotebookTaskConfig(proto.Message):
r"""Config for running scheduled notebooks.
Attributes:
notebook (str):
Required. Path to input notebook. This can be the GCS URI of
the notebook file or the path to a Notebook Content. The
execution args are accessible as environment variables
(``TASK_key=value``).
infrastructure_spec (google.cloud.dataplex_v1.types.Task.InfrastructureSpec):
Optional. Infrastructure specification for
the execution.
file_uris (Sequence[str]):
Optional. GCS URIs of files to be placed in
the working directory of each executor.
archive_uris (Sequence[str]):
Optional. GCS URIs of archives to be
extracted into the working directory of each
executor. Supported file types: .jar, .tar,
.tar.gz, .tgz, and .zip.
"""

notebook = proto.Field(
proto.STRING,
number=4,
)
infrastructure_spec = proto.Field(
proto.MESSAGE,
number=3,
message="Task.InfrastructureSpec",
)
file_uris = proto.RepeatedField(
proto.STRING,
number=5,
)
archive_uris = proto.RepeatedField(
proto.STRING,
number=6,
)

class ExecutionStatus(proto.Message):
r"""Status of the task execution (e.g. Jobs).
Expand Down Expand Up @@ -523,6 +573,12 @@ class ExecutionStatus(proto.Message):
oneof="config",
message=SparkTaskConfig,
)
notebook = proto.Field(
proto.MESSAGE,
number=302,
oneof="config",
message=NotebookTaskConfig,
)


class Job(proto.Message):
Expand Down

0 comments on commit 467f271

Please sign in to comment.