Feat: implement runtime CLI auth

Makefile

@@ -58,7 +58,7 @@ lint-core:
 	uv pip install docstring_parser_fork --reinstall
 	uv run ruff check
 	# NOTE: we exclude all D lint errors (docstrings)
-	uv run flake8 --extend-ignore=D --max-line-length=200 dlt
+	uv run flake8 --extend-ignore=D --max-line-length=200 dlt --exclude dlt/_workspace/runtime_clients
 	uv run flake8 --extend-ignore=D --max-line-length=200 tests --exclude tests/reflection/module_cases,tests/common/reflection/cases/modules/
 	uv run black dlt docs tests --check --diff --color --extend-exclude=".*syntax_error.py"
 
@@ -185,3 +185,15 @@ start-dlt-dashboard-e2e:
 # creates the dashboard test pipelines globally for manual testing of the dashboard app and cli
 create-test-pipelines:
 	uv run python tests/workspace/helpers/dashboard/example_pipelines.py
+
+## Generates python clients for runtime api and auth services
+# Before running, adjust the URL or run `make up` in runtime first
+# Uses pinned version of openapi-python-client to support python3.9's Optional syntax, pinned version to be removed when support for python3.9 is dropped
+generate-python-clients:
+	curl http://localhost:30000/schema/openapi.yaml -o dlt/_workspace/runtime_clients/api/openapi.yaml
+	python tools/clean_openapi_spec.py dlt/_workspace/runtime_clients/api/openapi.yaml
+	uvx openapi-python-client@0.26.2 generate  --meta none --path dlt/_workspace/runtime_clients/api/openapi.yaml --output-path dlt/_workspace/runtime_clients/api --overwrite
+
+	curl http://localhost:30001/schema/openapi.yaml -o dlt/_workspace/runtime_clients/auth/openapi.yaml
+	python tools/clean_openapi_spec.py dlt/_workspace/runtime_clients/auth/openapi.yaml
+	uvx openapi-python-client@0.26.2 generate  --meta none --path dlt/_workspace/runtime_clients/auth/openapi.yaml --output-path dlt/_workspace/runtime_clients/auth --overwrite

dlt/_workspace/_workspace_context.py

@@ -162,6 +162,11 @@ def plug(self) -> None:
     def unplug(self) -> None:
         pass
 
+    def reset_config(self) -> None:
+        # Drop resolved configuration to force re-resolve with refreshed providers
+        self._config = None
+        # no need to initialize the _config anew as it's done in .config property
+
     # SupportsProfilesOnContext
 
     @property

dlt/_workspace/cli/_plugins.py

@@ -22,6 +22,7 @@
     "plug_cli_deploy",
     "plug_cli_docs",
     "plug_cli_ai",
+    "plug_cli_runtime",
     "plug_cli_profile",
     "plug_cli_workspace",
 ]
@@ -83,6 +84,16 @@ def plug_cli_ai() -> Type[plugins.SupportsCliCommand]:
     return AiCommand
 
 
+@plugins.hookimpl(specname="plug_cli")
+def plug_cli_runtime() -> Type[plugins.SupportsCliCommand]:
+    if is_workspace_active():
+        from dlt._workspace.cli._runtime_command import RuntimeCommand
+
+        return RuntimeCommand
+    else:
+        return None
+
+
 @plugins.hookimpl(specname="plug_cli")
 def plug_cli_profile() -> Type[plugins.SupportsCliCommand]:
     if is_workspace_active():

dlt/_workspace/cli/_runtime_command.py

@@ -0,0 +1,127 @@
+import argparse
+import time
+from typing import Optional
+from dlt.common.configuration.plugins import SupportsCliCommand
+
+from dlt._workspace._workspace_context import active
+from dlt._workspace.exceptions import (
+    LocalWorkspaceIdNotSet,
+    RuntimeNotAuthenticated,
+    WorkspaceIdMismatch,
+)
+from dlt._workspace.runtime import RuntimeAuthService, get_auth_client
+from dlt._workspace.runtime_clients.auth.api.github import github_oauth_complete, github_oauth_start
+from dlt._workspace.cli import echo as fmt
+
+
+class RuntimeCommand(SupportsCliCommand):
+    command = "runtime"
+    help_string = "Connect to dltHub Runtime and run your code remotely"
+    description = """
+    Allows to connect to the dltHub Runtime, deploy and run local workspaces there. Requires dltHub license.
+    """
+
+    def configure_parser(self, parser: argparse.ArgumentParser) -> None:
+        self.parser = parser
+
+        subparsers = parser.add_subparsers(
+            title="Available subcommands", dest="runtime_command", required=False
+        )
+
+        subparsers.add_parser(
+            "login",
+            help="Login to the Runtime using Github OAuth",
+            description="Login to the Runtime using Github OAuth",
+        )
+
+        subparsers.add_parser(
+            "logout",
+            help="Logout from the Runtime",
+            description="Logout from the Runtime",
+        )
+
+    def execute(self, args: argparse.Namespace) -> None:
+        if args.runtime_command == "login":
+            login()
+        elif args.runtime_command == "logout":
+            logout()
+        else:
+            self.parser.print_usage()
+
+
+def login() -> None:
+    auth_service = RuntimeAuthService(run_context=active())
+    try:
+        auth_info = auth_service.authenticate()
+        fmt.echo("Already logged in as %s" % fmt.bold(auth_info.email))
+        connect(auth_service=auth_service)
+    except RuntimeNotAuthenticated:
+        fmt.echo("Logging in with Github OAuth")
+        client = get_auth_client()
+
+        # start device flow
+        login_request = github_oauth_start.sync(client=client)
+        if not isinstance(login_request, github_oauth_start.GithubDeviceFlowStartResponse):
+            raise RuntimeError("Failed to log in with Github OAuth")
+        fmt.echo(
+            "Please go to %s and enter the code %s"
+            % (fmt.bold(login_request.verification_uri), fmt.bold(login_request.user_code))
+        )
+        fmt.echo("Waiting for response from Github...")
+
+        while True:
+            time.sleep(login_request.interval)
+            token_response = github_oauth_complete.sync(
+                client=client,
+                body=github_oauth_complete.GithubDeviceFlowLoginRequest(
+                    device_code=login_request.device_code
+                ),
+            )
+            if isinstance(token_response, github_oauth_complete.LoginResponse):
+                auth_info = auth_service.login(token_response.jwt)
+                fmt.echo("Logged in as %s" % fmt.bold(auth_info.email))
+                connect(auth_service=auth_service)
+                break
+            elif isinstance(token_response, github_oauth_complete.ErrorResponse400):
+                raise RuntimeError("Failed to complete authentication with Github")
+
+
+def logout() -> None:
+    auth_service = RuntimeAuthService(run_context=active())
+    auth_service.logout()
+    fmt.echo("Logged out")
+
+
+def connect(auth_service: Optional[RuntimeAuthService] = None) -> None:
+    if auth_service is None:
+        auth_service = RuntimeAuthService(run_context=active())
+        auth_service.authenticate()
+
+    try:
+        auth_service.connect()
+    except LocalWorkspaceIdNotSet:
+        should_overwrite = fmt.confirm(
+            "No workspace id found in local config. Do you want to connect local workspace to the"
+            " remote one?",
+            default=True,
+        )
+        if should_overwrite:
+            auth_service.overwrite_local_workspace_id()
+            fmt.echo("Using remote workspace id")
+        else:
+            raise RuntimeError("Local workspace is not connected to the remote one")
+    except WorkspaceIdMismatch as e:
+        fmt.warning(
+            "Workspace id in local config (%s) is not the same as remote workspace id (%s)"
+            % (e.local_workspace_id, e.remote_workspace_id)
+        )
+        should_overwrite = fmt.confirm(
+            "Do you want to overwrite the local workspace id with the remote one?",
+            default=True,
+        )
+        if should_overwrite:
+            auth_service.overwrite_local_workspace_id()
+            fmt.echo("Local workspace id overwritten with remote workspace id")
+        else:
+            raise RuntimeError("Unable to synchronise remote and local workspaces")
+    fmt.echo("Authorized to workspace %s" % fmt.bold(auth_service.workspace_id))

dlt/_workspace/configuration.py

@@ -1,8 +1,8 @@
-from typing import ClassVar, Sequence
-from git import Optional
+from typing import ClassVar, Sequence, Optional
 from dlt.common.configuration.specs import known_sections
 from dlt.common.configuration.specs.base_configuration import BaseConfiguration, configspec
 from dlt.common.configuration.specs.runtime_configuration import RuntimeConfiguration
+from dlt.common.typing import TSecretStrValue
 
 
 @configspec
@@ -22,8 +22,14 @@ class WorkspaceSettings(BaseConfiguration):
 class WorkspaceRuntimeConfiguration(RuntimeConfiguration):
     """Extends runtime configuration with dlthub runtime"""
 
-    # TODO: connect workspace to runtime here
-    # TODO: optionally define scripts and other runtime settings
+    workspace_id: Optional[str] = None
+    """Id of the remote workspace that local one should be connected to"""
+    auth_token: Optional[TSecretStrValue] = None
+    """JWT token for Runtime API"""
+    auth_base_url: Optional[str] = "http://127.0.0.1:30001"
+    """Base URL for the dltHub Runtime authentication API"""
+    api_base_url: Optional[str] = "http://127.0.0.1:30000"
+    """Base URL for the dltHub Runtime API"""
 
 
 @configspec

dlt/_workspace/exceptions.py

@@ -1,7 +1,7 @@
 import os
 
 from dlt.common.exceptions import DltException
-from dlt.common.runtime.exceptions import RunContextNotAvailable
+from dlt.common.runtime.exceptions import RunContextNotAvailable, RuntimeException
 
 
 class WorkspaceException(DltException):
@@ -16,3 +16,24 @@ def __init__(self, run_dir: str):
             " folder."
         )
         super().__init__(run_dir, msg)
+
+
+class RuntimeNotAuthenticated(RuntimeException):
+    pass
+
+
+class RuntimeOperationNotAuthorized(WorkspaceException, RuntimeException):
+    pass
+
+
+class WorkspaceIdMismatch(RuntimeOperationNotAuthorized):
+    def __init__(self, local_workspace_id: str, remote_workspace_id: str):
+        self.local_workspace_id = local_workspace_id
+        self.remote_workspace_id = remote_workspace_id
+        super().__init__(local_workspace_id, remote_workspace_id)
+
+
+class LocalWorkspaceIdNotSet(RuntimeOperationNotAuthorized):
+    def __init__(self, remote_workspace_id: str):
+        self.remote_workspace_id = remote_workspace_id
+        super().__init__(remote_workspace_id)