documents deploy changes

Author: rudolfix

.github/workflows/test_common.yml

@@ -80,11 +80,11 @@ jobs:
         run: poetry install --no-interaction -E duckdb -E cli
 
       - run: |
-          poetry run pytest tests/extract tests/pipeline && poetry run pytest tests/cli/common
+          poetry run pytest tests/extract tests/pipeline tests/cli/common
         if: runner.os != 'Windows'
         name: Run extra tests Linux/MAC
       - run: |
-          poetry run pytest tests/extract tests/pipeline && poetry run pytest tests/cli/common
+          poetry run pytest tests/extract tests/pipeline tests/cli/common
         if: runner.os == 'Windows'
         name: Run extra tests Windows
         shell: cmd

Makefile

@@ -61,7 +61,7 @@ test-local:
 	DESTINATION__POSTGRES__CREDENTIALS=postgresql://loader:loader@localhost:5432/dlt_data DESTINATION__DUCKDB__CREDENTIALS=duckdb:///_storage/test_quack.duckdb  poetry run pytest tests -k '(postgres and duckdb)'
 
 test-common:
-	poetry run pytest tests/common tests/normalize tests/extract tests/pipeline tests/reflection tests/sources && poetry run pytest tests/cli/common
+	poetry run pytest tests/common tests/normalize tests/extract tests/pipeline tests/reflection tests/sources tests/cli/common
 
 reset-test-storage:
 	-rm -r _storage

dlt/cli/_dlt.py

@@ -19,7 +19,8 @@
 from dlt.cli.telemetry_command import DLT_TELEMETRY_DOCS_URL, change_telemetry_status_command, telemetry_status_command
 
 try:
-    from dlt.cli.deploy_command import PipelineWasNotRun, deploy_command, DLT_DEPLOY_DOCS_URL, DeploymentMethods, COMMAND_DEPLOY_REPO_LOCATION, SecretFormats
+    from dlt.cli import deploy_command
+    from dlt.cli.deploy_command import PipelineWasNotRun, DLT_DEPLOY_DOCS_URL, DeploymentMethods, COMMAND_DEPLOY_REPO_LOCATION, SecretFormats
 except ModuleNotFoundError:
     pass
 
@@ -57,7 +58,13 @@ def deploy_command_wrapper(pipeline_script_path: str, deployment_method: str, re
 
     from git import InvalidGitRepositoryError, NoSuchPathError
     try:
-        deploy_command(pipeline_script_path=pipeline_script_path, deployment_method=deployment_method, repo_location=repo_location, branch=branch, **kwargs)
+        deploy_command.deploy_command(
+            pipeline_script_path=pipeline_script_path,
+            deployment_method=deployment_method,
+            repo_location=repo_location,
+            branch=branch,
+            **kwargs
+        )
     except (CannotRestorePipelineException, PipelineWasNotRun) as ex:
         click.secho(str(ex), err=True, fg="red")
         fmt.note("You must run the pipeline locally successfully at least once in order to deploy it.")

docs/website/docs/general-usage/credentials.md

@@ -38,7 +38,7 @@ Once you have credentials for the source and destination, add them to the file a
 
 To add credentials to your deployment,
 
-- either use one of the dlt deploy commands;
+- either use one of the `dlt deploy` commands;
 - or follow the below instructions to pass credentials via code or environment.
 
 ### Passing credentials as code

docs/website/docs/getting-started/build-a-data-platform/building-data-warehouse.md

@@ -104,7 +104,7 @@ complexity.
 **This is one of the main reasons we created `dlt`:**
 
 - For common sources, `dlt`’s community-built and maintained pipelines can deploy to your airflow
-  with only two CLI commands (`dlt init source`, `dlt deploy source airflow`). Maintenance is partly
+  with only two CLI commands (`dlt init source`, `dlt deploy pipeline_script.pyu airflow-composer`). Maintenance is partly
   automated, and open sourced, while customisations are easy.
 - For simple sources, you can have an easy python function to just take any data and load it in a
   robust way.

docs/website/docs/reference/command-line-interface.md

@@ -33,7 +33,13 @@ Shows all available verified sources and their short descriptions. For each sour
 and prints the relevant warning.
 
 ## `dlt deploy`
-This command prepares your pipeline for deployment and gives you step by step instruction how to accomplish it.
+This command prepares your pipeline for deployment and gives you step by step instruction how to accomplish it. To enabled this functionality please first execute
+```sh
+pip install "dlt[cli]"
+```
+that will add additional packages to current environment.
+
+> 💡 We ask you to install those dependencies separately to keep our core library small and make it work everywhere.
 
 ### github-action
 

docs/website/docs/walkthroughs/deploy-a-pipeline/deploy-with-airflow-composer.md

@@ -26,9 +26,12 @@ python3 {pipeline_name}_pipeline.py
 This should successfully load data from the source to the destination once and allows `dlt` to gather required information for the deployment.
 
 ## 3. Initialize deployment
+First you need to add additional dependencies that `deploy` command requires:
+```bash
+pip install "dlt[cli]"
+```
 
-In the same `dlt` project as your working pipeline, you can run a deployment command.
-
+then:
 ```bash
 dlt deploy {pipeline_name}_pipeline.py airflow-composer
 ```

docs/website/docs/walkthroughs/deploy-a-pipeline/deploy-with-github-actions.md

@@ -29,16 +29,16 @@ python3 chess_pipeline.py # replace chess_pipeline.py with your pipeline file
 This should successfully load data from the source to the destination once.
 
 ## Initialize deployment
-
-In the same `dlt` project as your working pipeline, you can create a deployment using
-[GitHub Actions](https://github.com/features/actions) that will load data with the `chess_pipeline.py` script
-every 30 minutes by running:
-
+First you need to add additional dependencies that `deploy` command requires:
+```bash
+pip install "dlt[cli]"
+```
+then the command below will create a Github workflow that runs your pipeline script every 30 minutes:
 ```shell
-dlt deploy chess_pipeline.py github-action --schedule "*/30 * * * *" # replace chess_pipeline.py
+dlt deploy chess_pipeline.py github-action --schedule "*/30 * * * *"
 ```
 
-This command checks that your pipeline has run successfully before and creates a GitHub Actions
+It checks that your pipeline has run successfully before and creates a GitHub Actions
 workflow file `run_chess_workflow.yml` in `.github/workflows` with the necessary environment
 variables.
 

tests/cli/common/test_cli_invoke.py

@@ -89,7 +89,8 @@ def test_invoke_deploy_project(script_runner: ScriptRunner) -> None:
 
 
 def test_invoke_deploy_mock(script_runner: ScriptRunner) -> None:
-    with patch("dlt.cli._dlt.deploy_command") as _deploy_command:
+    # NOTE: you can mock only once per test with ScriptRunner !!
+    with patch("dlt.cli.deploy_command.deploy_command") as _deploy_command:
         script_runner.run(['dlt', 'deploy', 'debug_pipeline.py', 'github-action', '--schedule', '@daily'])
         assert _deploy_command.called
         assert _deploy_command.call_args[1] == {
@@ -102,7 +103,8 @@ def test_invoke_deploy_mock(script_runner: ScriptRunner) -> None:
             "run_manually": True,
             "run_on_push": False
         }
-    with patch("dlt.cli._dlt.deploy_command") as _deploy_command:
+
+        _deploy_command.reset_mock()
         script_runner.run(['dlt', 'deploy', 'debug_pipeline.py', 'github-action', '--schedule', '@daily', '--location', 'folder', '--branch', 'branch', '--run-on-push'])
         assert _deploy_command.called
         assert _deploy_command.call_args[1] == {
@@ -115,14 +117,14 @@ def test_invoke_deploy_mock(script_runner: ScriptRunner) -> None:
             "run_manually": True,
             "run_on_push": True
         }
-    # no schedule fails
-    with patch("dlt.cli._dlt.deploy_command") as _deploy_command:
+        # no schedule fails
+        _deploy_command.reset_mock()
         result = script_runner.run(['dlt', 'deploy', 'debug_pipeline.py', 'github-action'])
         assert not _deploy_command.called
         assert result.returncode != 0
         assert "the following arguments are required: --schedule" in result.stderr
-    # airflow without schedule works
-    with patch("dlt.cli._dlt.deploy_command") as _deploy_command:
+        # airflow without schedule works
+        _deploy_command.reset_mock()
         result = script_runner.run(['dlt', 'deploy', 'debug_pipeline.py', 'airflow-composer'])
         assert _deploy_command.called
         assert result.returncode == 0
@@ -134,8 +136,8 @@ def test_invoke_deploy_mock(script_runner: ScriptRunner) -> None:
             "command": "deploy",
             'secrets_format': 'toml'
         }
-    # env secrets format
-    with patch("dlt.cli._dlt.deploy_command") as _deploy_command:
+        # env secrets format
+        _deploy_command.reset_mock()
         result = script_runner.run(['dlt', 'deploy', 'debug_pipeline.py', 'airflow-composer', "--secrets-format", "env"])
         assert _deploy_command.called
         assert result.returncode == 0