tested and linted snippets (dlt-hub#559)

* add snippets prototype

* add tested snippet

* update lockfile

* create snippet test workflow

* disable dedenting

* fix indentation

* set and clean test storage on snippet execution

* pr comments

Author: sh-rp

.github/workflows/test_doc_snippets.yml

@@ -0,0 +1,52 @@
+
+name: lint
+
+on:
+  pull_request:
+    branches:
+      - master
+      - devel
+  workflow_dispatch:
+
+jobs:
+  get_docs_changes:
+    uses: ./.github/workflows/get_docs_changes.yml
+
+  run_lint:
+    name: Runs linter and tests on docs snippets
+    runs-on: ubuntu-latest
+
+    steps:
+
+      - name: Check out
+        uses: actions/checkout@master
+
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.10.x"
+
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+        with:
+          virtualenvs-create: true
+          virtualenvs-in-project: true
+          installer-parallel: true
+
+      - name: Load cached venv
+        id: cached-poetry-dependencies
+        uses: actions/cache@v3
+        with:
+          path: .venv
+          key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}
+
+      - name: Install dependencies
+        # if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
+        run: poetry install --no-interaction --all-extras
+
+      # - name: Install self
+      #   run: poetry install --no-interaction
+
+      - name: Run linter and tests
+        run: make test-and-lint-snippets
+

Makefile

@@ -46,11 +46,16 @@ dev: has-poetry
 
 lint:
 	./check-package.sh
-	poetry run mypy --config-file mypy.ini dlt docs/examples
-	poetry run flake8 --max-line-length=200 dlt docs/examples
+	poetry run mypy --config-file mypy.ini dlt
+	poetry run flake8 --max-line-length=200 dlt
 	poetry run flake8 --max-line-length=200 tests --exclude tests/reflection/module_cases
 	# $(MAKE) lint-security
 
+test-and-lint-snippets:
+	poetry run mypy --config-file mypy.ini docs/snippets docs/examples
+	poetry run flake8 --max-line-length=200 docs/snippets docs/examples
+	poetry run pytest docs/snippets
+
 lint-security:
 	poetry run bandit -r dlt/ -n 3 -l
 

docs/snippets/__init__.py

@@ -0,0 +1,3 @@
+from tests.utils import patch_home_dir, autouse_test_storage, preserve_environ
+
+

docs/snippets/conftest.py

@@ -0,0 +1,20 @@
+# @@@SNIPSTART intro_snippet
+import dlt
+import requests
+# Create a dlt pipeline that will load
+# chess player data to the DuckDB destination
+pipeline = dlt.pipeline(
+    pipeline_name='chess_pipeline',
+    destination='duckdb',
+    dataset_name='player_data'
+)
+# Grab some player data from Chess.com API
+data = []
+for player in ['magnuscarlsen', 'rpragchess']:
+    response = requests.get(f'https://api.chess.com/pub/player/{player}')
+    response.raise_for_status()
+    data.append(response.json())
+# Extract, normalize, and load the data
+info = pipeline.run(data, table_name='player')
+print(info)
+# @@@SNIPEND

docs/snippets/intro_snippet.py

@@ -0,0 +1,7 @@
+
+from tests.pipeline.utils import assert_load_info
+from docs.snippets.utils import run_snippet
+
+def test_intro_snippet() -> None:
+    variables = run_snippet("intro_snippet")
+    assert_load_info(variables["info"])

docs/snippets/intro_snippet_test.py

@@ -0,0 +1,14 @@
+from typing import Dict, Any
+
+from dlt.common.utils import set_working_dir
+
+from tests.utils import TEST_STORAGE_ROOT, test_storage
+
+BASEPATH = "docs/snippets"
+
+def run_snippet(filename: str) -> Dict[str, Any]:
+    with set_working_dir(BASEPATH):
+        code = open(f"{filename}.py", encoding="utf-8").read()
+        variables: Dict[str, Any] = {}
+        exec(code, variables)
+        return variables

docs/snippets/utils.py

@@ -12,4 +12,4 @@ Pick one of our high quality destinations and load your data to a local database
 
 Otherwise pick a destination below:
 
-<DocCardList />
+<DocCardList />

docs/website/docs/dlt-ecosystem/destinations/index.md

@@ -84,4 +84,4 @@ If you want to transform the data before loading, you can use Python. If you wan
 data after loading, you can use dbt or one of the following:
 
 1. [`dlt` SQL client.](sql.md)
-1. [Pandas.](pandas.md)
+1. [Pandas.](pandas.md)

docs/website/docs/dlt-ecosystem/transformations/dbt.md

@@ -44,4 +44,4 @@ If you want to transform the data before loading, you can use Python. If you wan
 data after loading, you can use SQL or one of the following:
 
 1. [dbt](dbt.md) (recommended).
-1. [Pandas.](pandas.md)
+1. [Pandas.](pandas.md)