Update dlt dependency from ^0.4.12 to latest

CHANGELOG.md

@@ -1,5 +1,14 @@
 [Go to GitHub Releases](https://github.com/dlt-hub/dlt-init-openapi/releases)
 
+## Unreleased
+
+* Bump dlt to 1.11.0
+
+## 0.1.0
+
+* Bump dlt to 0.4.12
+* First beta release of REST API client generator
+
 0.1.0 - Initial Release
 * Bump dlt to 0.4.12
 

Makefile

@@ -1,36 +1,29 @@
-# some useful commands for developing
-
-
-update-rest-api:
-	poetry run python dlt_init_openapi/utils/update_rest_api.py
-
 dev:
 	poetry install --all-extras
 
 # lint
-lint: update-rest-api
+lint:
 	rm -rf tests/_local
 	poetry run flake8 dlt_init_openapi tests
 	poetry run mypy dlt_init_openapi tests
 	poetry run black tests dlt_init_openapi --check
-	poetry run isort black tests dlt_init_openapi --check --diff
+	poetry run isort --profile black tests dlt_init_openapi --check --diff
 
 # format the whole project
-format: update-rest-api
+format:
 	rm -rf tests/_local
-	poetry run isort black tests dlt_init_openapi
+	poetry run isort --profile black tests dlt_init_openapi
 	poetry run black tests dlt_init_openapi
 
 # all tests excluding the checks on e2e tests
-test: update-rest-api
-	poetry run python dlt_init_openapi/utils/update_rest_api.py
+test:
 	poetry run pytest tests --ignore=tests/e2e
 
 # test without running all the specs through a source
-test-fast: update-rest-api
+test-fast:
 	poetry run pytest tests -m "not slow" --ignore=tests/e2e
 
-test-slow: update-rest-api
+test-slow:
 	poetry run pytest tests -m "slow" --ignore=tests/e2e
 
 # dev helpers
@@ -42,7 +35,7 @@ create-pokemon-pipeline-interactive:
 
 # e2e test helpers
 create-e2e-pokemon-pipeline:
-	poetry run dlt-init-openapi pokemon --path tests/cases/e2e_specs/pokeapi.yml --global-limit 2 --no-interactive
+	poetry run dlt-init-openapi pokemon --path tests/cases/e2e_specs/pokeapi.yml --no-interactive --global-limit 2
 
 run-pokemon-pipeline:
 	cd pokemon_pipeline && poetry run python pokemon_pipeline.py

dlt_init_openapi/__init__.py

@@ -1,4 +1,4 @@
-""" Generate modern Python clients from OpenAPI """
+"""Generate modern Python clients from OpenAPI"""
 
 from enum import Enum
 from importlib.metadata import version

dlt_init_openapi/cli/__init__.py

@@ -10,7 +10,6 @@
 from dlt_init_openapi.cli.cli_endpoint_selection import questionary_endpoint_selection
 from dlt_init_openapi.config import Config
 from dlt_init_openapi.exceptions import DltOpenAPITerminalException
-from dlt_init_openapi.utils import update_rest_api
 
 app = typer.Typer(add_completion=False)
 
@@ -50,7 +49,6 @@ def init(
         "--allow-openapi-2",
         help="Allow to use OpenAPI v2. specs. Migration of the spec to 3.0 is recommended though.",
     ),
-    update_rest_api_source: bool = typer.Option(False, help="Update the locally cached rest_api verified source."),
     version: bool = typer.Option(False, "--version", callback=_print_version, help="Print the version and exit"),
 ) -> None:
     """Generate a new dlt pipeline"""
@@ -64,7 +62,6 @@ def init(
         interactive=interactive,
         log_level=log_level,
         global_limit=global_limit,
-        update_rest_api_source=update_rest_api_source,
         allow_openapi_2=allow_openapi_2,
     )
 
@@ -100,7 +97,6 @@ def _init_command_wrapped(
     try:
 
         # sync rest api
-        update_rest_api.update_rest_api(force=update_rest_api_source)
 
         config = _load_config(
             path=config_path,

dlt_init_openapi/config.py

@@ -1,6 +1,5 @@
 import json
 import mimetypes
-import pathlib
 from pathlib import Path
 from typing import Any, List, Optional
 
@@ -11,8 +10,6 @@
 
 from .typing import TEndpointFilter
 
-REST_API_SOURCE_LOCATION = str(pathlib.Path(__file__).parent.resolve() / "../rest_api")
-
 
 class Config(BaseModel):
     """Contains any configurable values passed by the user."""

dlt_init_openapi/parser/properties/converter.py

@@ -1,4 +1,4 @@
-""" Utils for converting default values into valid Python """
+"""Utils for converting default values into valid Python"""
 
 __all__ = ["convert", "convert_chain"]
 

dlt_init_openapi/renderer/default/__init__.py

@@ -4,12 +4,11 @@
 
 import shutil
 import subprocess
-from distutils.dir_util import copy_tree
 
 from jinja2 import Environment, PackageLoader
 from loguru import logger
 
-from dlt_init_openapi.config import REST_API_SOURCE_LOCATION, Config
+from dlt_init_openapi.config import Config
 from dlt_init_openapi.parser.openapi_parser import OpenapiParser
 from dlt_init_openapi.renderer.base_renderer import BaseRenderer
 from dlt_init_openapi.utils import misc
@@ -68,9 +67,6 @@ def run(self, openapi: OpenapiParser, dry: bool = False) -> None:
         self._build_meta_files()
         self._run_post_hooks()
 
-        # copy rest api source into project dir
-        copy_tree(REST_API_SOURCE_LOCATION, str(self.config.project_dir / "rest_api"))
-
     def _build_meta_files(self) -> None:
         requirements_template = self.env.get_template("requirements.txt.j2")
         req_path = self.config.project_dir / "requirements.txt"

dlt_init_openapi/renderer/default/templates/requirements.txt.j2

@@ -1 +1 @@
-dlt>=0.4.12
+dlt>=1.11.0

dlt_init_openapi/renderer/default/templates/source.py.j2

@@ -3,8 +3,8 @@ from typing import List
 import dlt
 from dlt.extract.source import DltResource
 
-from rest_api.typing import RESTAPIConfig
-from rest_api import rest_api_source
+from dlt.sources.rest_api.typing import RESTAPIConfig
+from dlt.sources.rest_api import rest_api_source
 
 
 @dlt.source(name="{{source_name}}", max_table_nesting=2)

dlt_init_openapi/utils/paths.py

@@ -1,4 +1,3 @@
-import os.path
 from typing import Dict, Iterable, List, Optional, Tuple
 
 
@@ -15,12 +14,23 @@ def table_names_from_paths(paths: Iterable[str]) -> Dict[str, str]:
     # Remove common prefix for endpoints. For example  all paths might
     # start with /api/v2 and we don't want this to be part of the name
     paths = list(paths)
-    if not (paths := list(paths)):
+    if not paths:
         return {}
 
-    # normalize paths
-    api_prefix = os.path.commonpath(paths)
-    norm_paths = [p.removeprefix(api_prefix) for p in paths]
+    # Get path parts for all paths
+    path_parts = [tuple(get_path_parts(path)) for path in paths]
+    # Find the longest common prefix
+    common_prefix = find_longest_common_prefix(path_parts)
+    # Remove the common prefix from each path
+    norm_paths = []
+    for path, parts in zip(paths, path_parts):
+        if common_prefix:
+            # Remove the common prefix parts
+            remaining_parts = parts[len(common_prefix) :]
+            norm_path = "/" + "/".join(remaining_parts) if remaining_parts else "/"
+        else:
+            norm_path = path
+        norm_paths.append(norm_path)
 
     # Get all path components without slashes and without {parameters}
     split_paths = [get_non_var_path_parts(path) for path in norm_paths]