WIP: experiment scaffolding

Author: dat-a-man

dlt_init_openapi/config.py

@@ -11,8 +11,8 @@
 
 from .typing import TEndpointFilter
 
-# Vendored rest_api source should be located inside the dlt_init_openapi package directory
-# e.g., dlt_init_openapi/rest_api/
+# In dlt>=1.11.0, the rest_api is part of the main package
+# For backwards compatibility, we keep a stub directory
 REST_API_SOURCE_LOCATION = str(pathlib.Path(__file__).parent / "rest_api")
 
 # Placeholder for SecretsTomlConfig to resolve ImportError in tests

dlt_init_openapi/utils/update_rest_api.py

@@ -1,39 +1,24 @@
 import pathlib
 
-import requests
 from loguru import logger
 
-# REST_API_SOURCE_LOCATION from config might resolve to site-packages.
-# For this script, we explicitly want to write into the source tree.
-# Assuming this script is in dlt_init_openapi/utils/update_rest_api.py
-# The target is dlt_init_openapi/dlt_init_openapi/rest_api/
-SCRIPT_DIR = pathlib.Path(__file__).parent.resolve()
-# Target path: dlt_init_openapi/dlt_init_openapi/rest_api/
-VENDOR_PATH = SCRIPT_DIR.parent / "rest_api"
-
-BASEPATH = "https://raw.githubusercontent.com/dlt-hub/verified-sources/master/sources/rest_api/"
-FILES = ["__init__.py", "requirements.txt"]
-
 
 def update_rest_api(force: bool = False) -> None:
-    """updates local rest api"""
-    logger.info("Syncing rest_api verified source")
-
-    # Use the locally determined VENDOR_PATH for file operations
-    path = VENDOR_PATH
-    logger.info(f"Target absolute path for rest_api (script-determined): {path}")
-
-    path.mkdir(parents=True, exist_ok=True)
-    for file in FILES:
-        src_path = BASEPATH + file
-        dst_file_path = path / file
-        logger.info(f"Copying {src_path} to {dst_file_path}")
-        with requests.get(src_path, stream=True) as r:
-            r.raise_for_status()
-            with open(dst_file_path, "wb") as f:
-                for chunk in r.iter_content(chunk_size=8192):
-                    f.write(chunk)
-    logger.success("rest_api verified source synced")
+    """
+    This function is kept for backwards compatibility.
+    In dlt >=1.0.0, rest_api is part of the main package.
+    No need to vendor the files separately.
+    """
+    logger.info("Using built-in dlt.sources.rest_api (dlt >=1.11.0)")
+    # Create an empty rest_api directory to maintain compatibility
+    script_dir = pathlib.Path(__file__).parent.resolve()
+    vendor_path = script_dir.parent / "rest_api"
+    vendor_path.mkdir(parents=True, exist_ok=True)
+    # Create empty __init__.py to make it a proper package
+    init_file = vendor_path / "__init__.py"
+    if not init_file.exists():
+        with open(init_file, "w") as f:
+            f.write("# This is a compatibility package. Use dlt.sources.rest_api instead.\n")
 
 
 if __name__ == "__main__":

tests/integration/utils.py

@@ -61,15 +61,8 @@ def get_source_or_dict_from_open_api(
         source = source.replace("rest_api_source(source_config)", "source_config")
         source = source.replace("dlt.secrets.value", '"SECRET_VALUE"')
 
-    # Directly correct the known problematic imports from the renderer\'s template
-    source = source.replace(
-        "from rest_api.typing import RESTAPIConfig",
-        "from dlt.sources.rest_api.typing import RESTAPIConfig",
-    )
-    source = source.replace(
-        "from rest_api import rest_api_source",
-        "from dlt.sources.rest_api import rest_api_source",
-    )
+    # The template now correctly uses dlt.sources.rest_api imports
+    # No need to replace imports
 
     basename = os.path.basename(case).split(".")[0] + "_" + rt