Skip to content

Commit

Permalink
Merge branch 'main' into 2648-feat-add-a-generic-filestream-interface
Browse files Browse the repository at this point in the history
  • Loading branch information
edgarrmondragon committed Sep 26, 2024
2 parents 92f1522 + f1e8114 commit e5a4a4f
Show file tree
Hide file tree
Showing 16 changed files with 37 additions and 56 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,6 @@ jobs:
fail-fast: false
matrix:
python-version:
- "3.8"
- "3.9"
- "3.10"
- "3.11"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@ keywords = [
classifiers = [
"Intended Audience :: Developers",
"Operating System :: OS Independent",
"Programming Language :: Python :: 3.8",
"Programming Language :: Python :: 3.9",
"Programming Language :: Python :: 3.10",
"Programming Language :: Python :: 3.11",
Expand All @@ -30,7 +29,7 @@ packages = [
{%- endif %}

[tool.poetry.dependencies]
python = ">=3.8"
python = ">=3.9"
singer-sdk = { version="~=0.40.0"{{ ', extras = ["faker"]' if cookiecutter.faker_extra }} }
fs-s3fs = { version = "~=1.1.1", optional = true }

Expand All @@ -50,7 +49,7 @@ warn_unused_configs = true

[tool.ruff]
src = ["{{cookiecutter.library_name}}"]
target-version = "py38"
target-version = "py39"

[tool.ruff.lint]
ignore = [
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
# This file can be used to customize tox tests as well as other test frameworks like flake8 and mypy

[tox]
envlist = py{38,39,310,311,312}
envlist = py{39,310,311,312}
isolated_build = true

[testenv]
Expand All @@ -13,7 +13,7 @@ commands =
[testenv:pytest]
# Run the python tests.
# To execute, run `tox -e pytest`
envlist = py{38,39,310,311,312}
envlist = py{39,310,311,312}
commands =
poetry install -v
poetry run pytest
Original file line number Diff line number Diff line change
Expand Up @@ -3,14 +3,13 @@
from __future__ import annotations

import typing as t
from typing import TYPE_CHECKING

import singer_sdk.typing as th
from singer_sdk import _singerlib as singer
from singer_sdk.mapper import PluginMapper
from singer_sdk.mapper_base import InlineMapper

if TYPE_CHECKING:
if t.TYPE_CHECKING:
from pathlib import PurePath


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,6 @@ jobs:
fail-fast: false
matrix:
python-version:
- "3.8"
- "3.9"
- "3.10"
- "3.11"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@ keywords = [
classifiers = [
"Intended Audience :: Developers",
"Operating System :: OS Independent",
"Programming Language :: Python :: 3.8",
"Programming Language :: Python :: 3.9",
"Programming Language :: Python :: 3.10",
"Programming Language :: Python :: 3.11",
Expand All @@ -29,7 +28,7 @@ packages = [
{%- endif %}

[tool.poetry.dependencies]
python = ">=3.8"
python = ">=3.9"
importlib-resources = { version = "==6.4.*", python = "<3.9" }
singer-sdk = { version="~=0.40.0", extras = [
{%- if cookiecutter.auth_method == "JWT" -%}"jwt", {% endif -%}
Expand Down Expand Up @@ -65,7 +64,7 @@ plugins = "sqlmypy"

[tool.ruff]
src = ["{{cookiecutter.library_name}}"]
target-version = "py38"
target-version = "py39"

[tool.ruff.lint]
ignore = [
Expand Down
4 changes: 2 additions & 2 deletions cookiecutter/tap-template/{{cookiecutter.tap_id}}/tox.ini
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
# This file can be used to customize tox tests as well as other test frameworks like flake8 and mypy

[tox]
envlist = py{38,39,310,311,312}
envlist = py{39,310,311,312}
isolated_build = true

[testenv]
Expand All @@ -13,7 +13,7 @@ commands =
[testenv:pytest]
# Run the python tests.
# To execute, run `tox -e pytest`
envlist = py{38,39,310,311,312}
envlist = py{39,310,311,312}
commands =
poetry install -v
poetry run pytest
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

from __future__ import annotations

from typing import TYPE_CHECKING, Iterable
import typing as t

import requests # noqa: TCH002
from singer_sdk.streams import {{ cookiecutter.stream_type }}Stream
Expand All @@ -12,7 +12,7 @@
from {{ cookiecutter.library_name }}.auth import {{ cookiecutter.source_name }}Authenticator
{%- endif %}

if TYPE_CHECKING:
if t.TYPE_CHECKING:
from singer_sdk.helpers.types import Context


Expand Down Expand Up @@ -54,7 +54,7 @@ def http_headers(self) -> dict:
{%- endif %}
return headers

def parse_response(self, response: requests.Response) -> Iterable[dict]:
def parse_response(self, response: requests.Response) -> t.Iterable[dict]:
"""Parse the response and return an iterator of result records.
Args:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,11 @@

from __future__ import annotations

from typing import TYPE_CHECKING, Iterable
import typing as t

from singer_sdk.streams import Stream

if TYPE_CHECKING:
if t.TYPE_CHECKING:
from singer_sdk.helpers.types import Context


Expand All @@ -16,7 +16,7 @@ class {{ cookiecutter.source_name }}Stream(Stream):
def get_records(
self,
context: Context | None, # noqa: ARG002
) -> Iterable[dict]:
) -> t.Iterable[dict]:
"""Return a generator of record-type dictionary objects.
The optional `context` argument is used to identify a specific slice of the
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,11 @@

from __future__ import annotations

import sys
{%- if cookiecutter.auth_method in ("OAuth2", "JWT") %}
import typing as t
{% if cookiecutter.auth_method in ("OAuth2", "JWT") -%}
from functools import cached_property
{%- endif %}
from typing import TYPE_CHECKING, Any, Iterable
{% endif -%}
from importlib import resources

{% if cookiecutter.auth_method == "API Key" -%}
from singer_sdk.authenticators import APIKeyAuthenticator
Expand Down Expand Up @@ -40,12 +40,7 @@

{% endif -%}

if sys.version_info >= (3, 9):
import importlib.resources as importlib_resources
else:
import importlib_resources

if TYPE_CHECKING:
if t.TYPE_CHECKING:
import requests
{%- if cookiecutter.auth_method in ("OAuth2", "JWT") %}
from singer_sdk.helpers.types import Auth, Context
Expand All @@ -55,7 +50,7 @@


# TODO: Delete this is if not using json files for schema definition
SCHEMAS_DIR = importlib_resources.files(__package__) / "schemas"
SCHEMAS_DIR = resources.files(__package__) / "schemas"


class {{ cookiecutter.source_name }}Stream({{ cookiecutter.stream_type }}Stream):
Expand Down Expand Up @@ -164,8 +159,8 @@ def get_new_paginator(self) -> BaseAPIPaginator:
def get_url_params(
self,
context: Context | None, # noqa: ARG002
next_page_token: Any | None, # noqa: ANN401
) -> dict[str, Any]:
next_page_token: t.Any | None, # noqa: ANN401
) -> dict[str, t.Any]:
"""Return a dictionary of values to be used in URL parameterization.
Args:
Expand All @@ -186,7 +181,7 @@ def get_url_params(
def prepare_request_payload(
self,
context: Context | None, # noqa: ARG002
next_page_token: Any | None, # noqa: ARG002, ANN401
next_page_token: t.Any | None, # noqa: ARG002, ANN401
) -> dict | None:
"""Prepare the data payload for the REST API request.
Expand All @@ -202,7 +197,7 @@ def prepare_request_payload(
# TODO: Delete this method if no payload is required. (Most REST APIs.)
return None

def parse_response(self, response: requests.Response) -> Iterable[dict]:
def parse_response(self, response: requests.Response) -> t.Iterable[dict]:
"""Parse the response and return an iterator of result records.
Args:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@

from __future__ import annotations

from typing import Any, Iterable
import typing as t

import sqlalchemy # noqa: TCH002
from singer_sdk import SQLConnector, SQLStream
Expand Down Expand Up @@ -77,7 +77,7 @@ class {{ cookiecutter.source_name }}Stream(SQLStream):

connector_class = {{ cookiecutter.source_name }}Connector

def get_records(self, partition: dict | None) -> Iterable[dict[str, Any]]:
def get_records(self, partition: dict | None) -> t.Iterable[dict[str, t.Any]]:
"""Return a generator of record-type dictionary objects.
Developers may optionally add custom logic before calling the default
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,21 +2,15 @@

from __future__ import annotations

import sys
import typing as t
from importlib import resources

from singer_sdk import typing as th # JSON Schema typing helpers

from {{ cookiecutter.library_name }}.client import {{ cookiecutter.source_name }}Stream

if sys.version_info >= (3, 9):
import importlib.resources as importlib_resources
else:
import importlib_resources


# TODO: Delete this is if not using json files for schema definition
SCHEMAS_DIR = importlib_resources.files(__package__) / "schemas"
SCHEMAS_DIR = resources.files(__package__) / "schemas"


{%- if cookiecutter.stream_type == "GraphQL" %}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,6 @@ jobs:
fail-fast: false
matrix:
python-version:
- "3.8"
- "3.9"
- "3.10"
- "3.11"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@ keywords = [
classifiers = [
"Intended Audience :: Developers",
"Operating System :: OS Independent",
"Programming Language :: Python :: 3.8",
"Programming Language :: Python :: 3.9",
"Programming Language :: Python :: 3.10",
"Programming Language :: Python :: 3.11",
Expand All @@ -29,7 +28,7 @@ packages = [
{%- endif %}

[tool.poetry.dependencies]
python = ">=3.8"
python = ">=3.9"
singer-sdk = { version="~=0.40.0"{{ ', extras = ["faker"]' if cookiecutter.faker_extra }} }
fs-s3fs = { version = "~=1.1.1", optional = true }
{%- if cookiecutter.serialization_method != "SQL" %}
Expand All @@ -52,7 +51,7 @@ warn_unused_configs = true

[tool.ruff]
src = ["{{cookiecutter.library_name}}"]
target-version = "py38"
target-version = "py39"

[tool.ruff.lint]
ignore = [
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
# This file can be used to customize tox tests as well as other test frameworks like flake8 and mypy

[tox]
envlist = py{38,39,310,311,312}
envlist = py{39,310,311,312}
isolated_build = true

[testenv]
Expand All @@ -13,7 +13,7 @@ commands =
[testenv:pytest]
# Run the python tests.
# To execute, run `tox -e pytest`
envlist = py{38,39,310,311,312}
envlist = py{39,310,311,312}
commands =
poetry install -v
poetry run pytest
9 changes: 4 additions & 5 deletions samples/sample_tap_dummy_json/pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@ keywords = [
classifiers = [
"Intended Audience :: Developers",
"Operating System :: OS Independent",
"Programming Language :: Python :: 3.8",
"Programming Language :: Python :: 3.9",
"Programming Language :: Python :: 3.10",
"Programming Language :: Python :: 3.11",
Expand All @@ -20,13 +19,13 @@ classifiers = [
license = "Apache-2.0"

[tool.poetry.dependencies]
python = ">=3.8"
python = ">=3.9"
requests = "~=2.32.3"
singer-sdk = { version="~=0.38.0", extras = [] }
singer-sdk = {path = "../..", develop = true}

[tool.poetry.group.dev.dependencies]
pytest = ">=8"
singer-sdk = { version="~=0.38.0", extras = ["testing"] }
singer-sdk = {path = "../..", develop = true, extras = ["testing"]}

[tool.poetry.extras]
s3 = ["fs-s3fs"]
Expand All @@ -36,7 +35,7 @@ python_version = "3.12"
warn_unused_configs = true

[build-system]
requires = ["poetry-core==1.9.0"]
requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"

[tool.poetry.scripts]
Expand Down

0 comments on commit e5a4a4f

Please sign in to comment.