📝 Add missing reference docs (#1080)

* 🔥 Remove duplicate import

* ♻️ Clean up the imports mess

* 📝 Add missing reference docs & sort

* 🐛 Fix import shadowing 3rd party lib

.prettierignore

@@ -1 +1,2 @@
-docs/references/sources/sql.md
+docs/references/sources/database.md
+docs/references/sources/api.md

docs/references/orchestration/prefect/flows.md

@@ -1,24 +1,12 @@
-::: viadot.orchestration.prefect.flows.cloud_for_customers_to_adls
-
-::: viadot.orchestration.prefect.flows.cloud_for_customers_to_databricks
-
-::: viadot.orchestration.prefect.flows.exchange_rates_to_adls
-
-::: viadot.orchestration.prefect.flows.exchange_rates_to_databricks
-
-::: viadot.orchestration.prefect.flows.sap_to_redshift_spectrum
-
-::: viadot.orchestration.prefect.flows.sharepoint_to_adls
-
-::: viadot.orchestration.prefect.flows.sharepoint_to_databricks
+::: viadot.orchestration.prefect.flows.azure_sql_to_adls
 
-::: viadot.orchestration.prefect.flows.sharepoint_to_redshift_spectrum
+::: viadot.orchestration.prefect.flows.bigquery_to_adls
 
-::: viadot.orchestration.prefect.flows.sharepoint_to_s3
+::: viadot.orchestration.prefect.flows.cloud_for_customers_to_adls
 
-::: viadot.orchestration.prefect.flows.transform
+::: viadot.orchestration.prefect.flows.cloud_for_customers_to_databricks
 
-::: viadot.orchestration.prefect.flows.transform_and_catalog
+::: viadot.orchestration.prefect.flows.customer_gauge_to_adls
 
 ::: viadot.orchestration.prefect.flows.duckdb_to_parquet
 
@@ -28,20 +16,48 @@
 
 ::: viadot.orchestration.prefect.flows.epicor_to_parquet
 
+::: viadot.orchestration.prefect.flows.exchange_rates_to_adls
+
+::: viadot.orchestration.prefect.flows.exchange_rates_to_databricks
+
 ::: viadot.orchestration.prefect.flows.exchange_rates_api_to_redshift_spectrum
 
 ::: viadot.orchestration.prefect.flows.genesys_to_adls
 
 ::: viadot.orchestration.prefect.flows.hubspot_to_adls
 
+::: viadot.orchestration.prefect.flows.mediatool_to_adls
+
 ::: viadot.orchestration.prefect.flows.mindful_to_adls
 
 ::: viadot.orchestration.prefect.flows.outlook_to_adls
 
+::: viadot.orchestration.prefect.flows.salesforce_to_adls
+
+::: viadot.orchestration.prefect.flows.sap_bw_to_adls
+
 ::: viadot.orchestration.prefect.flows.sap_to_parquet
 
+::: viadot.orchestration.prefect.flows.sap_to_redshift_spectrum
+
+::: viadot.orchestration.prefect.flows.sftp_to_adls
+
+::: viadot.orchestration.prefect.flows.sharepoint_to_adls
+
+::: viadot.orchestration.prefect.flows.sharepoint_to_databricks
+
+::: viadot.orchestration.prefect.flows.sharepoint_to_redshift_spectrum
+
+::: viadot.orchestration.prefect.flows.sharepoint_to_s3
+
 ::: viadot.orchestration.prefect.flows.sql_server_to_minio
 
 ::: viadot.orchestration.prefect.flows.sql_server_to_parquet
 
-::: viadot.orchestration.prefect.flows.azure_sql_to_adls
+::: viadot.orchestration.prefect.flows.supermetrics_to_adls
+
+::: viadot.orchestration.prefect.flows.transform
+
+::: viadot.orchestration.prefect.flows.transform_and_catalog
+
+::: viadot.orchestration.prefect.flows.vid_club_to_adls

docs/references/orchestration/prefect/tasks.md

@@ -1,47 +1,67 @@
+::: viadot.orchestration.prefect.tasks.azure_sql_to_df
+
 ::: viadot.orchestration.prefect.tasks.adls_upload
 
-::: viadot.orchestration.prefect.tasks.df_to_adls
+::: viadot.orchestration.prefect.tasks.bcp
 
-::: viadot.orchestration.prefect.tasks.cloud_for_customers_to_df
+::: viadot.orchestration.prefect.tasks.clone_repo
 
-::: viadot.orchestration.prefect.tasks.df_to_databricks
+::: viadot.orchestration.prefect.tasks.bigquery_to_df
 
-::: viadot.orchestration.prefect.tasks.dbt_task
+::: viadot.orchestration.prefect.tasks.cloud_for_customers_to_df
 
-::: viadot.orchestration.prefect.tasks.exchange_rates_to_df
+::: viadot.orchestration.prefect.tasks.create_sql_server_table
 
-::: viadot.orchestration.prefect.tasks.clone_repo
+::: viadot.orchestration.prefect.tasks.customer_gauge_to_df
 
-::: viadot.orchestration.prefect.tasks.luma_ingest_task
+::: viadot.orchestration.prefect.tasks.dbt_task
 
-::: viadot.orchestration.prefect.tasks.df_to_redshift_spectrum
+::: viadot.orchestration.prefect.tasks.df_to_adls
 
-::: viadot.orchestration.prefect.tasks.s3_upload_file
+::: viadot.orchestration.prefect.tasks.df_to_databricks
 
-::: viadot.orchestration.prefect.tasks.sharepoint_download_file
+::: viadot.orchestration.prefect.tasks.df_to_minio
 
-::: viadot.orchestration.prefect.tasks.sharepoint_to_df
+::: viadot.orchestration.prefect.tasks.df_to_redshift_spectrum
 
 ::: viadot.orchestration.prefect.tasks.duckdb_query
 
 ::: viadot.orchestration.prefect.tasks.epicor_to_df
 
+::: viadot.orchestration.prefect.tasks.exchange_rates_to_df
+
 ::: viadot.orchestration.prefect.tasks.genesys_to_df
 
 ::: viadot.orchestration.prefect.tasks.hubspot_to_df
 
+::: viadot.orchestration.prefect.tasks.luma_ingest_task
+
+::: viadot.orchestration.prefect.tasks.mediatool_to_df
+
 ::: viadot.orchestration.prefect.tasks.mindful_to_df
 
 ::: viadot.orchestration.prefect.tasks.outlook_to_df
 
+::: viadot.orchestration.prefect.tasks.s3_upload_file
+
+::: viadot.orchestration.prefect.tasks.salesforce_to_df
+
+::: viadot.orchestration.prefect.tasks.sap_bw_to_df
+
 ::: viadot.orchestration.prefect.tasks.sap_rfc_to_df
 
-::: viadot.orchestration.prefect.tasks.bcp
+::: viadot.orchestration.prefect.tasks.sftp_list
 
-::: viadot.orchestration.prefect.tasks.create_sql_server_table
+::: viadot.orchestration.prefect.tasks.sftp_to_df
+
+::: viadot.orchestration.prefect.tasks.sharepoint_download_file
+
+::: viadot.orchestration.prefect.tasks.sharepoint_to_df
 
 ::: viadot.orchestration.prefect.tasks.sql_server_query
 
 ::: viadot.orchestration.prefect.tasks.sql_server_to_df
 
-::: viadot.orchestration.prefect.tasks.azure_sql_to_df
+::: viadot.orchestration.prefect.tasks.vid_club_to_df
+
+::: viadot.orchestration.prefect.tasks.supermetrics_to_df

docs/references/sources/api.md

@@ -1,23 +1,33 @@
-# API Sources
+# API sources
 
-::: viadot.sources.uk_carbon_intensity.UKCarbonIntensity
+::: viadot.sources.bigquery.BigQuery
 
 ::: viadot.sources.cloud_for_customers.CloudForCustomers
 
-::: viadot.sources.exchange_rates.ExchangeRates
+::: viadot.sources.customer_gauge.CustomerGauge
 
-::: viadot.sources.cloud_for_customers.CloudForCustomers
+::: viadot.sources.epicor.Epicor
 
-::: viadot.sources.sharepoint.Sharepoint
+::: viadot.sources.exchange_rates.ExchangeRates
 
 ::: viadot.sources.genesys.Genesys
 
-::: viadot.sources.outlook.Outlook
-
 ::: viadot.sources.hubspot.Hubspot
 
-::: viadot.sources.epicor.Epicor
+::: viadot.sources.mediatool.Mediatool
 
 ::: viadot.sources.mindful.Mindful
 
-::: viadot.sources.minio.MinIO
+::: viadot.sources._minio.MinIO
+
+::: viadot.sources.outlook.Outlook
+
+::: viadot.sources.salesforce.Salesforce
+
+::: viadot.sources.sharepoint.Sharepoint
+
+::: viadot.sources.supermetrics.Supermetrics
+
+::: viadot.sources.uk_carbon_intensity.UKCarbonIntensity
+
+::: viadot.sources.vid_club.VidClub

docs/references/sources/sql.md → docs/references/sources/database.md

@@ -1,27 +1,29 @@
-# SQL Sources
+# Database sources
 
-::: viadot.sources.base.Source
+::: viadot.sources.azure_data_lake.AzureDataLake
 
-::: viadot.sources.base.SQL
+::: viadot.sources.azure_sql.AzureSQL
 
-::: viadot.sources.azure_data_lake.AzureDataLake
+::: viadot.sources.databricks.Databricks
+
+::: viadot.sources._duckdb.DuckDB
 
 ::: viadot.sources.redshift_spectrum.RedshiftSpectrum
 
 ::: viadot.sources.s3.S3
 
-::: viadot.sources.sqlite.SQLite
+::: viadot.sources.sap_bw.SAPBW
 
-::: viadot.sources.sql_server.SQLServer
+::: viadot.sources.sap_rfc.SAPRFC
 
-::: viadot.sources.databricks.Databricks
+::: viadot.sources.sap_rfc.SAPRFCV2
 
-::: viadot.sources._trino.Trino
+::: viadot.sources.base.Source
 
-::: viadot.sources._duckdb.DuckDB
+::: viadot.sources.base.SQL
 
-::: viadot.sources.sap_rfc.SAPRFC
+::: viadot.sources.sqlite.SQLite
 
-::: viadot.sources.sap_rfc.SAPRFCV2
+::: viadot.sources.sql_server.SQLServer
 
-::: viadot.sources.azure_sql.AzureSQL
+::: viadot.sources._trino.Trino

docs/references/sources/other.md

@@ -0,0 +1,3 @@
+# Other sources
+
+::: viadot.sources.sftp.Sftp

mkdocs.yml

@@ -21,8 +21,9 @@ nav:
 
   - References:
       - Sources:
-          - SQL Sources: references/sources/sql.md
-          - API Sources: references/sources/api.md
+          - Database: references/sources/database.md
+          - API: references/sources/api.md
+          - Other: references/sources/other.md
       - Orchestration:
           - Prefect:
               - Tasks: references/orchestration/prefect/tasks.md

src/viadot/orchestration/prefect/tasks/__init__.py

@@ -43,7 +43,6 @@
     "dbt_task",
     "df_to_adls",
     "df_to_databricks",
-    "df_to_databricks",
     "df_to_minio",
     "df_to_redshift_spectrum",
     "duckdb_query",

src/viadot/sources/__init__.py

@@ -19,7 +19,8 @@
 from .sftp import Sftp
 from .sharepoint import Sharepoint
 from .sql_server import SQLServer
-from .supermetrics import Supermetrics, SupermetricsCredentials
+from .sqlite import SQLite
+from .supermetrics import Supermetrics
 from .uk_carbon_intensity import UKCarbonIntensity
 from .vid_club import VidClub
 
@@ -29,43 +30,41 @@
     "BigQuery",
     "CloudForCustomers",
     "CustomerGauge",
+    "DuckDB",
     "Epicor",
     "ExchangeRates",
     "Genesys",
     "Hubspot",
     "Mediatool",
     "Mindful",
     "Outlook",
+    "SQLite",
     "SQLServer",
     "Salesforce",
     "Sftp",
     "Sharepoint",
     "Supermetrics",
-    "SupermetricsCredentials",  # pragma: allowlist-secret
     "Trino",
     "UKCarbonIntensity",
     "VidClub",
 ]
+
 if find_spec("adlfs"):
     from viadot.sources.azure_data_lake import AzureDataLake  # noqa: F401
 
     __all__.extend(["AzureDataLake"])
-if find_spec("duckdb"):
-    from viadot.sources._duckdb import DuckDB  # noqa: F401
 
-    __all__.extend(["DuckDB"])
 if find_spec("redshift_connector"):
     from viadot.sources.redshift_spectrum import RedshiftSpectrum  # noqa: F401
 
     __all__.extend(["RedshiftSpectrum"])
+
 if find_spec("s3fs"):
+    from viadot.sources._minio import MinIO  # noqa: F401
     from viadot.sources.s3 import S3  # noqa: F401
 
-    __all__.extend(["S3"])
-if find_spec("s3fs"):
-    from viadot.sources.minio import MinIO  # noqa: F401
+    __all__.extend(["S3", "MinIO"])
 
-    __all__.extend(["MinIO"])
 if find_spec("pyrfc"):
     from viadot.sources.sap_bw import SAPBW  # noqa: F401
     from viadot.sources.sap_rfc import SAPRFC, SAPRFCV2  # noqa: F401

tests/unit/test_supermetrics.py

@@ -1,6 +1,7 @@
 import pytest
 
-from viadot.sources import Supermetrics, SupermetricsCredentials
+from viadot.sources import Supermetrics
+from viadot.sources.supermetrics import SupermetricsCredentials
 
 
 @pytest.fixture