feat: edk based dbt extension (#2)

Co-authored-by: Pat Nadolny <patnadolny@gmail.com>

.flake8

@@ -1,7 +1,12 @@
 [flake8]
 ignore = W503, C901, ANN101
 max-line-length = 88
-exclude = files_db_ext
+exclude = files_dbt_ext
 max-complexity = 10
 docstring-convention = google
 allow-star-arg-any = true
+per-file-ignores =
+    # Don't require docstrings or type annotations in tests
+    tests/*:D100,D102,D103,D104,DAR,ANN
+    # missing params in docstring
+    main.py: DAR101

.github/workflows/test.yml

@@ -32,4 +32,4 @@ jobs:
           poetry install
       - name: Test with pytest
         run: |
-          poetry run pytest
+          DBT_EXT_TYPE=postgres poetry run pytest

.pre-commit-config.yaml

@@ -36,9 +36,11 @@ repos:
     - darglint==1.8.1
     - flake8-annotations==2.9.0
     - flake8-docstrings==1.6.0
+    exclude: 'files_dbt_ext'
 
 - repo: https://github.com/asottile/pyupgrade
   rev: v3.1.0
   hooks:
   - id: pyupgrade
     args: [--py37-plus]
+    exclude: 'files_dbt_ext'

README.md

@@ -1,3 +1,240 @@
 # dbt-ext
 
 Meltano dbt utility extension
+
+## Configuration
+
+Note the new `DBT_EXT_TYPE` setting, used to indicate what dbt profile should be used, and new default executable of
+`dbt_invoker` instead of `dbt`.
+
+### dbt-postgres
+
+```yaml
+plugins:
+  utilities:
+    - name: dbt-postgres
+      label: dbt PostgreSQL extension
+      executable: dbt_invoker
+      namespace: dbt_ext
+      pip_url: dbt-core~=1.1.0 dbt-postgres~=1.1.0 git+https://github.com/meltano/dbt-ext.git@feat/working-dbt-ext
+      settings:
+      - name: skip_pre_invoke
+        kind: boolean
+        value: false
+        description: Whether to skip pre-invoke hooks which automatically run dbt clean and deps
+        env: DBT_EXT_SKIP_PRE_INVOKE
+      - name: type
+        env: DBT_EXT_TYPE
+        value: postgres
+      - name: project_dir
+        label: Projects Directory
+        value: $MELTANO_PROJECT_ROOT/transform
+      - name: profiles_dir
+        label: Profiles Directory
+        value: $MELTANO_PROJECT_ROOT/transform/profiles/postgres
+        env: DBT_PROFILES_DIR
+        # Postgres connection settings are set via `config:` blocks and mapped to `profiles.yml`
+      - name: host
+        label: Host
+        kind: string
+        description: |
+          The postgres host to connect to.
+      - name: user
+        label: User
+        kind: string
+        description: |
+          The user to connect as.
+      - name: password
+        label: Password
+        kind: password
+        description: |
+          The password to connect with.
+      - name: port
+        label: Port
+        kind: integer
+        description: |
+          The port to connect to.
+      - name: dbname
+        label: Database
+        aliases: ["database"]
+        kind: string
+        description: |
+          The db to connect to.
+      - name: schema
+        label: Schema
+        kind: string
+        description: |
+          The schema to use.
+      - name: keepalives_idle
+        label: Keep Alives Idle
+        kind: integer
+        description: |
+          Seconds between TCP keepalive packets.
+      - name: search_path
+        label: Search Path
+        kind: string
+        description: |
+          Overrides the default search path.
+      - name: role
+        label: Role
+        kind: string
+        description: |
+          Role for dbt to assume when executing queries.
+      - name: sslmode
+        label: SSL Mode
+        kind: array
+        description: |
+          SSL Mode used to connect to the database.
+      commands:
+        clean:
+          args: clean
+          description: Delete all folders in the clean-targets list (usually the dbt_modules and target directories.)
+        compile:
+          args: compile
+          description: Generates executable SQL from source model, test, and analysis files. Compiled SQL files are written to the target/ directory.
+        deps:
+          args: deps
+          description: Pull the most recent version of the dependencies listed in packages.yml
+        run:
+          args: run
+          description: Compile SQL and execute against the current target database.
+        seed:
+          args: seed
+          description: Load data from csv files into your data warehouse.
+        snapshot:
+          args: snapshot
+          description: Execute snapshots defined in your project.
+        test:
+          args: test
+          description: Runs tests on data in deployed models.
+        freshness:
+          args: source freshness
+          description: Check the freshness of your source data.
+        build:
+          args: build
+          description: Will run your models, tests, snapshots and seeds in DAG order.
+        docs-generate:
+          args: docs generate
+          description: Generate documentation for your project.
+        docs-serve:
+          args: docs serve
+          description: Serve documentation for your project. Make sure you ran `docs-generate` first.
+        debug:
+          args: debug
+          description: Debug your DBT project and warehouse connection.
+        describe:
+          args: describe
+          executable: dbt_extension
+        initialize:
+          args: initialize
+          executable: dbt_extension
+```
+
+### dbt-snowflake
+
+```yaml
+  - name: dbt-snowflake
+    label: dbt Snowflake
+    executable: dbt_invoker
+    namespace: dbt_ext
+    pip_url: dbt-core~=1.1.0 dbt-snowflake~=1.1.0 git+https://github.com/meltano/dbt-ext.git@feat/working-dbt-ext
+    settings:
+    - name: skip_pre_invoke
+      kind: boolean
+      value: false
+      description: Whether to skip pre-invoke hooks which automatically run dbt clean and deps
+      env: DBT_EXT_SKIP_PRE_INVOKE
+    - name: type
+      env: DBT_EXT_TYPE
+      value: snowflake
+    - name: project_dir
+      label: Projects Directory
+      value: $MELTANO_PROJECT_ROOT/transform
+    - name: profiles_dir
+      label: Profiles Directory
+      env: DBT_PROFILES_DIR
+      value: $MELTANO_PROJECT_ROOT/transform/profiles/snowflake
+    - name: account
+      label: Account
+      kind: string
+      description: The snowflake account to connect to.
+    - name: user
+      label: User
+      kind: string
+      description: The user to connect as.
+    - name: password
+      label: Password
+      kind: password
+      description: The user password to authenticate with.
+    - name: role
+      label: Role
+      kind: string
+      description: The user role to assume.
+    - name: warehouse
+      label: Warehouse
+      kind: string
+      description: The compute warehouse to use when building models.
+    - name: database
+      label: Database
+      kind: string
+      description: The database to create models in.
+    - name: schema
+      label: Schema
+      kind: string
+      description: The schema to build models into by default.
+    commands:
+      clean:
+        args: clean
+        description: Delete all folders in the clean-targets list (usually the dbt_modules
+          and target directories.)
+      compile:
+        args: compile
+        description: Generates executable SQL from source model, test, and analysis files.
+          Compiled SQL files are written to the target/ directory.
+      deps:
+        args: deps
+        description: Pull the most recent version of the dependencies listed in packages.yml
+      run:
+        args: run
+        description: Compile SQL and execute against the current target database.
+      seed:
+        args: seed
+        description: Load data from csv files into your data warehouse.
+      snapshot:
+        args: snapshot
+        description: Execute snapshots defined in your project.
+      test:
+        args: test
+        description: Runs tests on data in deployed models.
+      freshness:
+        args: source freshness
+        description: Check the freshness of your source data.
+      build:
+        args: build
+        description: Will run your models, tests, snapshots and seeds in DAG order.
+      docs-generate:
+        args: docs generate
+        description: Generate documentation for your project.
+      docs-serve:
+        args: docs serve
+        description: Serve documentation for your project. Make sure you ran `docs-generate` first.
+      debug:
+        args: debug
+        description: Debug your DBT project and warehouse connection.
+      describe:
+        args: describe
+        executable: dbt_extension
+      initialize:
+        args: initialize
+        executable: dbt_extension
+```
+
+
+## Installation
+
+```bash
+meltano install utility dbt-postgres
+meltano invoke dbt-postgres:initialize
+meltano invoke dbt-postgres list
+meltano invoke dbt-postgres test
+```