-
Notifications
You must be signed in to change notification settings - Fork 121
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
11 changed files
with
209 additions
and
1 deletion.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,19 @@ | ||
{% macro trino__prep_external() %} | ||
|
||
{% set external_schema = target.schema %} | ||
{% set external_schema_location = 's3://test_bucket/external_schema.db' %} | ||
|
||
{% set create_external_schema %} | ||
|
||
create schema if not exists | ||
hive.{{ external_schema }} | ||
with ( | ||
location = {{ external_schema_location }} | ||
) | ||
|
||
{% endset %} | ||
|
||
{% do log('Creating external schema ' ~ external_schema, info = true) %} | ||
{% do run_query(create_external_schema) %} | ||
|
||
{% endmacro %} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,54 @@ | ||
version: 2 | ||
|
||
sources: | ||
- name: trino_external | ||
schema: "{{ target.schema }}" | ||
tables: | ||
- name: people_csv_unpartitioned | ||
external: &csv-people | ||
location: "s3://dbt-external-tables-testing/csv/" | ||
file_format: CSV | ||
table_properties: | ||
skip_header_line_count: 1 | ||
columns: &cols-of-the-people | ||
- name: id | ||
data_type: int | ||
- name: first_name | ||
data_type: varchar(64) | ||
- name: last_name | ||
data_type: varchar(64) | ||
- name: email | ||
data_type: varchar(64) | ||
tests: &equal-to-the-people | ||
- dbt_utils.equality: | ||
compare_model: ref('people') | ||
compare_columns: | ||
- id | ||
- first_name | ||
- last_name | ||
|
||
- name: people_csv_partitioned | ||
external: | ||
<<: *csv-people | ||
partitions: &parts-of-the-people | ||
- name: section | ||
data_type: varchar(1) | ||
columns: *cols-of-the-people | ||
tests: *equal-to-the-people | ||
|
||
# ensure that all partitions are created | ||
- name: people_csv_multipartitioned | ||
external: | ||
<<: *csv-people | ||
location: "s3://dbt-external-tables-testing/" | ||
partitions: | ||
- name: file_format | ||
data_type: varchar(4) | ||
- name: section | ||
data_type: varchar(1) | ||
- name: some_date | ||
data_type: date | ||
- name: file_name | ||
data_type: varchar(10) | ||
columns: *cols-of-the-people |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,41 @@ | ||
{% macro trino__create_external_table(source_node) %} | ||
{%- set columns = source_node.columns.values() -%} | ||
{%- set external = source_node.external -%} | ||
|
||
create table {{source(source_node.source_name, source_node.name)}} ( | ||
|
||
{% for column in columns %} | ||
{{column.name}} {{column.data_type}} {{- ',' if not loop.last -}} | ||
{% endfor %} | ||
{%- if external.partitions %} {{- ',' }} | ||
{% for partition in external.partitions -%} | ||
{{ partition.name }} {{ partition.data_type }} {{- ',' if not loop.last }} | ||
{% endfor %} | ||
{%- endif %} | ||
|
||
) | ||
{% if external.comment -%} comment '{{external.comment}}' {%- endif %} | ||
with ( | ||
|
||
external_location = '{{external.location}}' | ||
|
||
{%- if external.file_format %} {{- ',' }} | ||
format = '{{external.file_format}}' | ||
{%- endif -%} | ||
|
||
{%- if external.partitions %} {{- ',' }} | ||
partitioned_by = ARRAY[ | ||
{%- for partition in external.partitions -%} | ||
'{{ partition.name }}' {{- ', ' if not loop.last }} | ||
{%- endfor -%} | ||
] | ||
{%- endif -%} | ||
|
||
{%- if external.table_properties %} {{- ',' }} | ||
{% for name, value in external.table_properties.items() -%} | ||
{{ name }}={{ '{!r}'.format(value) }} {{- ',' if not loop.last }} | ||
{% endfor -%} | ||
{%- endif %} | ||
|
||
) | ||
{% endmacro %} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,19 @@ | ||
{% macro trino__get_external_build_plan(source_node) %} | ||
{% set build_plan = [] %} | ||
{% set old_relation = adapter.get_relation( | ||
database = source_node.database, | ||
schema = source_node.schema, | ||
identifier = source_node.identifier | ||
) %} | ||
{% set create_or_replace = (old_relation is none or var('ext_full_refresh', false)) %} | ||
{% if create_or_replace %} | ||
{% set build_plan = [ | ||
dbt_external_tables.dropif(source_node), | ||
dbt_external_tables.create_external_table(source_node) | ||
] + dbt_external_tables.refresh_external_table(source_node) | ||
%} | ||
{% else %} | ||
{% set build_plan = dbt_external_tables.refresh_external_table(source_node) %} | ||
{% endif %} | ||
{% do return(build_plan) %} | ||
{% endmacro %} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,6 @@ | ||
{% macro trino__dropif(node) %} | ||
{% set ddl %} | ||
drop table if exists {{source(node.source_name, node.name)}} | ||
{% endset %} | ||
{{return(ddl)}} | ||
{% endmacro %} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,13 @@ | ||
{% macro trino__refresh_external_table(source_node) %} | ||
{%- set partitions = source_node.external.partitions -%} | ||
{%- if partitions -%} | ||
{% set drop_partitions -%} | ||
call system.sync_partition_metadata('{{ source_node.source_name }}', '{{ source_node.name }}', 'DROP', false) | ||
{%- endset %} | ||
{% set add_partitions -%} | ||
call system.sync_partition_metadata('{{ source_node.source_name }}', '{{ source_node.name }}', 'ADD', false) | ||
{%- endset %} | ||
{{ return([drop_partitions, add_partitions]) }} | ||
{% endif %} | ||
{% do return([]) %} | ||
{% endmacro %} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,25 @@ | ||
version: 2 | ||
|
||
sources: | ||
- name: trino | ||
tables: | ||
- name: snowplow | ||
description: "Snowplow events stored as CSV files in HDFS" | ||
external: | ||
location: "s3a://.../event.csv" # s3://, hdfs://, ... | ||
file_format: CSV # file format: CSV, ORC, PARQUET, ... | ||
table_properties: # as needed | ||
skip_header_line_count: 1 | ||
csv_separator: ";" | ||
csv_quote: '"' | ||
|
||
columns: | ||
- name: app_id | ||
data_type: varchar | ||
description: "Application ID" | ||
- name: domain_sessionidx | ||
data_type: int | ||
description: "A visit / session index" | ||
- name: etl_tstamp | ||
data_type: timestamp | ||
description: "Timestamp event began ETL" |