Merge pull request #30 from QuangTrinh1612/dev

Added Receivable Module

dbt-runner.py

@@ -0,0 +1,10 @@
+from typing import List
+from dbt.cli.main import dbtRunner, dbtRunnerResult
+
+class CustomRunner(dbtRunner):
+    def invoke(self, args: List[str], **kwargs) -> dbtRunnerResult: 
+        print("Here you can perform setup or authentication or modify args")
+        super().invoke(args, **kwargs)
+
+### patch the dbtRunner
+dbtRunner = CustomRunner

doc/dbt-models/dbt-docs.md

@@ -19,6 +19,11 @@ dbt run --profiles-dir config --project-dir transform/oracle_analytics
 dbt run --profiles-dir config --project-dir transform/sap_analytics
 ```
 
+dbt elementary commands
+```bash
+dbt run --select elementary --profiles-dir config --project-dir transform/oracle_analytics
+```
+
 dbt documentation commands
 ```bash
 dbt docs generate --profiles-dir config --project-dir transform/oracle_analytics

transform/oracle_analytics/dbt_project.yml

@@ -24,6 +24,9 @@ vars:
 models:
   +file_format: delta
 
+  elementary:
+    +schema: "test"
+
   oracle_analytics:
     staging:
       +materialized: ephemeral
@@ -38,11 +41,12 @@ models:
       finance:
         +schema: finance
 
-        # Temp Ignore
         account_payable:
-          +enabled: false
+          f_ap_line_item:
+            +enabled: false
         fixed_asset:
           +enabled: false
-        account_receivable:
-          d_customer:
-            +enabled: false
+
+flags:
+  require_explicit_package_overrides_for_builtin_materializations: False
+  source_freshness_run_project_hooks: True

transform/oracle_analytics/docs/table_descriptions.md

@@ -28,4 +28,21 @@ The `f_purchase_order` table is a fact table in the data warehouse designed to s
 - **Supplier Performance**: Evaluate supplier performance based on purchase orders and deliveries.
 - **Procurement Trends**: Identify trends in procurement activities, such as frequently ordered items or peak procurement periods.
 
+{% enddocs %}
+
+{% docs f_ar_line_item %}
+The 'f_ar_line_item' model is a comprehensive representation of
+Accounts Receivable (AR) line items. It is derived from multiple sources
+including customer transaction lines, headers, and distributions. The
+model includes a wide range of fields capturing various aspects of AR
+transactions such as transaction IDs, numbers, dates, classes, document
+types, currency codes, exchange rates, flags for prepayment, unpaid and
+on-hold status, total amounts, customer IDs, site IDs, line numbers,
+types, descriptions, quantities ordered, credited, invoiced, unit prices,
+sales order details, inventory item IDs, credit memo IDs, revenue amounts,
+remaining due amounts, tax rates, unit of measure codes, last update
+dates, and distribution items. The model also includes a unique identifier
+for each AR line item and a timestamp indicating the time of data
+ingestion. This model is particularly useful for financial analysis,
+reporting, and auditing purposes.
 {% enddocs %}

transform/oracle_analytics/models/intermediate/finance/account_payable/d_vendor.sql

@@ -0,0 +1,19 @@
+WITH supplier AS (
+    SELECT * FROM {{ ref('supplier') }}
+),
+supplier_site AS (
+    SELECT * FROM {{ ref('supplier_site') }}
+)
+SELECT
+    -- Surrogate Key
+    {{ dbt_utils.generate_surrogate_key(['s.vendor_id', 'ss.vendor_site_id']) }} AS vendor_surrg_key
+
+    ,s.vendor_id
+    ,s.business_relationship
+    ,s.organization_type_lookup_code
+    ,s.vendor_type_lookup_code
+    ,ss.vendor_site_id
+    ,ss.vendor_site_code
+FROM supplier s
+LEFT JOIN supplier_site ss
+    ON s.vendor_id = ss.vendor_id

transform/oracle_analytics/models/intermediate/finance/account_receivable/d_customer.sql

@@ -1,137 +1,46 @@
-{{ config(
-    materialized='incremental',
-    unique_key='customer_id'
-) }}
-
-WITH customer_parties AS (
-    SELECT 
-        party.party_id,
-        party.party_number,
-        party.party_name,
-        party.party_type,
-        party.taxpayer_identification_number,
-        party.creation_date AS party_creation_date,
-        party.last_update_date AS party_last_update_date
-    FROM 
-        {{ source('oracle_fusion_finance', 'HZ_PARTIES') }} party
-    WHERE party.party_type = 'ORGANIZATION'
+{{  
+    config(
+        materialized='incremental',
+        unique_key='customer_id',
+        incremental_strategy='merge'
+    )
+}}
+WITH customer_party AS (
+    SELECT * FROM {{ ref('customer_party') }}
 ),
 
 customer_accounts AS (
-    SELECT 
-        cust_acct.cust_account_id AS customer_id,
-        cust_acct.party_id,
-        cust_acct.account_number AS customer_account_number,
-        cust_acct.account_description AS customer_description,
-        cust_acct.status AS customer_status,
-        cust_acct.account_type AS customer_account_type,
-        cust_acct.creation_date AS account_creation_date,
-        cust_acct.last_update_date AS account_last_update_date
-    FROM 
-        {{ source('oracle_fusion_finance', 'HZ_CUST_ACCOUNTS') }} cust_acct
-),
-
-customer_sites AS (
-    SELECT 
-        sites.cust_acct_site_id AS customer_site_id,
-        sites.cust_account_id AS customer_id,
-        sites.site_use_code,
-        sites.status AS site_status,
-        sites.location_id,
-        sites.creation_date AS site_creation_date,
-        sites.last_update_date AS site_last_update_date
-    FROM 
-        {{ source('oracle_fusion_finance', 'HZ_CUST_ACCOUNT_SITES_ALL') }} sites
-),
-
-customer_locations AS (
-    SELECT 
-        loc.location_id,
-        loc.address1,
-        loc.address2,
-        loc.address3,
-        loc.address4,
-        loc.city,
-        loc.postal_code,
-        loc.state,
-        loc.province,
-        loc.country,
-        loc.creation_date AS location_creation_date,
-        loc.last_update_date AS location_last_update_date
-    FROM 
-        {{ source('oracle_fusion_finance', 'HZ_LOCATIONS') }} loc
-),
-
-customer_contacts AS (
-    SELECT 
-        contact.party_id,
-        contact.contact_point_id,
-        contact.contact_point_type,
-        contact.contact_point_purpose,
-        contact.email_address,
-        contact.phone_number,
-        contact.creation_date AS contact_creation_date,
-        contact.last_update_date AS contact_last_update_date
-    FROM 
-        {{ source('oracle_fusion_finance', 'HZ_CONTACT_POINTS') }} contact
-    WHERE contact.contact_point_type IN ('EMAIL', 'PHONE')
+    SELECT * FROM {{ ref('customer_account') }}
 )
-
 SELECT 
-    cust_acct.customer_id,
-    cust_acct.customer_account_number,
+    -- Surrogate Key
+    {{ dbt_utils.generate_surrogate_key(['party.party_id', 'cust_acct.cust_account_id']) }} AS customer_id,
+
+    cust_acct.cust_account_id AS cust_account_id,
+    cust_acct.account_number,
+    cust_acct.customer_type,
+    party.party_id,
     party.party_number,
     party.party_name AS customer_name,
-    cust_acct.customer_description,
-    cust_acct.customer_status,
-    cust_acct.customer_account_type,
-    party.taxpayer_identification_number,
-    cust_acct.account_creation_date,
-    cust_acct.account_last_update_date,
-    COALESCE(
-        MAX(CASE WHEN site.site_use_code = 'BILL_TO' THEN loc.address1 END),
-        MAX(CASE WHEN site.site_use_code = 'SHIP_TO' THEN loc.address1 END)
-    ) AS primary_address,
-    COALESCE(
-        MAX(CASE WHEN site.site_use_code = 'BILL_TO' THEN loc.city END),
-        MAX(CASE WHEN site.site_use_code = 'SHIP_TO' THEN loc.city END)
-    ) AS primary_city,
-    COALESCE(
-        MAX(CASE WHEN site.site_use_code = 'BILL_TO' THEN loc.state END),
-        MAX(CASE WHEN site.site_use_code = 'SHIP_TO' THEN loc.state END)
-    ) AS primary_state,
-    COALESCE(
-        MAX(CASE WHEN site.site_use_code = 'BILL_TO' THEN loc.country END),
-        MAX(CASE WHEN site.site_use_code = 'SHIP_TO' THEN loc.country END)
-    ) AS primary_country,
-    MAX(CASE WHEN contact.contact_point_type = 'EMAIL' THEN contact.email_address END) AS primary_email,
-    MAX(CASE WHEN contact.contact_point_type = 'PHONE' THEN contact.phone_number END) AS primary_phone,
-    party.party_creation_date,
-    party.party_last_update_date
+    cust_acct.status AS customer_status,
+    cust_acct.account_name,
+    cust_acct.tax_code,
+    party.address1,
+    party.address2,
+    party.address3,
+    party.address4,
+    party.city,
+    party.state,
+    party.country,
+    party.email_address,
+    party.primary_phone_number,
+    cust_acct.creation_date AS account_creation_date,
+    cust_acct.last_update_date AS account_last_update_date,
+    party.creation_date AS party_creation_date,
+    party.last_update_date AS party_last_update_date
 FROM 
     customer_accounts cust_acct
-LEFT JOIN 
-    customer_parties party ON cust_acct.party_id = party.party_id
-LEFT JOIN 
-    customer_sites site ON cust_acct.customer_id = site.customer_id
-LEFT JOIN 
-    customer_locations loc ON site.location_id = loc.location_id
-LEFT JOIN 
-    customer_contacts contact ON party.party_id = contact.party_id
-GROUP BY 
-    cust_acct.customer_id,
-    cust_acct.customer_account_number,
-    party.party_number,
-    party.party_name,
-    cust_acct.customer_description,
-    cust_acct.customer_status,
-    cust_acct.customer_account_type,
-    party.taxpayer_identification_number,
-    cust_acct.account_creation_date,
-    cust_acct.account_last_update_date,
-    party.party_creation_date,
-    party.party_last_update_date
-
+    LEFT JOIN customer_party party ON cust_acct.party_id = party.party_id
 {% if is_incremental() %}
-WHERE party.party_last_update_date > (SELECT MAX(party_last_update_date) FROM {{ this }})
+WHERE party.last_update_date > (SELECT COALESCE(MAX(party_last_update_date), '1970-01-01') FROM {{ this }})
 {% endif %}

transform/oracle_analytics/models/intermediate/finance/account_receivable/f_ar_line_item.sql

@@ -72,6 +72,7 @@ SELECT
     line.amount_due_remaining,
     line.tax_rate,
     line.uom_code,
+    line.last_update_date as line_last_update_date,
 
     dist.distribution_items, -- JSON string of distribution items for each line,
     NOW() AS _etl_ingestion_time
@@ -81,5 +82,5 @@ JOIN ar_line line ON hdr.customer_trx_id = line.customer_trx_id
 LEFT JOIN ar_distribution dist ON line.customer_trx_line_id = dist.customer_trx_line_id
 
 {% if is_incremental() %}
-WHERE l.last_update_date > (SELECT COALESCE(MAX(line_last_update_date), '1970-01-01') FROM {{ this }})
+WHERE line.last_update_date > (SELECT COALESCE(MAX(line_last_update_date), '1970-01-01') FROM {{ this }})
 {% endif %}