Allow parsing logs for all contracts e.g. Approve

blockchain-etl · Nov 14, 2020 · 9ca65b4 · 9ca65b4
1 parent e5313c9
commit 9ca65b4
Show file tree

Hide file tree

Showing 16 changed files with 140 additions and 49 deletions.
diff --git a/dags/ethereumetl_airflow/parse/parse_logic.py b/dags/ethereumetl_airflow/parse/parse_logic.py
@@ -266,7 +266,7 @@ def generate_parse_sql_template(
         parse_all_partitions,
         ds):
     contract_address = table_definition['parser']['contract_address']
-    if not contract_address.startswith('0x'):
+    if contract_address is not None and not contract_address.startswith('0x'):
         table_definition['parser']['contract_address_sql'] = replace_refs(
             contract_address, ref_regex, destination_project_id, dataset_name
         )

diff --git a/dags/resources/stages/parse/sqls/parse_logs.sql b/dags/resources/stages/parse/sqls/parse_logs.sql
@@ -7,13 +7,14 @@ WITH parsed_logs AS
     ,logs.address AS contract_address
     ,`{{internal_project_id}}.{{dataset_name}}.{{udf_name}}`(logs.data, logs.topics) AS parsed
 FROM `{{source_project_id}}.{{source_dataset_name}}.logs` AS logs
-WHERE address in (
-    {% if parser.contract_address_sql %}
-    {{parser.contract_address_sql}}
-    {% else %}
-    '{{parser.contract_address}}'
-    {% endif %}
-  )
+WHERE
+  {% if parser.contract_address_sql %}
+  address in ({{parser.contract_address_sql}})
+  {% elif parser.contract_address is none %}
+  true
+  {% else %}
+  address in ('{{parser.contract_address}}')
+  {% endif %}
   AND topics[SAFE_OFFSET(0)] = '{{selector}}'
   {% if parse_all_partitions is none %}
   -- pass
@@ -32,3 +33,4 @@ SELECT
      {% for column in table.schema %}
     ,parsed.{{ column.name }} AS `{{ column.name }}`{% endfor %}
 FROM parsed_logs
+WHERE parsed IS NOT NULL
diff --git a/dags/resources/stages/parse/sqls/parse_logs_udf.sql b/dags/resources/stages/parse/sqls/parse_logs_udf.sql
@@ -6,7 +6,14 @@ CREATE OR REPLACE FUNCTION
 
     var interface_instance = new ethers.utils.Interface([abi]);
 
-    var parsedLog = interface_instance.parseLog({topics: topics, data: data});
+    // A parsing error is possible for common abis that don't filter by contract address. Event signature is the same
+    // for ABIs that only differ by whether a field is indexed or not. E.g. if the ABI provided has an indexed field
+    // but the log entry has this field unindexed, parsing here will throw an exception.
+    try {
+      var parsedLog = interface_instance.parseLog({topics: topics, data: data});
+    } catch (e) {
+        return null;
+    }
 
     var parsedValues = parsedLog.values;
 

diff --git a/dags/resources/stages/parse/table_definitions/common/All_event_Approval.json b/dags/resources/stages/parse/table_definitions/common/All_event_Approval.json
@@ -0,0 +1,54 @@
+{
+    "parser": {
+        "abi": {
+            "anonymous": false,
+            "inputs": [
+                {
+                    "indexed": true,
+                    "internalType": "address",
+                    "name": "owner",
+                    "type": "address"
+                },
+                {
+                    "indexed": true,
+                    "internalType": "address",
+                    "name": "spender",
+                    "type": "address"
+                },
+                {
+                    "indexed": false,
+                    "internalType": "uint256",
+                    "name": "value",
+                    "type": "uint256"
+                }
+            ],
+            "name": "Approval",
+            "type": "event"
+        },
+        "contract_address": null,
+        "field_mapping": {},
+        "type": "log"
+    },
+    "table": {
+        "dataset_name": "common",
+        "schema": [
+            {
+                "description": "",
+                "name": "owner",
+                "type": "STRING"
+            },
+            {
+                "description": "",
+                "name": "spender",
+                "type": "STRING"
+            },
+            {
+                "description": "",
+                "name": "value",
+                "type": "STRING"
+            }
+        ],
+        "table_description": "",
+        "table_name": "All_event_Approval"
+    }
+}
diff --git a/...s/resources/ethereumetl_airflow/test_parse/dydx/SoloMargin_event_LogTrade.json_True_0.sql b/...s/resources/ethereumetl_airflow/test_parse/dydx/SoloMargin_event_LogTrade.json_True_0.sql
@@ -6,7 +6,14 @@ CREATE OR REPLACE FUNCTION
 
     var interface_instance = new ethers.utils.Interface([abi]);
 
-    var parsedLog = interface_instance.parseLog({topics: topics, data: data});
+    // A parsing error is possible for common abis that don't filter by contract address. Event signature is the same
+    // for ABIs that only differ by whether a field is indexed or not. E.g. if the ABI provided has an indexed field
+    // but the log entry has this field unindexed, parsing here will throw an exception.
+    try {
+      var parsedLog = interface_instance.parseLog({topics: topics, data: data});
+    } catch (e) {
+        return null;
+    }
 
     var parsedValues = parsedLog.values;
 

diff --git a/...s/resources/ethereumetl_airflow/test_parse/dydx/SoloMargin_event_LogTrade.json_True_1.sql b/...s/resources/ethereumetl_airflow/test_parse/dydx/SoloMargin_event_LogTrade.json_True_1.sql
@@ -7,11 +7,10 @@ WITH parsed_logs AS
     ,logs.address AS contract_address
     ,`blockchain-etl-internal.ethereum_dydx.parse_SoloMargin_event_LogTrade`(logs.data, logs.topics) AS parsed
 FROM `bigquery-public-data.crypto_ethereum.logs` AS logs
-WHERE address in (
+WHERE
 
-    '0x1e0447b19bb6ecfdae1e4ae1694b0c3659614e4e'
+  address in ('0x1e0447b19bb6ecfdae1e4ae1694b0c3659614e4e')
 
-  )
   AND topics[SAFE_OFFSET(0)] = '0x54d4cc60cf7d570631cc1a58942812cb0fc461713613400f56932040c3497d19'
 
   -- pass
@@ -35,4 +34,5 @@ SELECT
     ,parsed.makerInputUpdate AS `makerInputUpdate`
     ,parsed.makerOutputUpdate AS `makerOutputUpdate`
     ,parsed.autoTrader AS `autoTrader`
-FROM parsed_logs
+FROM parsed_logs
+WHERE parsed IS NOT NULL
diff --git a/...s/resources/ethereumetl_airflow/test_parse/dydx/SoloMargin_event_LogTrade.json_True_2.sql b/...s/resources/ethereumetl_airflow/test_parse/dydx/SoloMargin_event_LogTrade.json_True_2.sql
@@ -7,11 +7,10 @@ WITH parsed_logs AS
     ,logs.address AS contract_address
     ,`blockchain-etl-internal.ethereum_dydx.parse_SoloMargin_event_LogTrade`(logs.data, logs.topics) AS parsed
 FROM `bigquery-public-data.crypto_ethereum.logs` AS logs
-WHERE address in (
+WHERE
 
-    '0x1e0447b19bb6ecfdae1e4ae1694b0c3659614e4e'
+  address in ('0x1e0447b19bb6ecfdae1e4ae1694b0c3659614e4e')
 
-  )
   AND topics[SAFE_OFFSET(0)] = '0x54d4cc60cf7d570631cc1a58942812cb0fc461713613400f56932040c3497d19'
 
   AND DATE(block_timestamp) <= '2020-01-01'
@@ -35,4 +34,5 @@ SELECT
     ,parsed.makerInputUpdate AS `makerInputUpdate`
     ,parsed.makerOutputUpdate AS `makerOutputUpdate`
     ,parsed.autoTrader AS `autoTrader`
-FROM parsed_logs
+FROM parsed_logs
+WHERE parsed IS NOT NULL
diff --git a/tests/resources/ethereumetl_airflow/test_parse/ens/Registrar0_event_NewBid.json_False_0.sql b/tests/resources/ethereumetl_airflow/test_parse/ens/Registrar0_event_NewBid.json_False_0.sql
@@ -6,7 +6,14 @@ CREATE OR REPLACE FUNCTION
 
     var interface_instance = new ethers.utils.Interface([abi]);
 
-    var parsedLog = interface_instance.parseLog({topics: topics, data: data});
+    // A parsing error is possible for common abis that don't filter by contract address. Event signature is the same
+    // for ABIs that only differ by whether a field is indexed or not. E.g. if the ABI provided has an indexed field
+    // but the log entry has this field unindexed, parsing here will throw an exception.
+    try {
+      var parsedLog = interface_instance.parseLog({topics: topics, data: data});
+    } catch (e) {
+        return null;
+    }
 
     var parsedValues = parsedLog.values;
 

diff --git a/tests/resources/ethereumetl_airflow/test_parse/ens/Registrar0_event_NewBid.json_False_1.sql b/tests/resources/ethereumetl_airflow/test_parse/ens/Registrar0_event_NewBid.json_False_1.sql
@@ -7,11 +7,10 @@ WITH parsed_logs AS
     ,logs.address AS contract_address
     ,`blockchain-etl-internal.ethereum_ens.parse_Registrar0_event_NewBid`(logs.data, logs.topics) AS parsed
 FROM `bigquery-public-data.crypto_ethereum.logs` AS logs
-WHERE address in (
-
-    '0x6090a6e47849629b7245dfa1ca21d94cd15878ef'
-
-  )
+WHERE
+
+  address in ('0x6090a6e47849629b7245dfa1ca21d94cd15878ef')
+
   AND topics[SAFE_OFFSET(0)] = '0xb556ff269c1b6714f432c36431e2041d28436a73b6c3f19c021827bbdc6bfc29'
 
   -- pass
@@ -27,4 +26,5 @@ SELECT
     ,parsed.hash AS `hash`
     ,parsed.bidder AS `bidder`
     ,parsed.deposit AS `deposit`
-FROM parsed_logs
+FROM parsed_logs
+WHERE parsed IS NOT NULL
diff --git a/tests/resources/ethereumetl_airflow/test_parse/ens/Registrar0_event_NewBid.json_False_2.sql b/tests/resources/ethereumetl_airflow/test_parse/ens/Registrar0_event_NewBid.json_False_2.sql
@@ -7,11 +7,10 @@ WITH parsed_logs AS
     ,logs.address AS contract_address
     ,`blockchain-etl-internal.ethereum_ens.parse_Registrar0_event_NewBid`(logs.data, logs.topics) AS parsed
 FROM `bigquery-public-data.crypto_ethereum.logs` AS logs
-WHERE address in (
-
-    '0x6090a6e47849629b7245dfa1ca21d94cd15878ef'
-
-  )
+WHERE
+
+  address in ('0x6090a6e47849629b7245dfa1ca21d94cd15878ef')
+
   AND topics[SAFE_OFFSET(0)] = '0xb556ff269c1b6714f432c36431e2041d28436a73b6c3f19c021827bbdc6bfc29'
 
   AND DATE(block_timestamp) = '2020-01-01'
@@ -27,4 +26,5 @@ SELECT
     ,parsed.hash AS `hash`
     ,parsed.bidder AS `bidder`
     ,parsed.deposit AS `deposit`
-FROM parsed_logs
+FROM parsed_logs
+WHERE parsed IS NOT NULL
diff --git a/tests/resources/ethereumetl_airflow/test_parse/ens/Registrar0_event_NewBid.json_True_0.sql b/tests/resources/ethereumetl_airflow/test_parse/ens/Registrar0_event_NewBid.json_True_0.sql
@@ -6,7 +6,14 @@ CREATE OR REPLACE FUNCTION
 
     var interface_instance = new ethers.utils.Interface([abi]);
 
-    var parsedLog = interface_instance.parseLog({topics: topics, data: data});
+    // A parsing error is possible for common abis that don't filter by contract address. Event signature is the same
+    // for ABIs that only differ by whether a field is indexed or not. E.g. if the ABI provided has an indexed field
+    // but the log entry has this field unindexed, parsing here will throw an exception.
+    try {
+      var parsedLog = interface_instance.parseLog({topics: topics, data: data});
+    } catch (e) {
+        return null;
+    }
 
     var parsedValues = parsedLog.values;
 

diff --git a/tests/resources/ethereumetl_airflow/test_parse/ens/Registrar0_event_NewBid.json_True_1.sql b/tests/resources/ethereumetl_airflow/test_parse/ens/Registrar0_event_NewBid.json_True_1.sql
@@ -7,11 +7,10 @@ WITH parsed_logs AS
     ,logs.address AS contract_address
     ,`blockchain-etl-internal.ethereum_ens.parse_Registrar0_event_NewBid`(logs.data, logs.topics) AS parsed
 FROM `bigquery-public-data.crypto_ethereum.logs` AS logs
-WHERE address in (
+WHERE
 
-    '0x6090a6e47849629b7245dfa1ca21d94cd15878ef'
+  address in ('0x6090a6e47849629b7245dfa1ca21d94cd15878ef')
 
-  )
   AND topics[SAFE_OFFSET(0)] = '0xb556ff269c1b6714f432c36431e2041d28436a73b6c3f19c021827bbdc6bfc29'
 
   -- pass
@@ -27,4 +26,5 @@ SELECT
     ,parsed.hash AS `hash`
     ,parsed.bidder AS `bidder`
     ,parsed.deposit AS `deposit`
-FROM parsed_logs
+FROM parsed_logs
+WHERE parsed IS NOT NULL
diff --git a/tests/resources/ethereumetl_airflow/test_parse/ens/Registrar0_event_NewBid.json_True_2.sql b/tests/resources/ethereumetl_airflow/test_parse/ens/Registrar0_event_NewBid.json_True_2.sql
@@ -7,11 +7,10 @@ WITH parsed_logs AS
     ,logs.address AS contract_address
     ,`blockchain-etl-internal.ethereum_ens.parse_Registrar0_event_NewBid`(logs.data, logs.topics) AS parsed
 FROM `bigquery-public-data.crypto_ethereum.logs` AS logs
-WHERE address in (
+WHERE
 
-    '0x6090a6e47849629b7245dfa1ca21d94cd15878ef'
+  address in ('0x6090a6e47849629b7245dfa1ca21d94cd15878ef')
 
-  )
   AND topics[SAFE_OFFSET(0)] = '0xb556ff269c1b6714f432c36431e2041d28436a73b6c3f19c021827bbdc6bfc29'
 
   AND DATE(block_timestamp) <= '2020-01-01'
@@ -27,4 +26,5 @@ SELECT
     ,parsed.hash AS `hash`
     ,parsed.bidder AS `bidder`
     ,parsed.deposit AS `deposit`
-FROM parsed_logs
+FROM parsed_logs
+WHERE parsed IS NOT NULL
diff --git a/...sources/ethereumetl_airflow/test_parse/uniswap/Uniswap_event_AddLiquidity.json_True_0.sql b/...sources/ethereumetl_airflow/test_parse/uniswap/Uniswap_event_AddLiquidity.json_True_0.sql
@@ -6,7 +6,14 @@ CREATE OR REPLACE FUNCTION
 
     var interface_instance = new ethers.utils.Interface([abi]);
 
-    var parsedLog = interface_instance.parseLog({topics: topics, data: data});
+    // A parsing error is possible for common abis that don't filter by contract address. Event signature is the same
+    // for ABIs that only differ by whether a field is indexed or not. E.g. if the ABI provided has an indexed field
+    // but the log entry has this field unindexed, parsing here will throw an exception.
+    try {
+      var parsedLog = interface_instance.parseLog({topics: topics, data: data});
+    } catch (e) {
+        return null;
+    }
 
     var parsedValues = parsedLog.values;
 

diff --git a/...sources/ethereumetl_airflow/test_parse/uniswap/Uniswap_event_AddLiquidity.json_True_1.sql b/...sources/ethereumetl_airflow/test_parse/uniswap/Uniswap_event_AddLiquidity.json_True_1.sql
@@ -7,11 +7,10 @@ WITH parsed_logs AS
     ,logs.address AS contract_address
     ,`blockchain-etl-internal.ethereum_uniswap.parse_Uniswap_event_AddLiquidity`(logs.data, logs.topics) AS parsed
 FROM `bigquery-public-data.crypto_ethereum.logs` AS logs
-WHERE address in (
+WHERE
 
-    SELECT exchange FROM `blockchain-etl.ethereum_uniswap.Vyper_contract_event_NewExchange`
+  address in (SELECT exchange FROM `blockchain-etl.ethereum_uniswap.Vyper_contract_event_NewExchange`)
 
-  )
   AND topics[SAFE_OFFSET(0)] = '0x06239653922ac7bea6aa2b19dc486b9361821d37712eb796adfd38d81de278ca'
 
   -- pass
@@ -27,4 +26,5 @@ SELECT
     ,parsed.provider AS `provider`
     ,parsed.eth_amount AS `eth_amount`
     ,parsed.token_amount AS `token_amount`
-FROM parsed_logs
+FROM parsed_logs
+WHERE parsed IS NOT NULL
diff --git a/...sources/ethereumetl_airflow/test_parse/uniswap/Uniswap_event_AddLiquidity.json_True_2.sql b/...sources/ethereumetl_airflow/test_parse/uniswap/Uniswap_event_AddLiquidity.json_True_2.sql
@@ -7,11 +7,10 @@ WITH parsed_logs AS
     ,logs.address AS contract_address
     ,`blockchain-etl-internal.ethereum_uniswap.parse_Uniswap_event_AddLiquidity`(logs.data, logs.topics) AS parsed
 FROM `bigquery-public-data.crypto_ethereum.logs` AS logs
-WHERE address in (
+WHERE
 
-    SELECT exchange FROM `blockchain-etl.ethereum_uniswap.Vyper_contract_event_NewExchange`
+  address in (SELECT exchange FROM `blockchain-etl.ethereum_uniswap.Vyper_contract_event_NewExchange`)
 
-  )
   AND topics[SAFE_OFFSET(0)] = '0x06239653922ac7bea6aa2b19dc486b9361821d37712eb796adfd38d81de278ca'
 
   AND DATE(block_timestamp) <= '2020-01-01'
@@ -27,4 +26,5 @@ SELECT
     ,parsed.provider AS `provider`
     ,parsed.eth_amount AS `eth_amount`
     ,parsed.token_amount AS `token_amount`
-FROM parsed_logs
+FROM parsed_logs
+WHERE parsed IS NOT NULL