Add publisher feeds filtering.

oceanprotocol · Jan 15, 2024 · 5e1497e · 5e1497e
1 parent 3eae232
commit 5e1497e
Show file tree

Hide file tree

Showing 7 changed files with 107 additions and 35 deletions.
diff --git a/pdr_backend/ppss/base_ss.py b/pdr_backend/ppss/base_ss.py
@@ -1,3 +1,4 @@
+import copy
 from typing import Dict, List, Optional, Set, Tuple, Union
 
 from enforce_typing import enforce_types
@@ -33,7 +34,17 @@ def __init__(self, d: dict, assert_feed_attributes: Optional[List] = None):
     # yaml properties
     @property
     def feeds_strs(self) -> List[str]:
-        return self.d[self.__class__.FEEDS_KEY]  # eg ["binance BTC/USDT ohlcv",..]
+        nested_attrs = self.__class__.FEEDS_KEY.split(".")
+        lookup = copy.deepcopy(self.d)
+        for attr in nested_attrs:
+            try:
+                lookup = lookup[attr]
+            except KeyError:
+                raise ValueError(
+                    f"Could not find nested attribute {attr} in {nested_attrs}"
+                )
+
+        return lookup  # eg ["binance BTC/USDT ohlcv",..]
 
     # --------------------------------
 

diff --git a/pdr_backend/ppss/publisher_ss.py b/pdr_backend/ppss/publisher_ss.py
@@ -1,13 +1,20 @@
+from typing import Dict
+
 from enforce_typing import enforce_types
 
+from pdr_backend.ppss.base_ss import MultiFeedMixin
+from pdr_backend.subgraph.subgraph_feed import SubgraphFeed
 from pdr_backend.util.strutil import StrMixin
 
 
-class PublisherSS(StrMixin):
+class PublisherSS(MultiFeedMixin, StrMixin):
     @enforce_types
     def __init__(self, network: str, d: dict):
-        self.d = d  # yaml_dict["publisher_ss"]
         self.network = network  # e.g. "sapphire-testnet", "sapphire-mainnet"
+        self.__class__.FEEDS_KEY = network + ".feeds"
+        super().__init__(
+            d, assert_feed_attributes=["signal", "timeframe"]
+        )  # yaml_dict["publisher_ss"]
 
     # --------------------------------
     # yaml properties
@@ -18,8 +25,27 @@ def fee_collector_address(self) -> str:
         """
         return self.d[self.network]["fee_collector_address"]
 
+    @enforce_types
+    def filter_feeds_from_candidates(
+        self, cand_feeds: Dict[str, SubgraphFeed]
+    ) -> Dict[str, SubgraphFeed]:
+        raise NotImplementedError("PublisherSS should not filter subgraph feeds.")
+
 
 @enforce_types
-def mock_publisher_ss() -> PublisherSS:
-    d = {"developer": {"fee_collector_address": "0x1"}}
-    return PublisherSS("developer", d)
+def mock_publisher_ss(network) -> PublisherSS:
+    if network in ["development", "barge-pytest", "barge-predictoor-bot"]:
+        feeds = ["binance BTC/USDT ETH/USDT XRP/USDT c 5m"]
+    else:
+        # sapphire-testnet, sapphire-mainnet
+        feeds = [
+            "binance BTC/USDT ETH/USDT BNB/USDT XRP/USDT ADA/USDT DOGE/USDT SOL/USDT LTC/USDT TRX/USDT DOT/USDT c 5m,1h"
+        ]
+
+    d = {
+        network: {
+            "fee_collector_address": "0x1",
+            "feeds": feeds,
+        }
+    }
+    return PublisherSS(network, d)
diff --git a/pdr_backend/ppss/test/test_base_ss.py b/pdr_backend/ppss/test/test_base_ss.py
@@ -12,6 +12,10 @@ class MultiFeedMixinTest(MultiFeedMixin):
     FEEDS_KEY = "feeds"
 
 
+class NestedMultiFeedMixinTest(MultiFeedMixin):
+    FEEDS_KEY = "abc.xyz.feeds"
+
+
 class SingleFeedMixinTest(SingleFeedMixin):
     FEED_KEY = "feed"
 
@@ -47,6 +51,28 @@ def test_multi_feed():
     }
 
 
+@enforce_types
+def test_nested_multi_feed():
+    d = {
+        "abc": {
+            "xyz": {
+                "feeds": ["kraken ETH/USDT hc", "binanceus ETH/USDT,TRX/DAI h"],
+            }
+        }
+    }
+    ss = NestedMultiFeedMixinTest(d)
+    assert ss.n_feeds == 4
+
+    wrong_d = {
+        "abc": {
+            "feeds": ["kraken ETH/USDT hc", "binanceus ETH/USDT,TRX/DAI h"],
+        }
+    }
+
+    with pytest.raises(ValueError, match="Could not find nested attribute xyz"):
+        ss = NestedMultiFeedMixinTest(wrong_d)
+
+
 @enforce_types
 def test_multi_feed_filter():
     d = {

diff --git a/pdr_backend/ppss/test/test_publisher_ss.py b/pdr_backend/ppss/test/test_publisher_ss.py
@@ -1,3 +1,4 @@
+import pytest
 from enforce_typing import enforce_types
 
 from pdr_backend.ppss.publisher_ss import PublisherSS, mock_publisher_ss
@@ -6,17 +7,30 @@
 @enforce_types
 def test_publisher_ss():
     d = {
-        "sapphire-mainnet": {"fee_collector_address": "0x1"},
-        "sapphire-testnet": {"fee_collector_address": "0x2"},
+        "sapphire-mainnet": {
+            "fee_collector_address": "0x1",
+            "feeds": [
+                "binance BTC/USDT ETH/USDT BNB/USDT XRP/USDT ADA/USDT DOGE/USDT SOL/USDT LTC/USDT TRX/USDT DOT/USDT c 5m,1h"
+            ],
+        },
+        "sapphire-testnet": {
+            "fee_collector_address": "0x2",
+            "feeds": [
+                "binance BTC/USDT ETH/USDT BNB/USDT XRP/USDT ADA/USDT DOGE/USDT SOL/USDT LTC/USDT TRX/USDT DOT/USDT c 5m,1h"
+            ],
+        },
     }
     ss1 = PublisherSS("sapphire-mainnet", d)
     assert ss1.fee_collector_address == "0x1"
 
     ss2 = PublisherSS("sapphire-testnet", d)
     assert ss2.fee_collector_address == "0x2"
 
+    with pytest.raises(NotImplementedError):
+        ss1.filter_feeds_from_candidates({})
+
 
 @enforce_types
 def test_mock_publisher_ss():
-    publisher_ss = mock_publisher_ss()
+    publisher_ss = mock_publisher_ss("development")
     assert isinstance(publisher_ss, PublisherSS)
diff --git a/pdr_backend/publisher/publish_assets.py b/pdr_backend/publisher/publish_assets.py
@@ -1,7 +1,5 @@
 from enforce_typing import enforce_types
 
-from pdr_backend.cli.arg_feeds import ArgFeeds
-from pdr_backend.cli.timeframe import Timeframe
 from pdr_backend.ppss.publisher_ss import PublisherSS
 from pdr_backend.ppss.web3_pp import Web3PP
 from pdr_backend.publisher.publish_asset import publish_asset
@@ -11,9 +9,6 @@
 _RATE = 3 / (1 + _CUT + 0.001)  # token price
 _S_PER_SUBSCRIPTION = 60 * 60 * 24
 
-_SOME = "BTC/USDT ETH/USDT XRP/USDT"
-_ALL = "BTC/USDT ETH/USDT BNB/USDT XRP/USDT ADA/USDT DOGE/USDT SOL/USDT LTC/USDT TRX/USDT DOT/USDT"
-
 
 @enforce_types
 def publish_assets(web3_pp: Web3PP, publisher_ss: PublisherSS):
@@ -26,30 +21,24 @@ def publish_assets(web3_pp: Web3PP, publisher_ss: PublisherSS):
     if web3_pp.network == "development" or "barge" in web3_pp.network:
         trueval_submitter_addr = "0xe2DD09d719Da89e5a3D0F2549c7E24566e947260"
         fee_collector_addr = "0xe2DD09d719Da89e5a3D0F2549c7E24566e947260"
-        timeframe_strs = ["5m"]
-        feeds_strs = [f"binance {_SOME} c"]
     elif "sapphire" in web3_pp.network:
         trueval_submitter_addr = get_address(web3_pp, "PredictoorHelper")
         fee_collector_addr = publisher_ss.fee_collector_address
-        timeframe_strs = ["5m", "1h"]
-        feeds_strs = [f"binance {_ALL} c"]
     else:
         raise ValueError(web3_pp.network)
 
-    for timeframe_str in timeframe_strs:
-        feeds = ArgFeeds.from_strs(feeds_strs)
-        for feed in feeds:
-            publish_asset(
-                s_per_epoch=Timeframe(timeframe_str).s,
-                s_per_subscription=_S_PER_SUBSCRIPTION,
-                base=feed.pair.base_str,
-                quote=feed.pair.quote_str,
-                source=feed.exchange,
-                timeframe=timeframe_str,
-                trueval_submitter_addr=trueval_submitter_addr,
-                feeCollector_addr=fee_collector_addr,
-                rate=_RATE,
-                cut=_CUT,
-                web3_pp=web3_pp,
-            )
+    for feed in publisher_ss.feeds:
+        publish_asset(
+            s_per_epoch=feed.timeframe.s,
+            s_per_subscription=_S_PER_SUBSCRIPTION,
+            base=feed.pair.base_str,
+            quote=feed.pair.quote_str,
+            source=feed.exchange,
+            timeframe=str(feed.timeframe),
+            trueval_submitter_addr=trueval_submitter_addr,
+            feeCollector_addr=fee_collector_addr,
+            rate=_RATE,
+            cut=_CUT,
+            web3_pp=web3_pp,
+        )
     print("Done publishing.")
diff --git a/pdr_backend/publisher/test/test_publish_assets.py b/pdr_backend/publisher/test/test_publish_assets.py
@@ -65,7 +65,7 @@ def _setup_and_publish(network, monkeypatch):
         monkeypatch.delenv("NETWORK_OVERRIDE")
 
     web3_pp = mock_web3_pp(network)
-    publisher_ss = mock_publisher_ss()
+    publisher_ss = mock_publisher_ss(network)
 
     monkeypatch.setattr(f"{_PATH}.get_address", Mock())
 

diff --git a/ppss.yaml b/ppss.yaml
@@ -47,10 +47,16 @@ sim_ss: # sim only
 publisher_ss:
   sapphire-mainnet:
     fee_collector_address: 0x0000000000000000000000000000000000000000
+    feeds:
+      - binance BTC/USDT ETH/USDT BNB/USDT XRP/USDT ADA/USDT DOGE/USDT SOL/USDT LTC/USDT TRX/USDT DOT/USDT c 5m,1h
   sapphire-testnet:
     fee_collector_address: 0x0000000000000000000000000000000000000000
+    feeds:
+      - binance BTC/USDT ETH/USDT BNB/USDT XRP/USDT ADA/USDT DOGE/USDT SOL/USDT LTC/USDT TRX/USDT DOT/USDT c 5m,1h
   development:
     fee_collector_address: 0x0000000000000000000000000000000000000000
+    feeds:
+      - binance BTC/USDT ETH/USDT XRP/USDT c 5m
 
 trueval_ss:
   feeds: