Add publisher feeds filtering.

pdr_backend/ppss/base_ss.py

@@ -1,3 +1,4 @@
+import copy
 from typing import Dict, List, Optional, Set, Tuple, Union
 
 from enforce_typing import enforce_types
@@ -33,7 +34,21 @@ def __init__(self, d: dict, assert_feed_attributes: Optional[List] = None):
     # yaml properties
     @property
     def feeds_strs(self) -> List[str]:
-        return self.d[self.__class__.FEEDS_KEY]  # eg ["binance BTC/USDT ohlcv",..]
+        nested_attrs = self.__class__.FEEDS_KEY.split(".")
+        lookup = copy.deepcopy(self.d)
+
+        # Iterate over each attribute in the nesting
+        for attr in nested_attrs:
+            try:
+                # Attempt to access the next level in the dict
+                lookup = lookup[attr]
+            except KeyError as exc:
+                raise ValueError(
+                    f"Could not find nested attribute {attr} in {nested_attrs}"
+                ) from exc
+
+        assert isinstance(lookup, list)
+        return lookup  # eg ["binance BTC/USDT ohlcv",..]
 
     # --------------------------------
 

pdr_backend/ppss/publisher_ss.py

@@ -1,13 +1,20 @@
+from typing import Dict
+
 from enforce_typing import enforce_types
 
+from pdr_backend.ppss.base_ss import MultiFeedMixin
+from pdr_backend.subgraph.subgraph_feed import SubgraphFeed
 from pdr_backend.util.strutil import StrMixin
 
 
-class PublisherSS(StrMixin):
+class PublisherSS(MultiFeedMixin, StrMixin):
     @enforce_types
     def __init__(self, network: str, d: dict):
-        self.d = d  # yaml_dict["publisher_ss"]
         self.network = network  # e.g. "sapphire-testnet", "sapphire-mainnet"
+        self.__class__.FEEDS_KEY = network + ".feeds"
+        super().__init__(
+            d, assert_feed_attributes=["signal", "timeframe"]
+        )  # yaml_dict["publisher_ss"]
 
     # --------------------------------
     # yaml properties
@@ -18,8 +25,29 @@ def fee_collector_address(self) -> str:
         """
         return self.d[self.network]["fee_collector_address"]
 
+    @enforce_types
+    def filter_feeds_from_candidates(
+        self, cand_feeds: Dict[str, SubgraphFeed]
+    ) -> Dict[str, SubgraphFeed]:
+        raise NotImplementedError("PublisherSS should not filter subgraph feeds.")
+
 
 @enforce_types
-def mock_publisher_ss() -> PublisherSS:
-    d = {"developer": {"fee_collector_address": "0x1"}}
-    return PublisherSS("developer", d)
+def mock_publisher_ss(network) -> PublisherSS:
+    if network in ["development", "barge-pytest", "barge-predictoor-bot"]:
+        feeds = ["binance BTC/USDT ETH/USDT XRP/USDT c 5m"]
+    else:
+        # sapphire-testnet, sapphire-mainnet
+        feeds = [
+            "binance BTC/USDT ETH/USDT BNB/USDT XRP/USDT"
+            " ADA/USDT DOGE/USDT SOL/USDT LTC/USDT TRX/USDT DOT/USDT"
+            " c 5m,1h"
+        ]
+
+    d = {
+        network: {
+            "fee_collector_address": "0x1",
+            "feeds": feeds,
+        }
+    }
+    return PublisherSS(network, d)

pdr_backend/ppss/test/test_base_ss.py

@@ -12,6 +12,10 @@ class MultiFeedMixinTest(MultiFeedMixin):
     FEEDS_KEY = "feeds"
 
 
+class NestedMultiFeedMixinTest(MultiFeedMixin):
+    FEEDS_KEY = "abc.xyz.feeds"
+
+
 class SingleFeedMixinTest(SingleFeedMixin):
     FEED_KEY = "feed"
 
@@ -47,6 +51,28 @@ def test_multi_feed():
     }
 
 
+@enforce_types
+def test_nested_multi_feed():
+    d = {
+        "abc": {
+            "xyz": {
+                "feeds": ["kraken ETH/USDT hc", "binanceus ETH/USDT,TRX/DAI h"],
+            }
+        }
+    }
+    ss = NestedMultiFeedMixinTest(d)
+    assert ss.n_feeds == 4
+
+    wrong_d = {
+        "abc": {
+            "feeds": ["kraken ETH/USDT hc", "binanceus ETH/USDT,TRX/DAI h"],
+        }
+    }
+
+    with pytest.raises(ValueError, match="Could not find nested attribute xyz"):
+        ss = NestedMultiFeedMixinTest(wrong_d)
+
+
 @enforce_types
 def test_multi_feed_filter():
     d = {

pdr_backend/ppss/test/test_publisher_ss.py

@@ -1,22 +1,39 @@
+import pytest
 from enforce_typing import enforce_types
 
 from pdr_backend.ppss.publisher_ss import PublisherSS, mock_publisher_ss
 
 
 @enforce_types
 def test_publisher_ss():
+    sapphire_feeds = [
+        "binance BTC/USDT ETH/USDT BNB/USDT XRP/USDT"
+        " ADA/USDT DOGE/USDT SOL/USDT LTC/USDT TRX/USDT DOT/USDT"
+        " c 5m,1h"
+    ]
+
     d = {
-        "sapphire-mainnet": {"fee_collector_address": "0x1"},
-        "sapphire-testnet": {"fee_collector_address": "0x2"},
+        "sapphire-mainnet": {
+            "fee_collector_address": "0x1",
+            "feeds": sapphire_feeds,
+        },
+        "sapphire-testnet": {
+            "fee_collector_address": "0x2",
+            "feeds": sapphire_feeds,
+        },
     }
+
     ss1 = PublisherSS("sapphire-mainnet", d)
     assert ss1.fee_collector_address == "0x1"
 
     ss2 = PublisherSS("sapphire-testnet", d)
     assert ss2.fee_collector_address == "0x2"
 
+    with pytest.raises(NotImplementedError):
+        ss1.filter_feeds_from_candidates({})
+
 
 @enforce_types
 def test_mock_publisher_ss():
-    publisher_ss = mock_publisher_ss()
+    publisher_ss = mock_publisher_ss("development")
     assert isinstance(publisher_ss, PublisherSS)

pdr_backend/publisher/publish_assets.py

@@ -1,7 +1,5 @@
 from enforce_typing import enforce_types
 
-from pdr_backend.cli.arg_feeds import ArgFeeds
-from pdr_backend.cli.timeframe import Timeframe
 from pdr_backend.ppss.publisher_ss import PublisherSS
 from pdr_backend.ppss.web3_pp import Web3PP
 from pdr_backend.publisher.publish_asset import publish_asset
@@ -11,9 +9,6 @@
 _RATE = 3 / (1 + _CUT + 0.001)  # token price
 _S_PER_SUBSCRIPTION = 60 * 60 * 24
 
-_SOME = "BTC/USDT ETH/USDT XRP/USDT"
-_ALL = "BTC/USDT ETH/USDT BNB/USDT XRP/USDT ADA/USDT DOGE/USDT SOL/USDT LTC/USDT TRX/USDT DOT/USDT"
-
 
 @enforce_types
 def publish_assets(web3_pp: Web3PP, publisher_ss: PublisherSS):
@@ -26,30 +21,26 @@ def publish_assets(web3_pp: Web3PP, publisher_ss: PublisherSS):
     if web3_pp.network == "development" or "barge" in web3_pp.network:
         trueval_submitter_addr = "0xe2DD09d719Da89e5a3D0F2549c7E24566e947260"
         fee_collector_addr = "0xe2DD09d719Da89e5a3D0F2549c7E24566e947260"
-        timeframe_strs = ["5m"]
-        feeds_strs = [f"binance {_SOME} c"]
     elif "sapphire" in web3_pp.network:
         trueval_submitter_addr = get_address(web3_pp, "PredictoorHelper")
         fee_collector_addr = publisher_ss.fee_collector_address
-        timeframe_strs = ["5m", "1h"]
-        feeds_strs = [f"binance {_ALL} c"]
     else:
         raise ValueError(web3_pp.network)
 
-    for timeframe_str in timeframe_strs:
-        feeds = ArgFeeds.from_strs(feeds_strs)
-        for feed in feeds:
-            publish_asset(
-                s_per_epoch=Timeframe(timeframe_str).s,
-                s_per_subscription=_S_PER_SUBSCRIPTION,
-                base=feed.pair.base_str,
-                quote=feed.pair.quote_str,
-                source=str(feed.exchange),
-                timeframe=timeframe_str,
-                trueval_submitter_addr=trueval_submitter_addr,
-                feeCollector_addr=fee_collector_addr,
-                rate=_RATE,
-                cut=_CUT,
-                web3_pp=web3_pp,
-            )
+    for feed in publisher_ss.feeds:
+        publish_asset(
+            # timeframe is already asserted in PublisherSS
+            s_per_epoch=feed.timeframe.s,  # type: ignore[union-attr]
+            s_per_subscription=_S_PER_SUBSCRIPTION,
+            base=feed.pair.base_str,
+            quote=feed.pair.quote_str,
+            source=feed.exchange,
+            timeframe=str(feed.timeframe),
+            trueval_submitter_addr=trueval_submitter_addr,
+            feeCollector_addr=fee_collector_addr,
+            rate=_RATE,
+            cut=_CUT,
+            web3_pp=web3_pp,
+        )
+
     print("Done publishing.")

pdr_backend/publisher/test/test_publish_assets.py

@@ -61,7 +61,7 @@ def _test_sapphire(network, monkeypatch):
 
 def _setup_and_publish(network, monkeypatch):
     web3_pp = mock_web3_pp(network)
-    publisher_ss = mock_publisher_ss()
+    publisher_ss = mock_publisher_ss(network)
 
     monkeypatch.setattr(f"{_PATH}.get_address", Mock())
 

ppss.yaml

@@ -47,10 +47,16 @@ sim_ss: # sim only
 publisher_ss:
   sapphire-mainnet:
     fee_collector_address: 0x0000000000000000000000000000000000000000
+    feeds:
+      - binance BTC/USDT ETH/USDT BNB/USDT XRP/USDT ADA/USDT DOGE/USDT SOL/USDT LTC/USDT TRX/USDT DOT/USDT c 5m,1h
   sapphire-testnet:
     fee_collector_address: 0x0000000000000000000000000000000000000000
+    feeds:
+      - binance BTC/USDT ETH/USDT BNB/USDT XRP/USDT ADA/USDT DOGE/USDT SOL/USDT LTC/USDT TRX/USDT DOT/USDT c 5m,1h
   development:
     fee_collector_address: 0x0000000000000000000000000000000000000000
+    feeds:
+      - binance BTC/USDT ETH/USDT XRP/USDT c 5m
 
 trueval_ss:
   feeds: