refactor: sqs out-of-process (osmosis-labs#7242)

* refactor: sqs out-of-process * spell * godocs * go 1.20 * go work * updaets * updates * sqs deps * updates * latest * updates * updates * updates
Orchestra-Labs · Jan 9, 2024 · ff62dbd · ff62dbd
1 parent af010a0
commit ff62dbd
Show file tree

Hide file tree

Showing 99 changed files with 487 additions and 10,990 deletions.
diff --git a/app/app.go b/app/app.go
@@ -31,6 +31,9 @@ import (
 	"github.com/cosmos/ibc-go/v7/modules/apps/transfer"
 	ibc "github.com/cosmos/ibc-go/v7/modules/core"
 
+	"github.com/osmosis-labs/osmosis/v21/ingest/sqs"
+	"github.com/osmosis-labs/osmosis/v21/ingest/sqs/domain"
+
 	"github.com/osmosis-labs/osmosis/osmoutils"
 
 	nodeservice "github.com/cosmos/cosmos-sdk/client/grpc/node"
@@ -96,10 +99,6 @@ import (
 	_ "github.com/osmosis-labs/osmosis/v21/client/docs/statik"
 	"github.com/osmosis-labs/osmosis/v21/ingest"
 	"github.com/osmosis-labs/osmosis/v21/x/mint"
-
-	"github.com/osmosis-labs/osmosis/v21/ingest/sqs"
-
-	"github.com/osmosis-labs/osmosis/v21/ingest/sqs/pools/common"
 )
 
 const appName = "OsmosisApp"
@@ -260,7 +259,7 @@ func NewOsmosisApp(
 
 	// Initialize the SQS ingester if it is enabled.
 	if sqsConfig.IsEnabled {
-		sqsKeepers := common.SQSIngestKeepers{
+		sqsKeepers := domain.SQSIngestKeepers{
 			GammKeeper:         app.GAMMKeeper,
 			CosmWasmPoolKeeper: app.CosmwasmPoolKeeper,
 			BankKeeper:         app.BankKeeper,

diff --git a/app/apptesting/cosmwasmpool.go b/app/apptesting/cosmwasmpool.go
@@ -2,6 +2,7 @@ package apptesting
 
 import (
 	"encoding/json"
+	"fmt"
 	"os"
 	"strings"
 
@@ -22,20 +23,31 @@ const (
 	TransmuterContractName        = "transmuter"
 	TransmuterMigrateContractName = "transmuter_migrate"
 	DefaultCodeId                 = 1
+
+	osmosisRepository         = "osmosis"
+	osmosisRepoTransmuterPath = "x/cosmwasmpool/bytecode"
 )
 
 // PrepareCosmWasmPool sets up a cosmwasm pool with the default parameters.
 func (s *KeeperTestHelper) PrepareCosmWasmPool() cosmwasmpooltypes.CosmWasmExtension {
 	return s.PrepareCustomTransmuterPool(s.TestAccs[0], []string{DefaultTransmuterDenomA, DefaultTransmuterDenomB})
 }
 
-// PrepareCustomTransmuterPool sets up a transmuter pool with the custom parameters.
+// PrepareCustomTransmuterPool sets up a transmuter pool with the default parameters assuming that
+// the transmuter contract is stored under x/cosmwasmpool/bytecode in the Osmosis repository.
 func (s *KeeperTestHelper) PrepareCustomTransmuterPool(owner sdk.AccAddress, denoms []string) cosmwasmpooltypes.CosmWasmExtension {
+	return s.PrepareCustomTransmuterPoolCustomProject(owner, denoms, osmosisRepository, osmosisRepoTransmuterPath)
+}
+
+// PrepareCustomTransmuterPoolCustomProject sets up a transmuter pool with the custom parameters.
+// Gives flexibility for the helper to be reused outside of the Osmosis repository by providing custom
+// project name and bytecode path.
+func (s *KeeperTestHelper) PrepareCustomTransmuterPoolCustomProject(owner sdk.AccAddress, denoms []string, projectName, byteCodePath string) cosmwasmpooltypes.CosmWasmExtension {
 	// Mint some assets to the account.
 	s.FundAcc(s.TestAccs[0], DefaultAcctFunds)
 
 	// Upload contract code and get the code id.
-	codeId := s.StoreCosmWasmPoolContractCode(TransmuterContractName)
+	codeId := s.StoreCosmWasmPoolContractCodeCustomProject(TransmuterContractName, projectName, byteCodePath)
 
 	// Add code id to the whitelist.
 	s.App.CosmwasmPoolKeeper.WhitelistCodeId(s.Ctx, codeId)
@@ -79,6 +91,13 @@ func (s *KeeperTestHelper) GetTransmuterInstantiateMsgBytes(poolAssetDenoms []st
 // StoreCosmWasmPoolContractCode stores the cosmwasm pool contract code in the wasm keeper and returns the code id.
 // contractName is the name of the contract file in the x/cosmwasmpool/bytecode directory without the .wasm extension.
 func (s *KeeperTestHelper) StoreCosmWasmPoolContractCode(contractName string) uint64 {
+	return s.StoreCosmWasmPoolContractCodeCustomProject(contractName, osmosisRepository, osmosisRepoTransmuterPath)
+}
+
+// StoreCosmWasmPoolContractCodeCustomProject stores the cosmwasm pool contract code in the wasm keeper and returns the code id.
+// contractName is the name of the contract file in the x/cosmwasmpool/bytecode directory without the .wasm extension.
+// It has the flexibility of being used from outside the Osmosis repository by providing custom project name and bytecode path.
+func (s *KeeperTestHelper) StoreCosmWasmPoolContractCodeCustomProject(contractName, projectName, byteCodePath string) uint64 {
 	cosmwasmpoolModuleAddr := s.App.AccountKeeper.GetModuleAddress(cosmwasmpooltypes.ModuleName)
 	s.Require().NotNil(cosmwasmpoolModuleAddr)
 
@@ -93,7 +112,7 @@ func (s *KeeperTestHelper) StoreCosmWasmPoolContractCode(contractName string) ui
 	})
 	s.Require().NoError(err)
 
-	code := s.GetContractCode(contractName)
+	code := s.GetContractCodeCustomProject(contractName, projectName, byteCodePath)
 
 	instantiateConfig := wasmtypes.AccessConfig{Permission: wasmtypes.AccessTypeAnyOfAddresses, Addresses: []string{cosmwasmpoolModuleAddr.String()}}
 	codeID, _, err := s.App.ContractKeeper.Create(s.Ctx, cosmwasmpoolModuleAddr, code, &instantiateConfig)
@@ -102,16 +121,20 @@ func (s *KeeperTestHelper) StoreCosmWasmPoolContractCode(contractName string) ui
 	return codeID
 }
 
+func (s *KeeperTestHelper) GetContractCode(contractName string) []byte {
+	return s.GetContractCodeCustomProject(contractName, "osmosis", "x/cosmwasmpool/bytecode")
+}
+
 // GetContractCode returns the contract code for the given contract name.
 // Assumes that the contract code is stored under x/cosmwasmpool/bytecode.
-func (s *KeeperTestHelper) GetContractCode(contractName string) []byte {
+func (s *KeeperTestHelper) GetContractCodeCustomProject(contractName string, projectName string, path string) []byte {
 	workingDir, err := os.Getwd()
 	s.Require().NoError(err)
 
-	projectRootPath := "/osmosis/"
+	projectRootPath := fmt.Sprintf("/%s/", projectName)
 	projectRootIndex := strings.LastIndex(workingDir, projectRootPath) + len(projectRootPath)
 	workingDir = workingDir[:projectRootIndex]
-	code, err := os.ReadFile(workingDir + "x/cosmwasmpool/bytecode/" + contractName + ".wasm")
+	code, err := os.ReadFile(workingDir + path + "/" + contractName + ".wasm")
 	s.Require().NoError(err)
 
 	return code

diff --git a/app/apptesting/test_suite.go b/app/apptesting/test_suite.go
@@ -145,6 +145,13 @@ func (s *KeeperTestHelper) SetupWithCustomChainId(chainId string) {
 // PrepareAllSupportedPools creates all supported pools and returns their IDs.
 // Additionally, attaches an internal gauge ID for each pool.
 func (s *KeeperTestHelper) PrepareAllSupportedPools() SupportedPoolAndGaugeInfo {
+	return s.PrepareAllSupportedPoolsCustomProject(osmosisRepository, osmosisRepoTransmuterPath)
+}
+
+// PrepareAllSupportedPoolsCustomProject creates all supported pools and returns their IDs.
+// Additionally, attaches an internal gauge ID for each pool.
+// Allows the flexibility of being used from outside the Osmosis repository by providing custom project name and transmuter bytecode path.
+func (s *KeeperTestHelper) PrepareAllSupportedPoolsCustomProject(projectName, transmuterPath string) SupportedPoolAndGaugeInfo {
 	// This is the ID of the first gauge created next (concentrated).
 	nextGaugeID := s.App.IncentivesKeeper.GetLastGaugeID(s.Ctx) + 1
 
@@ -156,7 +163,7 @@ func (s *KeeperTestHelper) PrepareAllSupportedPools() SupportedPoolAndGaugeInfo
 		concentratedPoolID = concentratedPool.GetId()
 		balancerPoolID     = s.PrepareBalancerPool()
 		stableswapPoolID   = s.PrepareBasicStableswapPool()
-		cosmWasmPool       = s.PrepareCosmWasmPool()
+		cosmWasmPool       = s.PrepareCustomTransmuterPoolCustomProject(s.TestAccs[0], []string{DefaultTransmuterDenomA, DefaultTransmuterDenomB}, projectName, transmuterPath)
 		cosmWasmPoolID     = cosmWasmPool.GetId()
 	)
 	return SupportedPoolAndGaugeInfo{

diff --git a/cmd/osmosisd/cmd/root.go b/cmd/osmosisd/cmd/root.go
@@ -559,49 +559,6 @@ is-enabled = "false"
 db-host = "{{ .SidecarQueryServerConfig.StorageHost }}"
 db-port = "{{ .SidecarQueryServerConfig.StoragePort }}"
 
-# Defines the web server configuration.
-server-address = "{{ .SidecarQueryServerConfig.ServerAddress }}"
-timeout-duration-secs = "{{ .SidecarQueryServerConfig.ServerTimeoutDurationSecs }}"
-
-# Defines the logger configuration.
-logger-filename = "{{ .SidecarQueryServerConfig.LoggerFilename }}"
-logger-is-production = "{{ .SidecarQueryServerConfig.LoggerIsProduction }}"
-logger-level = "{{ .SidecarQueryServerConfig.LoggerLevel }}"
-
-# Defines the gRPC gateway endpoint of the chain.
-grpc-gateway-endpoint = "{{ .SidecarQueryServerConfig.ChainGRPCGatewayEndpoint }}"
-
-# The list of preferred poold IDs in the router.
-# These pools will be prioritized in the candidate route selection, ignoring all other
-# heuristics such as TVL.
-preferred-pool-ids = "{{ .SidecarQueryServerConfig.Router.PreferredPoolIDs }}"
-
-# The maximum number of pools to be included in a single route.
-max-pools-per-route = "{{ .SidecarQueryServerConfig.Router.MaxPoolsPerRoute }}"
-
-# The maximum number of routes to be returned in candidate route search.
-max-routes = "{{ .SidecarQueryServerConfig.Router.MaxRoutes }}"
-
-# The maximum number of routes to be split across. Must be smaller than or
-# equal to max-routes.
-max-split-routes = "{{ .SidecarQueryServerConfig.Router.MaxSplitRoutes }}"
-
-# The maximum number of iterations to split a route across.
-max-split-iterations = "{{ .SidecarQueryServerConfig.Router.MaxSplitIterations }}"
-
-# The minimum liquidity of a pool to be included in a route.
-min-osmo-liquidity = "{{ .SidecarQueryServerConfig.Router.MinOSMOLiquidity }}"
-
-# The height interval at which the candidate routes are recomputed and updated in
-# Redis
-route-update-height-interval = "{{ .SidecarQueryServerConfig.Router.RouteUpdateHeightInterval }}"
-
-# Whether to enable candidate route caching in Redis.
-route-cache-enabled = "{{ .SidecarQueryServerConfig.Router.RouteCacheEnabled }}"
-
-# The number of seconds to cache routes for before expiry.
-route-cache-expiry-seconds = "{{ .SidecarQueryServerConfig.Router.RouteCacheExpirySeconds }}"
-
 ###############################################################################
 ###              		       Wasm Configuration    					    ###
 ###############################################################################

diff --git a/go.mod b/go.mod
@@ -17,25 +17,24 @@ require (
 	github.com/cosmos/ibc-apps/middleware/packet-forward-middleware/v7 v7.1.1
 	github.com/cosmos/ibc-apps/modules/async-icq/v7 v7.1.1
 	github.com/cosmos/ibc-go/v7 v7.3.1
-	github.com/go-redis/redis v6.15.9+incompatible
 	github.com/golang/mock v1.6.0
 	github.com/golang/protobuf v1.5.3
 	github.com/golangci/golangci-lint v1.54.2
 	github.com/gorilla/mux v1.8.1
 	github.com/grpc-ecosystem/grpc-gateway v1.16.0
 	github.com/iancoleman/orderedmap v0.3.0
-	github.com/json-iterator/go v1.1.12
-	github.com/labstack/echo v3.3.10+incompatible
 	github.com/mattn/go-sqlite3 v1.14.17
 	github.com/ory/dockertest/v3 v3.10.0
 	github.com/osmosis-labs/go-mutesting v0.0.0-20221208041716-b43bcd97b3b3
-	github.com/osmosis-labs/osmosis/osmomath v0.0.7-0.20240108040604-9632b2b8af1b
-	github.com/osmosis-labs/osmosis/osmoutils v0.0.7-0.20240108040604-9632b2b8af1b
+	github.com/osmosis-labs/osmosis/osmomath v0.0.7-0.20240108180923-ddfef98789fd
+	github.com/osmosis-labs/osmosis/osmoutils v0.0.7-0.20240108180923-ddfef98789fd
 	github.com/osmosis-labs/osmosis/x/epochs v0.0.3-0.20240108040604-9632b2b8af1b
 	github.com/osmosis-labs/osmosis/x/ibc-hooks v0.0.9-0.20240108040604-9632b2b8af1b
+	github.com/osmosis-labs/sqs v0.0.0-20240108192026-6ccc0a29f77d
+	github.com/osmosis-labs/sqs/sqsdomain v0.0.0-20240108192026-6ccc0a29f77d
 	github.com/pkg/errors v0.9.1
 	github.com/rakyll/statik v0.1.7
-	github.com/redis/go-redis/v9 v9.3.1
+	github.com/redis/go-redis/v9 v9.4.0
 	github.com/spf13/cast v1.6.0
 	github.com/spf13/cobra v1.8.0
 	github.com/spf13/pflag v1.0.5
@@ -44,7 +43,6 @@ require (
 	github.com/tidwall/btree v1.6.0
 	github.com/tidwall/gjson v1.16.0
 	go.uber.org/multierr v1.11.0
-	go.uber.org/zap v1.26.0
 	golang.org/x/exp v0.0.0-20230905200255-921286631fa9
 	google.golang.org/genproto/googleapis/api v0.0.0-20231106174013-bbf56f31fb17
 	google.golang.org/grpc v1.60.1
@@ -70,15 +68,13 @@ require (
 	github.com/ChainSafe/go-schnorrkel v0.0.0-20200405005733-88cbf1b4c40d // indirect
 	github.com/Djarvur/go-err113 v0.1.0 // indirect
 	github.com/GaijinEntertainment/go-exhaustruct/v3 v3.1.0 // indirect
-	github.com/OneOfOne/xxhash v1.2.8 // indirect
 	github.com/OpenPeeDeeP/depguard/v2 v2.1.0 // indirect
 	github.com/alexkohler/nakedret/v2 v2.0.2 // indirect
 	github.com/alingse/asasalint v0.0.11 // indirect
 	github.com/aws/aws-sdk-go v1.44.224 // indirect
 	github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect
 	github.com/btcsuite/btcd/btcec/v2 v2.3.2 // indirect
 	github.com/btcsuite/btcd/btcutil v1.1.3 // indirect
-	github.com/btcsuite/btcd/chaincfg/chainhash v1.0.2 // indirect
 	github.com/butuzov/mirror v1.1.0 // indirect
 	github.com/ccojocar/zxcvbn-go v1.0.1 // indirect
 	github.com/chzyer/readline v1.5.1 // indirect
@@ -114,10 +110,10 @@ require (
 	github.com/hashicorp/go-safetemp v1.0.0 // indirect
 	github.com/huandu/skiplist v1.2.0 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/kkHAIKE/contextcheck v1.1.4 // indirect
 	github.com/kr/pretty v0.3.1 // indirect
 	github.com/kr/text v0.2.0 // indirect
-	github.com/labstack/gommon v0.4.0 // indirect
 	github.com/leodido/go-urn v1.2.4 // indirect
 	github.com/linxGnu/grocksdb v1.7.16 // indirect
 	github.com/manifoldco/promptui v0.9.0 // indirect
@@ -141,13 +137,12 @@ require (
 	github.com/tidwall/pretty v1.2.0 // indirect
 	github.com/timonwong/loggercheck v0.9.4 // indirect
 	github.com/ulikunitz/xz v0.5.11 // indirect
-	github.com/valyala/bytebufferpool v1.0.0 // indirect
-	github.com/valyala/fasttemplate v1.2.2 // indirect
 	github.com/xen0n/gosmopolitan v1.2.1 // indirect
 	github.com/ykadowak/zerologlint v0.1.3 // indirect
 	github.com/zimmski/go-mutesting v0.0.0-20210610104036-6d9217011a00 // indirect
 	github.com/zondax/ledger-go v0.14.3 // indirect
 	go.tmz.dev/musttag v0.7.2 // indirect
+	go.uber.org/zap v1.26.0 // indirect
 	golang.org/x/oauth2 v0.15.0 // indirect
 	golang.org/x/time v0.5.0 // indirect
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
@@ -326,7 +321,7 @@ require (
 	github.com/sasha-s/go-deadlock v0.3.1 // indirect
 	github.com/securego/gosec/v2 v2.17.0 // indirect
 	github.com/shazow/go-diff v0.0.0-20160112020656-b6b7b6733b8c // indirect
-	github.com/sirupsen/logrus v1.9.3
+	github.com/sirupsen/logrus v1.9.3 // indirect
 	github.com/sivchari/containedctx v1.0.3 // indirect
 	github.com/sivchari/tenv v1.7.1 // indirect
 	github.com/sonatard/noctx v0.0.2 // indirect
@@ -388,6 +383,10 @@ replace (
 	github.com/cosmos/gogoproto => github.com/cosmos/gogoproto v1.4.10
 	github.com/gogo/protobuf => github.com/regen-network/protobuf v1.3.3-alpha.regen.1
 
+	// We explicitly use tags from the `main` branch in https://github.com/osmosis-labs/sqs
+	github.com/osmosis-labs/sqs => github.com/osmosis-labs/sqs v0.0.0-20240108192026-6ccc0a29f77d
+	github.com/osmosis-labs/sqs/sqsdomain => github.com/osmosis-labs/sqs/sqsdomain v0.0.0-20240108192026-6ccc0a29f77d
+
 	// replace as directed by sdk upgrading.md https://github.com/cosmos/cosmos-sdk/blob/393de266c8675dc16cc037c1a15011b1e990975f/UPGRADING.md?plain=1#L713
 	github.com/syndtr/goleveldb => github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7
 
@@ -403,3 +402,7 @@ replace (
 
 // exclusion so we use v1.0.0
 exclude github.com/coinbase/rosetta-sdk-go v0.7.9
+
+exclude github.com/cosmos/cosmos-sdk v0.50.1
+
+exclude github.com/cometbft/cometbft v0.38.0