refactoring: Cleanup dependencies, port indexer utils, delete dead co…

…de. (#61)

.circleci/config.yml

@@ -2,7 +2,7 @@ version: 2.1
 
 orbs:
   go: circleci/go@1.7.0
-  codecov: codecov/codecov@3.1.1
+  codecov: codecov/codecov@3.2.4
 
 parameters:
   ubuntu_image:

cmd/conduit/internal/initialize/init.go

@@ -11,11 +11,10 @@ import (
 
 	"github.com/spf13/cobra"
 
-	algodimporter "github.com/algorand/indexer/conduit/plugins/importers/algod"
-
-	"github.com/algorand/conduit/conduit"
+	"github.com/algorand/conduit/conduit/data"
 	"github.com/algorand/conduit/conduit/pipeline"
 	"github.com/algorand/conduit/conduit/plugins/exporters/filewriter"
+	algodimporter "github.com/algorand/conduit/conduit/plugins/importers/algod"
 )
 
 // InitCommand is the init subcommand.
@@ -58,7 +57,7 @@ func runConduitInit(path string, importerFlag string, processorsFlag []string, e
 		return err
 	}
 
-	configFilePath := filepath.Join(path, conduit.DefaultConfigName)
+	configFilePath := filepath.Join(path, data.DefaultConfigName)
 	f, err := os.Create(configFilePath)
 	if err != nil {
 		return fmt.Errorf("runConduitInit(): failed to create %s", configFilePath)

cmd/conduit/internal/initialize/init_test.go

@@ -11,7 +11,7 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
-	"github.com/algorand/conduit/conduit/pipeline"
+	"github.com/algorand/conduit/conduit/data"
 	"github.com/algorand/conduit/conduit/plugins/exporters/filewriter"
 	noopExporter "github.com/algorand/conduit/conduit/plugins/exporters/noop"
 	algodimporter "github.com/algorand/conduit/conduit/plugins/importers/algod"
@@ -25,9 +25,9 @@ var defaultYml string
 
 // TestInitDataDirectory tests the initialization of the data directory
 func TestInitDataDirectory(t *testing.T) {
-	verifyYaml := func(data []byte, importer string, exporter string, processors []string) {
-		var cfg pipeline.Config
-		require.NoError(t, yaml.Unmarshal(data, &cfg))
+	verifyYaml := func(ydata []byte, importer string, exporter string, processors []string) {
+		var cfg data.Config
+		require.NoError(t, yaml.Unmarshal(ydata, &cfg))
 		assert.Equal(t, importer, cfg.Importer.Name)
 		assert.Equal(t, exporter, cfg.Exporter.Name)
 		require.Equal(t, len(processors), len(cfg.Processors))

cmd/conduit/main.go

@@ -13,7 +13,7 @@ import (
 
 	"github.com/algorand/conduit/cmd/conduit/internal/initialize"
 	"github.com/algorand/conduit/cmd/conduit/internal/list"
-	"github.com/algorand/conduit/conduit"
+	"github.com/algorand/conduit/conduit/data"
 	"github.com/algorand/conduit/conduit/loggers"
 	"github.com/algorand/conduit/conduit/pipeline"
 	_ "github.com/algorand/conduit/conduit/plugins/exporters/all"
@@ -40,7 +40,7 @@ func init() {
 }
 
 // runConduitCmdWithConfig run the main logic with a supplied conduit config
-func runConduitCmdWithConfig(args *conduit.Args) error {
+func runConduitCmdWithConfig(args *data.Args) error {
 	defer pipeline.HandlePanic(logger)
 
 	if args.ConduitDataDir == "" {
@@ -51,7 +51,7 @@ func runConduitCmdWithConfig(args *conduit.Args) error {
 		return fmt.Errorf("the data directory is required and must be provided with a command line option or the '%s' environment variable", conduitEnvVar)
 	}
 
-	pCfg, err := pipeline.MakePipelineConfig(args)
+	pCfg, err := data.MakePipelineConfig(args)
 	if err != nil {
 		return err
 	}
@@ -113,7 +113,7 @@ func runConduitCmdWithConfig(args *conduit.Args) error {
 
 // makeConduitCmd creates the main cobra command, initializes flags
 func makeConduitCmd() *cobra.Command {
-	cfg := &conduit.Args{}
+	cfg := &data.Args{}
 	var vFlag bool
 	cmd := &cobra.Command{
 		Use:   "conduit",

cmd/conduit/main_test.go

@@ -10,18 +10,17 @@ import (
 	"github.com/stretchr/testify/require"
 	"gopkg.in/yaml.v3"
 
-	"github.com/algorand/conduit/conduit"
-	"github.com/algorand/conduit/conduit/pipeline"
+	"github.com/algorand/conduit/conduit/data"
 )
 
 // Fills in a temp data dir and creates files
 // TODO: Refactor the code so that testing can be done without creating files and directories.
-func setupDataDir(t *testing.T, cfg pipeline.Config) *conduit.Args {
-	conduitArgs := &conduit.Args{ConduitDataDir: t.TempDir()}
-	data, err := yaml.Marshal(&cfg)
+func setupDataDir(t *testing.T, cfg data.Config) *data.Args {
+	conduitArgs := &data.Args{ConduitDataDir: t.TempDir()}
+	ydata, err := yaml.Marshal(&cfg)
 	require.NoError(t, err)
-	configFile := path.Join(conduitArgs.ConduitDataDir, conduit.DefaultConfigName)
-	os.WriteFile(configFile, data, 0755)
+	configFile := path.Join(conduitArgs.ConduitDataDir, data.DefaultConfigName)
+	os.WriteFile(configFile, ydata, 0755)
 	require.FileExists(t, configFile)
 	return conduitArgs
 }
@@ -39,12 +38,12 @@ func TestBanner(t *testing.T) {
 		defer f.Close()
 		os.Stdout = f
 
-		cfg := pipeline.Config{
-			ConduitArgs: &conduit.Args{ConduitDataDir: t.TempDir()},
+		cfg := data.Config{
+			ConduitArgs: &data.Args{ConduitDataDir: t.TempDir()},
 			HideBanner:  hideBanner,
-			Importer:    pipeline.NameConfigPair{Name: "test", Config: map[string]interface{}{"a": "a"}},
+			Importer:    data.NameConfigPair{Name: "test", Config: map[string]interface{}{"a": "a"}},
 			Processors:  nil,
-			Exporter:    pipeline.NameConfigPair{Name: "test", Config: map[string]interface{}{"a": "a"}},
+			Exporter:    data.NameConfigPair{Name: "test", Config: map[string]interface{}{"a": "a"}},
 		}
 		args := setupDataDir(t, cfg)
 
@@ -69,13 +68,13 @@ func TestBanner(t *testing.T) {
 }
 
 func TestEmptyDataDir(t *testing.T) {
-	args := conduit.Args{}
+	args := data.Args{}
 	err := runConduitCmdWithConfig(&args)
 	require.ErrorContains(t, err, conduitEnvVar)
 }
 
 func TestInvalidLogLevel(t *testing.T) {
-	cfg := pipeline.Config{
+	cfg := data.Config{
 		LogLevel: "invalid",
 	}
 	args := setupDataDir(t, cfg)
@@ -97,11 +96,11 @@ func TestLogFile(t *testing.T) {
 		defer f.Close()
 		os.Stdout = f
 
-		cfg := pipeline.Config{
+		cfg := data.Config{
 			LogFile:    logfile,
-			Importer:   pipeline.NameConfigPair{Name: "test", Config: map[string]interface{}{"a": "a"}},
+			Importer:   data.NameConfigPair{Name: "test", Config: map[string]interface{}{"a": "a"}},
 			Processors: nil,
-			Exporter:   pipeline.NameConfigPair{Name: "test", Config: map[string]interface{}{"a": "a"}},
+			Exporter:   data.NameConfigPair{Name: "test", Config: map[string]interface{}{"a": "a"}},
 		}
 		args := setupDataDir(t, cfg)
 

conduit/data/block_export_data.go

@@ -2,7 +2,6 @@ package data
 
 import (
 	sdk "github.com/algorand/go-algorand-sdk/v2/types"
-	"github.com/algorand/indexer/types"
 )
 
 // RoundProvider is the interface which all data types sent to Exporters should implement
@@ -35,70 +34,6 @@ type BlockData struct {
 	Certificate *map[string]interface{} `json:"cert,omitempty"`
 }
 
-// MakeBlockDataFromValidatedBlock makes BlockData from agreement.ValidatedBlock
-func MakeBlockDataFromValidatedBlock(input types.ValidatedBlock) BlockData {
-	blockData := BlockData{}
-	blockData.UpdateFromValidatedBlock(input)
-	return blockData
-}
-
-// UpdateFromValidatedBlock updates BlockData from ValidatedBlock input
-func (blkData *BlockData) UpdateFromValidatedBlock(input types.ValidatedBlock) {
-	blkData.BlockHeader = input.Block.BlockHeader
-	blkData.Payset = input.Block.Payset
-	delta := input.Delta
-	blkData.Delta = &delta
-}
-
-// UpdateFromEncodedBlockCertificate updates BlockData from EncodedBlockCert info
-func (blkData *BlockData) UpdateFromEncodedBlockCertificate(input *types.EncodedBlockCert) {
-	if input == nil {
-		return
-	}
-
-	blkData.BlockHeader = input.Block.BlockHeader
-	blkData.Payset = input.Block.Payset
-
-	cert := input.Certificate
-	blkData.Certificate = &cert
-}
-
-// ValidatedBlock returns a validated block from the BlockData object
-func (blkData BlockData) ValidatedBlock() types.ValidatedBlock {
-	tmpBlock := sdk.Block{
-		BlockHeader: blkData.BlockHeader,
-		Payset:      blkData.Payset,
-	}
-
-	tmpDelta := sdk.LedgerStateDelta{}
-	if blkData.Delta != nil {
-		tmpDelta = *blkData.Delta
-	}
-	vb := types.ValidatedBlock{
-		Block: tmpBlock,
-		Delta: tmpDelta,
-	}
-	return vb
-}
-
-// EncodedBlockCertificate returns an encoded block certificate from the BlockData object
-func (blkData BlockData) EncodedBlockCertificate() types.EncodedBlockCert {
-
-	tmpBlock := sdk.Block{
-		BlockHeader: blkData.BlockHeader,
-		Payset:      blkData.Payset,
-	}
-
-	tmpCert := make(map[string]interface{})
-	if blkData.Certificate != nil {
-		tmpCert = *blkData.Certificate
-	}
-	return types.EncodedBlockCert{
-		Block:       tmpBlock,
-		Certificate: tmpCert,
-	}
-}
-
 // Round returns the round to which the BlockData corresponds
 func (blkData BlockData) Round() uint64 {
 	return uint64(blkData.BlockHeader.Round)