Skip to content

Commit

Permalink
Better padding with init command sample files. (#94)
Browse files Browse the repository at this point in the history
  • Loading branch information
winder authored Jun 9, 2023
1 parent 6c45173 commit 93bd79c
Show file tree
Hide file tree
Showing 14 changed files with 87 additions and 88 deletions.
8 changes: 4 additions & 4 deletions conduit/plugins/exporters/filewriter/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,19 +4,19 @@ Write block data to files. This plugin works with the file rerader plugin to cre

## Configuration
```yml @sample.yaml
name: "file_writer"
config:
name: "file_writer"
config:
# BlocksDir is the path to a directory where block data should be stored.
# The directory is created if it doesn't exist. If no directory is provided
# blocks are written to the Conduit data directory.
#block-dir: "/path/to/block/files"

# FilenamePattern is the format used to write block files. It uses go
# string formatting and should accept one number for the round.
# If the file has a '.gz' extension, blocks will be gzipped.
# Default: "%[1]d_block.json"
filename-pattern: "%[1]d_block.json"

# DropCertificate is used to remove the vote certificate from the block data before writing files.
drop-certificate: true
```
8 changes: 4 additions & 4 deletions conduit/plugins/exporters/filewriter/sample.yaml
Original file line number Diff line number Diff line change
@@ -1,15 +1,15 @@
name: "file_writer"
config:
name: "file_writer"
config:
# BlocksDir is the path to a directory where block data should be stored.
# The directory is created if it doesn't exist. If no directory is provided
# blocks are written to the Conduit data directory.
#block-dir: "/path/to/block/files"

# FilenamePattern is the format used to write block files. It uses go
# string formatting and should accept one number for the round.
# If the file has a '.gz' extension, blocks will be gzipped.
# Default: "%[1]d_block.json"
filename-pattern: "%[1]d_block.json"

# DropCertificate is used to remove the vote certificate from the block data before writing files.
drop-certificate: true
6 changes: 3 additions & 3 deletions conduit/plugins/exporters/noop/sample.yaml
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
name: noop
# noop has no config
config:
name: noop
# noop has no config
config:
20 changes: 10 additions & 10 deletions conduit/plugins/exporters/postgresql/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -21,24 +21,24 @@ The delete-task prunes old transactions according to its configuration. This can

## Configuration
```yml @sample.yaml
name: postgresql
config:
name: postgresql
config:
# Pgsql connection string
# See https://github.com/jackc/pgconn for more details
connection-string: "host= port=5432 user= password= dbname="

# Maximum connection number for connection pool
# This means the total number of active queries that can be running
# concurrently can never be more than this
max-conn: 20

# The delete task prunes old transactions according to its configuration.
# By default transactions are not deleted.
delete-task:
# Interval used to prune the data. The values can be -1 to run at startup,
# 0 to disable, or N to run every N rounds.
interval: 0

# Rounds to keep
rounds: 100000
# Interval used to prune the data. The values can be -1 to run at startup,
# 0 to disable, or N to run every N rounds.
interval: 0
# Rounds to keep
rounds: 100000
```
20 changes: 10 additions & 10 deletions conduit/plugins/exporters/postgresql/sample.yaml
Original file line number Diff line number Diff line change
@@ -1,20 +1,20 @@
name: postgresql
config:
name: postgresql
config:
# Pgsql connection string
# See https://github.com/jackc/pgconn for more details
connection-string: "host= port=5432 user= password= dbname="

# Maximum connection number for connection pool
# This means the total number of active queries that can be running
# concurrently can never be more than this
max-conn: 20

# The delete task prunes old transactions according to its configuration.
# By default transactions are not deleted.
delete-task:
# Interval used to prune the data. The values can be -1 to run at startup,
# 0 to disable, or N to run every N rounds.
interval: 0

# Rounds to keep
rounds: 100000
# Interval used to prune the data. The values can be -1 to run at startup,
# 0 to disable, or N to run every N rounds.
interval: 0
# Rounds to keep
rounds: 100000
10 changes: 5 additions & 5 deletions conduit/plugins/importers/algod/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -18,22 +18,22 @@ When using a follower node, ledger state delta objects are provided to the proce

## Configuration
```yml @sample.yaml
name: algod
config:
name: algod
config:
# The mode of operation, either "archival" or "follower".
# * archival mode allows you to start processing on any round but does not
# contain the ledger state delta objects required for the postgres writer.
# * follower mode allows you to use a lightweight non-archival node as the
# data source. In addition, it will provide ledger state delta objects to
# the processors and exporter.
mode: "follower"

# Algod API address.
netaddr: "http://url:port"

# Algod API token.
token: ""

# Algod catchpoint catchup arguments
catchup-config:
# Algod Admin API Token. Set the admin token to use fast catchup during
Expand Down
12 changes: 6 additions & 6 deletions conduit/plugins/importers/algod/sample.yaml
Original file line number Diff line number Diff line change
@@ -1,19 +1,19 @@
name: algod
config:
name: algod
config:
# The mode of operation, either "archival" or "follower".
# * archival mode allows you to start processing on any round but does not
# contain the ledger state delta objects required for the postgres writer.
# * follower mode allows you to use a lightweight non-archival node as the
# data source. In addition, it will provide ledger state delta objects to
# the processors and exporter.
mode: "follower"

# Algod API address.
netaddr: "http://url:port"

# Algod API token.
token: ""

# Algod catchpoint catchup arguments
catchup-config:
# Algod Admin API Token. Set the admin token to use fast catchup during
Expand All @@ -27,4 +27,4 @@
# mainnet: https://algorand-catchpoints.s3.us-east-2.amazonaws.com/consolidated/mainnet_catchpoints.txt
# betanet: https://algorand-catchpoints.s3.us-east-2.amazonaws.com/consolidated/betanet_catchpoints.txt
# testnet: https://algorand-catchpoints.s3.us-east-2.amazonaws.com/consolidated/testnet_catchpoints.txt
catchpoint: ""
catchpoint: ""
10 changes: 5 additions & 5 deletions conduit/plugins/importers/filereader/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,22 +4,22 @@ Read files from a directory and import them as blocks. This plugin works with th

## Configuration
```yml @sample.yaml
name: file_reader
config:
name: file_reader
config:
# BlocksDir is the path to a directory where block data should be stored.
# The directory is created if it doesn't exist. If no directory is provided
# blocks are written to the Conduit data directory.
#block-dir: "/path/to/directory"

# RetryDuration controls the delay between checks when the importer has
# caught up and is waiting for new blocks to appear.
retry-duration: "5s"

# RetryCount controls the number of times to check for a missing block
# before generating an error. The retry count and retry duration should
# be configured according the expected round time.
retry-count: 5

# FilenamePattern is the format used to find block files. It uses go string
# formatting and should accept one number for the round.
filename-pattern: "%[1]d_block.json"
Expand Down
10 changes: 5 additions & 5 deletions conduit/plugins/importers/filereader/sample.yaml
Original file line number Diff line number Diff line change
@@ -1,19 +1,19 @@
name: file_reader
config:
name: file_reader
config:
# BlocksDir is the path to a directory where block data should be stored.
# The directory is created if it doesn't exist. If no directory is provided
# blocks are written to the Conduit data directory.
#block-dir: "/path/to/directory"

# RetryDuration controls the delay between checks when the importer has
# caught up and is waiting for new blocks to appear.
retry-duration: "5s"

# RetryCount controls the number of times to check for a missing block
# before generating an error. The retry count and retry duration should
# be configured according the expected round time.
retry-count: 5

# FilenamePattern is the format used to find block files. It uses go string
# formatting and should accept one number for the round.
filename-pattern: "%[1]d_block.json"
26 changes: 13 additions & 13 deletions conduit/plugins/processors/filterprocessor/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -63,19 +63,19 @@ expression as a regex.
```yml @sample.yaml
name: filter_processor
config:
# Whether or not the expression searches inner transactions for matches.
search-inner: true
# Whether or not to include the entire transaction group when the filter
# conditions are met.
omit-group-transactions: true
# The list of filter expressions to use when matching transactions.
filters:
- any:
- tag: txn.rcv
expression-type: exact
expression: "ADDRESS"
# Whether the expression searches inner transactions for matches.
search-inner: true
# Whether to include the entire transaction group when the filter
# conditions are met.
omit-group-transactions: true
# The list of filter expressions to use when matching transactions.
filters:
- any:
- tag: txn.rcv
expression-type: exact
expression: "ADDRESS"
```

## Examples
Expand Down
22 changes: 11 additions & 11 deletions conduit/plugins/processors/filterprocessor/sample.yaml
Original file line number Diff line number Diff line change
@@ -1,15 +1,15 @@
name: filter_processor
config:
# Whether or not the expression searches inner transactions for matches.
search-inner: true
# Whether the expression searches inner transactions for matches.
search-inner: true

# Whether or not to include the entire transaction group when the filter
# conditions are met.
omit-group-transactions: true
# Whether to include the entire transaction group when the filter
# conditions are met.
omit-group-transactions: true

# The list of filter expressions to use when matching transactions.
filters:
- any:
- tag: txn.rcv
expression-type: exact
expression: "ADDRESS"
# The list of filter expressions to use when matching transactions.
filters:
- any:
- tag: txn.rcv
expression-type: exact
expression: "ADDRESS"
2 changes: 1 addition & 1 deletion conduit/plugins/processors/noop/sample.yaml
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
name: noop
# noop has no config
config:
config:
9 changes: 4 additions & 5 deletions pkg/cli/internal/initialize/conduit.yml.example
Original file line number Diff line number Diff line change
Expand Up @@ -19,18 +19,17 @@ hide-banner: false

# When enabled prometheus metrics are available on '/metrics'
metrics:
mode: OFF
addr: ":9999"
prefix: "conduit"
mode: OFF
addr: ":9999"
prefix: "conduit"

# The importer is typically an algod follower node.
importer:
%s

# Zero or more processors may be defined to manipulate what data
# reaches the exporter.
processors:
%s
# An exporter is defined to do something with the data.
exporter:
%s
%s
12 changes: 6 additions & 6 deletions pkg/cli/internal/initialize/init.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,21 +27,19 @@ var errNoWriter = errors.New("configWriter is required")
//go:embed conduit.yml.example
var sampleConfig string

func formatArrayObject(obj string) string {

func formatObject(obj string, isArray bool) string {
var ret string
lines := strings.Split(obj, "\n")
for i, line := range lines {
if i == 0 {
if i == 0 && isArray {
ret += " - "
} else {
} else if len(line) > 0 {
ret += " "
}
ret += line + "\n"
}

return ret

}

func runConduitInit(path string, importerFlag string, processorsFlag []string, exporterFlag string) error {
Expand Down Expand Up @@ -99,6 +97,7 @@ func writeConfigFile(configWriter io.Writer, importerFlag string, processorsFlag
if importer == "" {
return fmt.Errorf("runConduitInit(): unknown importer name: %v", importerFlag)
}
importer = formatObject(importer, false)

var exporter string
if exporterFlag == "" {
Expand All @@ -113,13 +112,14 @@ func writeConfigFile(configWriter io.Writer, importerFlag string, processorsFlag
if exporter == "" {
return fmt.Errorf("runConduitInit(): unknown exporter name: %v", exporterFlag)
}
exporter = formatObject(exporter, false)

var processors string
for _, processorName := range processorsFlag {
found := false
for _, metadata := range pipeline.ProcessorMetadata() {
if metadata.Name == processorName {
processors = processors + formatArrayObject(metadata.SampleConfig)
processors = processors + formatObject(metadata.SampleConfig, true)
found = true
break
}
Expand Down

0 comments on commit 93bd79c

Please sign in to comment.