Skip to content

Commit

Permalink
chore: synced s3 storage provider
Browse files Browse the repository at this point in the history
  • Loading branch information
troykessler committed Jul 28, 2023
2 parents 4c32aa5 + 56dd229 commit 9263982
Show file tree
Hide file tree
Showing 63 changed files with 1,396 additions and 124 deletions.
4 changes: 4 additions & 0 deletions common/protocol/CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,10 @@
All notable changes to this project will be documented in this file.
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.

## [1.0.2](https://github.com/KYVENetwork/kyvejs/compare/@kyvejs/protocol@1.0.1...@kyvejs/protocol@1.0.2) (2023-07-14)

**Note:** Version bump only for package @kyvejs/protocol

## [1.0.1](https://github.com/KYVENetwork/kyvejs/compare/@kyvejs/protocol@1.0.0-beta.24...@kyvejs/protocol@1.0.1) (2023-06-02)

**Note:** Version bump only for package @kyvejs/protocol
Expand Down
11 changes: 7 additions & 4 deletions common/protocol/package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "@kyvejs/protocol",
"version": "1.0.1",
"version": "1.0.2",
"main": "./dist/src/index.js",
"types": "./dist/src/index.d.ts",
"repository": {
Expand All @@ -19,11 +19,12 @@
"doc": "typedoc"
},
"dependencies": {
"@aws-sdk/client-s3": "^3.370.0",
"@bundlr-network/client": "^0.8.9",
"@cosmjs/proto-signing": "^0.27.1",
"@cosmjs/stargate": "^0.27.1",
"@kyvejs/sdk": "1.0.1",
"@kyvejs/types": "1.0.1",
"@kyvejs/sdk": "1.0.2",
"@kyvejs/types": "1.0.2",
"@types/cli-progress": "^3.9.2",
"@types/jsonfile": "^6.0.1",
"arweave": "^1.10.17",
Expand All @@ -40,7 +41,8 @@
"seedrandom": "^3.0.5",
"semver": "^7.5.3",
"tslog": "^3.2.2",
"unique-names-generator": "^4.6.0"
"unique-names-generator": "^4.6.0",
"uuid": "^9.0.0"
},
"devDependencies": {
"@types/clone": "^2.1.1",
Expand All @@ -51,6 +53,7 @@
"@types/object-hash": "^2.2.1",
"@types/seedrandom": "^3.0.2",
"@types/semver": "^7.3.9",
"@types/uuid": "^9.0.2",
"jest": "^28.1.3",
"prettier": "^2.4.1",
"rimraf": "^3.0.2",
Expand Down
6 changes: 0 additions & 6 deletions common/protocol/src/commander/parser.ts
Original file line number Diff line number Diff line change
Expand Up @@ -29,12 +29,6 @@ export const parseValaccount = (value: string): string => {
};

export const parseStoragePriv = (value: string): string => {
if (!process.env[value]) {
throw new commander.InvalidArgumentError(
`Environment variable "${value}" has no value`
);
}

return process.env[value] || "";
};

Expand Down
4 changes: 2 additions & 2 deletions common/protocol/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ import {
isStorageBalanceLow,
} from "./methods";
import { ICacheProvider, IMetrics, IRuntime } from "./types";
import { standardizeJSON } from "./utils";
import { standardizeError } from "./utils";
import { SupportedChains } from "@kyvejs/sdk/dist/constants";
import { storageProviderFactory } from "./reactors/storageProviders";
import { compressionFactory } from "./reactors/compression";
Expand Down Expand Up @@ -335,7 +335,7 @@ export class Validator {
this.runCache();
} catch (err) {
this.logger.fatal(`Unexpected runtime error. Exiting ...`);
this.logger.fatal(standardizeJSON(err));
this.logger.fatal(standardizeError(err));

process.exit(1);
}
Expand Down
4 changes: 2 additions & 2 deletions common/protocol/src/methods/checks/isDataAvailable.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { Validator, standardizeJSON } from "../..";
import { Validator, standardizeError } from "../..";

/**
* isDataAvailable checks if the current next data item
Expand Down Expand Up @@ -46,7 +46,7 @@ export async function isDataAvailable(this: Validator): Promise<boolean> {
return true;
} catch (err) {
this.logger.fatal(`Data not available. Exiting ...`);
this.logger.fatal(standardizeJSON(err));
this.logger.fatal(standardizeError(err));

return false;
}
Expand Down
4 changes: 2 additions & 2 deletions common/protocol/src/methods/checks/isNodeValidator.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { Validator, standardizeJSON } from "../..";
import { Validator, standardizeError } from "../..";

/**
* isNodeValidator checks if the staker of the node is in the
Expand Down Expand Up @@ -31,7 +31,7 @@ export function isNodeValidator(this: Validator): boolean {
this.logger.fatal(
`Error while validating if node is a validator. Exiting ...`
);
this.logger.fatal(standardizeJSON(err));
this.logger.fatal(standardizeError(err));

return false;
}
Expand Down
12 changes: 10 additions & 2 deletions common/protocol/src/methods/checks/isStorageBalanceZero.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import BigNumber from "bignumber.js";
import { Validator, standardizeJSON } from "../..";
import { Validator, standardizeError } from "../..";

/**
* isStorageBalanceZero checks if the account of the storage provider
Expand All @@ -23,6 +23,14 @@ export async function isStorageBalanceZero(this: Validator): Promise<boolean> {
const address = await storageProvider.getAddress();
const balance = await storageProvider.getBalance();

// if storage provider has no balance we don't need to validate it
if (!balance) {
this.logger.info(
`StorageProvider:${storageProvider.name} has no balance. Continuing...\n`
);
return false;
}

this.logger.debug(`Account "${address}" has "${balance}" balance`);

if (new BigNumber(balance).isZero()) {
Expand All @@ -45,7 +53,7 @@ export async function isStorageBalanceZero(this: Validator): Promise<boolean> {
this.logger.fatal(
`Error while checking storage provider balance. Exiting ...`
);
this.logger.fatal(standardizeJSON(err));
this.logger.fatal(standardizeError(err));

return true;
}
Expand Down
4 changes: 2 additions & 2 deletions common/protocol/src/methods/checks/isValidRuntime.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { Validator, standardizeJSON } from "../..";
import { Validator, standardizeError } from "../..";

/**
* isValidRuntime checks if the runtime of the pool matches with the runtime of
Expand Down Expand Up @@ -28,7 +28,7 @@ export function isValidRuntime(this: Validator): boolean {
return true;
} catch (err) {
this.logger.fatal(`Error while validating runtime. Exiting ...`);
this.logger.fatal(standardizeJSON(err));
this.logger.fatal(standardizeError(err));

return false;
}
Expand Down
10 changes: 5 additions & 5 deletions common/protocol/src/methods/main/runCache.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { DataItem, Validator } from "../..";
import { callWithBackoffStrategy, sleep, standardizeJSON } from "../../utils";
import { callWithBackoffStrategy, sleep, standardizeError } from "../../utils";
import clone from "clone";

/**
Expand Down Expand Up @@ -111,7 +111,7 @@ export async function runCache(this: Validator): Promise<void> {
this.logger.error(
`Unexpected error deleting data item ${i.toString()} from local cache. Continuing ...`
);
this.logger.error(standardizeJSON(err));
this.logger.error(standardizeError(err));
continue;
}
}
Expand Down Expand Up @@ -172,7 +172,7 @@ export async function runCache(this: Validator): Promise<void> {
ctx.nextTimeoutInMs / 1000
).toFixed(2)}s ...`
);
this.logger.debug(standardizeJSON(err));
this.logger.debug(standardizeError(err));

this.m.runtime_get_data_item_failed.inc();
}
Expand Down Expand Up @@ -207,7 +207,7 @@ export async function runCache(this: Validator): Promise<void> {
this.logger.error(
`Unexpected error collecting data items to local cache. Continuing ...`
);
this.logger.error(standardizeJSON(err));
this.logger.error(standardizeError(err));

try {
// drop cache if an unexpected error occurs during caching
Expand All @@ -219,7 +219,7 @@ export async function runCache(this: Validator): Promise<void> {
this.logger.error(
`Unexpected error dropping local cache. Continuing ...`
);
this.logger.error(standardizeJSON(dropError));
this.logger.error(standardizeError(dropError));
}
}
}
Expand Down
8 changes: 4 additions & 4 deletions common/protocol/src/methods/queries/canPropose.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { Validator } from "../..";
import { callWithBackoffStrategy, sleep, standardizeJSON } from "../../utils";
import { callWithBackoffStrategy, sleep, standardizeError } from "../../utils";

const INFINITY_LOOP = true;

Expand Down Expand Up @@ -84,7 +84,7 @@ export async function canPropose(
}
} catch (err) {
this.logger.error(`REST call to "${this.rest[l]}" failed`);
this.logger.error(standardizeJSON(err));
this.logger.error(standardizeError(err));
}
}

Expand All @@ -97,7 +97,7 @@ export async function canPropose(
ctx.nextTimeoutInMs / 1000
).toFixed(2)}s ...`
);
this.logger.debug(standardizeJSON(err));
this.logger.debug(standardizeError(err));
this.m.query_can_propose_failed.inc();
}
);
Expand All @@ -116,7 +116,7 @@ export async function canPropose(
}
} catch (err) {
this.logger.error(`Failed to call canPropose`);
this.logger.error(standardizeJSON(err));
this.logger.error(standardizeError(err));

return false;
}
Expand Down
8 changes: 4 additions & 4 deletions common/protocol/src/methods/queries/canVote.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { Validator } from "../..";
import { callWithBackoffStrategy, standardizeJSON } from "../../utils";
import { callWithBackoffStrategy, standardizeError } from "../../utils";

/**
* canVote checks if the node is able to vote on the current
Expand Down Expand Up @@ -63,7 +63,7 @@ export async function canVote(
});
} catch (err) {
this.logger.error(`REST call to "${this.rest[l]}" failed`);
this.logger.error(standardizeJSON(err));
this.logger.error(standardizeError(err));
}
}

Expand All @@ -76,7 +76,7 @@ export async function canVote(
ctx.nextTimeoutInMs / 1000
).toFixed(2)}s ...`
);
this.logger.debug(standardizeJSON(err));
this.logger.debug(standardizeError(err));
this.m.query_can_vote_failed.inc();
}
);
Expand All @@ -93,7 +93,7 @@ export async function canVote(
}
} catch (err) {
this.logger.error(`Failed to call canVote`);
this.logger.error(standardizeJSON(err));
this.logger.error(standardizeError(err));

return false;
}
Expand Down
8 changes: 4 additions & 4 deletions common/protocol/src/methods/queries/getBalancesForMetrics.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import BigNumber from "bignumber.js";

import { Validator, standardizeJSON } from "../..";
import { Validator, standardizeError } from "../..";

/**
* getBalancesForMetrics tries to retrieve the $KYVE balance of the staker account, the $KYVE
Expand Down Expand Up @@ -36,7 +36,7 @@ export async function getBalancesForMetrics(this: Validator): Promise<void> {
break;
} catch (err) {
this.logger.error(`Failed to get $KYVE balance of staker`);
this.logger.error(standardizeJSON(err));
this.logger.error(standardizeError(err));
}
}

Expand Down Expand Up @@ -64,7 +64,7 @@ export async function getBalancesForMetrics(this: Validator): Promise<void> {
break;
} catch (err) {
this.logger.error(`Failed to get $KYVE balance of valaccount`);
this.logger.error(standardizeJSON(err));
this.logger.error(standardizeError(err));
}
}

Expand Down Expand Up @@ -95,6 +95,6 @@ export async function getBalancesForMetrics(this: Validator): Promise<void> {
this.m.balance_storage_provider.set(storageProviderBalance);
} catch (err) {
this.logger.error(`Failed to get balance of storage provider`);
this.logger.error(standardizeJSON(err));
this.logger.error(standardizeError(err));
}
}
8 changes: 4 additions & 4 deletions common/protocol/src/methods/queries/syncPoolState.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { Validator } from "../..";
import { callWithBackoffStrategy, standardizeJSON } from "../../utils";
import { callWithBackoffStrategy, standardizeError } from "../../utils";

/**
* syncPoolState fetches the state of the pool the node is running on.
Expand Down Expand Up @@ -55,7 +55,7 @@ export async function syncPoolState(
this.logger.fatal(
`Failed to sync runtime config. Either the config could not be parsed or was invalid.`
);
this.logger.fatal(standardizeJSON(err));
this.logger.fatal(standardizeError(err));

if (exitOnConfigError) process.exit(1);
}
Expand All @@ -64,7 +64,7 @@ export async function syncPoolState(
return;
} catch (err) {
this.logger.error(`REST call to "${this.rest[l]}" failed`);
this.logger.error(standardizeJSON(err));
this.logger.error(standardizeError(err));
}
}

Expand All @@ -77,7 +77,7 @@ export async function syncPoolState(
ctx.nextTimeoutInMs / 1000
).toFixed(2)}s ...`
);
this.logger.debug(standardizeJSON(err));
this.logger.debug(standardizeError(err));

this.m.query_pool_failed.inc();
}
Expand Down
4 changes: 2 additions & 2 deletions common/protocol/src/methods/setups/setupCacheProvider.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import path from "path";

import { Validator, standardizeJSON } from "../..";
import { Validator, standardizeError } from "../..";
import fse from "fs-extra";

/**
Expand Down Expand Up @@ -31,7 +31,7 @@ export async function setupCacheProvider(this: Validator): Promise<void> {
this.logger.info(`Using cache provider: ${this.cacheProvider.name}`);
} catch (err) {
this.logger.fatal(`Failed to setup cache provider. Exiting ...`);
this.logger.fatal(standardizeJSON(err));
this.logger.fatal(standardizeError(err));

process.exit(1);
}
Expand Down
4 changes: 2 additions & 2 deletions common/protocol/src/methods/setups/setupLogger.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import { appendFileSync, existsSync, mkdirSync } from "fs";
import path from "path";
import { ILogObject, Logger } from "tslog";

import { Validator, standardizeJSON } from "../..";
import { Validator, standardizeError } from "../..";

/**
* setupLogger creates the logger instance and defines the home and file
Expand Down Expand Up @@ -83,7 +83,7 @@ export function setupLogger(this: Validator): void {
this.logger.info("Starting node ...\n");
} catch (err) {
this.logger.fatal(`Failed to init logger. Exiting ...`);
this.logger.fatal(standardizeJSON(err));
this.logger.fatal(standardizeError(err));

process.exit(1);
}
Expand Down
4 changes: 2 additions & 2 deletions common/protocol/src/methods/setups/setupMetrics.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import http from "http";
import prom_client, { register } from "prom-client";
import url from "url";

import { IMetrics, Validator, standardizeJSON } from "../..";
import { IMetrics, Validator, standardizeError } from "../..";

/**
* setupMetrics initiates all metrics and starts if enabled a local prometheus
Expand Down Expand Up @@ -413,7 +413,7 @@ export function setupMetrics(this: Validator): void {
}
} catch (err) {
this.logger.fatal(`Failed to setup metrics. Exiting ...`);
this.logger.fatal(standardizeJSON(err));
this.logger.fatal(standardizeError(err));

process.exit(1);
}
Expand Down
Loading

0 comments on commit 9263982

Please sign in to comment.