Merge branch 'unstable' into dapplion/discovery-init

.eslintrc.js

@@ -68,6 +68,9 @@ module.exports = {
     "@typescript-eslint/no-floating-promises": "error",
     "@typescript-eslint/no-non-null-assertion": "error",
     "@typescript-eslint/no-require-imports": "error",
+    // We usually type-cast these standard types because the concerned function accepts any type
+    // and we want to TS detect error if original variable type changes
+    "@typescript-eslint/no-unnecessary-type-assertion": ["error", {typesToIgnore: ["string", "bigint", "number"]}],
     "@typescript-eslint/no-unsafe-assignment": "error",
     "@typescript-eslint/no-unsafe-call": "error",
     "@typescript-eslint/no-unsafe-member-access": "error",

CONTRIBUTING.md

@@ -5,7 +5,7 @@ Thanks for your interest in contributing to Lodestar. It's people like you that
 ## Prerequisites
 
 - :gear: [NodeJS](https://nodejs.org/) (LTS)
-- :toolbox: [Yarn](https://yarnpkg.com/)/[Lerna](https://lerna.js.org/)
+- :toolbox: [Yarn](https://yarnpkg.com/)
 
 ## Getting Started
 
@@ -18,10 +18,10 @@ Thanks for your interest in contributing to Lodestar. It's people like you that
 
 To run tests:
 
-- :test_tube: Run `lerna run test:unit` for unit tests.
-- :test_tube: Run `lerna run test:e2e` for end-to-end tests.
-- :test_tube: Run `lerna run test:spec` for spec tests.
-- :test_tube: Run `lerna run test` to run all tests.
+- :test_tube: Run `yarn test:unit` for unit tests.
+- :test_tube: Run `yarn test:e2e` for end-to-end tests.
+- :test_tube: Run `yarn test:spec` for spec tests.
+- :test_tube: Run `yarn test` to run all tests.
 - :test_tube: Run `yarn check-types` to check TypeScript types.
 - :test_tube: Run `yarn lint` to run the linter (ESLint).
 

README.md

@@ -30,12 +30,12 @@
 ## Prerequisites
 
 - :gear: [NodeJS](https://nodejs.org/) (LTS)
-- :toolbox: [Yarn](https://yarnpkg.com/)/[Lerna](https://lerna.js.org/)
+- :toolbox: [Yarn](https://yarnpkg.com/)
 
 ###### Developer Quickstart:
 
 ```bash
-lerna bootstrap
+yarn install
 yarn build
 ./lodestar --help
 ```

docs/install/source.md

@@ -43,12 +43,6 @@ Build across all packages.
 yarn run build
 ```
 
-Or if you are using [Lerna](https://lerna.js.org/):
-
-```bash
-lerna bootstrap
-```
-
 ## Lodestar CLI
 
 Lodestar should now be ready for use.

docs/usage/beacon-management.md

@@ -120,6 +120,15 @@ In case you really trust `checkpointSyncUrl` then you may skip providing `wssChe
 If possible, validate your `wssCheckpoint` from multiple places (e.g. different client distributions) or from other trusted sources. This will highly reduce the risk of starting off on a malicious chain.
 <!-- prettier-ignore-end -->
 
+**Taking too long to sync?**
+
+After your node has been offline for a while, it might be the case that it takes a long time to sync even though a `checkpointSyncUrl` is specified.
+This is due to the fact that the last db state is still within the weak subjectivity period (~15 days on mainnet) which causes the node
+to sync from the db state instead of the checkpoint state.
+
+It is possible to force syncing from checkpoint state by supplying the `--forceCheckpointSync` flag. This option is only recommended if it is absolutely
+necessary for the node to be synced right away to fulfill its duties as there is an inherent risk each time the state is obtained from an external source.
+
 ### Guide to the sync logs
 
 Lodestar beacon sync log aims to provide information of utmost importance  about your node and yet be suucint at the same time. You may see the sync logs in the following format:

package.json

@@ -15,9 +15,11 @@
     "build:watch": "lerna exec --parallel -- 'yarn run build:watch'",
     "build:ifchanged": "lerna exec -- ../../scripts/build_if_changed.sh",
     "lint": "eslint --color --ext .ts packages/*/src packages/*/test",
+    "lint:fix": "yarn lint --fix",
     "check-build": "lerna run check-build",
     "check-types": "lerna run check-types --no-bail",
     "coverage": "lerna run coverage --no-bail",
+    "test": "lerna run test --no-bail --concurrency 1",
     "test:unit": "lerna run test:unit --no-bail --concurrency 1",
     "test:browsers": "lerna run test:browsers",
     "test:e2e": "lerna run test:e2e --no-bail --concurrency 1",

packages/api/README.md

@@ -35,7 +35,7 @@ api.beacon
 
 ## Prerequisites
 
-- [Lerna](https://github.com/lerna/lerna)
+- [NodeJS](https://nodejs.org/) (LTS)
 - [Yarn](https://yarnpkg.com/)
 
 ## What you need

packages/api/src/beacon/client/beacon.ts

@@ -1,7 +1,6 @@
 import {ChainForkConfig} from "@lodestar/config";
 import {Api, ReqTypes, routesData, getReqSerializers, getReturnTypes} from "../routes/beacon/index.js";
 import {IHttpClient, generateGenericJsonClient} from "../../utils/client/index.js";
-import {ReturnTypes} from "../../utils/types.js";
 
 /**
  * REST HTTP client for beacon routes
@@ -10,10 +9,5 @@ export function getClient(config: ChainForkConfig, httpClient: IHttpClient): Api
   const reqSerializers = getReqSerializers(config);
   const returnTypes = getReturnTypes();
   // All routes return JSON, use a client auto-generator
-  return generateGenericJsonClient<Api, ReqTypes>(
-    routesData,
-    reqSerializers,
-    returnTypes as ReturnTypes<Api>,
-    httpClient
-  );
+  return generateGenericJsonClient<Api, ReqTypes>(routesData, reqSerializers, returnTypes, httpClient);
 }

packages/api/src/beacon/client/config.ts

@@ -1,6 +1,5 @@
 import {ChainForkConfig} from "@lodestar/config";
 import {generateGenericJsonClient, IHttpClient} from "../../utils/client/index.js";
-import {ReturnTypes} from "../../utils/types.js";
 import {Api, getReqSerializers, getReturnTypes, ReqTypes, routesData} from "../routes/config.js";
 
 /**
@@ -10,10 +9,5 @@ export function getClient(config: ChainForkConfig, httpClient: IHttpClient): Api
   const reqSerializers = getReqSerializers();
   const returnTypes = getReturnTypes();
   // All routes return JSON, use a client auto-generator
-  return generateGenericJsonClient<Api, ReqTypes>(
-    routesData,
-    reqSerializers,
-    returnTypes as ReturnTypes<Api>,
-    httpClient
-  );
+  return generateGenericJsonClient<Api, ReqTypes>(routesData, reqSerializers, returnTypes, httpClient);
 }

packages/api/src/beacon/client/debug.ts

@@ -2,7 +2,6 @@ import {ChainForkConfig} from "@lodestar/config";
 import {ApiClientResponse} from "../../interfaces.js";
 import {HttpStatusCode} from "../../utils/client/httpStatusCode.js";
 import {generateGenericJsonClient, getFetchOptsSerializers, IHttpClient} from "../../utils/client/index.js";
-import {ReturnTypes} from "../../utils/types.js";
 import {StateId} from "../routes/beacon/state.js";
 import {Api, getReqSerializers, getReturnTypes, ReqTypes, routesData, StateFormat} from "../routes/debug.js";
 
@@ -16,12 +15,7 @@ export function getClient(_config: ChainForkConfig, httpClient: IHttpClient): Ap
   const reqSerializers = getReqSerializers();
   const returnTypes = getReturnTypes();
   // Some routes return JSON, use a client auto-generator
-  const client = generateGenericJsonClient<Api, ReqTypes>(
-    routesData,
-    reqSerializers,
-    returnTypes as ReturnTypes<Api>,
-    httpClient
-  );
+  const client = generateGenericJsonClient<Api, ReqTypes>(routesData, reqSerializers, returnTypes, httpClient);
   // For `getState()` generate request serializer
   const fetchOptsSerializers = getFetchOptsSerializers<Api, ReqTypes>(routesData, reqSerializers);
 

packages/api/src/beacon/client/lightclient.ts

@@ -1,7 +1,6 @@
 import {ChainForkConfig} from "@lodestar/config";
 import {Api, ReqTypes, routesData, getReqSerializers, getReturnTypes} from "../routes/lightclient.js";
 import {IHttpClient, generateGenericJsonClient} from "../../utils/client/index.js";
-import {ReturnTypes} from "../../utils/types.js";
 
 /**
  * REST HTTP client for lightclient routes
@@ -10,10 +9,5 @@ export function getClient(_config: ChainForkConfig, httpClient: IHttpClient): Ap
   const reqSerializers = getReqSerializers();
   const returnTypes = getReturnTypes();
   // All routes return JSON, use a client auto-generator
-  return generateGenericJsonClient<Api, ReqTypes>(
-    routesData,
-    reqSerializers,
-    returnTypes as ReturnTypes<Api>,
-    httpClient
-  );
+  return generateGenericJsonClient<Api, ReqTypes>(routesData, reqSerializers, returnTypes, httpClient);
 }

packages/api/src/beacon/client/lodestar.ts

@@ -1,7 +1,6 @@
 import {ChainForkConfig} from "@lodestar/config";
 import {Api, ReqTypes, routesData, getReqSerializers, getReturnTypes} from "../routes/lodestar.js";
 import {IHttpClient, generateGenericJsonClient} from "../../utils/client/index.js";
-import {ReturnTypes} from "../../utils/types.js";
 
 /**
  * REST HTTP client for lodestar routes
@@ -10,10 +9,5 @@ export function getClient(_config: ChainForkConfig, httpClient: IHttpClient): Ap
   const reqSerializers = getReqSerializers();
   const returnTypes = getReturnTypes();
   // All routes return JSON, use a client auto-generator
-  return generateGenericJsonClient<Api, ReqTypes>(
-    routesData,
-    reqSerializers,
-    returnTypes as ReturnTypes<Api>,
-    httpClient
-  );
+  return generateGenericJsonClient<Api, ReqTypes>(routesData, reqSerializers, returnTypes, httpClient);
 }

packages/api/src/beacon/client/validator.ts

@@ -1,7 +1,6 @@
 import {ChainForkConfig} from "@lodestar/config";
 import {Api, ReqTypes, routesData, getReqSerializers, getReturnTypes} from "../routes/validator.js";
 import {IHttpClient, generateGenericJsonClient} from "../../utils/client/index.js";
-import {ReturnTypes} from "../../utils/types.js";
 
 /**
  * REST HTTP client for validator routes
@@ -10,10 +9,5 @@ export function getClient(_config: ChainForkConfig, httpClient: IHttpClient): Ap
   const reqSerializers = getReqSerializers();
   const returnTypes = getReturnTypes();
   // All routes return JSON, use a client auto-generator
-  return generateGenericJsonClient<Api, ReqTypes>(
-    routesData,
-    reqSerializers,
-    returnTypes as ReturnTypes<Api>,
-    httpClient
-  );
+  return generateGenericJsonClient<Api, ReqTypes>(routesData, reqSerializers, returnTypes, httpClient);
 }

packages/api/src/beacon/routes/beacon/block.ts

@@ -239,7 +239,7 @@ export function getReqSerializers(config: ChainForkConfig): ReqSerializers<Api,
     toJson: (data) =>
       isSignedBlockContents(data)
         ? AllForksSignedBlockContentsReqSerializer(getSignedBeaconBlockType).toJson(data)
-        : getSignedBeaconBlockType(data as allForks.SignedBeaconBlock).toJson(data as allForks.SignedBeaconBlock),
+        : getSignedBeaconBlockType(data).toJson(data),
 
     fromJson: (data) =>
       (data as {signed_block: unknown}).signed_block !== undefined
@@ -257,9 +257,7 @@ export function getReqSerializers(config: ChainForkConfig): ReqSerializers<Api,
       toJson: (data) =>
         isSignedBlindedBlockContents(data)
           ? AllForksSignedBlindedBlockContentsReqSerializer(getSignedBlindedBeaconBlockType).toJson(data)
-          : getSignedBlindedBeaconBlockType(data as allForks.SignedBlindedBeaconBlock).toJson(
-              data as allForks.SignedBlindedBeaconBlock
-            ),
+          : getSignedBlindedBeaconBlockType(data).toJson(data),
 
       fromJson: (data) =>
         (data as {signed_blinded_block: unknown}).signed_blinded_block !== undefined

packages/api/src/beacon/routes/events.ts

@@ -199,7 +199,7 @@ export function getTypeByEvent(config: ChainForkConfig): {[K in EventType]: Type
       fromJson: (data) =>
         getLightClientTypeFromHeader(
           // eslint-disable-next-line @typescript-eslint/naming-convention
-          (data as unknown as {attested_header: allForks.LightClientHeader}).attested_header
+          (data as {attested_header: allForks.LightClientHeader}).attested_header
         )["LightClientOptimisticUpdate"].fromJson(data),
     },
     [EventType.lightClientFinalityUpdate]: {
@@ -210,7 +210,7 @@ export function getTypeByEvent(config: ChainForkConfig): {[K in EventType]: Type
       fromJson: (data) =>
         getLightClientTypeFromHeader(
           // eslint-disable-next-line @typescript-eslint/naming-convention
-          (data as unknown as {attested_header: allForks.LightClientHeader}).attested_header
+          (data as {attested_header: allForks.LightClientHeader}).attested_header
         )["LightClientFinalityUpdate"].fromJson(data),
     },
     [EventType.lightClientUpdate]: {
@@ -221,7 +221,7 @@ export function getTypeByEvent(config: ChainForkConfig): {[K in EventType]: Type
       fromJson: (data) =>
         getLightClientTypeFromHeader(
           // eslint-disable-next-line @typescript-eslint/naming-convention
-          (data as unknown as {attested_header: allForks.LightClientHeader}).attested_header
+          (data as {attested_header: allForks.LightClientHeader}).attested_header
         )["LightClientUpdate"].fromJson(data),
     },
   };

packages/api/src/beacon/routes/lodestar.ts

@@ -77,6 +77,11 @@ export type LodestarNodePeer = NodePeer & {
 export type Api = {
   /** Trigger to write a heapdump to disk at `dirpath`. May take > 1min */
   writeHeapdump(dirpath?: string): Promise<ApiClientResponse<{[HttpStatusCode.OK]: {data: {filepath: string}}}>>;
+  /** Trigger to write 10m network thread profile to disk */
+  writeNetworkThreadProfile(
+    duration?: number,
+    dirpath?: string
+  ): Promise<ApiClientResponse<{[HttpStatusCode.OK]: {data: {filepath: string}}}>>;
   /** TODO: description */
   getLatestWeakSubjectivityCheckpointEpoch(): Promise<ApiClientResponse<{[HttpStatusCode.OK]: {data: Epoch}}>>;
   /** TODO: description */
@@ -124,27 +129,29 @@ export type Api = {
  * Define javascript values for each route
  */
 export const routesData: RoutesData<Api> = {
-  writeHeapdump: {url: "/eth/v1/lodestar/writeheapdump", method: "POST"},
+  writeHeapdump: {url: "/eth/v1/lodestar/write_heapdump", method: "POST"},
+  writeNetworkThreadProfile: {url: "/eth/v1/lodestar/write_network_thread_profile", method: "POST"},
   getLatestWeakSubjectivityCheckpointEpoch: {url: "/eth/v1/lodestar/ws_epoch", method: "GET"},
-  getSyncChainsDebugState: {url: "/eth/v1/lodestar/sync-chains-debug-state", method: "GET"},
-  getGossipQueueItems: {url: "/eth/v1/lodestar/gossip-queue-items/:gossipType", method: "GET"},
-  getRegenQueueItems: {url: "/eth/v1/lodestar/regen-queue-items", method: "GET"},
-  getBlockProcessorQueueItems: {url: "/eth/v1/lodestar/block-processor-queue-items", method: "GET"},
-  getStateCacheItems: {url: "/eth/v1/lodestar/state-cache-items", method: "GET"},
-  getGossipPeerScoreStats: {url: "/eth/v1/lodestar/gossip-peer-score-stats", method: "GET"},
-  getLodestarPeerScoreStats: {url: "/eth/v1/lodestar/lodestar-peer-score-stats", method: "GET"},
+  getSyncChainsDebugState: {url: "/eth/v1/lodestar/sync_chains_debug_state", method: "GET"},
+  getGossipQueueItems: {url: "/eth/v1/lodestar/gossip_queue_items/:gossipType", method: "GET"},
+  getRegenQueueItems: {url: "/eth/v1/lodestar/regen_queue_items", method: "GET"},
+  getBlockProcessorQueueItems: {url: "/eth/v1/lodestar/block_processor_queue_items", method: "GET"},
+  getStateCacheItems: {url: "/eth/v1/lodestar/state_cache_items", method: "GET"},
+  getGossipPeerScoreStats: {url: "/eth/v1/lodestar/gossip_peer_score_stats", method: "GET"},
+  getLodestarPeerScoreStats: {url: "/eth/v1/lodestar/lodestar_peer_score_stats", method: "GET"},
   runGC: {url: "/eth/v1/lodestar/gc", method: "POST"},
-  dropStateCache: {url: "/eth/v1/lodestar/drop-state-cache", method: "POST"},
+  dropStateCache: {url: "/eth/v1/lodestar/drop_state_cache", method: "POST"},
   connectPeer: {url: "/eth/v1/lodestar/connect_peer", method: "POST"},
   disconnectPeer: {url: "/eth/v1/lodestar/disconnect_peer", method: "POST"},
   getPeers: {url: "/eth/v1/lodestar/peers", method: "GET"},
-  discv5GetKadValues: {url: "/eth/v1/debug/discv5-kad-values", method: "GET"},
-  dumpDbBucketKeys: {url: "/eth/v1/debug/dump-db-bucket-keys/:bucket", method: "GET"},
-  dumpDbStateIndex: {url: "/eth/v1/debug/dump-db-state-index", method: "GET"},
+  discv5GetKadValues: {url: "/eth/v1/debug/discv5_kad_values", method: "GET"},
+  dumpDbBucketKeys: {url: "/eth/v1/debug/dump_db_bucket_keys/:bucket", method: "GET"},
+  dumpDbStateIndex: {url: "/eth/v1/debug/dump_db_state_index", method: "GET"},
 };
 
 export type ReqTypes = {
   writeHeapdump: {query: {dirpath?: string}};
+  writeNetworkThreadProfile: {query: {duration?: number; dirpath?: string}};
   getLatestWeakSubjectivityCheckpointEpoch: ReqEmpty;
   getSyncChainsDebugState: ReqEmpty;
   getGossipQueueItems: {params: {gossipType: string}};
@@ -170,6 +177,11 @@ export function getReqSerializers(): ReqSerializers<Api, ReqTypes> {
       parseReq: ({query}) => [query.dirpath],
       schema: {query: {dirpath: Schema.String}},
     },
+    writeNetworkThreadProfile: {
+      writeReq: (duration, dirpath) => ({query: {duration, dirpath}}),
+      parseReq: ({query}) => [query.duration, query.dirpath],
+      schema: {query: {dirpath: Schema.String}},
+    },
     getLatestWeakSubjectivityCheckpointEpoch: reqEmpty,
     getSyncChainsDebugState: reqEmpty,
     getGossipQueueItems: {
@@ -212,6 +224,7 @@ export function getReqSerializers(): ReqSerializers<Api, ReqTypes> {
 export function getReturnTypes(): ReturnTypes<Api> {
   return {
     writeHeapdump: sameType(),
+    writeNetworkThreadProfile: sameType(),
     getLatestWeakSubjectivityCheckpointEpoch: sameType(),
     getSyncChainsDebugState: jsonType("snake"),
     getGossipQueueItems: jsonType("snake"),

packages/api/src/builder/client.ts

@@ -1,6 +1,5 @@
 import {ChainForkConfig} from "@lodestar/config";
 import {IHttpClient, generateGenericJsonClient} from "../utils/client/index.js";
-import {ReturnTypes} from "../utils/types.js";
 import {Api, ReqTypes, routesData, getReqSerializers, getReturnTypes} from "./routes.js";
 
 /**
@@ -10,10 +9,5 @@ export function getClient(config: ChainForkConfig, httpClient: IHttpClient): Api
   const reqSerializers = getReqSerializers(config);
   const returnTypes = getReturnTypes();
   // All routes return JSON, use a client auto-generator
-  return generateGenericJsonClient<Api, ReqTypes>(
-    routesData,
-    reqSerializers,
-    returnTypes as ReturnTypes<Api>,
-    httpClient
-  );
+  return generateGenericJsonClient<Api, ReqTypes>(routesData, reqSerializers, returnTypes, httpClient);
 }

packages/api/src/keymanager/client.ts

@@ -1,16 +1,10 @@
 import {ChainForkConfig} from "@lodestar/config";
 import {IHttpClient, generateGenericJsonClient} from "../utils/client/index.js";
-import {ReturnTypes} from "../utils/types.js";
 import {Api, ReqTypes, routesData, getReqSerializers, getReturnTypes} from "./routes.js";
 
 export function getClient(_config: ChainForkConfig, httpClient: IHttpClient): Api {
   const reqSerializers = getReqSerializers();
   const returnTypes = getReturnTypes();
   // All routes return JSON, use a client auto-generator
-  return generateGenericJsonClient<Api, ReqTypes>(
-    routesData,
-    reqSerializers,
-    returnTypes as ReturnTypes<Api>,
-    httpClient
-  );
+  return generateGenericJsonClient<Api, ReqTypes>(routesData, reqSerializers, returnTypes, httpClient);
 }

packages/api/src/utils/serdes.ts

@@ -18,7 +18,7 @@ export function queryParseProofPathsArr(pathStrs: string | string[]): JsonPath[]
   if (Array.isArray(pathStrs)) {
     return pathStrs.map((pathStr) => queryParseProofPaths(pathStr));
   } else {
-    return [queryParseProofPaths(pathStrs) as JsonPath];
+    return [queryParseProofPaths(pathStrs)];
   }
 }
 

packages/api/src/utils/types.ts

@@ -211,7 +211,7 @@ export function jsonType<T extends Record<string, unknown> | Record<string, unkn
 export function sameType<T>(): TypeJson<T> {
   return {
     toJson: (val) => val as unknown,
-    fromJson: (json) => json as unknown as T,
+    fromJson: (json) => json as T,
   };
 }