From be86ef762a4bc450f6b2ac87204ffddba2dd61f3 Mon Sep 17 00:00:00 2001 From: Tobi Demeco Date: Tue, 8 Oct 2024 21:29:16 -0300 Subject: [PATCH] test: :test_tube: fix batch file confirm storing test --- client/blockchain-service/src/utils.rs | 1 + test/package.json | 167 +++++++++--------- test/suites/integration/bsp/volunteer.test.ts | 39 +++- 3 files changed, 119 insertions(+), 88 deletions(-) diff --git a/client/blockchain-service/src/utils.rs b/client/blockchain-service/src/utils.rs index 68a1eb1c4..268ccdc1b 100644 --- a/client/blockchain-service/src/utils.rs +++ b/client/blockchain-service/src/utils.rs @@ -544,6 +544,7 @@ impl BlockchainService { .pending_confirm_storing_request_deque() .pop_front() { + trace!(target: LOG_TARGET, "Processing confirm storing request for file [{:?}]", request.file_key); confirm_storing_requests.push(request); } else { break; diff --git a/test/package.json b/test/package.json index 869302179..efd374f62 100644 --- a/test/package.json +++ b/test/package.json @@ -1,85 +1,86 @@ { - "name": "@storagehub/test", - "version": "0.1.0", - "description": "Test package for storagehub", - "main": "index.js", - "type": "module", - "engines": { - "node": "22.x.x" - }, - "scripts": { - "bundle-types": "cd ../types-bundle && pnpm i && pnpm build && pnpm fmt:fix && cd ../test", - "typegen": "pnpm bundle-types && cd ../api-augment && pnpm scrape && pnpm generate:all && pnpm build && pnpm fmt:fix; cd ../test", - "fmt": "biome format .", - "fmt:fix": "biome format . --write", - "lint": "biome lint .", - "typecheck": "tsc --noEmit", - "crossbuild:mac": "DOCKER_DEFAULT_PLATFORM=linux/amd64 pnpm tsx scripts/crossBuildMac.ts", - "docker:build": "DOCKER_DEFAULT_PLATFORM=linux/amd64 pnpm tsx scripts/buildLocalDocker.ts", - "docker:start": "docker compose -f ../docker/local-node-compose.yml -p sh_dev_node up -d", - "docker:start:latest": "docker compose -f ../docker/latest-node-compose.yml -p sh_dev_node up -d", - "docker:stop": "docker compose -f ../docker/local-node-compose.yml -p sh_dev_node down", - "docker:stop:latest": "docker compose -f ../docker/latest-node-compose.yml -p sh_dev_node down", - "docker:start:bspnet": "NOISY=0 ROCKSDB=0 pnpm tsx scripts/bspNetBootstrap.ts", - "docker:stop:bspnet": "docker compose -f ../docker/local-dev-bsp-compose.yml down --remove-orphans && docker volume prune -f", - "docker:start:fullNet": "NOISY=0 ROCKSDB=0 pnpm tsx scripts/fullNetBootstrap.ts", - "docker:stop:fullNet": "docker compose -f ../docker/local-dev-full-compose.yml down --remove-orphans && docker volume prune -f", - "docker:start:bspnet:rocksdb": "NOISY=0 ROCKSDB=1 pnpm tsx scripts/bspNetBootstrap.ts", - "docker:stop:bspnet:rocksdb": "docker compose -f ../docker/local-dev-bsp-rocksdb-compose.yml down --remove-orphans && docker volume prune -f", - "docker:start:fullNet:rocksdb": "NOISY=0 ROCKSDB=1 pnpm tsx scripts/fullNetBootstrap.ts", - "docker:stop:fullNet:rocksdb": "docker compose -f ../docker/local-dev-full-rocksdb-compose.yml down --remove-orphans && docker volume prune -f", - "docker:start:noisynet": "NOISY=1 ROCKSDB=0 pnpm tsx scripts/bspNetBootstrap.ts", - "docker:stop:noisynet": "docker compose -f ../docker/noisy-bsp-compose.yml down --remove-orphans && docker volume prune -f", - "zombie:run:latest": "SH_IMAGE=docker.io/moonsonglabs/storage-hub:latest pnpm zombienet spawn configs/simple.toml", - "zombie:run:local": "DOCKER_BUILDKIT=0 SH_IMAGE=storage-hub:local pnpm zombienet spawn configs/simple.toml", - "zombie:run:native": "pnpm tsx scripts/downloadPolkadot.ts 1.9.0 && pnpm zombienet spawn configs/simpleNative.toml", - "zombie:run:full": "SH_IMAGE=docker.io/moonsonglabs/storage-hub:latest pnpm zombienet spawn configs/fullNetwork.toml", - "zombie:run:full:native": "pnpm tsx scripts/downloadPolkadot.ts 1.9.0 && pnpm zombienet spawn configs/fullNetworkNative.toml", - "zombie:setup:native": "pnpm tsx scripts/fullNetworkSetup.ts", - "zombie:test:native": "pnpm tsx scripts/downloadPolkadot.ts 1.9.0 && pnpm zombienet test --provider native configs/simpleNative.zndsl", - "zombie:test:local": "SH_IMAGE=storage-hub:local pnpm zombienet test configs/simple.zndsl", - "zombie:test:latest": "SH_IMAGE=docker.io/moonsonglabs/storage-hub:latest pnpm zombienet test configs/simple.zndsl", - "test:full": "node --no-deprecation --import tsx --test ./suites/zombie/**.spec.ts", - "test:bspnet": "NODE_OPTIONS='--no-deprecation' pnpm tsx scripts/checkRunning.ts && node --no-deprecation --test-concurrency 1 --import tsx --test ./suites/integration/bsp/**.test.ts", - "test:bspnet:only": "NODE_OPTIONS='--no-deprecation' pnpm tsx scripts/checkRunning.ts && node --no-deprecation --import tsx --test-concurrency 1 --test --test-only ./suites/integration/bsp/**.test.ts", - "test:user": "NODE_OPTIONS='--no-deprecation' pnpm tsx scripts/checkRunning.ts && node --no-deprecation --test-concurrency 1 --import tsx --test ./suites/integration/user/**.test.ts", - "test:user:only": "NODE_OPTIONS='--no-deprecation' pnpm tsx scripts/checkRunning.ts && node --no-deprecation --test-concurrency 1 --import tsx --test --test-only ./suites/integration/user/**.test.ts", - "test:node": "node --no-deprecation --import tsx --test ./suites/solo-node/**/**.test.ts", - "test:node:only": "node --no-deprecation --import tsx --test --test-only ./suites/solo-node/**/**.test.ts", - "test:node:single": "node --no-deprecation --import tsx --test --test-name-pattern=$FILTER ./suites/solo-node/**/**.test.ts" - }, - "keywords": [], - "author": "", - "license": "ISC", - "dependencies": { - "@polkadot/api": "12.4.2", - "@polkadot/api-augment": "12.4.2", - "@polkadot/keyring": "13.0.2", - "@polkadot/types": "12.4.2", - "@polkadot/util": "13.0.2", - "@polkadot/util-crypto": "13.0.2", - "@polkadot/wasm-crypto": "7.3.2", - "@reporters/github": "1.7.0", - "@storagehub/api-augment": "workspace:*", - "@storagehub/types-bundle": "workspace:*", - "@zombienet/cli": "1.3.109", - "@zombienet/utils": "0.0.25", - "docker-compose": "0.24.8", - "dockerode": "4.0.2", - "dotenv": "16.4.5", - "inquirer": "10.2.0", - "json-bigint": "^1.0.0", - "testcontainers": "10.13.0", - "tiny-invariant": "^1.3.3", - "tsx": "4.19.0", - "yaml": "^2.5.1" - }, - "devDependencies": { - "@biomejs/biome": "1.8.3", - "@types/dockerode": "3.3.31", - "@types/inquirer": "9.0.7", - "@types/json-bigint": "1.0.4", - "@types/node": "22.5.2", - "typescript": "5.5.4" - } + "name": "@storagehub/test", + "version": "0.1.0", + "description": "Test package for storagehub", + "main": "index.js", + "type": "module", + "engines": { + "node": "22.x.x" + }, + "scripts": { + "bundle-types": "cd ../types-bundle && pnpm i && pnpm build && pnpm fmt:fix && cd ../test", + "typegen": "pnpm bundle-types && cd ../api-augment && pnpm scrape && pnpm generate:all && pnpm build && pnpm fmt:fix; cd ../test", + "fmt": "biome format .", + "fmt:fix": "biome format . --write", + "lint": "biome lint .", + "typecheck": "tsc --noEmit", + "crossbuild:mac": "DOCKER_DEFAULT_PLATFORM=linux/amd64 pnpm tsx scripts/crossBuildMac.ts", + "docker:build": "DOCKER_DEFAULT_PLATFORM=linux/amd64 pnpm tsx scripts/buildLocalDocker.ts", + "docker:start": "docker compose -f ../docker/local-node-compose.yml -p sh_dev_node up -d", + "docker:start:latest": "docker compose -f ../docker/latest-node-compose.yml -p sh_dev_node up -d", + "docker:stop": "docker compose -f ../docker/local-node-compose.yml -p sh_dev_node down", + "docker:stop:latest": "docker compose -f ../docker/latest-node-compose.yml -p sh_dev_node down", + "docker:start:bspnet": "NOISY=0 ROCKSDB=0 pnpm tsx scripts/bspNetBootstrap.ts", + "docker:stop:bspnet": "docker compose -f ../docker/local-dev-bsp-compose.yml down --remove-orphans && docker volume prune -f", + "docker:start:fullNet": "NOISY=0 ROCKSDB=0 pnpm tsx scripts/fullNetBootstrap.ts", + "docker:stop:fullNet": "docker compose -f ../docker/local-dev-full-compose.yml down --remove-orphans && docker volume prune -f", + "docker:start:bspnet:rocksdb": "NOISY=0 ROCKSDB=1 pnpm tsx scripts/bspNetBootstrap.ts", + "docker:stop:bspnet:rocksdb": "docker compose -f ../docker/local-dev-bsp-rocksdb-compose.yml down --remove-orphans && docker volume prune -f", + "docker:start:fullNet:rocksdb": "NOISY=0 ROCKSDB=1 pnpm tsx scripts/fullNetBootstrap.ts", + "docker:stop:fullNet:rocksdb": "docker compose -f ../docker/local-dev-full-rocksdb-compose.yml down --remove-orphans && docker volume prune -f", + "docker:start:noisynet": "NOISY=1 ROCKSDB=0 pnpm tsx scripts/bspNetBootstrap.ts", + "docker:stop:noisynet": "docker compose -f ../docker/noisy-bsp-compose.yml down --remove-orphans && docker volume prune -f", + "zombie:run:latest": "SH_IMAGE=docker.io/moonsonglabs/storage-hub:latest pnpm zombienet spawn configs/simple.toml", + "zombie:run:local": "DOCKER_BUILDKIT=0 SH_IMAGE=storage-hub:local pnpm zombienet spawn configs/simple.toml", + "zombie:run:native": "pnpm tsx scripts/downloadPolkadot.ts 1.9.0 && pnpm zombienet spawn configs/simpleNative.toml", + "zombie:run:full": "SH_IMAGE=docker.io/moonsonglabs/storage-hub:latest pnpm zombienet spawn configs/fullNetwork.toml", + "zombie:run:full:native": "pnpm tsx scripts/downloadPolkadot.ts 1.9.0 && pnpm zombienet spawn configs/fullNetworkNative.toml", + "zombie:setup:native": "pnpm tsx scripts/fullNetworkSetup.ts", + "zombie:test:native": "pnpm tsx scripts/downloadPolkadot.ts 1.9.0 && pnpm zombienet test --provider native configs/simpleNative.zndsl", + "zombie:test:local": "SH_IMAGE=storage-hub:local pnpm zombienet test configs/simple.zndsl", + "zombie:test:latest": "SH_IMAGE=docker.io/moonsonglabs/storage-hub:latest pnpm zombienet test configs/simple.zndsl", + "test:full": "node --no-deprecation --import tsx --test ./suites/zombie/**.spec.ts", + "test:bspnet:vol": "NODE_OPTIONS='--no-deprecation' pnpm tsx scripts/checkRunning.ts && node --no-deprecation --test-concurrency 1 --import tsx --test ./suites/integration/bsp/volunteer.test.ts", + "test:bspnet": "NODE_OPTIONS='--no-deprecation' pnpm tsx scripts/checkRunning.ts && node --no-deprecation --test-concurrency 1 --import tsx --test ./suites/integration/bsp/**.test.ts", + "test:bspnet:only": "NODE_OPTIONS='--no-deprecation' pnpm tsx scripts/checkRunning.ts && node --no-deprecation --import tsx --test-concurrency 1 --test --test-only ./suites/integration/bsp/**.test.ts", + "test:user": "NODE_OPTIONS='--no-deprecation' pnpm tsx scripts/checkRunning.ts && node --no-deprecation --test-concurrency 1 --import tsx --test ./suites/integration/user/**.test.ts", + "test:user:only": "NODE_OPTIONS='--no-deprecation' pnpm tsx scripts/checkRunning.ts && node --no-deprecation --test-concurrency 1 --import tsx --test --test-only ./suites/integration/user/**.test.ts", + "test:node": "node --no-deprecation --import tsx --test ./suites/solo-node/**/**.test.ts", + "test:node:only": "node --no-deprecation --import tsx --test --test-only ./suites/solo-node/**/**.test.ts", + "test:node:single": "node --no-deprecation --import tsx --test --test-name-pattern=$FILTER ./suites/solo-node/**/**.test.ts" + }, + "keywords": [], + "author": "", + "license": "ISC", + "dependencies": { + "@polkadot/api": "12.4.2", + "@polkadot/api-augment": "12.4.2", + "@polkadot/keyring": "13.0.2", + "@polkadot/types": "12.4.2", + "@polkadot/util": "13.0.2", + "@polkadot/util-crypto": "13.0.2", + "@polkadot/wasm-crypto": "7.3.2", + "@reporters/github": "1.7.0", + "@storagehub/api-augment": "workspace:*", + "@storagehub/types-bundle": "workspace:*", + "@zombienet/cli": "1.3.109", + "@zombienet/utils": "0.0.25", + "docker-compose": "0.24.8", + "dockerode": "4.0.2", + "dotenv": "16.4.5", + "inquirer": "10.2.0", + "json-bigint": "^1.0.0", + "testcontainers": "10.13.0", + "tiny-invariant": "^1.3.3", + "tsx": "4.19.0", + "yaml": "^2.5.1" + }, + "devDependencies": { + "@biomejs/biome": "1.8.3", + "@types/dockerode": "3.3.31", + "@types/inquirer": "9.0.7", + "@types/json-bigint": "1.0.4", + "@types/node": "22.5.2", + "typescript": "5.5.4" + } } \ No newline at end of file diff --git a/test/suites/integration/bsp/volunteer.test.ts b/test/suites/integration/bsp/volunteer.test.ts index b0fde9a6d..588ae7442 100644 --- a/test/suites/integration/bsp/volunteer.test.ts +++ b/test/suites/integration/bsp/volunteer.test.ts @@ -1,5 +1,12 @@ import assert, { notEqual, strictEqual } from "node:assert"; -import { describeBspNet, shUser, sleep, type EnrichedBspApi } from "../../../util"; +import { + assertEventPresent, + describeBspNet, + shUser, + sleep, + type EnrichedBspApi +} from "../../../util"; +import invariant from "tiny-invariant"; describeBspNet("Single BSP Volunteering", ({ before, createBspApi, it, createUserApi }) => { let userApi: EnrichedBspApi; @@ -233,10 +240,34 @@ describeBspNet("Multiple BSPs volunteer ", ({ before, createBspApi, createUserAp await userApi.sealBlock(txs, shUser); + // Get the new storage request events, making sure we have 3 + const storageRequestEvents = await userApi.assert.eventMany("fileSystem", "NewStorageRequest"); + strictEqual(storageRequestEvents.length, 3); + + // Get the file keys from the storage request events + const fileKeys = storageRequestEvents.map((event) => { + const dataBlob = + userApi.events.fileSystem.NewStorageRequest.is(event.event) && event.event.data; + if (!dataBlob) { + throw new Error("Event doesn't match Type"); + } + return dataBlob.fileKey; + }); + // Wait for the BSP to volunteer await userApi.wait.bspVolunteer(source.length); - // Wait for the BSP to download the files and send a confirm transaction + // Wait for the BSP to receive and store all files + for (let i = 0; i < source.length; i++) { + const fileKey = fileKeys[i]; + await bspApi.assert.log({ + searchString: `File upload complete (${fileKey})`, + containerName: "docker-sh-bsp-1", + timeout: 1000 + }); + } + + // Wait and seal a block confirming the storage of the first file await userApi.wait.bspStored(1); const [ @@ -256,11 +287,9 @@ describeBspNet("Multiple BSPs volunteer ", ({ before, createBspApi, createUserAp const bspForestRootAfterConfirm = await bspApi.rpc.storagehubclient.getForestRoot(null); strictEqual(bspForestRootAfterConfirm.toString(), bspConfirmRes_newRoot.toString()); - // This block should trigger the next file to be confirmed. - await userApi.sealBlock(); - // Even though we didn't sent a new file, the BSP client should process the rest of the files. // We wait for the BSP to send the confirm transaction. + await userApi.sealBlock(); await userApi.wait.bspStored(1); const [