feat(cli): forge bulk upload ecs state script (#142)

* feat: wip forge bulkupload script

* feat: wip 2 forge bulkupload script

* feat: wip 3 forge bulkupload script

* feat: wip 4 forge bulkupload script

* feat: support prototype dev system in ecs map

* fix: use entity index to copy prototype components

* feat(std-contracts): bulk upload script using BulkSetState system

* feat(solecs): add getIndex function Set

* feat(cli): only send needed entities in BulkSetState transactions

* fix(cli): remove test ecs map

* fix(std-client): never error in getGameConfig

* feat(cli): add mud bulkupload script

* fix(std-client): explicit null check

Co-authored-by: alvarius <89248902+alvrs@users.noreply.github.com>

Co-authored-by: alvrs <alvarius@lattice.xyz>
Co-authored-by: alvarius <89248902+alvrs@users.noreply.github.com>

packages/cli/.gitignore

@@ -1 +1,4 @@
-dist
+dist
+out
+broadcast
+cache

packages/cli/.npmignore

@@ -1,5 +1,6 @@
 *
 
 !dist/**
+!src/**
 !package.json
 !README.md

packages/cli/foundry.toml

@@ -0,0 +1,9 @@
+[profile.default]
+ffi = false
+fuzz_runs = 256
+optimizer = true
+optimizer_runs = 1000000
+verbosity = 1
+libs = ["../../node_modules", "../solecs", "../std-contracts"]
+src = "src"
+out = "out"

packages/cli/git-install.sh

@@ -0,0 +1,5 @@
+#! usr/bin/bash
+giturl=https://github.com/$1.git
+head=$(git ls-remote $giturl HEAD | head -n1 | awk '{print $1;}')
+yarn add $giturl#$head
+echo "Installed $giturl#$head"

packages/cli/package.json

@@ -13,11 +13,12 @@
     "directory": "packages/cli"
   },
   "scripts": {
-    "prepare": "yarn build",
-    "globalinstall": "yarn build && npm link",
+    "prepare": "yarn build && chmod u+x git-install.sh",
+    "globalinstall": "yarn prepare && npm link",
     "build": "rimraf dist && tsc -p . && chmod u+x dist/index.js",
     "link": "yarn link",
-    "test": "echo 'todo: add tests'"
+    "test": "echo 'todo: add tests'",
+    "git:install": "bash git-install.sh"
   },
   "devDependencies": {
     "@types/clear": "^0.1.2",
@@ -36,13 +37,17 @@
   "dependencies": {
     "@latticexyz/solecs": "^0.11.1",
     "@latticexyz/utils": "^0.11.1",
+    "@latticexyz/std-contracts": "^0.11.1",
+
     "chalk": "^5.0.1",
     "clear": "^0.1.0",
     "commander": "^9.2.0",
+    "ds-test": "https://github.com/dapphub/ds-test.git#9310e879db8ba3ea6d5c6489a579118fd264a3f5",
     "esm": "^3.2.25",
     "ethers": "^5.6.7",
     "execa": "^6.1.0",
     "figlet": "^1.5.2",
+    "forge-std": "https://github.com/foundry-rs/forge-std.git#6b4ca42943f093642bac31783b08aa52a5a6ff64",
     "glob": "^8.0.3",
     "inquirer": "^8.2.4",
     "inquirer-prompt-suggest": "^0.1.0",
@@ -51,6 +56,7 @@
     "node-fetch": "^3.2.6",
     "openurl": "^1.1.1",
     "path": "^0.12.7",
+    "solmate": "https://github.com/Rari-Capital/solmate.git#9cf1428245074e39090dceacb0c28b1f684f584c",
     "typechain": "^8.1.0",
     "uuid": "^8.3.2",
     "yargs": "^17.5.1"

packages/cli/remappings.txt

@@ -0,0 +1,5 @@
+ds-test/=../../node_modules/ds-test/src/
+forge-std/=../../node_modules/forge-std/src/
+solmate/=../../node_modules/@rari-capital/solmate/src
+std-contracts/=../../node_modules/@latticexyz/std-contracts/src/
+solecs/=../../node_modules/@latticexyz/solecs/src/

packages/cli/src/commands/bulkupload.ts

@@ -0,0 +1,44 @@
+import { Arguments, CommandBuilder } from "yargs";
+
+const importExeca = eval('import("execa")') as Promise<typeof import("execa")>;
+
+type Options = {
+  statePath: string;
+  worldAddress: string;
+  rpc: string;
+};
+
+export const command = "bulkupload";
+export const desc = "Uploads the provided ECS state to the provided World";
+
+export const builder: CommandBuilder<Options, Options> = (yargs) =>
+  yargs.options({
+    statePath: { type: "string", demandOption: true, desc: "Path to the ECS state to upload" },
+    worldAddress: { type: "string", demandOption: true, desc: "Contract address of the World to upload to" },
+    rpc: { type: "string", demandOption: true, desc: "JSON RPC endpoint" },
+  });
+
+export const handler = async (argv: Arguments<Options>): Promise<void> => {
+  const { execa } = await importExeca;
+  const { statePath, worldAddress, rpc } = argv;
+  console.log("Uploading state at ", statePath, "to", worldAddress, "on", rpc);
+  const url = __dirname + "/../../src/contracts/BulkUpload.sol";
+  console.log("Using BulkUpload script from", url);
+
+  try {
+    await execa("forge", [
+      "script",
+      "--sig",
+      '"run(string, address)"',
+      "--rpc-url",
+      rpc,
+      `${url}:BulkUpload`,
+      statePath,
+      worldAddress,
+    ]);
+  } catch (e) {
+    console.error(e);
+  }
+
+  process.exit(0);
+};

packages/cli/src/contracts/BulkUpload.sol

@@ -0,0 +1,183 @@
+// SPDX-License-Identifier: Unlicense
+pragma solidity >=0.8.0;
+
+import "ds-test/test.sol";
+import {console} from "forge-std/console.sol";
+import {Cheats} from "./Cheats.sol";
+import {BulkSetStateSystem, ID as BulkSetStateSystemID, ECSEvent} from "std-contracts/systems/BulkSetStateSystem.sol";
+import {World} from "solecs/World.sol";
+import {System} from "solecs/System.sol";
+import {getAddressById} from "solecs/utils.sol";
+import {Set} from "solecs/Set.sol";
+
+struct ParsedState {
+  string[] componentIds;
+  string[] entities;
+  ParsedECSEvent[] state;
+}
+
+struct ParsedECSEvent {
+  uint8 component;
+  uint32 entity;
+  string value;
+}
+
+struct State {
+  uint256[] componentIds;
+  uint256[] entities;
+  ECSEvent[] state;
+}
+
+/**
+ * Usage:
+ * forge script --sig "run(string, address)" --rpc-url http://localhost:8545 src/contracts/BulkUpload.sol:BulkUpload path/to/ecs-map-test.json <WORLD_ADDRESS>
+ */
+contract BulkUpload is DSTest {
+  Cheats internal immutable vm = Cheats(HEVM_ADDRESS);
+
+  function run(
+    string memory path,
+    address worldAddress,
+    uint256 eventsPerTx
+  ) public {
+    vm.startBroadcast();
+
+    // Read JSON
+    console.log(path);
+    string memory json = vm.readFile(path);
+
+    console.log(worldAddress);
+
+    // Parse JSON
+    ParsedState memory parsedState = abi.decode(vm.parseJson(json), (ParsedState));
+
+    // Convert component ids
+    uint256[] memory componentIds = new uint256[](parsedState.componentIds.length);
+    for (uint256 i; i < componentIds.length; i++) {
+      componentIds[i] = hexToUint256(parsedState.componentIds[i]);
+    }
+
+    // Convert entitiy ids
+    uint256[] memory entities = new uint256[](parsedState.entities.length);
+    for (uint256 i; i < entities.length; i++) {
+      entities[i] = hexToUint256(parsedState.entities[i]);
+    }
+    World world = World(worldAddress);
+    System bulkSetStateSystem = System(getAddressById(world.systems(), BulkSetStateSystemID));
+
+    // Convert state
+    ECSEvent[] memory allEvents = new ECSEvent[](parsedState.state.length);
+    for (uint256 i; i < parsedState.state.length; i++) {
+      ParsedECSEvent memory p = parsedState.state[i];
+
+      // Convert value hex string to bytes
+      bytes memory value = hexToBytes(substring(p.value, 2, bytes(p.value).length));
+      allEvents[i] = ECSEvent(p.component, p.entity, value);
+    }
+
+    uint256 numTxs = allEvents.length / eventsPerTx;
+
+    for (uint256 i = 0; i < numTxs; i++) {
+      ECSEvent[] memory events = new ECSEvent[](eventsPerTx);
+      for (uint256 j = 0; j < eventsPerTx; j++) {
+        events[j] = allEvents[i * eventsPerTx + j];
+      }
+
+      (uint256[] memory newEntities, ECSEvent[] memory newEvents) = transformEventsToOnlyUseNeededEntities(
+        entities,
+        events
+      );
+
+      bulkSetStateSystem.execute(abi.encode(componentIds, newEntities, newEvents));
+    }
+
+    // overflow tx
+    uint256 overflowEventCount = allEvents.length - numTxs * eventsPerTx;
+
+    ECSEvent[] memory overflowEvents = new ECSEvent[](overflowEventCount);
+    for (uint256 j = 0; j < overflowEventCount; j++) {
+      overflowEvents[j] = allEvents[numTxs * eventsPerTx + j];
+    }
+
+    (
+      uint256[] memory newOverflowEntities,
+      ECSEvent[] memory newOverflowEvents
+    ) = transformEventsToOnlyUseNeededEntities(entities, overflowEvents);
+
+    bulkSetStateSystem.execute(abi.encode(componentIds, newOverflowEntities, newOverflowEvents));
+
+    vm.stopBroadcast();
+  }
+}
+
+function transformEventsToOnlyUseNeededEntities(uint256[] memory entities, ECSEvent[] memory events)
+  returns (uint256[] memory, ECSEvent[] memory)
+{
+  Set uniqueEntityIndices = new Set();
+
+  // Find unique entity indices
+  for (uint256 i = 0; i < events.length; i++) {
+    ECSEvent memory e = events[i];
+
+    uniqueEntityIndices.add(e.entity);
+  }
+
+  // Grab the Entity IDs from the big entities array and put them into our new array
+  uint256[] memory relevantEntities = new uint256[](uniqueEntityIndices.size());
+  for (uint256 i = 0; i < uniqueEntityIndices.size(); i++) {
+    relevantEntities[i] = entities[uniqueEntityIndices.getItems()[i]];
+  }
+
+  // Re-assign event entity indices to point to our new array
+  for (uint256 i = 0; i < events.length; i++) {
+    (, uint256 index) = uniqueEntityIndices.getIndex(events[i].entity);
+    events[i].entity = uint32(index);
+  }
+
+  return (relevantEntities, events);
+}
+
+function hexToUint8(bytes1 b) pure returns (uint8 res) {
+  if (b >= "0" && b <= "9") {
+    return uint8(b) - uint8(bytes1("0"));
+  } else if (b >= "A" && b <= "F") {
+    return 10 + uint8(b) - uint8(bytes1("A"));
+  } else if (b >= "a" && b <= "f") {
+    return 10 + uint8(b) - uint8(bytes1("a"));
+  }
+  return uint8(b); // or return error ...
+}
+
+function hexToUint256(string memory str) pure returns (uint256 value) {
+  bytes memory b = bytes(str);
+  uint256 number = 0;
+  for (uint256 i = 0; i < b.length; i++) {
+    number = number << 4; // or number = number * 16
+    number |= hexToUint8(b[i]); // or number += numberFromAscII(b[i]);
+  }
+  return number;
+}
+
+// Convert an hexadecimal string to raw bytes
+function hexToBytes(string memory s) pure returns (bytes memory) {
+  bytes memory ss = bytes(s);
+  require(ss.length % 2 == 0); // length must be even
+  bytes memory r = new bytes(ss.length / 2);
+  for (uint256 i = 0; i < ss.length / 2; ++i) {
+    r[i] = bytes1(hexToUint8(ss[2 * i]) * 16 + hexToUint8(ss[2 * i + 1]));
+  }
+  return r;
+}
+
+function substring(
+  string memory str,
+  uint256 start,
+  uint256 end
+) pure returns (string memory) {
+  bytes memory strBytes = bytes(str);
+  bytes memory result = new bytes(end - start);
+  for (uint256 i = start; i < end; i++) {
+    result[i - start] = strBytes[i];
+  }
+  return string(result);
+}