ops: Add bedrock-devnet script

.circleci/config.yml

@@ -508,6 +508,11 @@ jobs:
     machine:
       image: ubuntu-2004:2022.07.1
       docker_layer_caching: true
+    parameters:
+      deploy:
+        description: Deploy contracts
+        default: false
+        type: boolean
     environment:
       DOCKER_BUILDKIT: 1
     steps:
@@ -539,22 +544,48 @@ jobs:
           command: |
             yarn install
             yarn build
-      - run:
-          name: Bring up the stack
-          command: |
-            make devnet-up
-      - run:
-          name: Deposit ERC20 through the bridge
-          command: timeout 5m npx hardhat deposit-erc20 --network devnetL1
-          working_directory: packages/sdk
-      - run:
-          name: Deposit ETH through the bridge
-          command: timeout 5m npx hardhat deposit-eth --network devnetL1
-          working_directory: packages/sdk
-      - run:
-          name: Check the status
-          command: npx hardhat check-op-node
-          working_directory: packages/contracts-bedrock
+      - when:
+          condition:
+            and:
+              - equal: [ true, <<parameters.deploy>> ]
+          steps:
+            - run:
+                name: Bring up the stack
+                command: |
+                  make devnet-up-deploy
+            - run:
+                name: Deposit ERC20 through the bridge
+                command: timeout 5m npx hardhat deposit-erc20 --network devnetL1 --l1-contracts-json-path ../../.devnet/sdk-addresses.json
+                working_directory: packages/sdk
+            - run:
+                name: Deposit ETH through the bridge
+                command: timeout 5m npx hardhat deposit-eth --network devnetL1 --l1-contracts-json-path ../../.devnet/sdk-addresses.json
+                working_directory: packages/sdk
+            - run:
+                name: Check the status
+                command: npx hardhat check-op-node
+                working_directory: packages/contracts-bedrock
+      - when:
+          condition:
+            and:
+              - equal: [ false, <<parameters.deploy>> ]
+          steps:
+            - run:
+                name: Bring up the stack
+                command: |
+                  make devnet-up
+            - run:
+                name: Deposit ERC20 through the bridge
+                command: timeout 5m npx hardhat deposit-erc20 --network devnetL1
+                working_directory: packages/sdk
+            - run:
+                name: Deposit ETH through the bridge
+                command: timeout 5m npx hardhat deposit-eth --network devnetL1
+                working_directory: packages/sdk
+            - run:
+                name: Check the status
+                command: npx hardhat check-op-node
+                working_directory: packages/contracts-bedrock
 
   integration-tests:
     machine:
@@ -773,7 +804,12 @@ workflows:
       - bedrock-go-tests
       - fuzz-op-node
       - bedrock-markdown
-      - devnet
+      - devnet:
+          name: devnet (with deployed contracts)
+          deploy: true
+      - devnet:
+          name: devnet (with genesis contracts)
+          deploy: false
       - go-lint-test-build:
           name: batch-submitter-tests
           binary_name: batch-submitter

.gitignore

@@ -53,3 +53,6 @@ coverage.out
 # Ignore bedrock go bindings local output files
 op-bindings/bin
 op-exporter
+
+
+__pycache__

Makefile

@@ -68,6 +68,10 @@ devnet-up:
 	@bash ./ops-bedrock/devnet-up.sh
 .PHONY: devnet-up
 
+devnet-up-deploy:
+	PYTHONPATH=./bedrock-devnet python3 ./bedrock-devnet/main.py --monorepo-dir=.
+.PHONY: devnet-up-deploy
+
 devnet-down:
 	@(cd ./ops-bedrock && GENESIS_TIMESTAMP=$(shell date +%s) docker-compose stop)
 .PHONY: devnet-down
@@ -114,4 +118,4 @@ tag-bedrock-go-modules:
 
 update-op-geth:
 	./ops/scripts/update-op-geth.py
-.PHONY: update-op-geth
+.PHONY: update-op-geth

bedrock-devnet/README.md

@@ -0,0 +1,5 @@
+# bedrock-devnet
+
+This is a utility for running a local Bedrock devnet. It is designed to replace the legacy Bash-based devnet runner as part of a progressive migration away from Bash automation.
+
+The easiest way to invoke this script is to run `make devnet-up-deploy` from the root of this repository. Otherwise, to use this script run `python3 main.py --monorepo-path=<path to the monorepo>`. You may need to set `PYTHONPATH` to this directory if you are invoking the script from somewhere other than `bedrock-devnet`.

bedrock-devnet/devnet/__init__.py

@@ -0,0 +1,161 @@
+import argparse
+import logging
+import os
+import subprocess
+import json
+import socket
+
+import time
+
+import shutil
+
+import devnet.log_setup
+from devnet.genesis import GENESIS_TMPL
+
+parser = argparse.ArgumentParser(description='Bedrock devnet launcher')
+parser.add_argument('--monorepo-dir', help='Directory of the monorepo', default=os.getcwd())
+
+log = logging.getLogger()
+
+
+def main():
+    args = parser.parse_args()
+
+    pjoin = os.path.join
+    monorepo_dir = os.path.abspath(args.monorepo_dir)
+    devnet_dir = pjoin(monorepo_dir, '.devnet')
+    ops_bedrock_dir = pjoin(monorepo_dir, 'ops-bedrock')
+    contracts_bedrock_dir = pjoin(monorepo_dir, 'packages', 'contracts-bedrock')
+    deployment_dir = pjoin(contracts_bedrock_dir, 'deployments', 'devnetL1')
+    op_node_dir = pjoin(args.monorepo_dir, 'op-node')
+    genesis_l1_path = pjoin(devnet_dir, 'genesis-l1.json')
+    genesis_l2_path = pjoin(devnet_dir, 'genesis-l2.json')
+    addresses_json_path = pjoin(devnet_dir, 'addresses.json')
+    sdk_addresses_json_path = pjoin(devnet_dir, 'sdk-addresses.json')
+    rollup_config_path = pjoin(devnet_dir, 'rollup.json')
+    os.makedirs(devnet_dir, exist_ok=True)
+
+    if os.path.exists(genesis_l1_path):
+        log.info('L2 genesis already generated.')
+    else:
+        log.info('Generating L1 genesis.')
+        write_json(genesis_l1_path, GENESIS_TMPL)
+
+    log.info('Starting L1.')
+    run_command(['docker-compose', 'up', '-d', 'l1'], cwd=ops_bedrock_dir, env={
+        'PWD': ops_bedrock_dir
+    })
+    wait_up(8545)
+
+    log.info('Generating network config.')
+    devnet_cfg_orig = pjoin(contracts_bedrock_dir, 'deploy-config', 'devnetL1.json')
+    devnet_cfg_backup = pjoin(devnet_dir, 'devnetL1.json.bak')
+    shutil.copy(devnet_cfg_orig, devnet_cfg_backup)
+    deploy_config = read_json(devnet_cfg_orig)
+    deploy_config['l1GenesisBlockTimestamp'] = GENESIS_TMPL['timestamp']
+    deploy_config['l1StartingBlockTag'] = 'earliest'
+    write_json(devnet_cfg_orig, deploy_config)
+
+    if os.path.exists(addresses_json_path):
+        log.info('Contracts already deployed.')
+        addresses = read_json(addresses_json_path)
+    else:
+        log.info('Deploying contracts.')
+        run_command(['yarn', 'hardhat', '--network', 'devnetL1', 'deploy'], env={
+            'CHAIN_ID': '900',
+            'L1_RPC': 'http://localhost:8545',
+            'PRIVATE_KEY_DEPLOYER': 'ac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80'
+        }, cwd=contracts_bedrock_dir)
+        contracts = os.listdir(deployment_dir)
+        addresses = {}
+        for c in contracts:
+            if not c.endswith('.json'):
+                continue
+            data = read_json(pjoin(deployment_dir, c))
+            addresses[c.replace('.json', '')] = data['address']
+        sdk_addresses = {}
+        sdk_addresses.update({
+            'StateCommitmentChain': '0x0000000000000000000000000000000000000000',
+            'CanonicalTransactionChain': '0x0000000000000000000000000000000000000000',
+            'BondManager': '0x0000000000000000000000000000000000000000',
+        })
+        sdk_addresses['AddressManager'] = addresses['AddressManager']
+        sdk_addresses['L1CrossDomainMessenger'] = addresses['L1CrossDomainMessengerProxy']
+        sdk_addresses['L1StandardBridge'] = addresses['L1StandardBridgeProxy']
+        sdk_addresses['OptimismPortal'] = addresses['OptimismPortalProxy']
+        sdk_addresses['L2OutputOracle'] = addresses['L2OutputOracleProxy']
+        write_json(addresses_json_path, addresses)
+        write_json(sdk_addresses_json_path, sdk_addresses)
+
+    if os.path.exists(genesis_l2_path):
+        log.info('L2 genesis and rollup configs already generated.')
+    else:
+        log.info('Generating L2 genesis and rollup configs.')
+        run_command([
+            'go', 'run', 'cmd/main.go', 'genesis', 'l2',
+            '--l1-rpc', 'http://localhost:8545',
+            '--deploy-config', devnet_cfg_orig,
+            '--deployment-dir', deployment_dir,
+            '--outfile.l2', pjoin(devnet_dir, 'genesis-l2.json'),
+            '--outfile.rollup', pjoin(devnet_dir, 'rollup.json')
+        ], cwd=op_node_dir)
+
+    rollup_config = read_json(rollup_config_path)
+
+    if os.path.exists(devnet_cfg_backup):
+        shutil.move(devnet_cfg_backup, devnet_cfg_orig)
+
+    log.info('Bringing up L2.')
+    run_command(['docker-compose', 'up', '-d', 'l2'], cwd=ops_bedrock_dir, env={
+        'PWD': ops_bedrock_dir
+    })
+    wait_up(9545)
+
+    log.info('Bringing up everything else.')
+    run_command(['docker-compose', 'up', '-d', 'op-node', 'op-proposer', 'op-batcher'], cwd=ops_bedrock_dir, env={
+        'PWD': ops_bedrock_dir,
+        'L2OO_ADDRESS': addresses['L2OutputOracleProxy'],
+        'SEQUENCER_GENESIS_HASH': rollup_config['genesis']['l2']['hash'],
+        'SEQUENCER_BATCH_INBOX_ADDRESS': rollup_config['batch_inbox_address']
+    })
+
+    log.info('Devnet ready.')
+
+
+def run_command(args, check=True, shell=False, cwd=None, env=None):
+    env = env if env else {}
+    return subprocess.run(
+        args,
+        check=check,
+        shell=shell,
+        env={
+            **os.environ,
+            **env
+        },
+        cwd=cwd
+    )
+
+
+def wait_up(port, retries=10, wait_secs=1):
+    for i in range(0, retries):
+        log.info(f'Trying 127.0.0.1:{port}')
+        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        try:
+            s.connect(('127.0.0.1', int(port)))
+            s.shutdown(2)
+            log.info(f'Connected 127.0.0.1:{port}')
+            return True
+        except Exception:
+            time.sleep(wait_secs)
+
+    raise Exception(f'Timed out waiting for port {port}.')
+
+
+def write_json(path, data):
+    with open(path, 'w+') as f:
+        json.dump(data, f, indent='  ')
+
+
+def read_json(path):
+    with open(path, 'r') as f:
+        return json.load(f)

bedrock-devnet/devnet/genesis.py

@@ -0,0 +1,55 @@
+import time
+
+DEV_ACCOUNTS = [
+    '3c44cdddb6a900fa2b585dd299e03d12fa4293bc',
+    '70997970c51812dc3a010c7d01b50e0d17dc79c8',
+    'f39fd6e51aad88f6f4ce6ab8827279cfffb92266'
+]
+
+GENESIS_TMPL = {
+    'config': {
+        'chainId': 900,
+        "homesteadBlock": 0,
+        "eip150Block": 0,
+        "eip150Hash": "0x0000000000000000000000000000000000000000000000000000000000000000",
+        "eip155Block": 0,
+        "eip158Block": 0,
+        "byzantiumBlock": 0,
+        "constantinopleBlock": 0,
+        "petersburgBlock": 0,
+        "istanbulBlock": 0,
+        "muirGlacierBlock": 0,
+        "berlinBlock": 0,
+        "londonBlock": 0,
+        "arrowGlacierBlock": 0,
+        "grayGlacierBlock": 0,
+        "shanghaiBlock": None,
+        "cancunBlock": None,
+        'clique': {
+            'period': 15,
+            'epoch': 30000
+        }
+    },
+    'nonce': '0x0',
+    'timestamp': '{:#x}'.format(int(time.time())),
+    'extraData': '0x0000000000000000000000000000000000000000000000000000000000000000ca062b0fd91172d89bcd4bb084ac4e21972cc4670000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000',
+    'gasLimit': '0xE4E1C0',
+    'difficulty': '0x1',
+    'mixHash': '0x0000000000000000000000000000000000000000000000000000000000000000',
+    'coinbase': '0x0000000000000000000000000000000000000000',
+    'alloc': {
+        '{:x}'.format(i).ljust(40, '0'): {
+            'balance': '0x1'
+        } for i in range(0, 255)
+    },
+    'number': '0x0',
+    'gasUsed': '0x0',
+    'parentHash': '0x0000000000000000000000000000000000000000000000000000000000000000',
+    'baseFeePergas': '0x3B9ACA00'
+}
+
+GENESIS_TMPL['alloc'].update({
+    d: {
+        'balance': '0x200000000000000000000000000000000000000000000000000000000000000'
+    } for d in DEV_ACCOUNTS
+})

bedrock-devnet/devnet/log_setup.py

@@ -0,0 +1,30 @@
+import os
+
+from logging.config import dictConfig
+
+log_level = os.getenv('LOG_LEVEL')
+
+log_config = {
+    'version': 1,
+    'loggers': {
+        '': {
+            'handlers': ['console'],
+            'level': log_level if log_level is not None else 'INFO'
+        },
+    },
+    'handlers': {
+        'console': {
+            'formatter': 'stderr',
+            'class': 'logging.StreamHandler',
+            'stream': 'ext://sys.stdout'
+        }
+    },
+    'formatters': {
+        'stderr': {
+            'format': '[%(levelname)s|%(asctime)s] %(message)s',
+            'datefmt': '%m-%d-%Y %I:%M:%S'
+        }
+    },
+}
+
+dictConfig(log_config)

bedrock-devnet/main.py

@@ -0,0 +1,9 @@
+import devnet
+
+
+def main():
+    devnet.main()
+
+
+if __name__ == '__main__':
+    main()