Merge remote-tracking branch 'origin/master' into perf-nim

.github/workflows/add-new-impl-versions.yml

@@ -0,0 +1,71 @@
+name: Add new implementation versions
+
+on:
+  workflow_dispatch:
+  schedule:
+    # “At minute 36 past every 24th hour.” - https://crontab.guru/#36_*/24_*_*_*
+    - cron:  '36 */24 * * *'
+
+defaults:
+  run:
+    shell: bash
+
+permissions:
+  contents: write
+  pull-requests: write
+  actions: write
+
+jobs:
+  go:
+    runs-on: ubuntu-latest
+    env:
+      DIR: perf/impl/go-libp2p
+      REPO: libp2p/go-libp2p
+      BRANCH: perf/go-libp2p
+    steps:
+      - name: Checkout test-plans
+        uses: actions/checkout@v3
+      - name: Configure git
+        run: |
+          git fetch origin $BRANCH && git checkout $BRANCH || git checkout -b $BRANCH
+          git rebase $GITHUB_REF -X theirs || git rebase --abort
+          git config --global user.email $GITHUB_ACTOR@users.noreply.github.com
+          git config --global user.name $GITHUB_ACTOR
+      - id: go
+        uses: actions/setup-go@v3
+        with:
+          go-version: stable
+      - name: Get the latest version (local)
+        id: local
+        run: ls -d v* | sort -V | tail -n-1 | xargs -I{} echo "version={}" | tee -a $GITHUB_OUTPUT
+        working-directory: ${{ env.DIR }}
+      - name: Get the latest version (remote)
+        id: remote
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        run: gh api repos/$REPO/releases/latest --jq '.tag_name' | xargs -I{} echo "version={}" | tee -a $GITHUB_OUTPUT
+      - name: Add the latest version
+        if: ${{ !startsWith(steps.remote.outputs.version, steps.local.outputs.version) }}
+        env:
+          LOCAL_VERSION: ${{ steps.local.outputs.version }}
+          REMOTE_VERSION: ${{ steps.remote.outputs.version }}
+          GO_VERSION: ${{ steps.go.outputs.go-version }}
+          GITHUB_TOKEN: ${{ github.token }}
+        run: |
+          majorMinorRemoteVersion=$(echo $REMOTE_VERSION | sed 's/\.[0-9]*$//')
+          majorMinorGoVersion=$(echo $GO_VERSION | sed 's/\.[0-9]*$//')
+          cp -r $LOCAL_VERSION $majorMinorRemoteVersion
+          cd $majorMinorRemoteVersion
+          sed -i "1s/$LOCAL_VERSION/$majorMinorRemoteVersion/g" go.mod
+          go mod tidy -go=$majorMinorGoVersion
+          go mod tidy
+          go get github.com/libp2p/go-libp2p@$REMOTE_VERSION
+          git add .
+          git commit -m "chore: add go-libp2p@$REMOTE_VERSION to $DIR"
+          git push origin $BRANCH --force
+          # create a PR if an open one doesn't exist yet
+          if [[ $(gh pr list --state open --base $GITHUB_REF --head $BRANCH | wc -l) -eq 0 ]]; then
+            gh pr create --title "chore: add go-libp2p@$REMOTE_VERSION to $DIR" --body "This PR adds go-libp2p@$REMOTE_VERSION to $DIR" --head $BRANCH --base $GITHUB_REF
+          fi
+          gh workflow run perf.yml --ref $BRANCH
+        working-directory: ${{ env.DIR }}

.github/workflows/multidim-interop.yml

@@ -1,9 +1,13 @@
 on:
   workflow_dispatch:
   pull_request:
+    paths:
+      - 'multidim-interop/**'
   push:
     branches:
       - "master"
+    paths:
+      - 'multidim-interop/**'
 
 name: libp2p multidimensional interop test
 

.github/workflows/perf.yml

@@ -84,10 +84,13 @@ jobs:
         working-directory: perf/runner
       - name: Push
         if: github.event.inputs.push == 'true'
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
         run: |
           git add benchmark-results.json
           git commit -m "perf: update benchmark results"
           git push
+          gh pr comment --body "See new metrics at https://observablehq.com/@libp2p-workspace/performance-dashboard?branch=$(git rev-parse HEAD)" || true
         working-directory: perf/runner
       - name: Archive
         if: github.event.intputs.push == 'false'

README.md

@@ -1,21 +1,28 @@
-# Interoperabilty and end to end test-plans for libp2p
+# Interoperability/end to end test-plans & performance benchmarking for libp2p
 
-[![Interop Dashboard](https://github.com/libp2p/test-plans/workflows/libp2p%20multidimensional%20interop%20test/badge.svg?branch=master)](https://github.com/libp2p/test-plans/actions/runs/5836246193/attempts/2#summary-15833587518)
+[![Interop Dashboard](https://github.com/libp2p/test-plans/workflows/libp2p%20multidimensional%20interop%20test/badge.svg?branch=master)](https://github.com/libp2p/test-plans/actions/runs/6105882311/attempts/1#summary-16570044582)
 
 [![Made by Protocol Labs](https://img.shields.io/badge/made%20by-Protocol%20Labs-blue.svg?style=flat-square)](http://protocol.ai)
 
-This repository contains interoperability and end to end tests for libp2p modules across different implementations and versions.
+This repository contains:
+* interoperability and end to end tests for libp2p modules across different implementations and versions
+* components to run performance benchmarks for different libp2p implementations
 
-## Specs
+## Multidimensional Interop
+### Specs
 
-Please see our first specification for interoperability tests between transports, multiplexers, and secure channels here: [Interoperability Test Specs](https://github.com/libp2p/test-plans/blob/master/multidim-interop/README.md)
+Please see our first specification for interoperability tests between transports, multiplexers, and secure channels here: [Interoperability Test Specs](./multidim-interop/README.md)
 
 More specs to come soon!
 
 ## History
 
 These test-plans historically used Testground. To read why we're now using `docker compose` instead please see: [Why we're moving away from Testground](https://github.com/libp2p/test-plans/issues/103)
 
+## Performance Benchmarking
+
+Please see the [benchmarking README](./perf#libp2p-performance-benchmarking).
+
 ## Roadmap
 
 Our roadmap for test-plans can be found here: https://github.com/libp2p/test-plans/blob/master/ROADMAP.md

multidim-interop/README.md

@@ -30,6 +30,7 @@ The test should do two different things depending on if it's the dialer or
 listener.
 
 ## Running Locally
+
 In some cases you may want to run locally when debugging, such as modifying internal dependencies.
 
 1. To run the test locally, you'll also need to have docker installed in order to run the redis instance. Once docker is running, you can run the following command to start the redis instance:
@@ -43,20 +44,20 @@ This will start a redis instance on port 6379.
 2. Next, you'll need to install the dependencies and build the implementation for the test.  In this and the next step we are using a JS implementation as an example, so you would run the following command:
 
 ```bash
-cd impl/js/v0.xx.xx/ && npm i && npm run build
+cd impl/js/v0.xx.xx/ && make
 ```
 
-3. Then you can run a dialer by running the following command, ensure that you pass the required environment variables, as well as any that may be of use for debugging:
+3. Then you can run a listener by running the following command in this case we are running a rust listener:
 
 ```bash
-DEBUG=*:yamux:trace transport=tcp muxer=yamux security=noise is_dialer=true   npm run test -- -t node
-```
+ RUST_LOG=yamux=trace transport=tcp muxer=yamux security=noise is_dialer=false ip="0.0.0.0" redis_addr=localhost:6379  cargo run --package interop-tests
+ ```
 
-4. Finally you can run a listener by running the following command in this case where a running a rust listener:
+3. Finally you can run a dialer by running the following command, ensure that you pass the required environment variables, as well as any that may be of use for debugging:
 
 ```bash
- RUST_LOG=yamux=trace transport=tcp muxer=yamux security=noise is_dialer=false ip="0.0.0.0" redis_addr=localhost:6379  cargo run --package interop-tests
- ```
+DEBUG=*:yamux:trace transport=tcp muxer=yamux security=noise is_dialer=true   npm run test -- -t node
+```
 
 For more details on how to run a dialer vs a listener, see the sections below.
 

multidim-interop/impl/go/v0.28/Makefile → multidim-interop/impl/go/v0.30/Makefile

@@ -1,5 +1,5 @@
-image_name := go-v0.28
-version := 0.28.3
+image_name := go-v0.30
+version := 0.30.0
 
 all: image.json
 

multidim-interop/impl/go/v0.30/version.lock

@@ -0,0 +1 @@
+ce648ef713f6c68f9edf88e725605702ceb665ddc4c8a1b528f738a8689db1f3  go-libp2p-0.30.0.zip

multidim-interop/impl/go/v0.27/Makefile → multidim-interop/impl/go/v0.31/Makefile

@@ -1,5 +1,5 @@
-image_name := go-v0.27
-version := 0.27.9
+image_name := go-v0.31
+version := 0.31.0
 
 all: image.json
 
@@ -29,3 +29,4 @@ clean:
 	rm image.json
 	rm go-libp2p-*.zip
 	rm -rf go-libp2p-*
+

multidim-interop/impl/go/v0.31/version.lock

@@ -0,0 +1 @@
+fab332959e47bc8c7c5ed870f1f138d782c169f75ecf35eb274b46fefa83e29b  go-libp2p-0.31.0.zip

multidim-interop/impl/rust/v0.52/Makefile

@@ -1,35 +1,23 @@
 image_name := rust-v0.52
-version := 0.52.2
+commitSha := 54fa53af393adac83f9aab273c3d30efbc31ba57
 
 all: image.json chromium-image.json
 
-chromium-image.json: verify-checksum rust-libp2p-${version}
-	cd rust-libp2p-${version} && IMAGE_NAME=${image_name} ../../../../dockerBuildWrapper.sh -f interop-tests/Dockerfile.chromium .
+chromium-image.json: rust-libp2p-${commitSha}
+	cd rust-libp2p-${commitSha} && IMAGE_NAME=${image_name} ../../../../dockerBuildWrapper.sh -f interop-tests/Dockerfile.chromium .
 	docker image inspect ${image_name} -f "{{.Id}}" | \
 		xargs -I {} echo "{\"imageID\": \"{}\"}" > $@
 
-image.json: verify-checksum rust-libp2p-${version}
-	cd rust-libp2p-${version} && IMAGE_NAME=${image_name} ../../../../dockerBuildWrapper.sh -f interop-tests/Dockerfile.native .
+image.json: rust-libp2p-${commitSha}
+	cd rust-libp2p-${commitSha} && IMAGE_NAME=${image_name} ../../../../dockerBuildWrapper.sh -f interop-tests/Dockerfile.native .
 	docker image inspect ${image_name} -f "{{.Id}}" | \
 		xargs -I {} echo "{\"imageID\": \"{}\"}" > $@
 
-rust-libp2p-${version}: rust-libp2p-${version}.zip
-	unzip -o rust-libp2p-${version}.zip
-	mv rust-libp2p-libp2p-v${version} rust-libp2p-${version}
+rust-libp2p-${commitSha}: rust-libp2p-${commitSha}.zip
+	unzip -o rust-libp2p-${commitSha}.zip
 
-rust-libp2p-${version}.zip:
-	wget -O $@ "https://github.com/libp2p/rust-libp2p/archive/libp2p-v${version}.zip"
-
-# Run `make version.lock` to generate this lock file. This file should be commited.
-# This locks the exact contents of the specified version. This lets us use the
-# human readable name while still making sure the contents don't change.
-version.lock: rust-libp2p-${version}.zip
-	shasum -a 256 rust-libp2p-${version}.zip > $@
-
-verify-checksum: rust-libp2p-${version}.zip
-	shasum -a 256 -c version.lock
-
-.PHONY: clean all verify-checksum
+rust-libp2p-${commitSha}.zip:
+	wget -O $@ "https://github.com/libp2p/rust-libp2p/archive/${commitSha}.zip"
 
 clean:
 	rm image.json

multidim-interop/versions.ts

@@ -84,31 +84,31 @@ export const versions: Array<Version> = [
     {
         id: "chromium-js-v0.46",
         containerImageID: browserImageIDLookup,
-        transports: [{ name: "webtransport", onlyDial: true }, { name: "wss", onlyDial: true }, { name: "webrtc-direct", onlyDial: true }, "webrtc"],
+        transports: [{ name: "webtransport", onlyDial: true }, { name: "wss", onlyDial: true }, { name: "webrtc-direct", onlyDial: true }],
         secureChannels: ["noise"],
         muxers: ["mplex", "yamux"],
     },
     {
         id: "firefox-js-v0.46",
         containerImageID: browserImageIDLookup,
-        transports: [{ name: "wss", onlyDial: true }, { name: "webrtc-direct", onlyDial: true }, "webrtc"],
+        transports: [{ name: "wss", onlyDial: true }, { name: "webrtc-direct", onlyDial: true }],
         secureChannels: ["noise"],
         muxers: ["mplex", "yamux"],
     },
     {
-        id: "go-v0.29",
-        transports: ["tcp", "ws", "quic", "quic-v1", "webtransport"],
+        id: "go-v0.31",
+        transports: ["tcp", "ws", "quic-v1", "webtransport"],
         secureChannels: ["tls", "noise"],
-        muxers: ["mplex", "yamux"],
+        muxers: ["yamux"],
     },
     {
-        id: "go-v0.28",
-        transports: ["tcp", "ws", "quic", "quic-v1", "webtransport"],
+        id: "go-v0.30",
+        transports: ["tcp", "ws", "quic-v1", "webtransport"],
         secureChannels: ["tls", "noise"],
-        muxers: ["mplex", "yamux"],
+        muxers: ["yamux"],
     },
     {
-        id: "go-v0.27",
+        id: "go-v0.29",
         transports: ["tcp", "ws", "quic", "quic-v1", "webtransport"],
         secureChannels: ["tls", "noise"],
         muxers: ["mplex", "yamux"],

perf/README.md

@@ -15,7 +15,14 @@ Benchmark results can be visualized with https://observablehq.com/@libp2p-worksp
 3. Wait for action run to finish and to push a commit to your branch.
 4. Visualize results on https://observablehq.com/@libp2p-workspace/performance-dashboard.
 
-## Running via local machine
+## Running manually
+
+### Prerequisites
+
+- Terraform 1.5.4 or later
+- Node.js 18 or later
+- [an AWS IAM user](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_users.html)
+
 
 ### Provision infrastructure
 
@@ -26,13 +33,27 @@ Benchmark results can be visualized with https://observablehq.com/@libp2p-worksp
 5. `CLIENT_IP=$(terraform output -raw client_ip)`
 6. `SERVER_IP=$(terraform output -raw server_ip)`
 
-### Build and run implementations
+**Notes**
+- While running terraform you may encounter the following error:
+  ```bash
+    Error: collecting instance settings: reading EC2 Launch Template versions: couldn't find resource
+    │
+    │   with module.short_lived_server[0].aws_instance.perf,
+    │   on ../../modules/short_lived/main.tf line 15, in resource "aws_instance" "perf":
+    │   15: resource "aws_instance" "perf" {
+  ```
+- This implies that you haven't deployed the long-lived infrastructure on your AWS account. To do so along with each short-lived deployment, you can set *TF_VAR* [`long_lived_enabled`](./terraform/configs/local/terraform.tf#L42) env variable to default to `true`. Terraform should then spin up the long-lived resources that are required for the short-lived resources to be created.
+
+- It's best to destroy the infrastructure after you're done with your testing, you can do that by running `terraform destroy`.
+
+### Build and run libp2p implementations
 
-_WARNING_: Running the perf tests might take a while.
+Given you have provisioned your infrastructure, you can now build and run the libp2p implementations on the AWS instances.
 
 1. `cd runner`
 2. `npm ci`
-3. `npm run start -- --client-public-ip $CLIENT_IP --server-public-ip $SERVER_IP`
+3.  `npm run start -- --client-public-ip $CLIENT_IP --server-public-ip $SERVER_IP`
+   * Note: The default number of iterations that perf will run is 10; desired iterations can be set with the  `--iterations <value>` option.
 
 ### Deprovision infrastructure
 
@@ -41,27 +62,28 @@ _WARNING_: Running the perf tests might take a while.
 
 ## Adding a new implementation or a new version
 
-1. Add implementation to `impl/`.
-    - For a new implementation, create a folder `impl/<your-implementation-name>/`
+1. Add the implementation to new subdirectory in [`impl/*`](./impl/).
+    - For a new implementation, create a folder `impl/<your-implementation-name>/` e.g. `go-libp2p`
     - For a new version of an existing implementation, create a folder `impl/<your-implementation-name>/<your-implementation-version>`.
     - In that folder include a `Makefile` that builds an executable and stores it next to the `Makefile` under the name `perf`.
     - Requirements for the executable:
-      - Running as a libp2p-perf server
-        - Command line flags
-          - `--run-server`
+      - Running as a libp2p-perf server:
+        - The perf server must not exit as it will be closed by the test runner.
+        - The executable must accept the command flag `--run-server` which indicates it's running as server.
       - Running as a libp2p-perf client
+        - Given that perf is a client driven set of benchmarks, the performance will be measured by the client.
           - Input via command line
-            - `--server-ip-address`
-            - `--transport` (see `runner/versions.ts` for possible variants)
+            - `--server-address`
+            - `--transport` (see [`runner/versions.ts`](./runner/src/versions.ts#L7-L43) for possible variants)
             - `--upload-bytes` number of bytes to upload per stream.
             - `--download-bytes` number of bytes to download per stream.
           - Output
-            - Logging MUST go to stderr.
-            - Measurement output is printed to stdout as JSON in the form of:
+            - Logging MUST go to `stderr`.
+            - Measurement output is printed to `stdout` as JSON in the form of:
               ```json
               {"latency": 0.246442851}
               ```
               Note that the measurement includes the time to (1) establish the
               connection, (2) upload the bytes and (3) download the bytes.
-2. For a new implementation, in `impl/Makefile` include your implementation in the `all` target.
-3. For a new version, reference version in `runner/src/versions.ts`.
+2. For a new implementation, in [`impl/Makefile` include your implementation in the `all` target.](./impl/Makefile#L7)
+3. For a new version, reference version in [`runner/src/versions.ts`](./runner/src/versions.ts#L7-L43).