Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 15 additions & 9 deletions .github/actions/get-src/action.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,10 @@ name: GenVM get source
description: Checkout sources, optionally update submodules/third-party, and (optionally) set up Nix.
inputs:
with_nix:
description: if should setup nix
description: >-
if the *job* needs nix after this action. Nix is also installed whenever
`third_party` is not `none`, so it can be present without this being set;
an installation cannot be undone.
required: false
default: "false"
third_party:
Expand Down Expand Up @@ -38,6 +41,17 @@ outputs:
runs:
using: composite
steps:
# Before the checkout: `get-all-git.py` realizes `git-third-party` from the
# flake, so a third-party update needs nix here whether or not the job asked
# for one. The action itself only touches the runner, never the tree.
- name: setup nix
if: ${{ inputs.with_nix == 'true' || inputs.third_party != 'none' }}
# Pinned at every call site of these actions, `# <branch>` naming where the
# sha came from: a job re-run later has to run the action it first ran.
uses: genlayerlabs/github-actions/nix-setup@39b0a0d5e9bb27a1612d2e98b0f8509d31745157 # main
with:
github_token: ${{ inputs.github_token }}
cache_pull_token: ${{ inputs.nix_cache_pull_token }}
- name: checkout submodules
run: |
cd "$GITHUB_WORKSPACE"
Expand All @@ -54,11 +68,3 @@ runs:
run: |
cd "$GITHUB_WORKSPACE"
python3 support/scripts/ci-changes.py --github-output
- name: setup nix
if: ${{ inputs.with_nix == 'true' }}
# Pinned at every call site of these actions, `# <branch>` naming where the
# sha came from: a job re-run later has to run the action it first ran.
uses: genlayerlabs/github-actions/nix-setup@39b0a0d5e9bb27a1612d2e98b0f8509d31745157 # main
with:
github_token: ${{ inputs.github_token }}
cache_pull_token: ${{ inputs.nix_cache_pull_token }}
3 changes: 2 additions & 1 deletion .github/workflows/branch_run_tests_command.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ jobs:
actions: write
contents: read
issues: write
pull-requests: read
pull-requests: write
steps:
# issue_comment workflows run from the default branch. Keep that trusted
# checkout: this handler dispatches PR code with credentials in scope
Expand All @@ -57,6 +57,7 @@ jobs:
permissions:
contents: read
issues: write
pull-requests: write
steps:
# workflow_run also resolves this workflow from the default branch. Never
# check out or execute the completed run's potentially PR-authored tree
Expand Down
61 changes: 61 additions & 0 deletions .github/workflows/branch_wasmtime_watch.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
name: branch / wasmtime watch

# Daily sweep for published advisories against the upstream sources the manager
# vendors: each repo in an executor line's `manifest.json` by its pinned commit,
# and each crate that comes out of one by its locked version.
#
# It gates nothing and writes no branch: it keeps one `wasmtime-maintenance`
# issue in sync with what OSV currently says. Nothing PR-triggered could do this
# job — a pin becomes vulnerable while standing still, with no PR involved.
#
# Closing the issue is a human ruling on the findings it names; the tool never
# closes or reopens one. See support/ci/tools/wasmtime_watch.py.

on:
schedule:
# Daily, off the hour: a scheduled run on a busy minute is queued or dropped.
- cron: "17 5 * * *"
workflow_dispatch:
inputs:
dry_run:
description: Report findings without creating or editing the issue
type: boolean
required: false
default: false

permissions:
contents: read
issues: write

# Never cancel: a cancelled sweep can leave the issue describing a state that no
# longer holds, and the next run is a day away.
concurrency:
group: wasmtime-watch
cancel-in-progress: false

defaults:
run:
shell: bash -exo pipefail {0}

jobs:
watch:
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- uses: actions/checkout@v4
# Both files it reads are tracked inside the executor submodules, so this
# needs a checkout but not the materialized third-party trees — which is
# what lets it skip nix entirely.
- name: Get source
uses: ./.github/actions/get-src
with:
with_nix: "false"
third_party: none
github_token: ${{ secrets.GITHUB_TOKEN }}
- name: Sweep the vendored upstream sources
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
WASMTIME_REBASE_OWNER: ${{ vars.WASMTIME_REBASE_OWNER }}
run: >-
./support/ci/run.sh tool wasmtime-watch
${{ inputs.dry_run && '--dry-run' || '' }}
3 changes: 3 additions & 0 deletions .github/workflows/incl_initial.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,9 @@ jobs:
uses: ./.github/actions/get-src
with:
with_nix: "false"
# Commit messages need history, not vendored sources, and asking for
# them would install nix here for nothing.
third_party: none
pr: ${{ inputs.pr }}
# Needed even with with_nix=false: on a dispatched run there is no
# GITHUB_BASE_REF, so ci-changes.py resolves the base branch through
Expand Down
5 changes: 4 additions & 1 deletion .github/workflows/incl_release_build_test_cell_test.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,10 @@ jobs:
- name: Get source
uses: ./.github/actions/get-src
with:
third_party: --all
# A nixless cell has no dev shell to build anything in: it replays
# prebuilt tarballs and reads no vendored source, so asking for them
# would install a nix nothing else here uses.
third_party: ${{ inputs.with_nix && '--all' || 'none' }}
with_nix: ${{ inputs.with_nix && 'true' || 'false' }}
github_token: ${{ secrets.GITHUB_TOKEN }}
nix_cache_pull_token: ${{ secrets.NIX_CACHE_PULL_TOKEN }}
Expand Down
63 changes: 51 additions & 12 deletions docs/contributing/howto/committing/git-third-party.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,22 +5,31 @@ git-ignored and materialized on demand. Why patches rather than a fork:
[vendored-trees.md](../../explanation/vendored-trees.md). What is tracked, per
line, is `.git-third-party/`:

1. `config.json` — per repo: upstream `url`, pinned base `commit`, `patches`
count, optional `submodules` list
2. `patches/<repo-path>/<n>` — numbered `git format-patch` files, 1-based, mbox
format, applied on top of the base commit
1. `manifest.json` — per repo: upstream `url`, pinned base `commit`, the
ordered `patches` list, optional `submodules` list
2. `patches/<repo-path>/<name>` — `git format-patch` files in mbox format,
applied on top of the base commit in the order the manifest lists. `<name>`
is a digest of the patch bytes, so inserting a patch renames nothing else
3. `.gitattributes`, `.gitignore` — written by the tool. The first keeps the
enclosing repo from normalizing bytes the names are derived from

The tool is `support/tools/git-third-party/git-third-party`, invoked as
`git third-party`, on `PATH` in the dev shell and through `env.sh`. It has no
`--help`; running it bare prints usage
The tool comes from the `git-third-party` flake input, on `PATH` in the dev
shell and through `env.sh`, and is invoked as `git third-party`. It has no
`--help`; running it bare prints usage. Bump it with
`nix flake update git-third-party`

| Command | Effect |
|---|---|
| `add <PATH> <REPO_URL> <COMMIT>` | register a vendored repo (the path must already be git-ignored) and materialize it |
| `update {--all \| <path>...}` | materialize: nested `git init` + fetch of the pinned commit, submodule update, `git am` of the patches |
| `save {--all \| <path>...}` | regenerate the patches from the commits above the base commit, updating the count |
| `save {--all \| <path>...}` | regenerate the patches from the commits above the base commit, rewriting the manifest's list |

`update` refuses a dirty tree. The result is a real nested git repo: base commit
`update` refuses a dirty tree — untracked files and dirty submodules count —
and refuses to discard commits in a managed checkout that `save` has not
captured. `git am` runs with `--keep-cr --whitespace=nowarn --no-3way`, so
local git config cannot rewrite what a patch applies, and a patch that only
landed by 3-way fallback now fails instead. Materialized checkouts get their
push URL disabled. The result is a real nested git repo: base commit
plus one commit per patch. It always runs
`git submodule update --init --recursive --depth 1` first, so `submodules: []`
only skips the second pass and a list only adds a targeted one
Expand All @@ -35,15 +44,45 @@ only skips the second pass and a list only adds a targeted one
4. Bump the gitlink in the manager ([submodules.md](submodules.md))

The nested repo is never pushed, only the patches persist. To bump upstream,
edit `commit` in `config.json` and run `update`; on an `am` conflict fix up the
nested repo's commits and `save`
edit `commit` in `manifest.json` and run `update`; on an `am` conflict fix up
the nested repo's commits and `save`

## Build Integration

1. Local and CI checkouts run `git third-party update --all` **inside each
executor submodule** — the config resolves from the current git toplevel, so
running it at the manager root materializes nothing
2. Nix never calls the tool: `support/nix/git-third-party.nix` re-reads
`config.json` and uses `builtins.fetchGit` plus `pkgs.applyPatches`, ignoring
`manifest.json` and uses `builtins.fetchGit` plus `pkgs.applyPatches`, ignoring
`submodules`. So nix sees the committed base plus patches — `save` and commit
before building any flake package

## Advisory Monitoring

`branch / wasmtime watch` checks every vendored source of every active line
against [OSV](https://osv.dev) daily and keeps one `wasmtime-maintenance` issue
in sync with the result. It gates nothing and writes no branch. Run it by hand
with `./support/ci/run.sh tool wasmtime-watch --dry-run`

Two things are queried per line, because neither covers the other:

1. each repo in `manifest.json`, by its pinned upstream commit
2. each crate that comes out of one, by its locked version — a vendored tree is
dozens of crates (`cranelift-codegen`, `wasmparser`, `wiggle`), and that is
where most advisories against this stack are actually published

A crate counts as vendored when `executor/Cargo.lock` gives it no `source` and
no tracked `Cargo.toml` in the executor repo declares it. Registry crates are
excluded because they are covered upstream; first-party crates, because they are
ours. A first-party crate with no committed manifest falls through and is
queried too — a dismissible false positive, chosen over a silent gap

OSV knows nothing about the features GenVM disables or what the patch series
changes, so a hit is a finding to rule on, not a proven exposure

The sweep edits its issue in place and never closes or reopens one: closing it
is the ruling that its findings are handled. The advisory ids that issue covered
are recorded in its body marker, so later sweeps stay quiet until an id outside
that set appears, at which point a fresh issue supersedes it.
`WASMTIME_REBASE_OWNER` (an Actions variable) is the login the issue is assigned
to at creation; sweeps never reassign it
4 changes: 2 additions & 2 deletions docs/contributing/howto/setup.md
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,8 @@
`?submodules=1` is mandatory on every manager flake ref; without it nix
fails with `Path 'executors/v0.3.x' … is not tracked by Git`
3. Only outside the dev shell, and only for manual `git third-party` calls:
`source env.sh` puts `support/tools/git-third-party` on `PATH` and sources
`.env` if present
`source env.sh` realizes the pinned `git-third-party` flake package onto
`PATH` and sources `.env` if present

Next: [build.md](building/build.md). Vendoring mechanics:
[git-third-party.md](committing/git-third-party.md)
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ Requirements
~~~~~~~~~~~~

#. :term:`Sub-VM` must be in deterministic mode
#. index + buf_len must not overflow
#. index + buf_len must not exceed the :term:`Storage Slot` length

``storage_write``
-----------------
Expand All @@ -22,7 +22,7 @@ Requirements

#. :term:`Sub-VM` must be in deterministic mode
#. :term:`Sub-VM` must have write storage permission
#. index + buf_len must not overflow
#. index + buf_len must not exceed the :term:`Storage Slot` length
#. :term:`Sub-VM` Storage slot must not be locked, unless the sender is in ``upgraders``

``get_balance``
Expand Down
4 changes: 4 additions & 0 deletions docs/website/src/spec/changelog.rst
Original file line number Diff line number Diff line change
Expand Up @@ -72,3 +72,7 @@ Changed
code — e.g. exhausting the memory budget there reports
:ref:`gvm-def-str-trie-value-vm-error-out-of-memory-wasm-memory` — instead of
being reported as a bare ``invalid_contract``
#. ``storage_read`` and ``storage_write`` bound an access by the
:term:`Storage Slot` length instead of by ``u32`` overflow, so a slot's final
octet is addressable; every access naming it was previously refused. See
:doc:`02-execution-environment/03-wasi_genlayer_sdk/01-functions`
9 changes: 8 additions & 1 deletion env.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,14 @@

SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )

export PATH="$SCRIPT_DIR/support/tools/git-third-party:$PATH"
# The tool comes from the pinned flake input, so outside the dev shell it has to
# be realized first. Already on PATH inside any dev shell
if ! command -v git-third-party > /dev/null
then
git_third_party_out=$(nix build --no-link --print-out-paths "$SCRIPT_DIR?submodules=1#git-third-party")
export PATH="$git_third_party_out/bin:$PATH"
unset git_third_party_out
fi

if [ -f "$SCRIPT_DIR/.env" ]
then
Expand Down
2 changes: 1 addition & 1 deletion executors/v0.3.x
Submodule v0.3.x updated 32 files
+6 −0 .git-third-party/.gitattributes
+5 −0 .git-third-party/.gitignore
+0 −15 .git-third-party/config.json
+23 −0 .git-third-party/manifest.json
+3 −3 .git-third-party/patches/executor/third-party/wasm-tools/n359mz2sdabr2wsa
+0 −92 .git-third-party/patches/executor/third-party/wasmtime/4
+4 −6 .git-third-party/patches/executor/third-party/wasmtime/4rr93pqjtwm5m701
+0 −85 .git-third-party/patches/executor/third-party/wasmtime/5
+1,391 −0 .git-third-party/patches/executor/third-party/wasmtime/6xy06vg0d71czwmm
+262 −241 .git-third-party/patches/executor/third-party/wasmtime/gbm0xv16dc1fs2km
+104 −0 .git-third-party/patches/executor/third-party/wasmtime/r6518c8dxghpkhsj
+26 −11 .git-third-party/patches/executor/third-party/wasmtime/zsjxmza3dms0hr10
+129 −86 executor/Cargo.lock
+8 −0 executor/crates/common/src/expr/evaluator.rs
+23 −3 executor/crates/common/src/expr/value.rs
+1 −1 executor/src/rt/errors.rs
+5 −3 executor/src/rt/supervisor/mod.rs
+3 −0 executor/src/rt/vm/storage.rs
+13 −2 executor/src/wasi/genlayer_sdk/mod.rs
+18 −0 executor/src/wasi/genlayer_sdk/tests.rs
+0 −7 tests/integration/exploit/storage_rw_long/storage_r_long.py
+0 −1 tests/integration/exploit/storage_rw_long/storage_rw_long.0.stdout
+0 −1 tests/integration/exploit/storage_rw_long/storage_rw_long.0_0.stdout
+0 −12 tests/integration/exploit/storage_rw_long/storage_rw_long.jsonnet
+0 −6 tests/integration/exploit/storage_rw_long/storage_w_long.py
+16 −0 tests/integration/exploit/storage_slot_edge/storage_slot_edge.jsonnet
+74 −0 tests/integration/exploit/storage_slot_edge/storage_slot_edge.py
+2 −0 tests/integration/exploit/storage_slot_edge/storage_slot_edge.read_ends_at_end.stdout
+2 −0 tests/integration/exploit/storage_slot_edge/storage_slot_edge.read_last_byte.stdout
+2 −0 tests/integration/exploit/storage_slot_edge/storage_slot_edge.read_over_end.stdout
+3 −0 tests/integration/exploit/storage_slot_edge/storage_slot_edge.write_last_byte.stdout
+2 −0 tests/integration/exploit/storage_slot_edge/storage_slot_edge.write_over_end.stdout
63 changes: 63 additions & 0 deletions flake.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading
Loading