diff --git a/Makefile b/Makefile index 9d8fab1feab..497bedf42a2 100644 --- a/Makefile +++ b/Makefile @@ -49,6 +49,9 @@ shell-dev shell-prof shell-nix: shell shell-nix: ARGS += --arg 'workbenchDevMode' false ## Nix shell, (workbench from Nix store), vars: PROFILE, CMD, RUN shell-prof: ARGS += --arg 'profiled' true ## Nix shell, everything Haskell built profiled +analyse: RUN := wb analyse std ${TAG} +analyse: shell + list-profiles: ## List workbench profiles nix build .#workbench.profile-names-json --json | jq '.[0].outputs.out' -r | xargs jq . show-profile: ## NAME=profile-name diff --git a/nix/workbench/run.sh b/nix/workbench/run.sh index c6c19cf708d..019c43fd0a8 100644 --- a/nix/workbench/run.sh +++ b/nix/workbench/run.sh @@ -253,54 +253,56 @@ case "$op" in sort || true'" 2>/dev/null;; - allocate-from-aws | steal-from-aws | aws-get ) + allocate-from-aws | aws-get ) local usage="USAGE: wb run $op RUN [MACHINE] [DEPLOYMENT=bench-1] [ENV=bench]" local run=${1:?$usage} local mach=${2:-} local depl=${3:-bench-1} local env=${4:-bench} - local meta=$(ssh $env -- sh -c "'jq . $depl/runs/$run/meta.json'") - if ! jq . <<<$meta >/dev/null - then fail "allocate-from-aws: malformed $(yellow meta.json) in $(white $run) on $(white $depl)@$(white env)"; fi - - ## Minor validation passed, create & populate run with remote data: - local dir=$global_rundir/$run - mkdir -p "$dir" - jq . <<<$meta > $dir/meta.json - - local hosts=($(if test -n "$mach"; then echo $mach - else jq -r '.hostname | keys | .[]' <<<$meta; fi)) - local objects=( - ${hosts[*]} - genesis-shelley.json - genesis-alonzo.json - network-latency-matrix.json - machines.json + local args=( + "$run" + 'if test -f compressed/logs-$obj.tar.zst; then cat compressed/logs-$obj.tar.zst; else tar c $obj --zstd --ignore-failed-read; fi' + $mach + $depl + $env ) + run_aws_get "${args[@]}";; + + analysis-from-aws | aws-get-analysis | aws-geta ) + local usage="USAGE: wb run $op RUN [MACHINE] [DEPLOYMENT=bench-1] [ENV=bench]" + local run=${1:?$usage} + local mach=${2:-} + local depl=${3:-bench-1} + local env=${4:-bench} + + local args=( + "$run" + '{ ls analysis/*.{json,org,txt} | grep -v flt.json | grep -v flt.logobjs.json | xargs tar cz --ignore-failed-read; }' + "$mach" + $depl + $env + ) + if test "$(ssh $env -- sh -c "'ls -ld $depl/runs/$run | wc -l'")" = 0 + then fail "aws-analysis: run does not exist on AWS: $(white $run)" + elif test "$(ssh $env -- sh -c "'ls -ld $depl/runs/$run/analysis | wc -l'")" = 0 + then fail "aws-analysis: run has not been analysed on AWS: $(white $run)" + else run_aws_get "$run" '{ ls analysis/*.{json,org,txt} | grep -v flt.json | grep -v flt.logobjs.json | xargs tar cz --ignore-failed-read; }' "$mach" "$depl" "$env" + fi + ;; - local count=${#objects[*]} - progress "run | aws-get $(white $run)" "objects to fetch: $(white $count) total: $(yellow ${objects[*]})" - - local max_batch=9 - progress "run | aws-get $(white $run)" "fetching in batches" - - local base=0 batch - while test $base -lt $count - do local batch=(${objects[*]:$base:$max_batch}) - progress_ne "run | aws-get $(white $run)" "fetching batch: " - local obj= - for obj in ${batch[*]} - do { ssh $env -- \ - sh -c "'cd $depl/runs/$run && if test -f compressed/logs-$obj.tar.zst; then cat compressed/logs-$obj.tar.zst; else tar c $obj --zstd --ignore-failed-read; fi'" 2>/dev/null | - (cd $dir; tar x --zstd) - echo -ne " $(yellow $obj)" >&2 - } & - done - wait - echo >&2 - base=$((base + max_batch)) - done;; + analyse-aws | awsa ) + local usage="USAGE: wb run $op RUN [MACHINE] [DEPLOYMENT=bench-1] [ENV=bench]" + local run=${1:?$usage} + local mach=${2:-} + local depl=${3:-bench-1} + local env=${4:-bench} + + if test "$(ssh $env -- sh -c "'ls -ld $depl/runs/$run | wc -l'")" = 0 + then fail "aws-analysis: run does not exist on AWS: $(white $run)" + else ssh $env -- sh -c "'export WB_RUNDIR=../$depl/runs && cd cardano-node && echo env: $(yellow $env), rundir: $(color blue)\$WB_RUNDIR$(color reset), workbench: $(color yellow)\$(git log -n1)$(color reset) && make analyse TAG=$run'" + fi + ;; allocate ) local usage="USAGE: wb run $op BATCH-NAME PROFILE-NAME [ENV-CONFIG-OPTS..] [-- BACKEND-ARGS-AND-ENV-CONFIG-OPTS..]" @@ -631,3 +633,57 @@ EOF * ) usage_run;; esac } + +run_aws_get() { + local usage='USAGE: run_aws_get RUN REMOTE-TAR-CMD [MACHINE] [DEPLOYMENT] [ENV]' + local run=${1:?$usage} + local remote_tar_cmd=${2:?$usage} + local mach=${3:-} + local depl=${4:-bench-1} + local env=${5:-bench} + + progress "aws-get" "mach $(yellow $mach) depl $(yellow $depl) run $(white $run)" + progress "aws-get" "selector $(green $remote_tar_cmd)" + + local meta=$(ssh $env -- sh -c "'jq . $depl/runs/$run/meta.json'") + if ! jq . <<<$meta >/dev/null + then fail "allocate-from-aws: malformed $(yellow meta.json) in $(white $run) on $(white $depl)@$(white env)"; fi + + ## Minor validation passed, create & populate run with remote data: + local dir=$global_rundir/$run + mkdir -p "$dir" + jq . <<<$meta > $dir/meta.json + + local hosts=($(if test -n "$mach"; then echo $mach + else jq -r '.hostname | keys | .[]' <<<$meta; fi)) + local objects=( + ${hosts[*]} + genesis-shelley.json + genesis-alonzo.json + network-latency-matrix.json + machines.json + ) + + local count=${#objects[*]} + progress "run | aws-get $(white $run)" "objects to fetch: $(white $count) total: $(yellow ${objects[*]})" + + local max_batch=9 + progress "run | aws-get $(white $run)" "fetching in batches" + + local base=0 batch + while test $base -lt $count + do local batch=(${objects[*]:$base:$max_batch}) + progress_ne "run | aws-get $(white $run)" "fetching batch: " + local obj= + for obj in ${batch[*]} + do { ssh $env -- \ + sh -c "'cd $depl/runs/$run && ${remote_tar_cmd}'" 2>/dev/null | + (cd $dir; tar xz) + echo -ne " $(yellow $obj)" >&2 + } & + done + wait + echo >&2 + base=$((base + max_batch)) + done +}