Skip to content

Commit

Permalink
workbench: remote analysis & analysis fetching
Browse files Browse the repository at this point in the history
  • Loading branch information
@deepfire
deepfire committed Jun 22, 2022
1 parent 164b83e commit 034eb8d
Show file tree
Hide file tree
Showing 2 changed files with 99 additions and 40 deletions.
3 changes: 3 additions & 0 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,9 @@ shell-dev shell-prof shell-nix: shell
shell-nix: ARGS += --arg 'workbenchDevMode' false ## Nix shell, (workbench from Nix store), vars: PROFILE, CMD, RUN
shell-prof: ARGS += --arg 'profiled' true ## Nix shell, everything Haskell built profiled

analyse: RUN := wb analyse std ${TAG}
analyse: shell

list-profiles: ## List workbench profiles
nix build .#workbench.profile-names-json --json | jq '.[0].outputs.out' -r | xargs jq .
show-profile: ## NAME=profile-name
Expand Down
136 changes: 96 additions & 40 deletions nix/workbench/run.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -253,54 +253,56 @@ case "$op" in
sort ||
true'" 2>/dev/null;;

allocate-from-aws | steal-from-aws | aws-get )
allocate-from-aws | aws-get )
local usage="USAGE: wb run $op RUN [MACHINE] [DEPLOYMENT=bench-1] [ENV=bench]"
local run=${1:?$usage}
local mach=${2:-}
local depl=${3:-bench-1}
local env=${4:-bench}

local meta=$(ssh $env -- sh -c "'jq . $depl/runs/$run/meta.json'")
if ! jq . <<<$meta >/dev/null
then fail "allocate-from-aws: malformed $(yellow meta.json) in $(white $run) on $(white $depl)@$(white env)"; fi

## Minor validation passed, create & populate run with remote data:
local dir=$global_rundir/$run
mkdir -p "$dir"
jq . <<<$meta > $dir/meta.json

local hosts=($(if test -n "$mach"; then echo $mach
else jq -r '.hostname | keys | .[]' <<<$meta; fi))
local objects=(
${hosts[*]}
genesis-shelley.json
genesis-alonzo.json
network-latency-matrix.json
machines.json
local args=(
"$run"
'if test -f compressed/logs-$obj.tar.zst; then cat compressed/logs-$obj.tar.zst; else tar c $obj --zstd --ignore-failed-read; fi'
$mach
$depl
$env
)
run_aws_get "${args[@]}";;

analysis-from-aws | aws-get-analysis | aws-geta )
local usage="USAGE: wb run $op RUN [MACHINE] [DEPLOYMENT=bench-1] [ENV=bench]"
local run=${1:?$usage}
local mach=${2:-}
local depl=${3:-bench-1}
local env=${4:-bench}

local args=(
"$run"
'{ ls analysis/*.{json,org,txt} | grep -v flt.json | grep -v flt.logobjs.json | xargs tar cz --ignore-failed-read; }'
"$mach"
$depl
$env
)
if test "$(ssh $env -- sh -c "'ls -ld $depl/runs/$run | wc -l'")" = 0
then fail "aws-analysis: run does not exist on AWS: $(white $run)"
elif test "$(ssh $env -- sh -c "'ls -ld $depl/runs/$run/analysis | wc -l'")" = 0
then fail "aws-analysis: run has not been analysed on AWS: $(white $run)"
else run_aws_get "$run" '{ ls analysis/*.{json,org,txt} | grep -v flt.json | grep -v flt.logobjs.json | xargs tar cz --ignore-failed-read; }' "$mach" "$depl" "$env"
fi
;;

local count=${#objects[*]}
progress "run | aws-get $(white $run)" "objects to fetch: $(white $count) total: $(yellow ${objects[*]})"

local max_batch=9
progress "run | aws-get $(white $run)" "fetching in batches"

local base=0 batch
while test $base -lt $count
do local batch=(${objects[*]:$base:$max_batch})
progress_ne "run | aws-get $(white $run)" "fetching batch: "
local obj=
for obj in ${batch[*]}
do { ssh $env -- \
sh -c "'cd $depl/runs/$run && if test -f compressed/logs-$obj.tar.zst; then cat compressed/logs-$obj.tar.zst; else tar c $obj --zstd --ignore-failed-read; fi'" 2>/dev/null |
(cd $dir; tar x --zstd)
echo -ne " $(yellow $obj)" >&2
} &
done
wait
echo >&2
base=$((base + max_batch))
done;;
analyse-aws | awsa )
local usage="USAGE: wb run $op RUN [MACHINE] [DEPLOYMENT=bench-1] [ENV=bench]"
local run=${1:?$usage}
local mach=${2:-}
local depl=${3:-bench-1}
local env=${4:-bench}

if test "$(ssh $env -- sh -c "'ls -ld $depl/runs/$run | wc -l'")" = 0
then fail "aws-analysis: run does not exist on AWS: $(white $run)"
else ssh $env -- sh -c "'export WB_RUNDIR=../$depl/runs && cd cardano-node && echo env: $(yellow $env), rundir: $(color blue)\$WB_RUNDIR$(color reset), workbench: $(color yellow)\$(git log -n1)$(color reset) && make analyse TAG=$run'"
fi
;;

allocate )
local usage="USAGE: wb run $op BATCH-NAME PROFILE-NAME [ENV-CONFIG-OPTS..] [-- BACKEND-ARGS-AND-ENV-CONFIG-OPTS..]"
Expand Down Expand Up @@ -631,3 +633,57 @@ EOF

* ) usage_run;; esac
}

run_aws_get() {
local usage='USAGE: run_aws_get RUN REMOTE-TAR-CMD [MACHINE] [DEPLOYMENT] [ENV]'
local run=${1:?$usage}
local remote_tar_cmd=${2:?$usage}
local mach=${3:-}
local depl=${4:-bench-1}
local env=${5:-bench}

progress "aws-get" "mach $(yellow $mach) depl $(yellow $depl) run $(white $run)"
progress "aws-get" "selector $(green $remote_tar_cmd)"

local meta=$(ssh $env -- sh -c "'jq . $depl/runs/$run/meta.json'")
if ! jq . <<<$meta >/dev/null
then fail "allocate-from-aws: malformed $(yellow meta.json) in $(white $run) on $(white $depl)@$(white env)"; fi

## Minor validation passed, create & populate run with remote data:
local dir=$global_rundir/$run
mkdir -p "$dir"
jq . <<<$meta > $dir/meta.json

local hosts=($(if test -n "$mach"; then echo $mach
else jq -r '.hostname | keys | .[]' <<<$meta; fi))
local objects=(
${hosts[*]}
genesis-shelley.json
genesis-alonzo.json
network-latency-matrix.json
machines.json
)

local count=${#objects[*]}
progress "run | aws-get $(white $run)" "objects to fetch: $(white $count) total: $(yellow ${objects[*]})"

local max_batch=9
progress "run | aws-get $(white $run)" "fetching in batches"

local base=0 batch
while test $base -lt $count
do local batch=(${objects[*]:$base:$max_batch})
progress_ne "run | aws-get $(white $run)" "fetching batch: "
local obj=
for obj in ${batch[*]}
do { ssh $env -- \
sh -c "'cd $depl/runs/$run && ${remote_tar_cmd}'" 2>/dev/null |
(cd $dir; tar xz)
echo -ne " $(yellow $obj)" >&2
} &
done
wait
echo >&2
base=$((base + max_batch))
done
}

0 comments on commit 034eb8d

Please sign in to comment.