gm: Add option to download chain binary from a URL

.changelog/unreleased/features/1371-gm-features.md

@@ -0,0 +1,5 @@
+- [gm](scripts/gm)
+  - Binaries in the config can be defined as URLs now.
+  - Add the option to set gm-lib path via the $GM_LIB environment variable ([#1365])
+
+[#1365]: https://github.com/informalsystems/ibc-rs/issues/1365

scripts/gm/CHANGELOG.md

@@ -3,15 +3,21 @@
 ## v0.0.9
 
 ### FEATURES
+- Binaries in the config can be defined as URLs now.
 - Add the option to set gm-lib path via the $GM_LIB environment variable ([#1365])
 
+### BUGFIXES
+- Fixed debug messages not printing to stdout properly.
+- Minor cosmetic fixes.
+
 ## v0.0.8
 
 ### BUGFIXES
 - Fixed gaiad 0.43 keys add printing key to stderr instead of stdout issue ([#1312])
 - Bumped default rpc_timeout in Hermes config to 5s ([#1312])
 
 [#1312]: https://github.com/informalsystems/ibc-rs/issues/1312
+[#1365]: https://github.com/informalsystems/ibc-rs/issues/1365
 
 ## v0.0.7
 

scripts/gm/bin/gm

@@ -7,6 +7,7 @@ SCRIPT_0="${0:-$HOME/.gm/bin/gm}"
 export SCRIPT_DIR="${SCRIPT_0%%gm}"
 export LOCAL_LIB_GM="${SCRIPT_DIR}lib-gm"
 if [ -f "$LOCAL_LIB_GM" ]; then
+  # shellcheck source=lib-gm
   . "$LOCAL_LIB_GM"
 elif [ -f "$HOME/.gm/bin/lib-gm" ]; then
   . "$HOME/.gm/bin/lib-gm"
@@ -37,9 +38,9 @@ ports       print the ports of a (running) node
 start       start one or more nodes (starts all nodes if no parameter is given)
 status      print the status of nodes
 stop        stop one or more nodes (stops all nodes if no parameter is given)
-reset       reset one or more nodes' database (resets all nodes if no parameter is given)"
-rm          delete the configuration of a node"
-version     print the application version"
+reset       reset one or more nodes' database (resets all nodes if no parameter is given)
+rm          delete the configuration of a node
+version     print the application version
 
 EOF
 }
@@ -61,6 +62,19 @@ keys         add keys to $HOME/.hermes
 EOF
 }
 
+if [ "${OUTPUT:-}" == "json" ]; then
+  case "${1:-version}" in
+    version)
+      version
+      exit 0
+      ;;
+    *)
+      exit_with_error "not implemented"
+      exit 1
+      ;;
+  esac
+fi
+
 case "${1:-help}" in
   help)
     usage
@@ -95,7 +109,7 @@ case "${1:-help}" in
         ;;
       *)
         hermes_usage
-        exit_with_error "could not understand the command hermes \"$1\""
+        exit_with_error "could not understand the command hermes '$1'"
         ;;
     esac
     ;;
@@ -175,8 +189,7 @@ case "${1:-help}" in
     ;;
   ss)
     # For debug purposes
-    # Todo: either make this a supported command or remove it
-    lsof -i4:27000-28000 -P
+    (which lsof && lsof -i:27000-28000 -P) || (which ss && ss -tulpn sport '>=' :27000 '&&' sport '<=' :28000) || (which netstat && netstat -tulpn) || echo "ERROR: no app found to list open ports"
     ;;
   start)
     parse_config_file
@@ -252,6 +265,6 @@ case "${1:-help}" in
     ;;
   *)
     usage
-    exit_with_error "could not understand the command \"$1\""
+    exit_with_error "could not understand the command '$1'"
     ;;
 esac

scripts/gm/bin/lib-gm

@@ -5,14 +5,37 @@ if [ "${DEBUG:-}" = "2" ]; then
 fi
 
 version() {
-  echo "v0.0.8"
+  VERSION="v0.0.9"
+  if [ "$OUTPUT" == "json" ]; then
+    printf '{"status": "success", "message": "%s"}' "$VERSION"
+  else
+    echo "$VERSION"
+  fi
 }
 
 # Configuration Management
 ###
 
+# Get the operating system in lowercase letters. Works for linux and darwin.
+get_operating_system() {
+  # uname is optional, we will assume linux if it's missing in some streamlined docker images.
+  OP_SYS="$(uname -s || echo)"
+  case "${OP_SYS}" in
+    "Linux" )
+      echo linux
+      ;;
+    "Darwin" )
+      echo darwin
+      ;;
+    * )
+      echo linux
+      ;;
+  esac
+}
+
 set_config_defaults() {
-    GLOBAL_GAIAD_BINARY="$(which gaiad || echo "${GOPATH:-./gaiad}")"
+    OP_SYS="$(get_operating_system)"
+    GLOBAL_GAIAD_BINARY="$(which gaiad || echo "https://github.com/cosmos/gaia/releases/download/v5.0.6/gaiad-v5.0.6-${OP_SYS}-amd64")"
     #GLOBAL_PORTS_START_AT is deliberately not exported because it is updated sometimes during execution
     DEFAULT_PORTS_START_AT=27000
     GLOBAL_HOME_DIR="${HOME}/.gm"
@@ -153,16 +176,15 @@ EOF
 
 find_config_file() {
   if [ -f "${SCRIPT_DIR}gm.toml" ]; then
-    export CONFIG_FILE="${SCRIPT_DIR}gm.toml"
+    export CONFIG_DIR="${SCRIPT_DIR}"
   elif [ -f "${HOME}/.gm/gm.toml" ]; then
-    export CONFIG_FILE="${HOME}/.gm/gm.toml"
+    export CONFIG_DIR="${HOME}/.gm/"
+  elif [ -d "${HOME}/.gm" ]; then
+    export CONFIG_DIR="${HOME}/.gm/"
   else
-    if [ -d "${HOME}/.gm" ]; then
-      export CONFIG_FILE="${HOME}/.gm/gm.toml"
-    else
-      export CONFIG_FILE="${SCRIPT_DIR}gm.toml"
-    fi
+    export CONFIG_DIR="${SCRIPT_DIR}"
   fi
+  export CONFIG_FILE="${CONFIG_DIR}gm.toml"
 }
 
 # End Configuration Management
@@ -213,6 +235,9 @@ enforce_requirements() {
   if [ -z "$(which tr || echo)" ]; then
     exit_with_error "missing tr, please install it"
   fi
+  if [ -z "$(which basename || echo)" ]; then
+    exit_with_error "missing basename, please install it"
+  fi
   if [ -z "$(which dirname || echo)" ]; then
     exit_with_error "missing dirname, please install it"
   fi
@@ -231,7 +256,7 @@ enforce_requirements() {
 
 debug() {
   if [ -n "${DEBUG:-}" ]; then
-    echo "DEBUG: $*"
+    echo "DEBUG: $*" 1>&2
   fi
 }
 
@@ -254,7 +279,11 @@ warn_unknown_hermes_node() {
 }
 
 exit_with_error() {
-  echo "ERROR: $*, exiting..."
+  if [ "${OUTPUT:-}" == "json" ]; then
+    printf '{"status": "error", "message": "%s"}' "$*"
+  else
+    echo "ERROR: $*, exiting..."
+  fi
   return 1
 }
 
@@ -298,15 +327,72 @@ load_all_sections() {
   grep '^ *\[.\+\] *$' "$CONFIG_FILE" | sed 's/^ *\[\([^]]*\)\] *$/\1/'
 }
 
+get_url_id() {
+  SUMMARIZER="$(which md5sum || which shasum || which sha256sum)"
+  if [ -z "${SUMMARIZER}" ]; then
+    echo "${1}" | tr -d ":/?=.&"
+  else
+    SUMMARY="$(echo "${1}" | "${SUMMARIZER}")"
+    SUMMARY="${SUMMARY%%  -}"
+    echo "${SUMMARY}"
+  fi
+}
+
+eval_binary() {
+  RAW_BINARY="$(eval echo "${1}")"
+  if [ "${RAW_BINARY##http://}" != "$RAW_BINARY" ] || [ "${RAW_BINARY##https://}" != "$RAW_BINARY" ] || [ "${RAW_BINARY##ftp://}" != "$RAW_BINARY" ]; then
+    FILE_URL_PATH="$(dirname "${RAW_BINARY}")"
+    FILE_URL_PATH_ID="$(get_url_id "${FILE_URL_PATH}")"
+    if [ ! -d "${CONFIG_DIR}${FILE_URL_PATH_ID}" ]; then
+      mkdir "${CONFIG_DIR}${FILE_URL_PATH_ID}" || exit_with_error "could not create temporary folder ${CONFIG_DIR}${FILE_URL_PATH_ID} to download binary ${RAW_BINARY}"
+    fi
+    FILE_NAME="$(basename "${RAW_BINARY}" | tr -d "?=&")"
+    BINARY="${CONFIG_DIR}${FILE_URL_PATH_ID}/${FILE_NAME}"
+    if [ -x "${BINARY}" ]; then
+      echo "${BINARY}"
+    else
+      DOWNLOADER="$(which wget)"
+      if [ -n "${DOWNLOADER}" ]; then
+        debug "downloading ${RAW_BINARY} using wget..."
+        # shellcheck disable=SC2086
+        "${DOWNLOADER}" -q -nd ${WGET_DOWNLOAD_OPTIONS:-} "${RAW_BINARY}" -O "${BINARY}"
+        chmod +x "${BINARY}"
+        echo "${BINARY}"
+      else
+        DOWNLOADER="$(which curl)"
+        if [ -n "${DOWNLOADER}" ]; then
+          debug "downloading ${RAW_BINARY} using curl..."
+          # shellcheck disable=SC2086
+          "${DOWNLOADER}" -s -f -L ${CURL_DOWNLOAD_OPTIONS:-} "${RAW_BINARY}" -o "${BINARY}"
+          chmod +x "${BINARY}"
+          echo "${BINARY}"
+        else
+          exit_with_error "found URL ${RAW_BINARY} but no downloader wget or curl"
+        fi
+      fi
+    fi
+  else
+    if [ -x "${RAW_BINARY}" ]; then
+      echo "${RAW_BINARY}"
+    else
+      exit_with_error "binary \"${RAW_BINARY}\" not found or not executable, check your gm.toml config"
+    fi
+  fi
+}
+
 get_gaiad_binary() {
   RESULT="$(stoml -q "$CONFIG_FILE" "${1}.gaiad_binary")"
   if [ -z "$RESULT" ]; then
-    echo "$GLOBAL_GAIAD_BINARY"
+    eval_binary "$GLOBAL_GAIAD_BINARY"
   else
-    eval echo "$RESULT"
+    eval_binary "$RESULT"
   fi
 }
 
+get_hermes_binary() {
+  eval_binary "$GLOBAL_HERMES_BINARY"
+}
+
 get_ports_start_at() {
   RESULT="$(stoml -q "$CONFIG_FILE" "${1}.ports_start_at")"
   if [ -z "$RESULT" ]; then
@@ -364,7 +450,7 @@ get_auto_maintain_config() {
 get_network() {
   RESULT="$(stoml -q "$CONFIG_FILE" "${1}.network")"
   if [ -z "$RESULT" ]; then
-    exit_on_error "Network not found for node ${1}"
+    exit_with_error "network not found for node ${1}"
   else
     if ! a_in_b "$RESULT" "$VALIDATORS"; then
       return 1
@@ -786,16 +872,16 @@ EOF
 hermes_keys() {
   ID="$(get_chain_id "$1")"
   NETWORK_HOME_DIR="$(get_home_dir "$ID")"
-  test -x "$GLOBAL_HERMES_BINARY" || exit_with_error "hermes binary \"${GLOBAL_HERMES_BINARY}\" not found, check your gm.toml config"
+  HERMES_BINARY="$(get_hermes_binary)"
   HDPATH="$(get_hdpath "$1")"
   if [ -z "$GLOBAL_HERMES_CONFIG" ] && [ -z "$HDPATH" ]; then
-      "$GLOBAL_HERMES_BINARY" keys add "$ID" -f "${NETWORK_HOME_DIR}/wallet_seed.json"
+      "$HERMES_BINARY" keys add "$ID" -f "${NETWORK_HOME_DIR}/wallet_seed.json"
   elif [ -n "$GLOBAL_HERMES_CONFIG" ] && [ -z "$HDPATH" ]; then
-      "$GLOBAL_HERMES_BINARY" -c "$GLOBAL_HERMES_CONFIG" keys add "$ID" -f "${NETWORK_HOME_DIR}/wallet_seed.json"
+      "$HERMES_BINARY" -c "$GLOBAL_HERMES_CONFIG" keys add "$ID" -f "${NETWORK_HOME_DIR}/wallet_seed.json"
   elif [ -z "$GLOBAL_HERMES_CONFIG" ] && [ -n "$HDPATH" ]; then
-      "$GLOBAL_HERMES_BINARY" keys add "$ID" --hd-path "$HDPATH" -f "${NETWORK_HOME_DIR}/wallet_seed.json"
+      "$HERMES_BINARY" keys add "$ID" --hd-path "$HDPATH" -f "${NETWORK_HOME_DIR}/wallet_seed.json"
   elif [ -n "$GLOBAL_HERMES_CONFIG" ] && [ -n "$HDPATH" ]; then
-      "$GLOBAL_HERMES_BINARY" -c "$GLOBAL_HERMES_CONFIG" keys add "$ID" --hd-path "$HDPATH" -f "${NETWORK_HOME_DIR}/wallet_seed.json"
+      "$HERMES_BINARY" -c "$GLOBAL_HERMES_CONFIG" keys add "$ID" --hd-path "$HDPATH" -f "${NETWORK_HOME_DIR}/wallet_seed.json"
   fi
 }
 
@@ -810,11 +896,12 @@ hermes_cc() {
   done
   CHAINS="${CHAINS## }"
   N="$(($(echo "$CHAINS" | wc -w)))"
+  HERMES_BINARY="$(get_hermes_binary)"
   for i in $(seq 1 $((N-1)))
   do
     for j in $(seq $((i+1)) $N)
     do
-      echo "\"${GLOBAL_HERMES_BINARY}\" create channel $(n_from_a "$i" "$CHAINS") $(n_from_a "$j" "$CHAINS") --port-a transfer --port-b transfer"
+      echo "\"${HERMES_BINARY}\" create channel $(n_from_a "$i" "$CHAINS") $(n_from_a "$j" "$CHAINS") --port-a transfer --port-b transfer"
     done
   done
 }

scripts/gm/gm.toml

@@ -8,7 +8,9 @@
 [global]
 
 # Path to the `gaiad` (or other SDK-based) binary to use.
-gaiad_binary="$GOPATH/bin/gaiad"
+# You can add a local path here or a ftp/http/https URL.
+# In the case of a URL, a local copy will be downloaded into a temporary folder within the GM folder.
+gaiad_binary="https://github.com/cosmos/gaia/releases/download/v5.0.5/gaiad-v5.0.5-linux-amd64"
 
 # The first free port to use for newly created nodes.
 # The value will be incremented (by 10) when a new node requires ports.
@@ -50,6 +52,8 @@ add_to_hermes=false
 [global.hermes]
 
 # Path to the `hermes` binary to use.
+# You can add a local path here or a ftp/http/https URL.
+# In the case of a URL, a local copy will be downloaded into a temporary folder within the GM folder.
 binary="./hermes"
 
 # Hermes configuration file path.