In tests/dev.sh: upgrade ruamel to support python 3.7. Closes #3312

Tested on python 3.5 and 3.7

We make light use of pyenv to set an appropriate python version if
installed. We could easily install a correct version too if we wanted
but that seemed invasive.

The newer ruamel was an annoying upgrade but also offers some
improvements that exposed some test suite issues (fixed later).

.gitignore

@@ -1,4 +1,5 @@
 .hasura-dev-python-venv
+server/tests-py/.devsh_version
 npm-debug.log
 *.temp
 *.DS_Store

scripts/dev.sh

@@ -10,6 +10,12 @@ set -euo pipefail
 echo_pretty() {
     echo ">>> $(tput setaf 2)$1$(tput sgr0)"
 }
+echo_error() {
+    echo ">>> $(tput setaf 1)$1$(tput sgr0)"
+}
+echo_warn() {
+    echo ">>> $(tput setaf 3)$1$(tput sgr0)"
+}
 
 die_usage() {
 cat <<EOL
@@ -37,6 +43,10 @@ EOL
 exit 1
 }
 
+# Bump this to:
+#  - force a reinstall of dependencies
+DEVSH_VERSION=1.2
+
 case "${1-}" in
   graphql-engine)
     case "${2-}" in
@@ -68,6 +78,20 @@ MODE="$1"
 PROJECT_ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." >/dev/null 2>&1 && pwd )"   # ... https://stackoverflow.com/a/246128/176841
 cd "$PROJECT_ROOT"
 
+# Use pyenv if available to set an appropriate python version that will work with pytests etc.
+if command -v pyenv ; then
+  # For now I guess use the greatest python3 >= 3.5
+  v=$(pyenv versions --bare | (grep  '^ *3' || true) | awk '{if($1>=3.5)print$1}' | tail -n1) 
+  if [ -z "$v" ]; then
+    echo_error 'Please `pyenv install` a version of python >= 3.5 so we can use it'
+    exit 2
+  fi
+  echo_pretty "Pyenv found. Using python version: $v"
+  export PYENV_VERSION=$v
+  python3 --version
+else
+  echo_warn "Pyenv not installed. Proceeding with system python version: $(python3 --version)"
+fi
 
 ####################################
 ###   Shared environment stuff   ###
@@ -299,14 +323,33 @@ elif [ "$MODE" = "test" ]; then
   done
   echo " Ok"
 
+  ### Check for and install dependencies in venv
   cd "$PROJECT_ROOT/server/tests-py"
   PY_VENV=.hasura-dev-python-venv
+  DEVSH_VERSION_FILE=.devsh_version
+  # Do we need to force reinstall?
+  if [ "$DEVSH_VERSION" = "$(cat $DEVSH_VERSION_FILE 2>/dev/null || true)" ]; then
+    true # ok
+  else
+    echo_warn 'dev.sh version was bumped. Forcing reinstallation of dependencies.'
+    rm -r "$PY_VENV"
+    echo "$DEVSH_VERSION" > "$DEVSH_VERSION_FILE"
+  fi
   set +u  # for venv activate
   if [ ! -d "$PY_VENV" ]; then
     python3 -m venv "$PY_VENV"
     source "$PY_VENV/bin/activate"
     pip3 install wheel
-    pip3 install -r requirements.txt
+    # If the maintainer of this script or pytests needs to change dependencies:
+    #  - alter requirements-top-level.txt as needed
+    #  - delete requirements.txt
+    #  - run this script, then check in the new frozen requirements.txt
+    if [ -f requirements.txt ]; then
+      pip3 install -r requirements.txt
+    else
+      pip3 install -r requirements-top-level.txt
+      pip3 freeze > requirements.txt
+    fi
   else
     echo_pretty "It looks like python dependencies have been installed already. Skipping."
     echo_pretty "If things fail please run this and try again"
@@ -315,6 +358,7 @@ elif [ "$MODE" = "test" ]; then
     source "$PY_VENV/bin/activate"
   fi
 
+
   # TODO MAYBE: fix deprecation warnings, make them an error
   if pytest -W ignore::DeprecationWarning --hge-urls http://127.0.0.1:$HASURA_GRAPHQL_SERVER_PORT --pg-urls "$DB_URL" $PYTEST_ARGS; then
     PASSED=true

server/CONTRIBUTING.md

@@ -11,6 +11,7 @@ own machine and how to contribute.
 - brotli
 - libpq-dev
 - python >= 3.5 with pip3
+- [pyenv](https://github.com/pyenv/pyenv) (Recommended, not required)
 
 The Brotli can be installed from source using `git`, `cmake` and `pkgconf` on Debian with:
 

server/tests-py/context.py

@@ -309,8 +309,9 @@ def v1q(self, q, headers = {}):
 
     def v1q_f(self, fn):
         with open(fn) as f:
-            # NOTE: preserve ordering with RoundTripLoader:
-            return self.v1q(yaml.load(f, yaml.RoundTripLoader))
+            # NOTE: preserve ordering with ruamel
+            yml = yaml.YAML()
+            return self.v1q(yml.load(f))
 
     def teardown(self):
         self.http.close()

server/tests-py/requirements-top-level.txt

@@ -1,6 +1,6 @@
 psycopg2-binary
 sqlalchemy
-pytest
+pytest==4.5.0
 pytest-xdist
 requests
 pyyaml
@@ -10,5 +10,5 @@ jsondiff
 cryptography
 graphene
 brotlipy
-ruamel.yaml < 0.15
+ruamel.yaml > 0.15
 graphql-core

server/tests-py/requirements.txt

@@ -1,37 +1,37 @@
-aniso8601==3.0.2
+aniso8601==7.0.0
 apipkg==1.5
-asn1crypto==0.24.0
 atomicwrites==1.3.0
-attrs==19.1.0
-certifi==2019.3.9
-cffi==1.12.3
+attrs==19.3.0
+brotlipy==0.7.0
+certifi==2019.9.11
+cffi==1.13.2
 chardet==3.0.4
-cryptography==2.7
-execnet==1.6.0
-graphene==2.1.3
-graphql-core==2.1
-graphql-relay==0.4.5
+cryptography==2.8
+execnet==1.7.1
+graphene==2.1.8
+graphql-core==2.2.1
+graphql-relay==2.0.0
 idna==2.8
-importlib-metadata==0.17
-jsondiff==1.1.2
-more-itertools==7.0.0
-pluggy==0.12.0
+importlib-metadata==0.23
+jsondiff==1.2.0
+more-itertools==7.2.0
+pluggy==0.13.0
 promise==2.2.1
-psycopg2-binary==2.8.2
+psycopg2-binary==2.8.4
 py==1.8.0
 pycparser==2.19
 PyJWT==1.7.1
 pytest==4.5.0
-pytest-forked==1.0.2
-pytest-xdist==1.28.0
-PyYAML==5.1
+pytest-forked==1.1.3
+pytest-xdist==1.30.0
+PyYAML==5.1.2
 requests==2.22.0
+ruamel.yaml==0.16.5
+ruamel.yaml.clib==0.2.0
 Rx==1.6.1
-six==1.12.0
-SQLAlchemy==1.3.4
-urllib3==1.25.3
+six==1.13.0
+SQLAlchemy==1.3.11
+urllib3==1.25.7
 wcwidth==0.1.7
 websocket-client==0.56.0
-zipp==0.5.1
-brotlipy==0.7.0
-ruamel.yaml==0.14.12
+zipp==0.6.0

server/tests-py/validate.py

@@ -2,6 +2,8 @@
 
 import pytest
 import ruamel.yaml as yaml
+from ruamel.yaml.compat import ordereddict, StringIO
+from ruamel.yaml.comments import CommentedMap
 import json
 import copy
 import graphql
@@ -250,22 +252,46 @@ def assert_graphql_resp_expected(resp_orig, exp_response_orig, query):
     # consideration only the ordering that we care about:
     resp         = collapse_order_not_selset(resp_orig,         query)
     exp_response = collapse_order_not_selset(exp_response_orig, query)
-    matched = resp == exp_response
+    matched = equal_CommentedMap(resp, exp_response)
 
     if pytest.config.getoption("--accept"):
         print('skipping assertion since we chose to --accept new output')
     else:
-        assert matched, '\n' + yaml.dump({
+        yml = yaml.YAML()
+        # https://yaml.readthedocs.io/en/latest/example.html#output-of-dump-as-a-string  :
+        dump_str = StringIO()
+        yml.dump({
             # Keep strict received order when displaying errors:
             'response': resp_orig,
             'expected': exp_response_orig,
             'diff': 
               (lambda diff: 
                  "(results differ only in their order of keys)" if diff == {} else diff)
               (stringify_keys(jsondiff.diff(exp_response, resp)))
-        }, Dumper=yaml.RoundTripDumper )
+        }, stream=dump_str)
+        assert matched, dump_str.getvalue()
     return resp, matched  # matched always True unless --accept
 
+# This really sucks; newer ruamel made __eq__ ignore ordering:
+#   https://bitbucket.org/ruamel/yaml/issues/326/commentedmap-equality-no-longer-takes-into
+def equal_CommentedMap(m1, m2):
+    if isinstance(m1, list) and isinstance(m2, list):
+        return (len(m1) == len(m2) and all(equal_CommentedMap(x,y) for x,y in zip(m1,m2)))
+    elif isinstance(m1, dict) and isinstance(m2, dict):
+        # (see collapse_order_not_selset):
+        if isinstance(m1, (ordereddict, CommentedMap)) and \
+           isinstance(m2, (ordereddict, CommentedMap)):
+            m1_l = list(m1.items())
+            m2_l = list(m2.items())
+        else:
+            m1_l = sorted(list(m1.items()))
+            m2_l = sorted(list(m2.items()))
+        return (len(m1_l) == len(m2_l) and 
+                all(k1 == k2 and equal_CommentedMap(v1,v2) 
+                    for (k1,v1),(k2,v2) in zip(m1_l,m2_l)))
+    # else this is a scalar:
+    else:
+        return m1 == m2
 
 def check_query_f(hge_ctx, f, transport='http', add_auth=True):
     print("Test file: " + f)
@@ -275,10 +301,11 @@ def check_query_f(hge_ctx, f, transport='http', add_auth=True):
         # For `--accept`:
         should_write_back = False
 
-        # ruamel RoundTripLoader will preserve order so that we can test the
-        # JSON ordering property conforms to YAML spec.
-        # It also lets us write back the yaml nicely when we --accept.
-        conf = yaml.load(c, yaml.RoundTripLoader)
+        # ruamel will preserve order so that we can test the JSON ordering
+        # property conforms to YAML spec.  It also lets us write back the yaml
+        # nicely when we `--accept.`
+        yml = yaml.YAML()
+        conf = yml.load(c)
         if isinstance(conf, list):
             for ix, sconf in enumerate(conf):
                 actual_resp, matched = check_query(hge_ctx, sconf, transport, add_auth)
@@ -305,7 +332,7 @@ def check_query_f(hge_ctx, f, transport='http', add_auth=True):
                 "\n   NOTE: if this case was marked 'xfail' this won't be correct!"
             )
             c.seek(0)
-            c.write(yaml.dump(conf, Dumper=yaml.RoundTripDumper))
+            c.write(yml.dump(conf))
             c.truncate()