Add some more type annotations to clients.py

.editorconfig

@@ -0,0 +1,9 @@
+root = true
+
+[*]
+charset = utf-8
+indent_style = space
+indent_size = 4
+
+[yaml]
+indent_size = 2

.github/workflows/main.yml

@@ -16,15 +16,18 @@ jobs:
         python-version: [3.7, 3.8, 3.9, '3.10', '3.11', '3.12']
 
     steps:
-      - uses: actions/checkout@master
+      - uses: actions/checkout@v4
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
+          cache: "pip"
+          cache-dependency-path: setup.py
 
       - name: Set up Java
-        uses: actions/setup-java@v1
+        uses: actions/setup-java@v4
         with:
+          distribution: "temurin"
           java-version: 11
 
       - name: Install dependencies
@@ -43,19 +46,21 @@ jobs:
         env:
           WACKLIG_TOKEN: ${{ secrets.WACKLIG_TOKEN }}
         run: |
-          curl -s https://raw.githubusercontent.com/pipifein/wacklig-uploader/master/wacklig.py | python - --token $WACKLIG_TOKEN || echo "Upload to wacklig failed"
+          curl -s https://raw.githubusercontent.com/pipifein/wacklig-uploader/master/wacklig.py | python - --token "$WACKLIG_TOKEN" || echo "Upload to wacklig failed"
 
   publish:
     name: Build & publish package to pypi
     needs: test
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@master
+      - uses: actions/checkout@v4
 
       - name: Set up python 3.7
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v5
         with:
           python-version: 3.7
+          cache: "pip"
+          cache-dependency-path: setup.py
 
       - name: Build package
         run: |

.vscode/launch.json

@@ -0,0 +1,22 @@
+{
+   "$schema": "https://raw.githubusercontent.com/mfussenegger/dapconfig-schema/master/dapconfig-schema.json",
+   "version": "0.2.0",
+   "configurations": [
+       {
+           "type": "debugpy",
+           "request": "launch",
+           "name": "run_crate",
+           "module": "cr8.run_crate",
+           "args": ["latest-nightly"],
+           "console": "integratedTerminal"
+       },
+       {
+           "type": "debugpy",
+           "request": "launch",
+           "name": "run_spec",
+           "module": "cr8.run_spec",
+           "args": ["specs/sample.toml", "localhost:4200"],
+           "console": "integratedTerminal"
+       }
+   ]
+}

README.rst

@@ -75,13 +75,10 @@ Python >= 3.7 is required to use the command line tools.
 
 Install them using `pip <https://pip.pypa.io/en/stable/>`_::
 
-    python3.7 -m pip install --user cr8
-
-This will install ``cr8`` into ``~/.local/bin``. Either use
-``~/.local/bin/cr8`` to launch it or add ``~/.local/bin`` to your ``$PATH``
-environment variable.
-
+    python3 -m venv venv
+    venv/bin/python -m pip install cr8
 
+This will install ``cr8`` into ``venv/bin``
 An alternative is to download a single ``zipapp`` file from the `releases page
 <https://github.com/mfussenegger/cr8/releases>`_.
 

cr8/clients.py

@@ -7,17 +7,17 @@
 import time
 from urllib.parse import urlparse, parse_qs, urlunparse
 from datetime import datetime, date
-from typing import List, Union, Iterable, Dict
+from typing import List, Union, Iterable, Dict, Optional, Any
 from decimal import Decimal
 from cr8.aio import asyncio  # import via aio for uvloop setup
 
 try:
     import asyncpg
 except ImportError:
-    asyncpg = None
+    asyncpg = None  # type: ignore
 
 try:
-    import simdjson
+    import simdjson  # type: ignore
     dumps = simdjson.dumps
 except ImportError:
     dumps = json.dumps
@@ -137,7 +137,7 @@ def _plain_or_callable(obj):
         return obj
 
 
-def _date_or_none(d: str) -> str:
+def _date_or_none(d: str) -> Optional[str]:
     """Return a date as if, if valid, otherwise None
 
     >>> _date_or_none('2017-02-27')
@@ -152,7 +152,7 @@ def _date_or_none(d: str) -> str:
         return None
 
 
-def _to_dsn(hosts):
+def _to_dsn(hosts: str) -> str:
     """Convert a host URI into a dsn for aiopg.
 
     >>> _to_dsn('aiopg://myhostname:4242/mydb')
@@ -177,15 +177,15 @@ def _to_dsn(hosts):
         host, port = netloc.split(':', maxsplit=1)
     except ValueError:
         host = netloc
-        port = 5432
+        port = "5432"
     dbname = p.path[1:] if p.path else 'doc'
     dsn = f'postgres://{user_and_pw}@{host}:{port}/{dbname}'
     if p.query:
         dsn += '?' + '&'.join(k + '=' + v[0] for k, v in parse_qs(p.query).items())
     return dsn
 
 
-def _to_boolean(v):
+def _to_boolean(v: str) -> bool:
     if str(v).lower() in ("true"):
         return True
     elif str(v).lower() in ("false"):
@@ -194,7 +194,7 @@ def _to_boolean(v):
         raise ValueError('not a boolean value')
 
 
-def _verify_ssl_from_first(hosts):
+def _verify_ssl_from_first(hosts: List[str]) -> bool:
     """Check if SSL validation parameter is passed in URI
 
     >>> _verify_ssl_from_first(['https://myhost:4200/?verify_ssl=false'])
@@ -295,7 +295,7 @@ def __exit__(self, *exs):
         self.close()
 
 
-def _append_sql(host):
+def _append_sql(host: str) -> str:
     """ Append `/_sql` to the host, dropping any query parameters.
 
     >>> _append_sql('http://n1:4200')
@@ -316,12 +316,15 @@ def _append_sql(host):
 
 
 class HttpClient:
-    def __init__(self, hosts, conn_pool_limit=25, session_settings=None):
+    def __init__(self,
+                 hosts: List[str],
+                 conn_pool_limit=25,
+                 session_settings: Optional[Dict[str, Any]] = None):
         self.hosts = hosts
         self.urls = itertools.cycle(list(map(_append_sql, hosts)))
         self.conn_pool_limit = conn_pool_limit
         self.is_cratedb = True
-        self._pools = {}
+        self._pools: Dict[str, asyncio.Queue] = {}
         self.session_settings = session_settings or {}
 
     async def _session(self, url):

cr8/engine.py

@@ -5,7 +5,7 @@
 
 from cr8 import aio
 from cr8.metrics import Stats, get_sampler
-from cr8.clients import client, HttpClient
+from cr8.clients import client
 
 TimedStats = namedtuple('TimedStats', ['started', 'ended', 'stats'])
 

setup.py

@@ -32,7 +32,8 @@
         'asyncpg'
     ],
     extras_require={
-        'extra': ['uvloop', 'pysimdjson']
+        'extra': ['uvloop', 'pysimdjson'],
+        "dev": ["asyncpg-stubs", "mypy"]
     },
     python_requires='>=3.7',
     classifiers=[

specs/count_countries.json

@@ -15,8 +15,8 @@
         ]
     },
     "session_settings": {
-            "application_name": "my_app",
-            "timezone": "UTC"
+        "application_name": "my_app",
+        "timezone": "UTC"
     },
     "queries": [{
         "iterations": 1000,

tests/test_reindex.py

@@ -19,43 +19,43 @@ def setUp(self):
             'http.port': '44200-44250'
         }
 
-    def teardown(self):
+    def tearDown(self):
         for node in self._to_stop:
             node.stop()
         self._to_stop = []
         shutil.rmtree(self.data_path, ignore_errors=True)
 
     def test_reindex(self):
-        crate_v3 = CrateNode(
-            crate_dir=get_crate('3.x.x'),
+        crate_v4 = CrateNode(
+            crate_dir=get_crate('4.x.x'),
             keep_data=True,
             settings=self.crate_settings
         )
-        self._to_stop.append(crate_v3)
-        crate_v3.start()
-        with client(crate_v3.http_url) as c:
+        self._to_stop.append(crate_v4)
+        crate_v4.start()
+        with client(crate_v4.http_url) as c:
             aio.run(c.execute, "create table t (x int)")
             args = (
                 (1,),
                 (2,),
                 (3,),
             )
             aio.run(c.execute_many, "insert into t (x) values (?)", args)
-        crate_v3.stop()
-        self._to_stop.remove(crate_v3)
+        crate_v4.stop()
+        self._to_stop.remove(crate_v4)
 
-        crate_v4 = CrateNode(
-            crate_dir=get_crate('4.0.3'),
+        crate_v5 = CrateNode(
+            crate_dir=get_crate('5.0.3'),
             keep_data=True,
             settings=self.crate_settings
         )
-        self._to_stop.append(crate_v4)
-        crate_v4.start()
-        reindex(hosts=crate_v4.http_url)
-        with client(crate_v4.http_url) as c:
+        self._to_stop.append(crate_v5)
+        crate_v5.start()
+        reindex(hosts=crate_v5.http_url)
+        with client(crate_v5.http_url) as c:
             result = aio.run(c.execute, "SELECT version FROM information_schema.tables WHERE table_name = 't'")
             version = result['rows'][0][0]
-            self.assertEqual(version, {'upgraded': None, 'created': '4.0.3'})
+            self.assertEqual(version, {'upgraded': None, 'created': '5.0.3'})
 
             cnt = aio.run(c.execute, 'SELECT count(*) FROM t')['rows'][0][0]
             self.assertEqual(cnt, 3)