chore: e2e + ci

.github/workflows/build.yaml

@@ -0,0 +1,52 @@
+name: Build
+permissions:
+  statuses: write
+  checks: write
+on:
+  pull_request:
+    branches:
+      - main
+  push:
+    branches:
+      - main
+    paths-ignore:
+      - '*.md'
+      - '*.puml'
+      - '.github/workflows/release.yaml'
+jobs:
+  build_job:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 20
+      - uses: wagoid/commitlint-github-action@v5
+        with:
+          configFile: .commitlintrc.yml
+          token: ${{ secrets.GITHUB_TOKEN }}
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+      - uses: Gr1N/setup-poetry@v8
+      - uses: actions/cache@v3
+        with:
+          path: ~/.cache/pypoetry/virtualenvs
+          key: ${{ runner.os }}-poetry-${{ hashFiles('poetry.lock') }}
+      - run: |
+          poetry install
+      - uses: extractions/setup-just@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - run: just test
+      - name: test report junit
+        uses: dorny/test-reporter@v1
+        if: ${{ success() || failure() }}
+        with:
+          name: test report
+          path: ./test-*.xml
+          reporter: java-junit
+      - name: failure logs
+        if: ${{ failure() }}
+        run:
+          docker ps
+

.gitignore

@@ -158,3 +158,5 @@ cython_debug/
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 #.idea/
+
+test-*.xml

README.md

@@ -26,6 +26,7 @@ pip install -U git+https://github.com/eshepelyuk/pykli@main
 
 ## TODO
 
+* Session wide variables with `DEFINE`
 * Metadata autocompletion
     * table ans stream names
     * column names and functions in queries

justfile

@@ -6,13 +6,15 @@ up:
   docker-compose -f tests/docker-compose.yaml up -d
 
 down:
-  docker-compose -f tests/docker-compose.yaml down
+  docker-compose -f tests/docker-compose.yaml down -v --remove-orphans
 
-test filter='':
-  poetry run pytest -m 'not e2e' --capture=tee-sys -k '{{filter}}'
+test-unit filter='':
+  poetry run pytest -o junit_suite_name='Unit tests' -m 'not e2e' --capture=tee-sys --junit-xml=test-unit.xml -k '{{filter}}'
 
-e2e filter='': up
-  poetry run pytest -m e2e --capture=tee-sys -k '{{filter}}'
+test-e2e filter='': up
+  poetry run pytest -o junit_suite_name='E2E tests' -m e2e --capture=tee-sys --junit-xml=test-e2e.xml -k '{{filter}}'
+
+test: test-unit test-e2e
 
 run srv="http://localhost:28088":
   poetry run pykli {{srv}}

pykli/__main__.py

@@ -1,31 +1,36 @@
 import sys
 import click
+import pathlib
+
 from . import LOG
 from .ksqldb import KsqlDBClient
-from .repl_read import pykli_prompt, pykli_read
+from .repl_read import pykli_prompt, pykli_read, file_prompt
 from .repl_eval import pykli_eval
 from .repl_print import pykli_print
-from .tokens import Info, ErrorMessage, initialize_sqlparse
+from .tokens import Info, ErrMsg, initialize_sqlparse
 
 
 @click.command()
+@click.option('-f', '--file', help="execute commands from file, then exit",
+    type=click.Path(exists=True, path_type=pathlib.Path))
 @click.argument("server", default = "http://localhost:8088")
-def main(server):
+def main(server, file):
     """
     pyKLI - ksqlDB command line client.
 
     \b SERVER The address of the ksqlDB server."""
 
-    LOG.info(f"pyKLI started: server={server}")
+    LOG.info(f"pyKLI started: server={server}, file={file}")
 
     initialize_sqlparse()
 
     eval = pykli_eval(KsqlDBClient(server))
+    prompt = pykli_prompt() if file is None else file_prompt(file)
 
-    if isinstance(pykli_print(eval(Info(server))), ErrorMessage):
+    if isinstance(pykli_print(eval(Info(server))), ErrMsg):
         sys.exit(1)
 
-    for token in pykli_read(pykli_prompt()):
+    for token in pykli_read(prompt):
         pykli_print(eval(token))
 
     sys.exit(0)

pykli/ksqldb.py

@@ -5,6 +5,7 @@
 class KsqlDBClient:
     def __init__(self, url, api_key=None, api_secret=None):
         # TODO introduce common headers
+        self._url = url
         self._headers = {"Content-Type": "application/vnd.ksql.v1+json"}
         self._client = httpx.Client(base_url=url, http2=True)
 
@@ -13,6 +14,11 @@ def __init__(self, url, api_key=None, api_secret=None):
             self._headers["Authorization"] = f"Basic {b64string}"
 
 
+    @property
+    def url(self):
+        return self._url
+
+
     def info(self):
         r = self._client.get("/info", headers=self._headers)
         r.raise_for_status()
@@ -40,6 +46,31 @@ def pull_query(self, ksql_str, stream_props={}):
         r.raise_for_status()
         return r.json()
 
+
+    def list_topic_names(self) -> list[str]:
+        json = self.stmt("show topics;")[0]
+        return [t["name"] for t in json["topics"]]
+
+
+    def list_stream_names(self) -> list[str]:
+        json = self.stmt("show streams;")[0]
+        return [t["name"] for t in json["streams"]]
+
+
+    def list_type_names(self) -> list[str]:
+        json = self.stmt("show types;")[0]
+        return json["types"].keys()
+
+
+    def list_table_names(self) -> list[str]:
+        json = self.stmt("show tables;")[0]
+        return [t["name"] for t in json["tables"]]
+
+
+    def list_connector_names(self) -> list[str]:
+        json = self.stmt("show connectors;")[0]
+        return [t["name"] for t in json["connectors"]]
+
     # async def query_async(self, query_string, stream_properties=None, timeout=10):
     #     async for x in self.api.query(
     #         query_string=query_string,

pykli/repl_eval.py

@@ -1,5 +1,5 @@
 from . import LOG
-from .tokens import Stmt, ErrorMessage, KResponse, Info, PullQuery, QueryResponse
+from .tokens import Stmt, ErrMsg, StmtResponse, Info, PullQuery, QueryResponse
 
 import httpx
 from pprint import pformat
@@ -14,20 +14,20 @@ def __call__(self, token):
             match token:
                 case Info(srv):
                     info = self._ksqldb.info()
-                    return KResponse([info | {"@type": "info", "server": srv}])
+                    return StmtResponse([info | {"@type": "info", "server": srv}])
                 case Stmt(ksql):
                     resp = self._ksqldb.stmt(ksql)
                     LOG.debug(f"KSQL={ksql}, response={pformat(resp)}")
-                    return KResponse(resp)
+                    return StmtResponse(resp)
                 case PullQuery(ksql):
                     resp = self._ksqldb.pull_query(ksql)
                     LOG.debug(f"KSQL={ksql}, response={pformat(resp)}")
                     return QueryResponse(resp)
-                case ErrorMessage():
+                case ErrMsg():
                     return token
                 case _:
-                    return ErrorMessage(f"not yet implemented: {token}")
+                    return ErrMsg(f"not yet implemented: {token}")
         except httpx.HTTPStatusError as e:
-            return ErrorMessage(e.response.json()["message"])
+            return ErrMsg(e.response.json()["message"])
         except httpx.TransportError as e:
-            return ErrorMessage(f"Transport error: {pformat(e)}")
+            return ErrMsg(f"Transport error: {pformat(e)}")

pykli/repl_print.py

@@ -9,7 +9,7 @@
 from cli_helpers.tabular_output.preprocessors import style_output
 
 from . import MONOKAI_STYLE, LOG
-from .tokens import KResponse, ErrorMessage, QueryResponse
+from .tokens import StmtResponse, ErrMsg, QueryResponse
 
 
 DESCRIBE_SRC_HEADERS = ("Field", "Type")
@@ -199,11 +199,11 @@ def print_query(json_arr):
 
 def pykli_print(token):
     match token:
-        case KResponse(resp):
+        case StmtResponse(resp):
             print_stmt(resp)
         case QueryResponse(resp):
             print_query(resp)
-        case ErrorMessage(msg):
+        case ErrMsg(msg):
             perr(msg)
         case _:
             perr(f"output not implemented: {token}")

pykli/repl_read.py

@@ -13,7 +13,20 @@
 from . import MONOKAI_STYLE, HISTORY_FILE, LOG
 from .completer import pykli_completer
 from .keybindgings import pykli_keys
-from .tokens import Stmt, ErrorMessage, KRunScript, PullQuery
+from .tokens import Stmt, ErrMsg, KRunScript, PullQuery
+
+
+class file_prompt:
+    def __init__(self, path):
+        self._path = path
+
+    def __call__(self):
+        if  self._path:
+            ksql = self._path.read_text()
+            self._path = None
+            return ksql
+
+        return "exit"
 
 
 class pykli_prompt:
@@ -47,9 +60,9 @@ def tokenize_script(stmt):
             for stmt in sqlparse.parse(path.read_text()):
                 yield from tokenize_sql_stmt(stmt)
         else:
-            yield ErrorMessage(f"'{path}' not found")
+            yield ErrMsg(f"'{path}' not found")
     else:
-        yield ErrorMessage(f"syntax error: {stmt}")
+        yield ErrMsg(f"syntax error: {stmt}")
 
 
 def tokenize_sql_stmt(stmt):