add docker/compose for bytewax/hackernews example

Author: jovezhong

example/bytewax/.dockerignore

@@ -0,0 +1,34 @@
+# Include any files or directories that you don't want to be copied to your
+# container here (e.g., local build artifacts, temporary files, etc.).
+#
+# For more help, visit the .dockerignore file reference guide at
+# https://docs.docker.com/engine/reference/builder/#dockerignore-file
+
+**/.DS_Store
+**/__pycache__
+**/.venv
+**/.classpath
+**/.dockerignore
+**/.env
+**/.git
+**/.gitignore
+**/.project
+**/.settings
+**/.toolstarget
+**/.vs
+**/.vscode
+**/*.*proj.user
+**/*.dbmdl
+**/*.jfm
+**/bin
+**/charts
+**/docker-compose*
+**/compose*
+**/Dockerfile*
+**/node_modules
+**/npm-debug.log
+**/obj
+**/secrets.dev.yaml
+**/values.dev.yaml
+LICENSE
+README.md

example/bytewax/Dockerfile

@@ -0,0 +1,41 @@
+# syntax=docker/dockerfile:1
+
+FROM python:3.10 as base
+
+# Prevents Python from writing pyc files.
+ENV PYTHONDONTWRITEBYTECODE=1
+
+# Keeps Python from buffering stdout and stderr to avoid situations where
+# the application crashes without emitting any logs due to buffering.
+ENV PYTHONUNBUFFERED=1
+
+#WORKDIR /app
+
+# Create a non-privileged user that the app will run under.
+# See https://docs.docker.com/develop/develop-images/dockerfile_best-practices/#user
+ARG UID=10001
+RUN adduser \
+    --disabled-password \
+    --gecos "" \
+    --home "/nonexistent" \
+    --shell "/sbin/nologin" \
+    --no-create-home \
+    --uid "${UID}" \
+    appuser
+
+# Download dependencies as a separate step to take advantage of Docker's caching.
+# Leverage a cache mount to /root/.cache/pip to speed up subsequent builds.
+# Leverage a bind mount to requirements.txt to avoid having to copy them into
+# into this layer.
+RUN --mount=type=cache,target=/root/.cache/pip \
+    --mount=type=bind,source=requirements.txt,target=requirements.txt \
+    python -m pip install -r requirements.txt
+
+# Switch to the non-privileged user to run the application.
+USER appuser
+
+# Copy the source code into the container.
+COPY . .
+
+# Run the application.
+CMD python -m bytewax.run "hackernews:run_hn_flow(0)"

example/bytewax/compose.yaml

@@ -0,0 +1,11 @@
+services:
+  proton:
+    image: ghcr.io/timeplus-io/proton:latest
+  hn_datagen:
+    build:
+      context: .  
+    image: timeplus/hackernews_bytewax:latest
+    environment:
+      - PROTON_HOST=proton
+    depends_on:
+      - proton

example/bytewax/hackernews.py

@@ -1,6 +1,6 @@
 import requests
 from datetime import datetime, timedelta
-import time, json
+import time, json, os
 from typing import Any, Optional
 
 from bytewax.connectors.periodic import SimplePollingInput
@@ -76,5 +76,5 @@ def run_hn_flow(init_item):
     # traversal function to get the ultimate parent
     flow.map(key_on_parent)
 
-    flow.output("out", ProtonOutput("hn"))
+    flow.output("out", ProtonOutput("hn",os.environ["PROTON_HOST"]))
     return flow

example/bytewax/proton.py

@@ -10,8 +10,8 @@
 # logger.setLevel(logging.DEBUG)
 
 class _ProtonSink(StatelessSink):
-    def __init__(self, stream: str):
-        self.client=client.Client(host='127.0.0.1', port=8463)
+    def __init__(self, stream: str, host: str):
+        self.client=client.Client(host=host, port=8463)
         self.stream=stream
         sql=f"CREATE STREAM IF NOT EXISTS `{stream}` (raw string)"
         logger.debug(sql)
@@ -26,8 +26,9 @@ def write_batch(self, items):
         self.client.execute(sql,rows)
 
 class ProtonOutput(DynamicOutput):
-    def __init__(self, stream: str):
-        self.stream=stream
+    def __init__(self, stream: str, host: str):
+        self.stream = stream
+        self.host = host if host is not None and host != "" else "127.0.0.1"
 
     """Write each output item to Proton on that worker.
 
@@ -43,4 +44,4 @@ def __init__(self, stream: str):
 
     def build(self, worker_index, worker_count):
         """See ABC docstring."""
-        return _ProtonSink(self.stream)
+        return _ProtonSink(self.stream, self.host)

example/bytewax/requirements.txt

@@ -0,0 +1,3 @@
+bytewax
+requests
+proton-driver