diff --git a/Makefile b/Makefile index b53704b..650ccf6 100644 --- a/Makefile +++ b/Makefile @@ -10,8 +10,8 @@ build-m1: rebuild: docker build --no-cache --tag opentopodata:$(VERSION) --file docker/Dockerfile . -rebuild-m1-no-cache: - docker build --tag opentopodata:$(VERSION) --file docker/apple-silicon.Dockerfile . +rebuild-m1: + docker build --no-cache --tag opentopodata:$(VERSION) --file docker/apple-silicon.Dockerfile . run: docker run --rm -it --volume "$(shell pwd)/data:/app/data:ro" -p 5000:5000 opentopodata:$(VERSION) @@ -29,7 +29,7 @@ run-local: FLASK_APP=opentopodata/api.py FLASK_DEBUG=1 flask run --port 5000 black: - black --target-version py311 tests opentopodata + black --target-version py311 tests opentopodata docker black-check: docker run --rm opentopodata:$(VERSION) python -m black --check --target-version py311 tests opentopodata diff --git a/docker/Dockerfile b/docker/Dockerfile index e7dd3f9..b722280 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -15,6 +15,8 @@ FROM python:3.11.10-slim-bookworm RUN set -e && \ apt-get update && \ apt-get install -y --no-install-recommends \ + inotify-tools \ + nano \ nginx \ memcached \ supervisor && \ diff --git a/docker/config_watcher.py b/docker/config_watcher.py new file mode 100644 index 0000000..a45dc65 --- /dev/null +++ b/docker/config_watcher.py @@ -0,0 +1,110 @@ +import logging +import time +from pathlib import Path +import subprocess +import sys + +from watchdog.observers import Observer +from watchdog.events import FileSystemEventHandler + + +# Paths. +CONFIG_DIR = Path("/app/") +CONFIG_PATH = Path("/app/config.yaml") +EXAMPLE_CONFIG_PATH = Path("/app/example-config.yaml") + +# Debouncing: once the config has been reloaded, any queued unprocessed events should be discarded. +LAST_INVOCATION_TIME = time.time() + + +# Logger setup. +logger = logging.getLogger("configwatcher") +LOG_FORMAT = "%(asctime)s %(levelname)-8s %(message)s" +formatter = logging.Formatter(LOG_FORMAT) +logger.setLevel(logging.INFO) +handler = logging.StreamHandler(sys.stdout) +handler.setLevel(logging.INFO) +handler.setFormatter(formatter) +logger.addHandler(handler) + + +def run_cmd(cmd, shell=False): + r = subprocess.run(cmd, shell=shell, capture_output=True) + is_error = r.returncode != 0 + stdout = r.stdout.decode("utf-8") + if is_error: + logger.error(f"Error running command, returncode: {r.returncode}") + logger.error("cmd:") + logger.error(" ".join(cmd)) + if r.stdout: + logger.error("stdout:") + logger.error(stdout) + if r.stderr: + logger.error("stderr:") + logger.error(r.stderr.decode("utf-8")) + raise ValueError + return stdout + + +def reload_config(): + global LAST_INVOCATION_TIME + LAST_INVOCATION_TIME = time.time() + logger.info("Restarting OTD due to config change.") + run_cmd(["supervisorctl", "-c", "/app/docker/supervisord.conf", "stop", "uwsgi"]) + run_cmd( + ["supervisorctl", "-c", "/app/docker/supervisord.conf", "restart", "memcached"] + ) + run_cmd(["supervisorctl", "-c", "/app/docker/supervisord.conf", "start", "uwsgi"]) + run_cmd( + ["supervisorctl", "-c", "/app/docker/supervisord.conf", "start", "warm_cache"] + ) + LAST_INVOCATION_TIME = time.time() + logger.info("Restarted OTD due to config change.") + + +class Handler(FileSystemEventHandler): + + def on_any_event(self, event): + watch_paths_str = [ + EXAMPLE_CONFIG_PATH.as_posix(), + CONFIG_PATH.as_posix(), + ] + + # Filter unwanted events. + if event.event_type not in {"modified", "created"}: + logger.info(f"Dropping event with type {event.event_type=}") + return + if event.is_directory: + logger.info(f"Dropping dir event") + return + if event.src_path not in watch_paths_str: + logger.info(f"Dropping event with path {event.src_path=}") + return + if not Path(event.src_path).exists(): + logger.info(f"Dropping event for nonexistent path {event.src_path=}") + return + + # Debouncing. + mtime = Path(event.src_path).lstat().st_mtime + if mtime < LAST_INVOCATION_TIME: + msg = f"Dropping event for file that hasn't been modified since the last run. {event.src_path=}" + logger.info(msg) + return + + logger.info(f"Dispatching event on {event.src_path=}") + reload_config() + + +if __name__ == "__main__": + + event_handler = Handler() + observer = Observer() + observer.schedule(event_handler, CONFIG_DIR, recursive=False) + observer.start() + + try: + while True: + time.sleep(1) + finally: + observer.stop() + observer.join() diff --git a/docker/supervisord.conf b/docker/supervisord.conf index 9c1d09e..57f6577 100644 --- a/docker/supervisord.conf +++ b/docker/supervisord.conf @@ -2,6 +2,19 @@ nodaemon=true user=root + +# Supervisorctl config/ +[unix_http_server] +file=/var/run/supervisor.sock +chmod=0700 +[rpcinterface:supervisor] +supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface +[supervisorctl] +serverurl=unix:///var/run/supervisor.sock + + +# OTD services. + [program:uwsgi] command=/usr/local/bin/uwsgi --ini /app/docker/uwsgi.ini --processes %(ENV_N_UWSGI_THREADS)s stdout_logfile=/dev/stdout @@ -33,10 +46,9 @@ stderr_logfile=/dev/stderr stderr_logfile_maxbytes=0 autorestart=false -[program:watchdog] -command=python /app/docker/watcher.py +[program:watch_config] +command=python /app/docker/config_watcher.py stdout_logfile=/dev/stdout stdout_logfile_maxbytes=0 stderr_logfile=/dev/stderr -stderr_logfile_maxbytes=0 - +stderr_logfile_maxbytes=0 \ No newline at end of file diff --git a/docker/warm_cache.py b/docker/warm_cache.py index c5bd92d..1644313 100644 --- a/docker/warm_cache.py +++ b/docker/warm_cache.py @@ -34,5 +34,7 @@ break else: - logging.error("Timeout while trying to pre-populate the cache. This probably means that Open Topo Data isn't working.") + logging.error( + "Timeout while trying to pre-populate the cache. This probably means that Open Topo Data isn't working." + ) sys.exit(1) diff --git a/docker/watcher.py b/docker/watcher.py deleted file mode 100644 index d235e38..0000000 --- a/docker/watcher.py +++ /dev/null @@ -1,48 +0,0 @@ -import logging -import time -import warm_cache - -from watchdog.observers import Observer -from watchdog.events import FileSystemEventHandler - -from pymemcache.client.base import Client - - -#Connect to memcached socket. Since this is hardcoded in the supervisored.conf it shouldn't change that much -memcacheClient = Client("/tmp/memcached.sock") - - -def reload_config(): - logging.error("config.yaml changes. Restarting uswgi and memcached to make the changes") - #Restart uwsgi - #TODO - #Flush memcache using pymemcached - memcacheClient.flush_all() - warm_cache - -class Handler(FileSystemEventHandler): - - def on_created(self, event): - reload_config() - def on_modified(self, event): - reload_config() - - - - - -CONFIG_PATH = "/app/config.yaml" -EXAMPLE_CONFIG_PATH = "/app/example-config.yaml" - -if __name__ == "__main__": - event_handler = Handler() - observer = Observer() - observer.schedule(event_handler, CONFIG_PATH) - observer.schedule(event_handler, EXAMPLE_CONFIG_PATH) - observer.start() - try: - while True: - time.sleep(1) - finally: - observer.stop() - observer.join() diff --git a/example-config.yaml b/example-config.yaml index 6e5014a..6c356e5 100644 --- a/example-config.yaml +++ b/example-config.yaml @@ -1,5 +1,5 @@ # An example of a config.yaml file showing all possible options. If no -# config.yaml file exists, opentopodata will load example-config.yaml instead. +# config.yaml file exists, opentopodata will load example-config.yaml instead.. # 400 error will be thrown above this limit. diff --git a/requirements.in b/requirements.in index 1e13c5b..914d5a7 100644 --- a/requirements.in +++ b/requirements.in @@ -13,5 +13,4 @@ pytest-timeout PyYAML rasterio>=1.3.8,<1.4.0 # 1.3.8+ avoids memory leak https://github.com/ajnisbet/opentopodata/issues/68; 1.4.0 introduces some bugs in rowcol/xy (as of 2024-10-11). requests -watchdog # Monitor changes on the config.yaml -pymemcache # Handle cache flushing without messing \ No newline at end of file +watchdog diff --git a/requirements.txt b/requirements.txt index c8e524f..6cb80ac 100644 --- a/requirements.txt +++ b/requirements.txt @@ -81,14 +81,10 @@ polyline==2.0.2 # via -r requirements.in pylibmc==1.6.3 # via -r requirements.in - -pymemcache==4.0.0 - # via -r requirements.in pyparsing==3.1.4 - # via rasterio + # via snuggs pyproj==3.7.0 # via -r requirements.in - pyproject-hooks==1.2.0 # via # build @@ -104,15 +100,16 @@ pytest-timeout==2.3.1 # via -r requirements.in pyyaml==6.0.2 # via -r requirements.in -rasterio==1.4.1 +rasterio==1.3.11 # via -r requirements.in requests==2.32.3 # via -r requirements.in +snuggs==1.4.7 + # via rasterio urllib3==2.2.3 # via requests watchdog==5.0.3 # via -r requirements.in - werkzeug==3.0.4 # via flask wheel==0.44.0