-
Notifications
You must be signed in to change notification settings - Fork 6
Refactor daemon with memory-safe loop and logging #40
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -7,10 +7,14 @@ | |
| """ | ||
|
|
||
| import argparse | ||
| import sys | ||
| import datetime | ||
| import gc | ||
| import logging | ||
| import os | ||
| import sys | ||
| import time | ||
|
|
||
| from config import config, validate_config, ConfigError, update_config_path | ||
| from drivers import DriverError, create_driver | ||
| from media_sync import ( | ||
| create_mergin_client, | ||
|
|
@@ -19,10 +23,33 @@ | |
| mc_pull, | ||
| MediaSyncError, | ||
| ) | ||
| from config import config, validate_config, ConfigError, update_config_path | ||
| from version import __version__ | ||
|
|
||
|
|
||
| def setup_logger(): | ||
| logger = logging.getLogger("media-sync") | ||
| logger.setLevel(logging.INFO) | ||
| handler = logging.StreamHandler(sys.stdout) | ||
| formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s") | ||
| handler.setFormatter(formatter) | ||
| logger.addHandler(handler) | ||
| return logger | ||
|
|
||
|
|
||
| def run_sync_cycle(mc, driver, logger): | ||
| try: | ||
| logger.info("Pulling changes from Mergin...") | ||
| files_to_sync = mc_pull(mc) | ||
| media_sync_push(mc, driver, files_to_sync) | ||
| logger.info("Sync complete.") | ||
|
|
||
| # Force garbage collection | ||
| gc.collect() | ||
|
|
||
| except MediaSyncError as e: | ||
| logger.error(f"Media sync error: {e}") | ||
|
|
||
|
|
||
| def main(): | ||
| parser = argparse.ArgumentParser( | ||
| prog="media_sync_daemon.py", | ||
|
|
@@ -34,66 +61,54 @@ def main(): | |
| "config_file", | ||
| nargs="?", | ||
| default="config.yaml", | ||
| help="Path to file with configuration. Default value is config.yaml in current working directory.", | ||
| help="Path to file with configuration. Default is ./config.yaml", | ||
| ) | ||
|
|
||
| args = parser.parse_args() | ||
|
|
||
| print(f"== Starting Mergin Media Sync daemon version {__version__} ==") | ||
| logger = setup_logger() | ||
| logger.info(f"== Starting Mergin Media Sync daemon v{__version__} ==") | ||
|
|
||
| try: | ||
| update_config_path(args.config_file) | ||
| except IOError as e: | ||
| print("Error:" + str(e)) | ||
| validate_config(config) | ||
| except (IOError, ConfigError) as e: | ||
| logger.error(f"Configuration error: {e}") | ||
| sys.exit(1) | ||
|
|
||
| sleep_time = config.as_int("daemon.sleep_time") | ||
|
|
||
| try: | ||
| validate_config(config) | ||
| except ConfigError as e: | ||
| print("Error: " + str(e)) | ||
| return | ||
|
|
||
| try: | ||
| driver = create_driver(config) | ||
| except DriverError as e: | ||
| print("Error: " + str(e)) | ||
| return | ||
| logger.error(f"Driver error: {e}") | ||
| sys.exit(1) | ||
|
|
||
| print("Logging in to Mergin...") | ||
| logger.info("Logging in to Mergin...") | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I suggest to update |
||
| try: | ||
| mc = create_mergin_client() | ||
|
|
||
| # initialize or pull changes to sync with latest project version | ||
| if not os.path.exists(config.project_working_dir): | ||
| logger.info("Project directory not found. Downloading from Mergin...") | ||
| files_to_sync = mc_download(mc) | ||
| media_sync_push(mc, driver, files_to_sync) | ||
| except MediaSyncError as e: | ||
| print("Error: " + str(e)) | ||
| return | ||
| logger.error(f"Initial sync error: {e}") | ||
| sys.exit(1) | ||
|
|
||
| # keep running until killed by ctrl+c: | ||
| # - sleep N seconds | ||
| # - pull | ||
| # - push | ||
| logger.info("Entering sync loop...") | ||
| while True: | ||
| print(datetime.datetime.now()) | ||
| try: | ||
| files_to_sync = mc_pull(mc) | ||
| media_sync_push(mc, driver, files_to_sync) | ||
| logger.info(f"Heartbeat: {datetime.datetime.utcnow().isoformat()} UTC") | ||
| run_sync_cycle(mc, driver, logger) | ||
|
|
||
| # check mergin client token expiration | ||
| delta = mc._auth_session["expire"] - datetime.datetime.now( | ||
| datetime.timezone.utc | ||
| ) | ||
| # Check token expiry | ||
| try: | ||
| delta = mc._auth_session["expire"] - datetime.datetime.now(datetime.timezone.utc) | ||
| if delta.total_seconds() < 3600: | ||
| logger.info("Refreshing Mergin auth token...") | ||
| mc = create_mergin_client() | ||
| except Exception as e: | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This seems to me too broad, can you narrow it down to MerginClient error or something similar? |
||
| logger.warning(f"Error checking token expiration: {e}") | ||
|
|
||
| except MediaSyncError as e: | ||
| print("Error: " + str(e)) | ||
|
|
||
| print("Going to sleep") | ||
| logger.info(f"Sleeping for {sleep_time} seconds...") | ||
| time.sleep(sleep_time) | ||
|
|
||
|
|
||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Do we know what was the issue with memory (maybe we could fix the source of the issue)? As gc should be automatic we should probably know why to force it.