Skip to content
Open
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
87 changes: 51 additions & 36 deletions media_sync_daemon.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,14 @@
"""

import argparse
import sys
import datetime
import gc
import logging
import os
import sys
import time

from config import config, validate_config, ConfigError, update_config_path
from drivers import DriverError, create_driver
from media_sync import (
create_mergin_client,
Expand All @@ -19,10 +23,33 @@
mc_pull,
MediaSyncError,
)
from config import config, validate_config, ConfigError, update_config_path
from version import __version__


def setup_logger():
logger = logging.getLogger("media-sync")
logger.setLevel(logging.INFO)
handler = logging.StreamHandler(sys.stdout)
formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s")
handler.setFormatter(formatter)
logger.addHandler(handler)
return logger


def run_sync_cycle(mc, driver, logger):
try:
logger.info("Pulling changes from Mergin...")
files_to_sync = mc_pull(mc)
media_sync_push(mc, driver, files_to_sync)
logger.info("Sync complete.")

# Force garbage collection
gc.collect()
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we know what was the issue with memory (maybe we could fix the source of the issue)? As gc should be automatic we should probably know why to force it.


except MediaSyncError as e:
logger.error(f"Media sync error: {e}")


def main():
parser = argparse.ArgumentParser(
prog="media_sync_daemon.py",
Expand All @@ -34,66 +61,54 @@ def main():
"config_file",
nargs="?",
default="config.yaml",
help="Path to file with configuration. Default value is config.yaml in current working directory.",
help="Path to file with configuration. Default is ./config.yaml",
)

args = parser.parse_args()

print(f"== Starting Mergin Media Sync daemon version {__version__} ==")
logger = setup_logger()
logger.info(f"== Starting Mergin Media Sync daemon v{__version__} ==")

try:
update_config_path(args.config_file)
except IOError as e:
print("Error:" + str(e))
validate_config(config)
except (IOError, ConfigError) as e:
logger.error(f"Configuration error: {e}")
sys.exit(1)

sleep_time = config.as_int("daemon.sleep_time")

try:
validate_config(config)
except ConfigError as e:
print("Error: " + str(e))
return

try:
driver = create_driver(config)
except DriverError as e:
print("Error: " + str(e))
return
logger.error(f"Driver error: {e}")
sys.exit(1)

print("Logging in to Mergin...")
logger.info("Logging in to Mergin...")
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I suggest to update Mergin... to Mergin Maps server ... in all occurrences for better clarity

try:
mc = create_mergin_client()

# initialize or pull changes to sync with latest project version
if not os.path.exists(config.project_working_dir):
logger.info("Project directory not found. Downloading from Mergin...")
files_to_sync = mc_download(mc)
media_sync_push(mc, driver, files_to_sync)
except MediaSyncError as e:
print("Error: " + str(e))
return
logger.error(f"Initial sync error: {e}")
sys.exit(1)

# keep running until killed by ctrl+c:
# - sleep N seconds
# - pull
# - push
logger.info("Entering sync loop...")
while True:
print(datetime.datetime.now())
try:
files_to_sync = mc_pull(mc)
media_sync_push(mc, driver, files_to_sync)
logger.info(f"Heartbeat: {datetime.datetime.utcnow().isoformat()} UTC")
run_sync_cycle(mc, driver, logger)

# check mergin client token expiration
delta = mc._auth_session["expire"] - datetime.datetime.now(
datetime.timezone.utc
)
# Check token expiry
try:
delta = mc._auth_session["expire"] - datetime.datetime.now(datetime.timezone.utc)
if delta.total_seconds() < 3600:
logger.info("Refreshing Mergin auth token...")
mc = create_mergin_client()
except Exception as e:
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This seems to me too broad, can you narrow it down to MerginClient error or something similar?

logger.warning(f"Error checking token expiration: {e}")

except MediaSyncError as e:
print("Error: " + str(e))

print("Going to sleep")
logger.info(f"Sleeping for {sleep_time} seconds...")
time.sleep(sleep_time)


Expand Down