Skip to content

Commit bcfc1e8

Browse files
committed
Add logging
1 parent 8b98fc7 commit bcfc1e8

File tree

7 files changed

+156
-35
lines changed

7 files changed

+156
-35
lines changed

cleaner/cleaner.py

Lines changed: 44 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -11,15 +11,17 @@ class Cleaner:
1111
Files are cleaned based on last modification time, which have to be in interval given in configuration.
1212
"""
1313

14-
def __init__(self, config):
14+
def __init__(self, config, logger):
1515
"""
1616
Constructor which takes configuration of cleaner.
1717
1818
:param config: have to be instance of ConfigManager class
19+
:param logger: System logger instance
1920
:raises Exception: if values from configuration are not valid
2021
"""
2122
self._cache_dir = config.get_cache_dir()
2223
self._file_age = config.get_file_age()
24+
self._logger = logger
2325

2426
self.check_cache_dir()
2527
self.check_file_age()
@@ -47,10 +49,8 @@ def check_file_age(self):
4749
if self._file_age is None:
4850
raise Exception("File age not specified!")
4951

50-
if not self._file_age.isdigit():
51-
raise Exception("File age is not a number!")
52-
53-
self._file_age = int(self._file_age)
52+
if self._file_age <= 0:
53+
raise Exception("File age is not positive number!")
5454

5555
def clean(self):
5656
"""
@@ -61,41 +61,61 @@ def clean(self):
6161

6262
# some general debug information about this time execution
6363
now = round(time.time())
64-
print("****************************************")
65-
print("Cleaning files from \"" + self._cache_dir + "\"")
66-
print("With maximum file age: " + str(self._file_age) + " seconds")
67-
print("Timestamp now: " + str(now) + " seconds")
68-
print("****************************************")
64+
self._logger.info("Cleaning files from \"{}\"".format(self._cache_dir))
65+
self._logger.info("Maximum file age: {} seconds".format(self._file_age))
6966

70-
def process_path(root, file, action):
67+
def process_file(root, file):
7168
"""
72-
Process given path, path can be file or directory.
73-
Given action is executed on constructed full path
69+
Process given file. If modification timestamp is too old, the file will be removed.
7470
7571
:param root: path to root directory, used as base
7672
:param file: name of file itself, root is used as base and joined with this
77-
:param action: action which will be performed if modification timestamp is older than given interval
7873
:return: Nothing
7974
"""
8075

8176
full_path = os.path.join(root, file)
8277
last_modification = round(os.stat(full_path).st_mtime)
8378
difference = now - last_modification
8479

85-
print(full_path)
86-
print(" last modification: " + str(last_modification))
87-
print(" difference: " + str(difference))
80+
self._logger.debug("last modification: {}".format(last_modification))
81+
self._logger.debug("age from now: {} seconds".format(difference))
8882

8983
if difference > self._file_age:
84+
self._logger.debug("file \"{}\" marked for deletion".format(full_path))
9085
try:
91-
action(full_path)
92-
print(" >>> REMOVED <<<")
86+
os.remove(full_path)
87+
self._logger.info("file \"{}\" removed".format(full_path))
9388
except Exception as ex:
94-
print(" >>> Exception occured: " + str(ex))
89+
self._logger.warning("removing file \"{}\" failed: {}".format(full_path, ex))
90+
else:
91+
self._logger.debug("file \"{}\" will be kept".format(full_path))
92+
93+
def process_directory(root, dir):
94+
"""
95+
Process given directory. If it is empty, the directory will be removed.
9596
96-
# iterate recursively through given directory
97-
for root, dirs, files in os.walk(self._cache_dir):
97+
:param root: path to root directory, used as base
98+
:param file: name of directory itself, root is used as base and joined with this
99+
:return: Nothing
100+
"""
101+
102+
full_path = os.path.join(root, dir)
103+
if not os.listdir(full_path):
104+
self._logger.debug("directory \"{}\" is empty".format(full_path))
105+
try:
106+
os.rmdir(full_path)
107+
self._logger.info("directory \"{}\" removed".format(full_path))
108+
except Exception as ex:
109+
self._logger.warning("removing directory \"{}\" failed: {}".format(full_path, ex))
110+
else:
111+
self._logger.debug("directory \"{}\" is not empty and will be kept".format(full_path))
112+
113+
# iterate recursively through given directory (in DFS order)
114+
for root, dirs, files in os.walk(self._cache_dir, topdown=False):
98115
for file in files:
99-
process_path(root, file, os.remove)
116+
self._logger.info("Processing file: {}".format(file))
117+
process_file(root, file)
100118
for dir in dirs:
101-
process_path(root, dir, os.rmdir)
119+
self._logger.info("Processing directory: {}".format(dir))
120+
process_directory(root, dir)
121+

cleaner/config_manager.py

Lines changed: 92 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
#!/usr/bin/env python3
22

33
import yaml
4+
import logging
5+
import logging.handlers
46

57

68
class ConfigManager:
@@ -35,7 +37,95 @@ def get_file_age(self):
3537
"""
3638
Get maximum file age in seconds.
3739
38-
:return: Single textual value
40+
:return: Single integer value
3941
"""
40-
return self._config.get('file-age')
42+
return int(self._config['file-age']) if 'file-age' in self._config else 604800
43+
44+
def get_logger_settings(self):
45+
"""
46+
Get path to system log file.
47+
48+
:return: List with 4 items - string path, logging level, integer maximum size
49+
of logfile and integer number of rotations kept.
50+
"""
51+
result = ["/tmp/recodex-cleaner.log", logging.INFO, 1024*1024, 3]
52+
if 'logger' in self._config:
53+
sect = self._config['logger']
54+
if 'file' in sect:
55+
result[0] = sect['file']
56+
if 'level' in sect:
57+
result[1] = self._get_loglevel_from_string(sect['level'])
58+
if 'max-size' in sect:
59+
try:
60+
result[2] = int(sect['max-size'])
61+
except:
62+
pass
63+
if 'rotations' in sect:
64+
try:
65+
result[3] = int(sect['rotations'])
66+
except:
67+
pass
68+
return result
69+
70+
def _get_loglevel_from_string(self, str_level):
71+
"""
72+
Convert logging level from string to logging module type.
73+
74+
:param str_level: string representation of logging level
75+
:return: logging level (defaults to logging.INFO)
76+
"""
77+
level_mapping = {
78+
"debug": logging.DEBUG,
79+
"info": logging.INFO,
80+
"warning": logging.WARNING,
81+
"error": logging.ERROR,
82+
"critical": logging.CRITICAL
83+
}
84+
if str_level in level_mapping:
85+
return level_mapping[str_level]
86+
else:
87+
return logging.INFO
88+
89+
90+
def init_logger(logfile, level, max_size, rotations):
91+
"""
92+
Initialize new system logger for cleaner. If arguments are invalid,
93+
empty logger will be created.
94+
95+
:param logfile: Path to file with log.
96+
:param level: Log level as logging.<LEVEL>
97+
:param max_size: Maximum size of log file.
98+
:param rotations: Number of log files kept.
99+
:return: Initialized logger.
100+
"""
101+
try:
102+
# create logger
103+
logger = logging.getLogger('recodex-cleaner')
104+
logger.setLevel(level)
105+
106+
# create rotating file handler
107+
ch = logging.handlers.RotatingFileHandler(logfile, maxBytes=max_size, backupCount=rotations)
108+
ch.setLevel(level)
109+
110+
# create formatter
111+
formatter = logging.Formatter('[%(asctime)s] [%(name)s] [%(levelname)s] %(message)s')
112+
113+
# add formatter to ch
114+
ch.setFormatter(formatter)
115+
116+
# add ch to logger
117+
logger.addHandler(ch)
118+
except Exception as e:
119+
# create empty logger
120+
print("Invalid logger configuration. Creating null logger. Error: {}".format(e))
121+
logger = logging.getLogger('recodex-cleaner-dummy')
122+
logging.disable(logging.CRITICAL)
123+
124+
# print welcome message to log file
125+
logger.critical("-------------------------")
126+
logger.critical(" ReCodEx Cleaner started")
127+
logger.critical("-------------------------")
128+
129+
# return created logger
130+
return logger
41131

cleaner/install/config.yml

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,9 @@
11
---
2-
cache-dir: "/tmp"
3-
file-age: "604800" # in seconds
2+
cache-dir: "/var/recodex-worker-wd"
3+
file-age: 604800 # in seconds
4+
logger:
5+
file: "/var/log/recodex/cleaner.log"
6+
level: "debug" # level of logging - one of "debug", "info", "warning", "error", "critical"
7+
max-size: 1048576 # 1 MB; max size of file before log rotation
8+
rotations: 3 # number of rotations kept
49
...

cleaner/install/recodex-cleaner.service

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,3 +21,4 @@ ExecStart=/usr/bin/recodex-cleaner -c /etc/recodex/cleaner/config.yml
2121

2222
[Install]
2323
WantedBy=multi-user.target
24+

cleaner/install/recodex-cleaner.timer

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,8 @@ DefaultDependencies=true
1212

1313
[Timer]
1414
OnCalendar=daily
15-
Persistent=true
16-
15+
Persistent=true
16+
1717
[Install]
1818
WantedBy=timers.target
19+

cleaner/main.py

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
#!/usr/bin/env python3
22

3-
from .config_manager import ConfigManager
3+
from .config_manager import ConfigManager, init_logger
44
from .cleaner import Cleaner
55
import argparse
66
import sys
@@ -21,10 +21,14 @@ def main():
2121
try:
2222
# get configuration
2323
config = ConfigManager(args.config)
24-
cleaner = Cleaner(config)
24+
logger = init_logger(*config.get_logger_settings())
25+
cleaner = Cleaner(config, logger)
2526
cleaner.clean()
27+
logger.info("Cleaning finished, quiting")
2628
except Exception as ex:
27-
print("Exception occured: " + str(ex), file=sys.stderr)
29+
error_text = "Exception occured: {}".format(str(ex))
30+
logger.warning(error_text)
31+
print(error_text, file=sys.stderr)
2832

2933
if __name__ == "__main__":
3034
main()

recodex-cleaner.spec

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
%define name recodex-cleaner
22
%define short_name cleaner
33
%define version 1.1.0
4-
%define unmangled_version 0131021222d8c8a93b4eed71091be6cfbe1a765f
5-
%define release 2
4+
%define unmangled_version 8b98fc783723e764ad89e2e3e91fde6055a330fb
5+
%define release 3
66

77
Summary: Clean cache which is used by ReCodEx workers
88
Name: %{name}

0 commit comments

Comments
 (0)