Skip to content

Commit

Permalink
Reformat, clarify comments
Browse files Browse the repository at this point in the history
  • Loading branch information
deanishe committed Jul 5, 2015
1 parent c6d8690 commit a3f1670
Show file tree
Hide file tree
Showing 2 changed files with 46 additions and 42 deletions.
69 changes: 35 additions & 34 deletions tests/test_workflow.py
Original file line number Diff line number Diff line change
Expand Up @@ -1502,25 +1502,25 @@ def fake(wf):


class AtomicWriterTests(unittest.TestCase):
"""Test for atomic writer"""
"""Tests for atomic writer"""

def setUp(self):
self.tempdir = tempfile.mkdtemp()
self.settings_file = os.path.join(self.tempdir, 'settings.json')

def tearDown(self):
if os.path.exists(self.tempdir):
shutil.rmtree(self.tempdir)

def test_write_file_succeed(self):
"""succeed, no temp file left"""
"""AtomicWriter: Succeed, no temp file left"""
with atomic_writer(self.settings_file, 'wb') as file_obj:
json.dump(DEFAULT_SETTINGS, file_obj)
self.assertEqual(len(os.listdir(self.tempdir)), 1)
self.assertTrue(os.path.exists(self.settings_file))

def test_failed_before_writing(self):
"""throw exception before writing"""
"""AtomicWriter: Exception before writing"""
def write_function():
with atomic_writer(self.settings_file, 'wb'):
raise Exception()
Expand All @@ -1529,17 +1529,17 @@ def write_function():
self.assertEqual(len(os.listdir(self.tempdir)), 0)

def test_failed_after_writing(self):
"""throw exception in the end"""
"""AtomicWriter: Exception after writing"""
def write_function():
with atomic_writer(self.settings_file, 'wb') as file_obj:
json.dump(DEFAULT_SETTINGS, file_obj)
raise Exception()

self.assertRaises(Exception, write_function)
self.assertEqual(len(os.listdir(self.tempdir)), 0)

def test_failed_without_overwriting(self):
"""throw exception after writing won't overwrite the old file"""
"""AtomicWriter: Exception after writing won't overwrite the old file"""
mockSettings = {}

def write_function():
Expand All @@ -1566,58 +1566,59 @@ class AtomicMultipleFilesWriterTester(unittest.TestCase):

@atomic_multiple_files_writer
def _mock_write_function(self):
if self.isKilledFlag:
self.isKilledFlag = False
if self.kill_flag:
self.kill_flag = False
self._kill()
self.resultFlag = True
self.result_flag = True

def _old_signal_handler(self, s, f):
self.oldSignalFlag = True
self.old_signal_flag = True

def _kill(self):
os.kill(os.getpid(), signal.SIGTERM)

def setUp(self):
signal.signal(signal.SIGTERM, signal.SIG_DFL)
self.resultFlag = False # flag for if the write_function is executed or not
self.oldSignalFlag = False # flag for old signal handler is executed or not
self.isKilledFlag = True # flag for if process need to be killed or not

self.result_flag = False # True if the write_function is executed
self.old_signal_flag = False # True if old signal handler is executed
self.kill_flag = True # True if process should be killed

def test_normal(self):
"""Normal writing operator"""
self.isKilledFlag = False
"""AtomicMultiple: Normal writing operator"""
self.kill_flag = False

self._mock_write_function()

self.assertTrue(self.resultFlag)
self.assertTrue(self.result_flag)

def test_sigterm_signal(self):
"""The process is killed"""
"""AtomicMultiple: Process is killed"""
self.assertRaises(SystemExit, self._mock_write_function)

self.assertTrue(self.resultFlag)
self.assertFalse(self.isKilledFlag)
self.assertTrue(self.result_flag)
self.assertFalse(self.kill_flag)

def test_old_signal_handler(self):
"""The process is killed with some other signal handler"""
"""AtomicMultiple: Process killed with some other signal handler"""
signal.signal(signal.SIGTERM, self._old_signal_handler)

self._mock_write_function()

self.assertTrue(self.resultFlag)
self.assertTrue(self.oldSignalFlag)
self.assertFalse(self.isKilledFlag)
self.assertTrue(self.result_flag)
self.assertTrue(self.old_signal_flag)
self.assertFalse(self.kill_flag)

def test_old_signal_handler_recover(self):
"""The previous signal handler is put back after writing"""
def test_old_signal_handler_restore(self):
"""AtomicMultiple: Previous signal handler is restored after write"""
signal.signal(signal.SIGTERM, self._old_signal_handler)
self.isKilledFlag = False
self.kill_flag = False

self._mock_write_function()

self.assertTrue(self.resultFlag)
self.assertEqual(signal.getsignal(signal.SIGTERM), self._old_signal_handler)

self.assertTrue(self.result_flag)
self.assertEqual(signal.getsignal(signal.SIGTERM),
self._old_signal_handler)


class SettingsTests(unittest.TestCase):
Expand Down
19 changes: 11 additions & 8 deletions workflow/workflow.py
Original file line number Diff line number Diff line change
Expand Up @@ -815,20 +815,22 @@ def atomic_writer(file_path, mode):


class atomic_multiple_files_writer(object):
"""Atomic file writer decorator, for multiple files
"""Atomic file writer decorator for writing multiple files
The docorator will finsih the writing process once it started,
The decorator will finish the writing process once it started,
even after the SIGTERM signal.
Note:
- This decorator does NOT support multiple threads.
- Write function may be executed twice.
"""
def __init__(self, write_function, clazz_name=""):

def __init__(self, write_function, class_name=''):
self.write_function = write_function

"""Signal handler, called by system when SIGTERM is received by process."""
def signal_handler(self, signal_number, frame):
"""Called when process receives SIGTERM."""
self.write_function(*self.args, **self.kwargs)
if callable(self.old_signal_handler):
self.old_signal_handler(signal_number, frame)
Expand All @@ -844,7 +846,8 @@ def __call__(self, *args, **kwargs):
signal.signal(signal.SIGTERM, self.old_signal_handler)

def __get__(self, obj=None, clazz=None):
return self.__class__(self.write_function.__get__(obj, clazz), clazz.__name__)
return self.__class__(self.write_function.__get__(obj, clazz),
clazz.__name__)


class Settings(dict):
Expand Down Expand Up @@ -1639,11 +1642,11 @@ def store_data(self, name, data, serializer=None):
self._store_data_to_file(metadata_path, serializer_name, data_path, serializer, data)

self.logger.debug('Stored data saved at : {0}'.format(data_path))

@atomic_multiple_files_writer
def _store_data_to_file(self, metadata_path, serializer_name, data_path, serializer, data):
"""atomic write metadata and data to file
:param: medata_path
:type: ``unicode``
:param: serializer_name
Expand Down

0 comments on commit a3f1670

Please sign in to comment.