Persistent results is a Python class that ensures the results will be available even if interruptions occur.
pip install pers
If the code below breaks for some reason (let's say the computer shuts down because of a power outage or some exception), you can restart it to continue from the step it finished.
from pers import PersistentResults
import pandas as pd # pandas is not required
results = PersistentResults(
'test.pickle', # filename for result caching
interval=1, # how often dump the results
tmpfilename='~test.pickle' # tmp cache file (optional)
)
fun = lambda x, y, a, b: x**2 + y
for x in range(10):
for y in range(11):
results.append(fun, x, y, a=x, b=y)
results.save()
print(pd.DataFrame(results.data))
Output:
{'result': 66, 'x': 8, 'y': 2, 'a': 8, 'b': 2}
110
| | result | x | y | a | b |
|----:|---------:|----:|----:|----:|----:|
| 0 | 0 | 0 | 0 | 0 | 0 |
| 1 | 1 | 0 | 1 | 0 | 1 |
| 2 | 2 | 0 | 2 | 0 | 2 |
[..]
| 107 | 89 | 9 | 8 | 9 | 8 |
| 108 | 90 | 9 | 9 | 9 | 9 |
| 109 | 91 | 9 | 10 | 9 | 10 |
from pers import PersistentResults
import pandas as pd
results = PersistentResults(
'test2.pickle',
interval=1,
result_prefix='' # we do not want a prefix _result_
)
def fun(x, y, a, b):
print(f'x: {x}, y:{y}')
return { # yes, we can return dictionary
'out': x**2 + y,
'x': x, # yes, we can return input in dict
'a': a,
}
try:
for x in range(10):
for y in range(11):
results.append(fun, x=x, y=y, a=x, b=y)
if x == 5 and y ==5: # simulate reboot
raise Exception('Unexpected reboot...')
except:
pass
# RERUN the tests
# will skip already processed elements
for x in range(10):
for y in range(11):
results.append(fun, x=x, y=y, a=x, b=y)
results.save()
print(pd.DataFrame(results.data).to_markdown())
x: 0, y:0
x: 0, y:1
[..]
x: 5, y:4
x: 5, y:5
Unexpected reboot...
x: 5, y:6
x: 5, y:7
[..]
x: 9, y:9
x: 9, y:10
| | out | x | a | y | b |
|----:|------:|----:|----:|----:|----:|
| 0 | 0 | 0 | 0 | 0 | 0 |
| 1 | 1 | 0 | 0 | 1 | 1 |
| 2 | 2 | 0 | 0 | 2 | 2 |
[..]
| 107 | 89 | 9 | 9 | 8 | 8 |
| 108 | 90 | 9 | 9 | 9 | 9 |
| 109 | 91 | 9 | 9 | 10 | 10 |