Merge pull request Rolf-Hempel#53 from IIlllII/prototype

explicit memory maximum logic

Author: Rolf-Hempel

planetary_system_stacker/configuration.py

@@ -25,6 +25,8 @@
 from copy import deepcopy
 from os.path import expanduser, join, isfile, dirname
 from os.path import splitext
+
+import psutil
 from numpy import uint8
 
 from exceptions import ArgumentError
@@ -49,6 +51,8 @@ def __init__(self):
         self.global_parameters_write_protocol_to_file = None
         self.global_parameters_store_protocol_with_result = None
         self.global_parameters_buffering_level = None
+        self.global_parameters_maximum_memory_active = None
+        self.global_parameters_maximum_memory_amount = None
         self.global_parameters_include_postprocessing = None
         self.global_parameters_image_format = None
         self.global_parameters_parameters_in_filename = None
@@ -89,6 +93,9 @@ def set_defaults(self):
         self.global_parameters_write_protocol_to_file = False
         self.global_parameters_store_protocol_with_result = False
         self.global_parameters_buffering_level = -1
+        self.global_parameters_maximum_memory_active = False
+        self.global_parameters_maximum_memory_amount = \
+            max(1, int(dict(psutil.virtual_memory()._asdict())['available'] / 1e9))
         self.global_parameters_include_postprocessing = True
         self.global_parameters_image_format = "png"
         self.global_parameters_parameters_in_filename = False
@@ -142,6 +149,10 @@ def copy_from_config_object(self, configuration_object):
             configuration_object.global_parameters_store_protocol_with_result
         self.global_parameters_buffering_level = \
             configuration_object.global_parameters_buffering_level
+        self.global_parameters_maximum_memory_active = \
+            configuration_object.global_parameters_maximum_memory_active
+        self.global_parameters_maximum_memory_amount = \
+            configuration_object.global_parameters_maximum_memory_amount
         self.global_parameters_include_postprocessing = \
             configuration_object.global_parameters_include_postprocessing
         self.global_parameters_image_format = \
@@ -334,6 +345,10 @@ def import_from_configuration_parameters(self, configuration_parameters):
             configuration_parameters.global_parameters_store_protocol_with_result
         self.global_parameters_buffering_level = \
             configuration_parameters.global_parameters_buffering_level
+        self.global_parameters_maximum_memory_active = \
+            configuration_parameters.global_parameters_maximum_memory_active
+        self.global_parameters_maximum_memory_amount = \
+            configuration_parameters.global_parameters_maximum_memory_amount
         self.global_parameters_include_postprocessing = \
             configuration_parameters.global_parameters_include_postprocessing
         self.global_parameters_image_format = \
@@ -495,6 +510,10 @@ def get_all_parameters_from_configparser(self, conf):
             'store protocol with result', default_conf_obj.global_parameters_store_protocol_with_result)
         self.global_parameters_buffering_level = get_from_conf(conf, 'Global parameters',
             'buffering level', default_conf_obj.global_parameters_buffering_level)
+        self.global_parameters_maximum_memory_active = get_from_conf(conf, 'Global parameters',
+            'maximum memory active', default_conf_obj.global_parameters_maximum_memory_active)
+        self.global_parameters_maximum_memory_amount = get_from_conf(conf, 'Global parameters',
+            'maximum memory amount', default_conf_obj.global_parameters_maximum_memory_amount)
         self.global_parameters_include_postprocessing = get_from_conf(conf, 'Global parameters',
             'include postprocessing', default_conf_obj.global_parameters_include_postprocessing)
         self.global_parameters_image_format = get_from_conf(conf, 'Global parameters',
@@ -590,6 +609,10 @@ def store_all_parameters_to_config_parser(self):
                            str(self.global_parameters_store_protocol_with_result))
         self.set_parameter('Global parameters', 'buffering level',
                            str(self.global_parameters_buffering_level))
+        self.set_parameter('Global parameters', 'maximum memory active',
+                           str(self.global_parameters_maximum_memory_active))
+        self.set_parameter('Global parameters', 'maximum memory amount',
+                           str(self.global_parameters_maximum_memory_amount))
         self.set_parameter('Global parameters', 'include postprocessing',
                            str(self.global_parameters_include_postprocessing))
         self.set_parameter('Global parameters', 'image format',

planetary_system_stacker/configuration_editor.py

@@ -21,6 +21,8 @@
 """
 
 from PyQt5 import QtWidgets, QtCore
+from PyQt5.QtGui import QIntValidator
+from PyQt5.QtWidgets import QDialog, QDialogButtonBox, QVBoxLayout, QLabel
 
 from configuration import ConfigurationParameters
 from parameter_configuration import Ui_ConfigurationDialog
@@ -93,6 +95,9 @@ def __init__(self, parent_gui, parent=None):
         self.gpbl_combobox.addItem('3')
         self.gpbl_combobox.addItem('4')
         self.gpbl_combobox.activated[str].connect(self.gpbl_changed)
+        self.mr_checkBox.stateChanged.connect(self.mr_changed)
+        self.mr_lineEdit.setValidator(QIntValidator(1, 2147483647))
+        self.mr_lineEdit.textChanged.connect(self.mr_text_changed)
         self.gpif_comboBox.addItem('png')
         self.gpif_comboBox.addItem('tiff')
         self.gpif_comboBox.addItem('fits')
@@ -162,6 +167,10 @@ def initialize_widgets_and_local_parameters(self):
             self.gpbl_combobox.setCurrentIndex(self.config_copy.global_parameters_buffering_level+1)
         else:
             self.gpbl_combobox.setCurrentIndex(0)
+
+        self.mr_lineEdit.setText(str(self.config_copy.global_parameters_maximum_memory_amount))
+        self.mr_checkBox.setChecked(self.config_copy.global_parameters_maximum_memory_active)
+
         index = self.gpif_comboBox.findText(self.config_copy.global_parameters_image_format,
                                            QtCore.Qt.MatchFixedString)
         if index >= 0:
@@ -303,6 +312,37 @@ def gpbl_changed(self, value):
         else:
             self.config_copy.global_parameters_buffering_level = int(value)
 
+    def mr_activate(self):
+        self.config_copy.global_parameters_maximum_memory_active = True
+        self.gpbl_combobox.setCurrentText('auto')  # With a fixed memory limit buffering is set to 'auto'
+        self.gpbl_combobox.setEnabled(False)
+        self.mr_lineEdit.setEnabled(True)
+
+    def mr_deactivate(self):
+        self.config_copy.global_parameters_maximum_memory_active = False
+        self.gpbl_combobox.setEnabled(True)
+        self.mr_lineEdit.setEnabled(False)
+
+    def mr_changed(self, state):
+        max_memory_active = (state == QtCore.Qt.Checked)
+        if max_memory_active:
+            if max_memory_active != self.config_copy.global_parameters_maximum_memory_active:  # Only dialog on change
+                warning = ExplicitMemoryWarning()
+                if warning.exec():
+                    self.mr_activate()
+                else:
+                    self.mr_checkBox.setCheckState(QtCore.Qt.Unchecked)  # Go back to unchecked if cancelled
+            else:
+                self.mr_activate()
+        else:
+            self.mr_deactivate()
+
+    def mr_text_changed(self, state):
+        try:
+            self.config_copy.global_parameters_maximum_memory_amount = int(state)
+        except ValueError:
+            return
+
     def gpif_changed(self, value):
         self.config_copy.global_parameters_image_format = value
 
@@ -538,6 +578,18 @@ def accept(self):
                 self.config_copy.global_parameters_buffering_level
             self.configuration.configuration_changed = True
 
+        if self.config_copy.global_parameters_maximum_memory_active != \
+                self.configuration.global_parameters_maximum_memory_active:
+            self.configuration.global_parameters_maximum_memory_active = \
+                self.config_copy.global_parameters_maximum_memory_active
+            self.configuration.configuration_changed = True
+
+        if self.config_copy.global_parameters_maximum_memory_amount != \
+                self.configuration.global_parameters_maximum_memory_amount:
+            self.configuration.global_parameters_maximum_memory_amount = \
+                self.config_copy.global_parameters_maximum_memory_amount
+            self.configuration.configuration_changed = True
+
         if self.config_copy.global_parameters_include_postprocessing != \
                 self.configuration.global_parameters_include_postprocessing:
             self.configuration.global_parameters_include_postprocessing = \
@@ -613,3 +665,21 @@ def closeEvent(self, event):
 
         self.parent_gui.display_widget(None, display=False)
         self.close()
+
+
+class ExplicitMemoryWarning(QDialog):
+    def __init__(self):
+        super().__init__()
+
+        self.setWindowTitle("Memory limit warning")
+
+        self.buttonBox = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
+        self.buttonBox.accepted.connect(self.accept)
+        self.buttonBox.rejected.connect(self.reject)
+
+        self.layout = QVBoxLayout()
+        message = QLabel("PlanetarySystemStacker takes no responsibility for enough memory being available.\r\n"
+                         "Setting the memory limit too high may cause degraded OS performance and/or process crashes")
+        self.layout.addWidget(message)
+        self.layout.addWidget(self.buttonBox)
+        self.setLayout(self.layout)

planetary_system_stacker/workflow.py

@@ -274,9 +274,12 @@ def execute_frames(self, job):
                 Miscellaneous.print_stacking_parameters(self.configuration, self.attached_log_file)
 
             try:
-                # Look up the available RAM (without paging)
-                virtual_memory = dict(psutil.virtual_memory()._asdict())
-                available_ram = virtual_memory['available'] / 1e9
+                if self.configuration.global_parameters_maximum_memory_active:
+                    available_ram = float(self.configuration.global_parameters_maximum_memory_amount)
+                else:
+                    # Look up the available RAM (without paging)
+                    virtual_memory = dict(psutil.virtual_memory()._asdict())
+                    available_ram = virtual_memory['available'] / 1e9
 
                 self.frames = Frames(self.configuration, names, type=self.job.type,
                                      bayer_option_selected=self.job.bayer_option_selected,