Mantid versions: latest & nightly (#1463)

* Create a Dockerfile with Python 3.8, qp from the local repo, and currently the latest Mantid (6.4.0)

* Update PYTHONPATH to reflect where the Mantid packages and scripts are located

* Add Dockerfile to package queue-processor without Mantid

* Pass required Mantid version in as an arg, reducing need for multiple similar Dockerfiles

* Not sure if required currently, part of the old Dockerfile so should be seen if it is needed

* Delete old Dockerfile

* Move copy to builder

* Move conda-pack to root (base) environment - the conda-pack  command will then be available in all sub-environments as well

* Pypi package of mysqlclient needed the libs on the OS in order to install - therefore these are installed on both the builder and the runner

* Add ARG for whether to download nightly version of Mantid. Use if statement to download latest stable or latest nightly.

* Refacot references from Mantid 6.2.0 to Mantid Latest

* Fix pylint spacing

Author: james0209

autoreduce_qp/queue_processor/reduction/tests/common.py

@@ -27,7 +27,7 @@ def add_data_and_message():
         'description': 'This is a test',
         'software': {
             "name": "Mantid",
-            "version": "6.2.0",
+            "version": "latest",
         },
     }
 
@@ -36,40 +36,6 @@ def add_data_and_message():
     return data, message
 
 
-def expected_return_data_and_message():
-    """
-    Makes and returns some test data and message
-    """
-    data = {
-        'data': '\\\\isis\\inst$\\NDXTESTINSTRUMENT\\Instrument\\data\\cycle_21_1\\data.nxs',
-        'facility': 'ISIS',
-        'instrument': 'TESTINSTRUMENT',
-        'rb_number': '1234',
-        'run_number': '4321',
-        'run_version': 1,
-        'reduction_script': 'print("hello")',  # not actually used for the reduction
-        'reduction_arguments': {
-            "standard_vars": {
-                "arg1": "differentvalue",
-                "arg2": 321
-            },
-            "advanced_vars": {
-                "adv_arg1": "advancedvalue2",
-                "adv_arg2": ""
-            }
-        },
-        'description': 'This is a test',
-        'software': "6.2.0",
-        "reduction_data": "/instrument/TESTINSTRUMENT/RBNumber/RB1234/autoreduced/Test run name/run-version-1",
-        "reduction_log":
-        "Running reduction script: /isis/NDXTESTINSTRUMENT/user/scripts/autoreduction/reduce.py\nHello\n"
-    }
-
-    message = Message()
-    message.populate(data)
-    return data, message
-
-
 def add_bad_data_and_message():
     """
     Makes and returns some test data and message

autoreduce_qp/queue_processor/reduction/tests/test_process_manager.py

@@ -12,18 +12,16 @@
 from autoreduce_db.reduction_viewer.models import Software
 
 from autoreduce_qp.queue_processor.reduction.process_manager import ReductionProcessManager
-from autoreduce_qp.queue_processor.reduction.tests.common import (add_bad_data_and_message, add_data_and_message,
-                                                                  expected_return_data_and_message)
+from autoreduce_qp.queue_processor.reduction.tests.common import (add_bad_data_and_message, add_data_and_message)
 
 
 class TestReductionProcessManager(unittest.TestCase):
 
     def setUp(self) -> None:
         self.data, self.message = add_data_and_message()
-        self.expected_data, self.expected_message = expected_return_data_and_message()
         self.bad_data, self.bad_message = add_bad_data_and_message()
         self.run_name = "Test run name"
-        self.software = Software(name="Mantid", version="6.2.0")
+        self.software = Software(name="Mantid", version="latest")
 
     def test_init(self):
         """Test that the constructor is doing what's expected"""

autoreduce_qp/queue_processor/reduction/tests/test_runner.py

@@ -184,7 +184,7 @@ def test_reduce_throws_reductionscripterror(self, reduce: Mock, _get_mantid_vers
         assert runner.message.reduction_data is None
         assert runner.message.software == {
             "name": "Mantid",
-            "version": "6.2.0",
+            "version": "latest",
         }
         assert "Error encountered when running the reduction script" in runner.message.message
 
@@ -206,7 +206,7 @@ def test_reduce_throws_any_exception(self, reduce: Mock, _get_mantid_version: Mo
         assert runner.message.reduction_data is None
         assert runner.message.software == {
             "name": "Mantid",
-            "version": "6.2.0",
+            "version": "latest",
         }
         assert "REDUCTION Error:" in runner.message.message
 
@@ -233,7 +233,7 @@ def test_reduce_ok(self, datafile_type: str, reduce: Mock, _get_mantid_version:
         assert runner.message.message is None
         assert runner.message.software == {
             "name": "Mantid",
-            "version": "6.2.0",
+            "version": "latest",
         }
 
     @staticmethod

autoreduce_qp/queue_processor/reduction/tests/test_utilities.py

@@ -61,7 +61,7 @@ def test_get_correct_image(self):
         client = docker.from_env()
         test_software = MagicMock()
         test_software.name = "Mantid"
-        test_software.version = "6.2.0"
+        test_software.version = "latest"
         image = get_correct_image(client, test_software)
         self.assertIsNotNone(image)
 

autoreduce_qp/queue_processor/tests/test_handle_message.py

@@ -38,7 +38,7 @@ def make_test_message(instrument: str) -> Message:
         "data": "/path",
         'software': {
             "name": "Mantid",
-            "version": "6.2.0",
+            "version": "latest",
         },
         "description": "This is a fake description",
         "instrument": instrument,  # Autoreduction Mock Instrument
@@ -67,7 +67,7 @@ def setUp(self):
 
         self.experiment, _ = Experiment.objects.get_or_create(reference_number=1231231)
         self.instrument, _ = Instrument.objects.get_or_create(name=self.instrument_name, is_active=True)
-        self.software, _ = Software.objects.get_or_create(name="Mantid", version="6.2.0")
+        self.software, _ = Software.objects.get_or_create(name="Mantid", version="latest")
         status = Status.get_queued()
         self.reduction_run, self.message = create_reduction_run_record(self.experiment, self.instrument, self.msg, 0,
                                                                        status, self.software)

autoreduce_qp/systemtests/base_systemtest.py

@@ -68,7 +68,7 @@ def setUp(self):
         self.run_title = "test title"
         self.software = {
             "name": "Mantid",
-            "version": "6.2.0",
+            "version": "latest",
         }
 
         # Create test archive and add data

container/qp_mantid_python36.D

@@ -0,0 +1,7 @@
+FROM ghcr.io/autoreduction/base
+
+# Installs queue-processor from your local repository
+ADD . .
+RUN python3 -m pip install --user --no-cache-dir .
+
+CMD autoreduce-qp-start

container/qp_py38.Dockerfile

@@ -0,0 +1,77 @@
+FROM continuumio/miniconda3 AS build
+
+ARG NIGHTLY=False
+WORKDIR /app
+ADD . .
+
+RUN export DEBIAN_FRONTEND=noninteractive && apt-get update &&\
+    apt-get install -y \
+    wget \
+    gnupg \
+    git \
+    software-properties-common \
+    python3-dev \
+    default-libmysqlclient-dev \
+    build-essential \
+    gcc
+
+# Install conda-pack into the base environment
+RUN conda install conda-pack
+
+# Create the conda environment
+RUN conda create -n py38 python=3.8.12
+
+# Make RUN, CMD, and ENTRYPOINT commands use the new environment with the SHELL command
+SHELL [ "conda", "run", "-n", "py38", "/bin/bash", "-c" ]
+
+# Install Mantid and install autoreduce-qp from local directory
+RUN conda config --add channels conda-forge
+RUN if [ "$NIGHTLY" = "False" ]; then \
+    conda install mantid -c mantid; \
+    else \
+    conda install -c mantid/label/nightly mantid; \
+    fi
+
+RUN python3 -m pip install --no-cache-dir . 
+RUN python3 -m pip install --no-cache-dir mysqlclient debugpy
+
+# Use conda-pack to create a standalone enviornment in /venv:
+RUN conda-pack -n py38 -o /tmp/env.tar && \
+    mkdir /venv && cd /venv && tar xf /tmp/env.tar && \
+    rm /tmp/env.tar
+
+RUN cp /venv/bin/Mantid.properties /venv/lib/Mantid.properties
+
+# We've put venv in same path it'll be in final image,
+# so now fix up paths:
+RUN /venv/bin/conda-unpack
+
+
+
+FROM debian:buster AS runtime
+
+RUN export DEBIAN_FRONTEND=noninteractive && apt-get update &&\
+    apt-get install -y \
+    wget \
+    gnupg \
+    git \
+    software-properties-common \
+    python3-dev \
+    default-libmysqlclient-dev \
+    build-essential \
+    gcc
+
+RUN useradd -m --no-log-init -s /bin/bash -u 880844730 isisautoreduce
+USER isisautoreduce
+WORKDIR /home/isisautoreduce
+
+COPY --from=build /venv /venv
+
+ENV PYTHONPATH=/venv/scripts/:/venv/scripts/Diffraction/:/venv/scripts/Engineering/:/venv/bin:/venv/lib:/venv/plugins:/venv/scripts/SANS/:/venv/scripts/Inelastic/:/venv/scripts/ExternalInterfaces:/venv/scripts/Interface
+
+# Add the environment to the PATH (equivalent of running source /venv/bin/activate)
+ENV PATH="/venv/bin:$PATH"
+
+# When the image is run, run the code with the environment
+SHELL [ "/bin/bash", "-c" ]
+CMD autoreduce-qp-start