Update Docker and Compose files. Add pytest fixture.

Dockerfile.dev → docker/Dockerfile.dev

@@ -2,7 +2,7 @@ FROM datajoint/djbase:py3.9-debian-fcd8909
 
 USER anaconda:anaconda
 
-COPY ./workflow-array-ephys/apt_requirements.txt /tmp/
+COPY ./workflow-array-ephys/docker/apt_requirements.txt /tmp/
 RUN /entrypoint.sh echo "Installed dependencies."
 
 RUN mkdir /main/element-lab \

Dockerfile.test → docker/Dockerfile.test

@@ -2,7 +2,7 @@ FROM datajoint/djbase:py3.9-debian-fcd8909
 
 USER anaconda:anaconda
 
-COPY ./workflow-array-ephys/apt_requirements.txt /tmp/
+COPY ./workflow-array-ephys/docker/apt_requirements.txt /tmp/
 RUN /entrypoint.sh echo "Installed dependencies."
 
 WORKDIR /main/workflow-array-ephys
@@ -19,20 +19,23 @@ WORKDIR /main/workflow-array-ephys
 # RUN git clone https://github.com/<user>/workflow-array-ephys.git /main/workflow-array-ephys
 
 # Option 3 - Install user's local fork of element and workflow
-RUN mkdir /main/element-lab
+RUN mkdir /main/element-lab \
+          /main/element-animal \
+          /main/element-session \
+          /main/element-array-ephys \
+          /main/workflow-array-ephys
+
 COPY --chown=anaconda:anaconda ./element-lab /main/element-lab
-RUN pip install -e /main/element-lab
-RUN mkdir /main/element-animal
 COPY --chown=anaconda:anaconda ./element-animal /main/element-animal
-RUN pip install -e /main/element-animal
-RUN mkdir /main/element-session
 COPY --chown=anaconda:anaconda ./element-session /main/element-session
-RUN pip install -e /main/element-session
-RUN mkdir /main/element-array-ephys
 COPY --chown=anaconda:anaconda ./element-array-ephys /main/element-array-ephys
-RUN pip install -e /main/element-array-ephys
 COPY --chown=anaconda:anaconda ./workflow-array-ephys /main/workflow-array-ephys
-# RUN rm -f /main/workflow-array-ephys/dj_local_conf.json
+
+RUN pip install -e /main/element-lab
+RUN pip install -e /main/element-animal
+RUN pip install -e /main/element-session
+RUN pip install -e /main/element-array-ephys
+RUN rm -f /main/workflow-array-ephys/dj_local_conf.json
 
 # Install the workflow
 RUN pip install /main/workflow-array-ephys

docker/docker-compose-dev.yaml

@@ -0,0 +1,35 @@
+# docker-compose -f ./docker/docker-compose-dev.yaml up -d --build
+# docker-compose -f ./docker/docker-compose-dev.yaml down
+
+version: "2.4"
+x-net: &net
+  networks:
+      - main
+services:
+  array-ephys-dev-db:
+    <<: *net
+    image: datajoint/mysql:5.7
+    environment:
+      - MYSQL_ROOT_PASSWORD=simple
+  array-ephys-dev-workflow:
+    <<: *net
+    build:
+      context: ../../
+      dockerfile: ./workflow-array-ephys/docker/Dockerfile.dev
+    env_file: .env
+    image: workflow_array_ephys_dev:0.1.0a4
+    environment:
+      - EPHYS_ROOT_DATA_DIR=/main/test_data/workflow_ephys_data1/,/main/test_data/workflow_ephys_data2/
+    volumes:
+      - ${TEST_DATA_DIR}:/main/test_data
+      - ./apt_requirements.txt:/tmp/apt_requirements.txt
+      - ../../element-lab:/main/element-lab
+      - ../../element-animal:/main/element-animal
+      - ../../element-session:/main/element-session
+      - ../../element-array-ephys:/main/element-array-ephys
+      - ..:/main/workflow-array-ephys
+    depends_on:
+      array-ephys-dev-db:
+        condition: service_healthy
+networks:
+  main:

docker-compose-test.yaml → docker/docker-compose-test.yaml

@@ -1,24 +1,24 @@
 # export COMPOSE_DOCKER_CLI_BUILD=0 # some machines need for smooth --build
 # .env file: TEST_DATA_DIR=<local parent directory of workflow_ephys_data{1,2}>
-# docker-compose -f docker-compose-test.yaml up --build
+# docker-compose -f ./docker/docker-compose-test.yaml up --build
 # docker exec -it workflow-array-ephys_workflow_1 /bin/bash
-# docker-compose -f docker-compose-test.yaml down
+# docker-compose -f ./docker/docker-compose-test.yaml down
 
 version: "2.4"
 x-net: &net
   networks:
       - main
 services:
-  db:
+  array-ephys-test-db:
     <<: *net
     image: datajoint/mysql:5.7
     environment:
       - MYSQL_ROOT_PASSWORD=simple
-  workflow:
+  array-ephys-test-workflow:
     <<: *net
     build:
-      context: ../
-      dockerfile: ./workflow-array-ephys/Dockerfile.test
+      context: ../../
+      dockerfile: ./workflow-array-ephys/docker/Dockerfile.test
     env_file: .env
     image: workflow_array_ephys_test:0.1.0a4
     environment:
@@ -37,13 +37,13 @@ services:
     volumes:
       - ${TEST_DATA_DIR}:/main/test_data
       - ./apt_requirements.txt:/tmp/apt_requirements.txt
-      - ../element-lab:/main/element-lab
-      - ../element-animal:/main/element-animal
-      - ../element-session:/main/element-session
-      - ../element-array-ephys:/main/element-array-ephys
-      - .:/main/workflow-array-ephys
+      - ../../element-lab:/main/element-lab
+      - ../../element-animal:/main/element-animal
+      - ../../element-session:/main/element-session
+      - ../../element-array-ephys:/main/element-array-ephys
+      - ..:/main/workflow-array-ephys
     depends_on:
-      db:
+      array-ephys-test-db:
         condition: service_healthy
 networks:
   main:

tests/__init__.py

@@ -1,7 +1,7 @@
-# run all tests: pytest -sv --cov-report term-missing \
-#                --cov=workflow_array_ephys -p no:warnings tests/
-# run one test, debug: pytest [above options] --pdb tests/tests_name.py -k \
-#                      function_name
+# run all tests:
+# pytest -sv --cov-report term-missing --cov=workflow_array_ephys -p no:warnings tests/
+# run one test, debug:
+# pytest [above options] --pdb tests/tests_name.py -k function_name
 
 import os
 import sys
@@ -56,8 +56,7 @@ def dj_config():
     dj.config['custom'] = {
         'database.prefix': (os.environ.get('DATABASE_PREFIX')
                             or dj.config['custom']['database.prefix']),
-        'ephys_root_data_dir': (os.environ.get('EPHYS_ROOT_DATA_DIR').split(',')
-                                or dj.config['custom']['ephys_root_data_dir'])
+        'ephys_root_data_dir': (os.environ.get('EPHYS_ROOT_DATA_DIR').split(',') if os.environ.get('EPHYS_ROOT_DATA_DIR') else dj.config['custom']['ephys_root_data_dir'])
     }
     return
 
@@ -198,6 +197,28 @@ def testdata_paths():
         'npx3B-p1-ks': 'subject6/session1/towersTask_g0_imec0'
     }
 
+@pytest.fixture
+def ephys_insertionlocation(pipeline, ingest_sessions):
+    """Insert probe location into ephys.InsertionLocation"""
+    ephys = pipeline['ephys']
+
+    for probe_insertion_key in ephys.ProbeInsertion.fetch('KEY'):
+        ephys.InsertionLocation.insert1(dict(**probe_insertion_key,
+                                             skull_reference='Bregma',
+                                             ap_location=0,
+                                             ml_location=0,
+                                             depth=0,
+                                             theta=0,
+                                             phi=0,
+                                             beta=0))
+    yield
+
+    if _tear_down:
+        if verbose:
+            ephys.InsertionLocation.delete()
+        else:
+            with QuietStdOut():
+                ephys.InsertionLocation.delete()
 
 @pytest.fixture
 def kilosort_paramset(pipeline):

workflow_array_ephys/pipeline.py

@@ -6,7 +6,7 @@
 
 from element_animal.subject import Subject
 from element_lab.lab import Source, Lab, Protocol, User, Project
-from element_session.session import Session
+from element_session.session_with_datetime import Session
 
 from .paths import get_ephys_root_data_dir, get_session_directory