Merge branch 'main' into facemap

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -60,9 +60,11 @@ body:
     attributes:
       label: Python Version
       options:
-        - 3.7
-        - 3.8
-        - 3.9
+        - "3.8"
+        - "3.9"
+        - "3.10"
+        - "3.11"
+        - "3.12"
     validations:
       required: true
   - type: textarea

.github/workflows/dailies.yml

@@ -3,32 +3,48 @@ name: Daily workflows
 on:
   workflow_dispatch:
   schedule:
-    - cron: "0 16 * * *"  # Daily at noon EST
+    - cron: "0 16 * * *" # Daily at noon EST
 
 jobs:
-
   run-daily-tests:
-    uses: catalystneuro/neuroconv/.github/workflows/testing.yml@main
+    uses: ./.github/workflows/testing.yml
     secrets:
       AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
       AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
       S3_GIN_BUCKET: ${{ secrets.S3_GIN_BUCKET }}
+      CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
 
   run-daily-doc-link-checks:
-    uses: catalystneuro/neuroconv/.github/workflows/test-external-links.yml@main
+    uses: ./.github/workflows/test-external-links.yml
+
+  notify-test-failure:
+    runs-on: ubuntu-latest
+    needs: [run-daily-tests]
+    if: ${{ always() && needs.run-daily-tests.result == 'failure' }}
+    steps:
+      - uses: dawidd6/action-send-mail@v3
+        with:
+          server_address: smtp.gmail.com
+          server_port: 465
+          username: ${{ secrets.MAIL_USERNAME }}
+          password: ${{ secrets.MAIL_PASSWORD }}
+          subject: NeuroConv Daily Test Failure
+          to: ${{ secrets.DAILY_FAILURE_EMAIL_LIST }}
+          from: NeuroConv
+          body: "The daily test workflow failed, please check status at https://github.com/catalystneuro/neuroconv/actions/workflows/dailies.yml"
 
-  notify:
+  notify-link-check-failure:
     runs-on: ubuntu-latest
-    needs: [run-daily-tests, run-daily-doc-link-checks]
-    if: ${{ always() && contains(needs.*.result, 'failure') }}
+    needs: [run-daily-doc-link-checks]
+    if: ${{ always() && needs.run-daily-doc-link-checks.result == 'failure' }}
     steps:
-    - uses: dawidd6/action-send-mail@v3
-      with:
-        server_address: smtp.gmail.com
-        server_port: 465  # TSL
-        username: ${{ secrets.MAIL_USERNAME }}
-        password: ${{ secrets.MAIL_PASSWORD }}
-        subject: NeuroConv Daily Failure
-        to: ${{ secrets.DAILY_FAILURE_EMAIL_LIST }}  # add more with commas, no separation
-        from: NeuroConv
-        body: "The daily workflow for the NWB Inspector failed: please check status at https://github.com/catalystneuro/neuroconv/actions/workflows/dailies.yml"
+      - uses: dawidd6/action-send-mail@v3
+        with:
+          server_address: smtp.gmail.com
+          server_port: 465
+          username: ${{ secrets.MAIL_USERNAME }}
+          password: ${{ secrets.MAIL_PASSWORD }}
+          subject: NeuroConv Daily Doc Link Check Failure
+          to: ${{ secrets.DAILY_FAILURE_EMAIL_LIST }}
+          from: NeuroConv
+          body: "The daily check for working links in the documentation failed, please check status at https://github.com/catalystneuro/neuroconv/actions/workflows/dailies.yml"

.github/workflows/deploy-tests.yml

@@ -12,7 +12,7 @@ concurrency:  # Cancel previous workflows on the same pull request
 jobs:
 
   assess-file-changes:
-    uses: catalystneuro/neuroconv/.github/workflows/assess-file-changes.yml@main
+    uses: ./.github/workflows/assess-file-changes.yml
 
   detect-changelog-updates:
     needs: assess-file-changes
@@ -30,24 +30,25 @@ jobs:
   run-tests:
     needs: assess-file-changes
     if: ${{ needs.assess-file-changes.outputs.SOURCE_CHANGED == 'true' }}
-    uses: catalystneuro/neuroconv/.github/workflows/testing.yml@main
+    uses: ./.github/workflows/testing.yml
     secrets:
       AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
       AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
       S3_GIN_BUCKET: ${{ secrets.S3_GIN_BUCKET }}
+      CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
 
 
   run-live-service-tests:
     needs: assess-file-changes
     if: ${{ needs.assess-file-changes.outputs.SOURCE_CHANGED == 'true' }}
-    uses: catalystneuro/neuroconv/.github/workflows/live-service-testing.yml@main
+    uses: ./.github/workflows/live-service-testing.yml
     secrets:
       DANDI_API_KEY: ${{ secrets.DANDI_API_KEY }}
 
   run-dev-tests:
     needs: assess-file-changes
     if: ${{ needs.assess-file-changes.outputs.SOURCE_CHANGED == 'true' }}
-    uses: catalystneuro/neuroconv/.github/workflows/dev-testing.yml@main
+    uses: ./.github/workflows/dev-testing.yml
     secrets:
       DANDI_API_KEY: ${{ secrets.DANDI_API_KEY }}
       AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
@@ -57,7 +58,7 @@ jobs:
   run-doctests-only:
     needs: assess-file-changes
     if: ${{ needs.assess-file-changes.outputs.CONVERSION_GALLERY_CHANGED == 'true' || needs.assess-file-changes.outputs.SOURCE_CHANGED != 'true' }}
-    uses: catalystneuro/neuroconv/.github/workflows/doctests.yml@main
+    uses: ./.github/workflows/doctests.yml
 
   check-final-status:
     name: All tests passing

.github/workflows/dev-testing.yml

@@ -23,9 +23,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11"]
+        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
     steps:
-      - uses: s-weigand/setup-conda@v1
       - uses: actions/checkout@v4
       - run: git fetch --prune --unshallow --tags
       - name: Setup Python
@@ -43,8 +42,8 @@ jobs:
 
 
 
-      - name: Install full requirements (-e needed for codecov report)
-        run: pip install --no-cache-dir -e .[full,test]
+      - name: Install full requirements
+        run: pip install --no-cache-dir .[full,test]
 
       - name: Dev gallery - ROIExtractors
         run: pip install --no-cache-dir git+https://github.com/CatalystNeuro/roiextractors@main
@@ -63,7 +62,6 @@ jobs:
 
       - name: Display installed packages and their sources
         run: |
-          conda list
           pip list
 
       - name: Get ephy_testing_data current head hash

.github/workflows/doctests.yml

@@ -9,10 +9,9 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11"]
+        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
         os: [ubuntu-latest, macos-latest, windows-latest]
     steps:
-      - uses: s-weigand/setup-conda@v1
       - uses: actions/checkout@v4
       - run: git fetch --prune --unshallow --tags
       - name: Setup Python ${{ matrix.python-version }}

.github/workflows/live-service-testing.yml

@@ -17,10 +17,9 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11"]
+        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
         os: [ubuntu-latest, macos-latest, windows-latest]
     steps:
-      - uses: s-weigand/setup-conda@v1
       - uses: actions/checkout@v4
       - run: git fetch --prune --unshallow --tags
       - name: Setup Python ${{ matrix.python-version }}
@@ -36,8 +35,8 @@ jobs:
           git config --global user.name "CI Almighty"
           pip install wheel # Needed for scan image
 
-      - name: Install full requirements (-e needed for codecov report)
-        run: pip install -e .[full]
+      - name: Install full requirements
+        run: pip install .[full]
 
       - name: Run subset of tests that use S3 live services
         run: pytest -rsx -n auto tests/test_minimal/test_tools/s3_tools.py

.github/workflows/test-external-links.yml

@@ -18,6 +18,6 @@ jobs:
         with:
           python-version: "3.10"
       - name: Install package for API docs
-        run: pip install -e .[docs]
+        run: pip install .[docs]
       - name: Test External Links
         run: sphinx-build -b linkcheck ./docs ./docs/_build/

.github/workflows/testing.yml

@@ -1,7 +1,5 @@
 name: Minimal and Full Tests
 on:
-  schedule:
-    - cron: "0 16 * * *"  # Daily at noon EST
   workflow_call:
     secrets:
       AWS_ACCESS_KEY_ID:
@@ -10,6 +8,8 @@ on:
         required: true
       S3_GIN_BUCKET:
         required: true
+      CODECOV_TOKEN:
+        required: true
   workflow_dispatch:
 
 jobs:
@@ -19,10 +19,9 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11"]
+        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
         os: [ubuntu-latest, macos-latest, windows-latest]
     steps:
-      - uses: s-weigand/setup-conda@v1
       - uses: actions/checkout@v4
       - run: git fetch --prune --unshallow --tags
       - name: Setup Python ${{ matrix.python-version }}
@@ -77,7 +76,7 @@ jobs:
 
 
       - name: Install full requirements
-        run: pip install -e .[full]  # -e needed for codecov report
+        run: pip install .[full]
 
 
 
@@ -127,10 +126,8 @@ jobs:
         name: Download behavior dataset from S3
         run: aws s3 cp --recursive ${{ secrets.S3_GIN_BUCKET }}/behavior_testing_data ./behavior_testing_data
 
-
-
       - name: Run full pytest with coverage
-        run: pytest -vv -rsx -n auto --dist loadscope --cov=./ --cov-report xml:./codecov.xml
+        run: pytest -vv -rsx -n auto --dist loadscope --cov=neuroconv --cov-report xml:./codecov.xml
       - name: Upload full coverage to Codecov
         if: ${{ matrix.python-version == '3.9' && matrix.os == 'ubuntu-latest' }}
         uses: codecov/codecov-action@v4
@@ -139,4 +136,6 @@ jobs:
           file: ./codecov.xml
           flags: unittests
           name: codecov-umbrella
-          yml: ./codecov.yml
+          codecov_yml_path: ./codecov.yml
+          fail_ci_if_error: true
+          verbose: true

.github/workflows/update-s3-testing-data.yml

@@ -11,7 +11,6 @@ jobs:
     strategy:
       fail-fast: false
     steps:
-      - uses: s-weigand/setup-conda@v1
       - uses: actions/checkout@v4
       - run: git fetch --prune --unshallow --tags
       - name: Setup Python

.gitignore

@@ -124,3 +124,6 @@ venv.bak/
 # NWB
 .nwb
 .pre-commit-config.yaml
+
+# Spyder
+.spyproject

.pre-commit-config.yaml

@@ -6,7 +6,7 @@ repos:
     -   id: end-of-file-fixer
     -   id: trailing-whitespace
 -   repo: https://github.com/psf/black
-    rev: 24.2.0
+    rev: 24.3.0
     hooks:
     -   id: black
         exclude: ^docs/

CHANGELOG.md

@@ -1,17 +1,46 @@
 # Upcoming
 
+### Bug fixes
+* Fixed writing waveforms directly to file [PR #799](https://github.com/catalystneuro/neuroconv/pull/799)
+
+### Deprecations
+* Removed `stream_id` as an argument from `IntanRecordingInterface` [PR #794](https://github.com/catalystneuro/neuroconv/pull/794)
+* Replaced `waveform_extractor.is_extension` with `waveform_extractor.has_extension`[PR #799](https://github.com/catalystneuro/neuroconv/pull/799)
+
+# v0.4.8 (March 20, 2024)
+
+### Bug fixes
+* Fixed writing the `electrodes` field in `add_electrical_series` when multiple groups are present. [PR #784](https://github.com/catalystneuro/neuroconv/pull/784)
+
+### Improvements
+* Upgraded Pydantic support to `>v2.0.0`. [PR #767](https://github.com/catalystneuro/neuroconv/pull/767)
+* Absorbed the `DatasetInfo` model into the `DatasetIOConfiguration` model. [PR #767](https://github.com/catalystneuro/neuroconv/pull/767)
+* Keyword argument `field_name` of the `DatasetIOConfiguration.from_neurodata_object` method has been renamed to `dataset_name` to be more consistent with its usage. This only affects direct initialization of the model; usage via the `BackendConfiguration` constructor and its associated helper functions in `neuroconv.tools.nwb_helpers` is unaffected. [PR #767](https://github.com/catalystneuro/neuroconv/pull/767)
+* Manual construction of a `DatasetIOConfiguration` now requires the field `dataset_name`, and will be validated to match the final path of `location_in_file`. Usage via the automated constructors is unchanged. [PR #767](https://github.com/catalystneuro/neuroconv/pull/767)
+* Enhance `get_schema_from_method_signature` to extract descriptions from the method docval. [PR #771](https://github.com/catalystneuro/neuroconv/pull/771)
+* Avoid writing `channel_to_uV` and `offset_to_uV` in `add_electrodes`  [PR #803](https://github.com/catalystneuro/neuroconv/pull/803)
+
+
+# v0.4.7 (February 21, 2024)
+
+### Deprecation
+* Removed `.get_electrode_table_json()` on the `BaseRecordingExtractorInterface` in favor of GUIDE specific interactions. [PR #431](https://github.com/catalystneuro/neuroconv/pull/431)
+* Removed the `SIPickleRecordingInterface` and `SIPickleSortingInterface` interfaces. [PR #757](https://github.com/catalystneuro/neuroconv/pull/757)
+* Removed the `SpikeGLXLFPInterface` interface. [PR #757](https://github.com/catalystneuro/neuroconv/pull/757)
+
 ### Bug fixes
 * LocalPathExpander matches only `folder_paths` or `file_paths` if that is indicated in the passed specification. [PR #679](https://github.com/catalystneuro/neuroconv/pull/675) and [PR #675](https://github.com/catalystneuro/neuroconv/pull/679
 * Fixed depth consideration in partial chunking pattern for the ROI data buffer. [PR #677](https://github.com/catalystneuro/neuroconv/pull/677)
-* Fix mapping between channel names and the electrode table when writing more than one `ElectricalSeries` to the NWBFile. This fixes an issue when the converter pipeline of `SpikeGLXConverterPipe` was writing the electrode table region of the NIDQ stream incorrectly [PR #678](https://github.com/catalystneuro/neuroconv/pull/678)
+* Fix mapping between channel names and the electrode table when writing more than one `ElectricalSeries` to the NWBFile. This fixes an issue when the converter pipeline of `SpikeGLXConverterPipe` was writing the electrode table region of the NIDQ stream incorrectly. [PR #678](https://github.com/catalystneuro/neuroconv/pull/678)
+* Fix `configure_backend` when applied to `TimeSeries` contents that leverage internal links for `data` or `timestamps`. [PR #732](https://github.com/catalystneuro/neuroconv/pull/732)
 
 ### Features
 * Changed the `Suite2pSegmentationInterface` to support multiple plane segmentation outputs. The interface now has a `plane_name` and `channel_name` arguments to determine which plane output and channel trace add to the NWBFile. [PR #601](https://github.com/catalystneuro/neuroconv/pull/601)
 * Added `create_path_template` and corresponding tests [PR #680](https://github.com/catalystneuro/neuroconv/pull/680)
 * Added tool function `configure_datasets` for configuring all datasets of an in-memory `NWBFile` to be backend specific. [PR #571](https://github.com/catalystneuro/neuroconv/pull/571)
 * Added `LightningPoseConverter` to add pose estimation data and the original and the optional labeled video added as ImageSeries to NWB. [PR #633](https://github.com/catalystneuro/neuroconv/pull/633)
 * Added gain as a required `__init__` argument for `TdtRecordingInterface`. [PR #704](https://github.com/catalystneuro/neuroconv/pull/704)
-* Extract session_start_time from Plexon plx recording file. [PR #723](https://github.com/catalystneuro/neuroconv/pull/723)
+* Extract session_start_time from Plexon `plx` recording file. [PR #723](https://github.com/catalystneuro/neuroconv/pull/723)
 * Add `FacemapInterface`, which currently only handles eye tracking [PR #752](https://github.com/catalystneuro/neuroconv/pull/752)
 
 ### Improvements
@@ -23,14 +52,13 @@
 * Refactored `ScanImagingInterface` to reference ROIExtractors' version of `extract_extra_metadata`. [PR #731](https://github.com/catalystneuro/neuroconv/pull/731)
 * Added support for Long NHP probe types for the `SpikeGLXRecorddingInterfacce`. [PR #701](https://github.com/catalystneuro/neuroconv/pull/701)
 * Remove unnecessary duplication of probe setting in `SpikeGLXRecordingInterface`. [PR #696](https://github.com/catalystneuro/neuroconv/pull/696)
-
-### Deprecation
-* Removed `.get_electrode_table_json()` on the `BaseRecordingExtractorInterface` in favor of GUIDE specific interactions. [PR #431](https://github.com/catalystneuro/neuroconv/pull/431)
+* Added associated suffixes to all interfaces and converters. [PR #734](https://github.com/catalystneuro/neuroconv/pull/734)
+* Added convenience function `get_format_summaries` to `tools.importing` (and exposed at highest level). [PR #734](https://github.com/catalystneuro/neuroconv/pull/734)
 
 ### Testing
 * `RecordingExtractorInterfaceTestMixin` now compares either `group_name`, `group` or a default value of  `ElectrodeGroup` to the `group` property in the `NWBRecordingExtractor` instead of comparing `group` to `group` as it was done before [PR #736](https://github.com/catalystneuro/neuroconv/pull/736)
 * `TestScanImageImagingInterfaceRecent` now checks metadata against new roiextractors implementation [PR #741](https://github.com/catalystneuro/neuroconv/pull/741).
-
+* Removed editable installs from the CI workflow. [PR #756](https://github.com/catalystneuro/neuroconv/pull/756)
 
 
 # v0.4.6 (November 30, 2023)
@@ -62,6 +90,7 @@
 * The `CEDRecordingInterface` has now been removed; use the `Spike2RecordingInterface` instead. [PR #602](https://github.com/catalystneuro/neuroconv/pull/602)
 
 ### Features
+* Added support for python 3.12 [PR #626](https://github.com/catalystneuro/neuroconv/pull/626)
 * Added `session_start_time` extraction to `FicTracDataInterface`. [PR #598](https://github.com/catalystneuro/neuroconv/pull/598)
 * Added `imaging_plane_name` keyword argument to `add_imaging_plane` function to determine which imaging plane to add from the metadata by name instead of `imaging_plane_index`.
 * Added reference for `imaging_plane` to default plane segmentation metadata. [PR #594](https://github.com/catalystneuro/neuroconv/pull/594)