Skip to content

Commit

Permalink
Merge branch 'main' into inscopix
Browse files Browse the repository at this point in the history
  • Loading branch information
bendichter authored May 4, 2024
2 parents 9d17d52 + 5f231d9 commit e409322
Show file tree
Hide file tree
Showing 192 changed files with 4,752 additions and 1,532 deletions.
8 changes: 5 additions & 3 deletions .github/ISSUE_TEMPLATE/bug_report.yml
Original file line number Diff line number Diff line change
Expand Up @@ -60,9 +60,11 @@ body:
attributes:
label: Python Version
options:
- 3.7
- 3.8
- 3.9
- "3.8"
- "3.9"
- "3.10"
- "3.11"
- "3.12"
validations:
required: true
- type: textarea
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/assess-file-changes.yml
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ jobs:

name: Test changed-files
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
with:
fetch-depth: 0

Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/auto-publish.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,9 @@ jobs:
deploy:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v4
uses: actions/setup-python@v5
with:
python-version: "3.10"
- name: Install dependencies
Expand Down
41 changes: 41 additions & 0 deletions .github/workflows/build_and_upload_docker_image_dev.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
name: Build and Upload Docker Image of Current Dev Branch to GHCR

on:
schedule:
- cron: "0 16 * * 1" # Weekly at noon EST on Monday
workflow_dispatch:

concurrency: # Cancel previous workflows on the same pull request
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true

jobs:
release-image:
name: Build and Upload Docker Image of Current Dev Branch to GHCR
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ secrets.DOCKER_UPLOADER_USERNAME }}
password: ${{ secrets.DOCKER_UPLOADER_PASSWORD }}
- name: Get current date
id: date
run: |
date_tag="$(date +'%Y-%m-%d')"
echo "date_tag=$date_tag" >> $GITHUB_OUTPUT
- name: Build and push
uses: docker/build-push-action@v5
with:
push: true # Push is a shorthand for --output=type=registry
tags: ghcr.io/catalystneuro/neuroconv:dev,ghcr.io/catalystneuro/neuroconv:${{ steps.date.outputs.date_tag }}
context: .
file: dockerfiles/neuroconv_dev_dockerfile
provenance: false
46 changes: 46 additions & 0 deletions .github/workflows/build_and_upload_docker_image_latest_release.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
name: Build and Upload Docker Image of Latest Release to GHCR

on:
workflow_run:
workflows: [auto-publish]
types: [completed]
workflow_dispatch:

concurrency: # Cancel previous workflows on the same pull request
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true

jobs:
release-image:
name: Build and Upload Docker Image of Latest Release to GHCR
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Parse the version from the GitHub latest release tag
id: parsed_version
run: |
git fetch --prune --unshallow --tags
tags="$(git tag --list)"
version_tag=${tags: -6 : 6}
echo "version_tag=$version_tag" >> $GITHUB_OUTPUT
- name: Printout parsed version for GitHub Action log
run: echo ${{ steps.parsed_version.outputs.version_tag }}
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ secrets.DOCKER_UPLOADER_USERNAME }}
password: ${{ secrets.DOCKER_UPLOADER_PASSWORD }}
- name: Build and push
uses: docker/build-push-action@v5
with:
push: true # Push is a shorthand for --output=type=registry
tags: ghcr.io/catalystneuro/neuroconv:latest,ghcr.io/catalystneuro/neuroconv:${{ steps.parsed_version.outputs.version_tag }}
context: .
file: dockerfiles/neuroconv_latest_release_dockerfile
provenance: false
37 changes: 37 additions & 0 deletions .github/workflows/build_and_upload_docker_image_yaml_variable.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
name: Build and Upload Docker Image of latest with YAML variable to GHCR

on:
workflow_run:
workflows: [build_and_upload_docker_image_latest_release]
types: [completed]
workflow_dispatch:

concurrency: # Cancel previous workflows on the same pull request
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true

jobs:
release-image:
name: Build and Upload Docker Image of latest with YAML variable to GHCR
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ secrets.DOCKER_UPLOADER_USERNAME }}
password: ${{ secrets.DOCKER_UPLOADER_PASSWORD }}
- name: Build and push YAML variable image based on latest
uses: docker/build-push-action@v5
with:
push: true # Push is a shorthand for --output=type=registry
tags: ghcr.io/catalystneuro/neuroconv:yaml_variable
context: .
file: dockerfiles/neuroconv_latest_yaml_variable
provenance: false
2 changes: 1 addition & 1 deletion .github/workflows/codespell.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,6 @@ jobs:

steps:
- name: Checkout
uses: actions/checkout@v3
uses: actions/checkout@v4
- name: Codespell
uses: codespell-project/actions-codespell@v1
50 changes: 33 additions & 17 deletions .github/workflows/dailies.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,32 +3,48 @@ name: Daily workflows
on:
workflow_dispatch:
schedule:
- cron: "0 16 * * *" # Daily at noon EST
- cron: "0 16 * * *" # Daily at noon EST

jobs:

run-daily-tests:
uses: catalystneuro/neuroconv/.github/workflows/testing.yml@main
uses: ./.github/workflows/testing.yml
secrets:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
S3_GIN_BUCKET: ${{ secrets.S3_GIN_BUCKET }}
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

run-daily-doc-link-checks:
uses: catalystneuro/neuroconv/.github/workflows/test-external-links.yml@main
uses: ./.github/workflows/test-external-links.yml

notify-test-failure:
runs-on: ubuntu-latest
needs: [run-daily-tests]
if: ${{ always() && needs.run-daily-tests.result == 'failure' }}
steps:
- uses: dawidd6/action-send-mail@v3
with:
server_address: smtp.gmail.com
server_port: 465
username: ${{ secrets.MAIL_USERNAME }}
password: ${{ secrets.MAIL_PASSWORD }}
subject: NeuroConv Daily Test Failure
to: ${{ secrets.DAILY_FAILURE_EMAIL_LIST }}
from: NeuroConv
body: "The daily test workflow failed, please check status at https://github.com/catalystneuro/neuroconv/actions/workflows/dailies.yml"

notify:
notify-link-check-failure:
runs-on: ubuntu-latest
needs: [run-daily-tests, run-daily-doc-link-checks]
if: ${{ always() && contains(needs.*.result, 'failure') }}
needs: [run-daily-doc-link-checks]
if: ${{ always() && needs.run-daily-doc-link-checks.result == 'failure' }}
steps:
- uses: dawidd6/action-send-mail@v3
with:
server_address: smtp.gmail.com
server_port: 465 # TSL
username: ${{ secrets.MAIL_USERNAME }}
password: ${{ secrets.MAIL_PASSWORD }}
subject: NeuroConv Daily Failure
to: ${{ secrets.DAILY_FAILURE_EMAIL_LIST }} # add more with commas, no separation
from: NeuroConv
body: "The daily workflow for the NWB Inspector failed: please check status at https://github.com/catalystneuro/neuroconv/actions/workflows/dailies.yml"
- uses: dawidd6/action-send-mail@v3
with:
server_address: smtp.gmail.com
server_port: 465
username: ${{ secrets.MAIL_USERNAME }}
password: ${{ secrets.MAIL_PASSWORD }}
subject: NeuroConv Daily Doc Link Check Failure
to: ${{ secrets.DAILY_FAILURE_EMAIL_LIST }}
from: NeuroConv
body: "The daily check for working links in the documentation failed, please check status at https://github.com/catalystneuro/neuroconv/actions/workflows/dailies.yml"
11 changes: 6 additions & 5 deletions .github/workflows/deploy-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ concurrency: # Cancel previous workflows on the same pull request
jobs:

assess-file-changes:
uses: catalystneuro/neuroconv/.github/workflows/assess-file-changes.yml@main
uses: ./.github/workflows/assess-file-changes.yml

detect-changelog-updates:
needs: assess-file-changes
Expand All @@ -30,24 +30,25 @@ jobs:
run-tests:
needs: assess-file-changes
if: ${{ needs.assess-file-changes.outputs.SOURCE_CHANGED == 'true' }}
uses: catalystneuro/neuroconv/.github/workflows/testing.yml@main
uses: ./.github/workflows/testing.yml
secrets:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
S3_GIN_BUCKET: ${{ secrets.S3_GIN_BUCKET }}
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}


run-live-service-tests:
needs: assess-file-changes
if: ${{ needs.assess-file-changes.outputs.SOURCE_CHANGED == 'true' }}
uses: catalystneuro/neuroconv/.github/workflows/live-service-testing.yml@main
uses: ./.github/workflows/live-service-testing.yml
secrets:
DANDI_API_KEY: ${{ secrets.DANDI_API_KEY }}

run-dev-tests:
needs: assess-file-changes
if: ${{ needs.assess-file-changes.outputs.SOURCE_CHANGED == 'true' }}
uses: catalystneuro/neuroconv/.github/workflows/dev-testing.yml@main
uses: ./.github/workflows/dev-testing.yml
secrets:
DANDI_API_KEY: ${{ secrets.DANDI_API_KEY }}
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
Expand All @@ -57,7 +58,7 @@ jobs:
run-doctests-only:
needs: assess-file-changes
if: ${{ needs.assess-file-changes.outputs.CONVERSION_GALLERY_CHANGED == 'true' || needs.assess-file-changes.outputs.SOURCE_CHANGED != 'true' }}
uses: catalystneuro/neuroconv/.github/workflows/doctests.yml@main
uses: ./.github/workflows/doctests.yml

check-final-status:
name: All tests passing
Expand Down
24 changes: 13 additions & 11 deletions .github/workflows/dev-testing.yml
Original file line number Diff line number Diff line change
Expand Up @@ -23,13 +23,12 @@ jobs:
strategy:
fail-fast: false
matrix:
python-version: ["3.8", "3.9", "3.10", "3.11"]
python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
steps:
- uses: s-weigand/setup-conda@v1
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- run: git fetch --prune --unshallow --tags
- name: Setup Python
uses: actions/setup-python@v4
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}

Expand All @@ -43,32 +42,35 @@ jobs:
- name: Install full requirements (-e needed for codecov report)
run: pip install --no-cache-dir -e .[full,test]
- name: Install full requirements
run: pip install --no-cache-dir .[full,test]

- name: Dev gallery - ROIExtractors
run: pip install --no-cache-dir git+https://github.com/CatalystNeuro/roiextractors@main
- name: Dev gallery - DANDI
run: pip install --no-cache-dir git+https://github.com/dandi/dandi-cli@master
- name: Dev gallery - PyNWB
run: pip install --no-cache-dir git+https://github.com/NeurodataWithoutBorders/pynwb@dev
- name: Dev gallery - ProbeInterface
run: pip install --no-cache-dir git+https://github.com/spikeinterface/probeinterface@main
- name: Dev gallery - SpikeInterface
run: pip install --no-cache-dir git+https://github.com/spikeinterface/spikeinterface@main
run: pip install --no-cache-dir "spikeinterface[test_core] @ git+https://github.com/spikeinterface/spikeinterface@main"
- name: Dev gallery - NEO
run: pip install --no-cache-dir git+https://github.com/NeuralEnsemble/python-neo@master
- name: Dev gallery - HDMF
run: pip install --no-cache-dir git+https://github.com/hdmf-dev/hdmf@dev
- name: Dev gallery - HDMF-ZARR
run : pip install --no-cache-dir git+https://github.com/hdmf-dev/hdmf-zarr@dev

- name: Display installed packages and their sources
run: |
conda list
pip list
- name: Get ephy_testing_data current head hash
id: ephys
run: echo "::set-output name=HASH_EPHY_DATASET::$(git ls-remote https://gin.g-node.org/NeuralEnsemble/ephy_testing_data.git HEAD | cut -f1)"
- name: Cache ephys dataset - ${{ steps.ephys.outputs.HASH_EPHY_DATASET }}
uses: actions/cache@v3
uses: actions/cache@v4
id: cache-ephys-datasets
with:
path: ./ephy_testing_data
Expand All @@ -77,7 +79,7 @@ jobs:
id: ophys
run: echo "::set-output name=HASH_OPHYS_DATASET::$(git ls-remote https://gin.g-node.org/CatalystNeuro/ophys_testing_data.git HEAD | cut -f1)"
- name: Cache ophys dataset - ${{ steps.ophys.outputs.HASH_OPHYS_DATASET }}
uses: actions/cache@v3
uses: actions/cache@v4
id: cache-ophys-datasets
with:
path: ./ophys_testing_data
Expand All @@ -86,7 +88,7 @@ jobs:
id: behavior
run: echo "::set-output name=HASH_BEHAVIOR_DATASET::$(git ls-remote https://gin.g-node.org/CatalystNeuro/behavior_testing_data.git HEAD | cut -f1)"
- name: Cache behavior dataset - ${{ steps.behavior.outputs.HASH_BEHAVIOR_DATASET }}
uses: actions/cache@v3
uses: actions/cache@v4
id: cache-behavior-datasets
with:
path: ./behavior_testing_data
Expand Down
Loading

0 comments on commit e409322

Please sign in to comment.