Skip to content

Commit 6573c55

Browse files
joshmoorejoshmoore
committed
Merge remote-tracking branch 'origin/master' into key-sep
2 parents 6cac054 + 23422fc commit 6573c55

20 files changed

+777
-114
lines changed

‎.coveragerc

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
[run]
2-
omit = zarr/meta_v1.py
2+
omit =
3+
zarr/meta_v1.py
4+
bench/compress_normal.py
35

46
[report]
57
exclude_lines =

‎.github/workflows/python-package.yml

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
2+
# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
3+
4+
name: Linux Testing
5+
6+
on:
7+
push:
8+
branches: [ master ]
9+
pull_request:
10+
branches: [ master ]
11+
12+
jobs:
13+
build:
14+
15+
runs-on: ubuntu-latest
16+
strategy:
17+
matrix:
18+
python-version: ['3.6.9' , 3.7, 3.8]
19+
numpy_version: ['==1.16.4', '']
20+
services:
21+
redis:
22+
image: redis
23+
# Set health checks to wait until redis has started
24+
options: >-
25+
--health-cmd "redis-cli ping"
26+
--health-interval 10s
27+
--health-timeout 5s
28+
--health-retries 5
29+
ports:
30+
- 6379:6379
31+
mongodb:
32+
image: mongo:3.4.23
33+
ports:
34+
- 27017:27017
35+
steps:
36+
- uses: actions/checkout@v2
37+
with:
38+
fetch-depth: 0
39+
- name: Setup Miniconda
40+
uses: conda-incubator/setup-miniconda@master
41+
with:
42+
channels: conda-forge
43+
python-version: ${{ matrix.python-version }}
44+
env:
45+
ACTIONS_ALLOW_UNSECURE_COMMANDS: true
46+
# Runs a set of commands using the runners shell
47+
- name: Create Conda environment with the rights deps
48+
shell: "bash -l {0}"
49+
run: |
50+
conda create -n zarr-env python==${{matrix.python-version}} bsddb3 numcodecs==0.6.4 lmdb pip nodejs flake8 mypy
51+
conda activate zarr-env
52+
npm install -g azurite
53+
- name: Install dependencies
54+
shell: "bash -l {0}"
55+
run: |
56+
conda activate zarr-env
57+
python -m pip install --upgrade pip
58+
python -m pip install -U pip setuptools wheel codecov
59+
python -m pip install -rrequirements_dev_minimal.txt numpy${{ matrix.numpy_version}} -rrequirements_dev_optional.txt pymongo redis
60+
python -m pip install -e .
61+
python -m pip freeze
62+
- name: Tests
63+
shell: "bash -l {0}"
64+
env:
65+
COVERAGE_FILE: .coverage.${{matrix.python-version}}.${{matrix.numpy_version}}
66+
ZARR_TEST_ABS: 1
67+
ZARR_TEST_MONGO: 1
68+
ZARR_TEST_REDIS: 1
69+
run: |
70+
conda activate zarr-env
71+
mkdir ~/blob_emulator
72+
azurite -l ~/blob_emulator --debug debug.log 2>&1 > stdouterr.log &
73+
pytest --cov=zarr --cov-config=.coveragerc --doctest-plus --cov-report xml --cov=./
74+
- uses: codecov/codecov-action@v1
75+
with:
76+
#token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos
77+
#files: ./coverage1.xml,./coverage2.xml # optional
78+
#flags: unittests # optional
79+
#name: codecov-umbrella # optional
80+
#fail_ci_if_error: true # optional (default = false)
81+
verbose: true # optional (default = false)
82+
- name: Linting
83+
shell: "bash -l {0}"
84+
run: |
85+
conda activate zarr-env
86+
flake8 zarr
87+
mypy zarr
88+
89+

‎.github/workflows/windows-testing.yml

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
2+
# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
3+
4+
name: Python package
5+
6+
on:
7+
push:
8+
branches: [ master ]
9+
pull_request:
10+
branches: [ master ]
11+
12+
jobs:
13+
windows:
14+
name: Windows Tests
15+
runs-on: "windows-latest"
16+
strategy:
17+
fail-fast: True
18+
matrix:
19+
python-version: ["3.8"]
20+
steps:
21+
- uses: actions/checkout@v2
22+
with:
23+
fetch-depth: 0
24+
- uses: conda-incubator/setup-miniconda@v2
25+
with:
26+
auto-update-conda: true
27+
python-version: ${{ matrix.python-version }}
28+
channels: conda-forge
29+
env:
30+
ACTIONS_ALLOW_UNSECURE_COMMANDS: true
31+
- name: Create Conda environment with the rights deps
32+
shell: bash -l {0}
33+
run: |
34+
conda create -n zarr-env python==${{matrix.python-version}} numcodecs==0.6.4 pip nodejs
35+
- name: Install dependencies
36+
shell: bash -l {0}
37+
run: |
38+
conda activate zarr-env
39+
python -m pip install --upgrade pip
40+
python -m pip install -U pip setuptools wheel
41+
python -m pip install -r requirements_dev_numpy.txt -r requirements_dev_minimal.txt -r requirements_dev_optional.txt
42+
python -m pip install -e .
43+
python -m pip freeze
44+
npm install -g azurite
45+
- name: Run Tests
46+
shell: bash -l {0}
47+
run: |
48+
conda activate zarr-env
49+
mkdir ~/blob_emulator
50+
azurite -l ~/blob_emulator --debug debug.log 2>&1 > stdouterr.log &
51+
pytest
52+
env:
53+
ZARR_TEST_ABS: 1
54+
- name: Conda info
55+
shell: bash -l {0}
56+
run: conda info
57+
- name: Conda list
58+
shell: pwsh
59+
run: conda list
60+

‎.travis.yml

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -38,13 +38,12 @@ before_script:
3838
- mongo mydb_test --eval 'db.createUser({user:"travis",pwd:"test",roles:["readWrite"]});'
3939

4040
install:
41-
- pip install -U pip setuptools wheel tox-travis coveralls mypy codecov
41+
- pip install -U pip setuptools wheel tox-travis mypy codecov
4242

4343
script:
4444
- tox
4545
- if [[ $BUILD_DOCS == 'true' ]]; then tox -e docs; fi
4646
- if [[ $MYPY_CHECK == 'true' ]]; then mypy zarr; fi
4747

4848
after_success:
49-
- coveralls --service=travis-pro
5049
- codecov

‎README.md

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -47,8 +47,8 @@
4747
<tr>
4848
<td>Coverage</td>
4949
<td>
50-
<a href="https://coveralls.io/github/zarr-developers/zarr-python?branch=master">
51-
<img src="https://coveralls.io/repos/github/zarr-developers/zarr-python/badge.svg?branch=master" alt="coverage" />
50+
<a href="https://codecov.io/gh/zarr-developers/zarr-python">
51+
<img src="https://codecov.io/gh/zarr-developers/zarr-python/branch/master/graph/badge.svg"/ alt="coverage">
5252
</a>
5353
</td>
5454
</tr>

‎appveyor.yml

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,18 +14,19 @@ environment:
1414
# See: http://stackoverflow.com/a/13751649/163740
1515
CMD_IN_ENV: "cmd /E:ON /V:ON /C .\\build.cmd"
1616
EMULATOR_LOC: C:\\Program Files (x86)\\Microsoft SDKs\\Azure\\Storage Emulator\\AzureStorageEmulator.exe
17-
ZARR_TEST_ABS: 1
1817

1918
matrix:
20-
2119
- PYTHON: "C:\\Python36-x64"
2220
PYTHON_VERSION: "3.6"
21+
ZARR_TEST_ABS: 0
2322

2423
- PYTHON: "C:\\Python37-x64"
2524
PYTHON_VERSION: "3.7"
25+
ZARR_TEST_ABS: 0
2626

2727
- PYTHON: "C:\\Python38-x64"
2828
PYTHON_VERSION: "3.8"
29+
ZARR_TEST_ABS: 1
2930

3031
install:
3132
- "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%"
@@ -45,3 +46,6 @@ test_script:
4546

4647
after_test:
4748
- '"%EMULATOR_LOC%" stop'
49+
50+
matrix:
51+
fast_finish: true

‎docs/release.rst

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,18 @@ This release od Zarr Python is is the first release of Zarr to not supporting Py
3030
See `this link <https://github.com/zarr-developers/zarr-python/milestone/11?closed=1>` for the full list of closed and
3131
merged PR tagged with the 2.6 milestone.
3232

33+
* Add ability to partially read and decompress arrays, see :issue:`667`. It is
34+
only available to chunks stored using fs-spec and using bloc as a compressor.
35+
36+
For certain analysis case when only a small portion of chunks is needed it can
37+
be advantageous to only access and decompress part of the chunks. Doing
38+
partial read and decompression add high latency to many of the operation so
39+
should be used only when the subset of the data is small compared to the full
40+
chunks and is stored contiguously (that is to say either last dimensions for C
41+
layout, firsts for F). Pass ``partial_decompress=True`` as argument when
42+
creating an ``Array``, or when using ``open_array``. No option exists yet to
43+
apply partial read and decompress on a per-operation basis.
44+
3345
2.5.0
3446
-----
3547

‎requirements_dev_minimal.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
# library requirements
22
asciitree==0.3.3
33
fasteners==0.15
4-
numcodecs==0.6.4
4+
numcodecs==0.7.2
55
msgpack-python==0.5.6
66
setuptools-scm==3.3.3
77
# test requirements

‎requirements_dev_optional.txt

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
# optional library requirements
2-
bsddb3==6.2.6; sys_platform != 'win32'
2+
# bsddb3==6.2.6; sys_platform != 'win32'
33
lmdb==0.97; sys_platform != 'win32'
44
# optional library requirements for Jupyter
55
ipytree==0.1.3
@@ -11,12 +11,10 @@ redis==3.3.8
1111
pymongo==3.9.0
1212
# optional test requirements
1313
tox==3.14.0
14-
coverage==5.0.3
15-
coveralls==1.11.1
14+
coverage
1615
flake8==3.8.3
1716
pytest-cov==2.7.1
1817
pytest-doctestplus==0.4.0
19-
pytest-remotedata==0.3.2
2018
h5py==2.10.0
2119
s3fs==0.5.1; python_version > '3.6'
2220
fsspec==0.8.4; python_version > '3.6'

‎windows_conda_dev.txt

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
coverage
2-
coveralls
32
fasteners
43
flake8
54
monotonic

0 commit comments

Comments
 (0)