Skip to content

Commit

Permalink
fix: np.float128 not found in macOS ARM (#15)
Browse files Browse the repository at this point in the history
* fix: np.float128 not found in macOS ARM

Signed-off-by: Keming <kemingy94@gmail.com>

* add windows test

Signed-off-by: Keming <kemingy94@gmail.com>

* release 0.5.0

Signed-off-by: Keming <kemingy94@gmail.com>

---------

Signed-off-by: Keming <kemingy94@gmail.com>
  • Loading branch information
kemingy authored Feb 28, 2024
1 parent 41a91ad commit dab6d2f
Show file tree
Hide file tree
Showing 8 changed files with 94 additions and 52 deletions.
56 changes: 40 additions & 16 deletions .github/workflows/python-check.yml
Original file line number Diff line number Diff line change
Expand Up @@ -22,22 +22,46 @@ on:
permissions:
contents: read

jobs:
build:
concurrency:
group: ${{ github.ref }}-${{ github.workflow }}
cancel-in-progress: true

jobs:
lint:
runs-on: ubuntu-latest

steps:
- uses: actions/checkout@v3
- name: Set up Python 3.10
uses: actions/setup-python@v3
with:
python-version: "3.10"
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install -e .[dev]
- name: Lint
run: make lint
- name: Test with pytest
run: make test
- uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: "3.11"
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install -e .[dev]
- name: Lint
run: make lint
test:
runs-on: ${{ matrix.os }}
strategy:
max-parallel: 20
matrix:
python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
os: [ubuntu-latest , macos-13, macos-14, windows-latest]
exclude:
- python-version: "3.8"
os: "macos-14"
- python-version: "3.9"
os: "macos-14"
steps:
- uses: actions/checkout@v4
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install -e .[dev]
- name: Test with pytest
run: make test
2 changes: 1 addition & 1 deletion .github/workflows/python-publish.yml
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ jobs:
- name: Set up Python
uses: actions/setup-python@v3
with:
python-version: '3.10'
python-version: '3.11'
- name: Install dependencies
run: |
python -m pip install --upgrade pip
Expand Down
8 changes: 3 additions & 5 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -15,13 +15,11 @@ dev:
@pip install -q .[dev]

lint:
@black --check --diff ${PY_SOURCE_FILES}
@isort --check --diff --project=${PROJECT} ${PY_SOURCE_FILES}
@ruff check ${PY_SOURCE_FILES}

format:
@autoflake --in-place --recursive ${PY_SOURCE_FILES}
@isort --project=${PROJECT} ${PY_SOURCE_FILES}
@black ${PY_SOURCE_FILES}
@ruff check --fix ${PY_SOURCE_FILES}
@ruff format ${PY_SOURCE_FILES}

install:
@pip install -q .[msgpack]
Expand Down
2 changes: 1 addition & 1 deletion benchmark/bench.py
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ def time_record(func, data: Data, threshold=1):
t0 = perf_counter()
try:
func(data)
except:
except Exception:
print(traceback.format_exc())
finally:
res.append(perf_counter() - t0)
Expand Down
2 changes: 2 additions & 0 deletions numbin/__init__.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
"""NumBin is a binary serialization format for numerical data."""

from .codec import NumBin, dump, dumps, load, loads

__all__ = ["NumBin", "dumps", "dump", "loads", "load"]
41 changes: 21 additions & 20 deletions numbin/codec.py
Original file line number Diff line number Diff line change
@@ -1,26 +1,28 @@
"""Serde for NumPy ndarray."""

import struct
from io import BytesIO
from typing import IO

import numpy as np

NP_TYPE = [
np.dtype("bool"),
np.dtype("int8"),
np.dtype("int16"),
np.dtype("int32"),
np.dtype("int64"),
np.dtype("uint8"),
np.dtype("uint16"),
np.dtype("uint32"),
np.dtype("uint64"),
np.dtype("float16"),
np.dtype("float32"),
np.dtype("float64"),
np.dtype("float128"),
np.dtype("complex64"),
np.dtype("complex128"),
np.dtype("complex256"),
"bool",
"int8",
"int16",
"int32",
"int64",
"uint8",
"uint16",
"uint32",
"uint64",
"float16",
"float32",
"float64",
"float128",
"complex64",
"complex128",
"complex256",
]
TYPE_TO_INDEX = dict((t, i) for (i, t) in enumerate(NP_TYPE, start=1))
INDEX_TO_TYPE = dict((i, t) for (i, t) in enumerate(NP_TYPE, start=1))
Expand All @@ -38,13 +40,12 @@ class NumBin:
It only supports NumPy ndarray type.
"""

def __init__(self) -> None:
pass

def dump(self, array: np.ndarray, fp: IO[bytes]):
"""Serialize NumPy array to binary and write to the file."""
fp.write(self.dumps(array))

def load(self, fp: IO[bytes]) -> np.ndarray:
"""Read binary from the file and deserialize to NumPy array."""
return self.loads(fp.read())

def dumps(self, array: np.ndarray) -> bytes:
Expand All @@ -53,7 +54,7 @@ def dumps(self, array: np.ndarray) -> bytes:
| index_type | dim | shape | data |
"""
shape = array.shape
index = TYPE_TO_INDEX[array.dtype]
index = TYPE_TO_INDEX[array.dtype.name]
binary = array.tobytes()
data = BytesIO()
data.write(struct.pack(f"{BYTE_ORDER}{INDEX_FORMAT}", index))
Expand Down
14 changes: 9 additions & 5 deletions numbin/msg_ext.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
"""Extend msgpack to support NumPy ndarray."""

from typing import IO, Any

import msgpack
Expand All @@ -10,6 +12,7 @@


def ext_hook(code, data):
"""Msgapck hook to handle NumPy ndarray."""
if code == EXT_CODE:
return _nb.loads(data)
return msgpack.ExtType(code, data)
Expand All @@ -29,20 +32,21 @@ class NumBinMessage(NumBin):
This requires extra dependencies: `pip install numbin[msgpack]`
"""

def __init__(self) -> None:
super().__init__()

def dump(self, obj, fp: IO[bytes]):
"""Serialize to binary and write to the file."""
return super().dump(obj, fp)

def load(self, fp: IO[bytes]) -> Any:
"""Read binary from the file and deserialize to NumPy array."""
return super().load(fp)

def dumps(self, obj: Any) -> bytes:
"""Serialize to binary."""
return msgpack.packb(obj, default=_encode)

def loads(self, bytes):
return msgpack.unpackb(bytes, ext_hook=ext_hook, use_list=False)
def loads(self, buf: bytes):
"""Deserialize binary data to NumPy array."""
return msgpack.unpackb(buf, ext_hook=ext_hook, use_list=False)


# alias
Expand Down
21 changes: 17 additions & 4 deletions pyproject.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[project]
name = "numbin"
version = "0.4.1"
version = "0.5.0"
description = "an efficient binary serialization format for numerical data"
authors = [
{name = "Keming", email = "kemingy94@gmail.com"},
Expand Down Expand Up @@ -39,9 +39,7 @@ build-backend = "setuptools.build_meta"

[project.optional-dependencies]
dev = [
"black>=22.8",
"isort>=5.10",
"autoflake>=1.6.1",
"ruff>=0.2.2",
"pytest>=7.1",
]
msgpack = ["msgpack>=1.0.4"]
Expand All @@ -56,3 +54,18 @@ bench = [

[tool.setuptools.package-data]
numbin = ["py.typed"]

[tool.ruff]
target-version = "py38"
[tool.ruff.lint]
select = ["E", "F", "G", "B", "I", "SIM", "TID", "PL", "RUF", "D"]
ignore = ["E501", "D203", "D213"]
[tool.ruff.lint.isort]
known-first-party = ["numbin"]
[tool.ruff.lint.pylint]
max-args = 10
[tool.ruff.lint.per-file-ignores]
"test/*" = ["D"]
"benchmark/*" = ["D"]
[tool.ruff.lint.pydocstyle]
convention = "google"

0 comments on commit dab6d2f

Please sign in to comment.