C-Extension written from scratch

- Typically with 7-10x speed improvement (over old Cython generated one)

CHANGELOG

@@ -1,3 +1,8 @@
+0.9.0
+- C extension re-implemented (without Cython) with major speedup (7-10x)
+- object_pairs_hook now works like built-in json module
+- Minor pure version improvements
+
 0.8.5
 - Added Python 3.5 to classifiers list
 - Fix index in argv (command line utility)

NOTICE

@@ -4,6 +4,9 @@ Copyright 2016 Iotic Labs Ltd
 Universal Binary JSON Specification
 Copyright 2015 ubjson.org
 
+Floating point related functions (python_funcs.c) and integer encode/decode logic
+Copyright (c) 2001-2016 Python Software Foundation
+
 six.py
 Copyright (c) 2010-2015 Benjamin Peterson
 

coverage_test.sh

@@ -2,6 +2,6 @@
 
 # coverage currently only measured without extension, so clean out first
 git clean -fXd ubjson
-python3 -mcoverage run -m unittest discover test/ -v
+python3 -mcoverage run --omit=ubjson/compat.py -m unittest discover test/ -v
 python3 -mcoverage html
 echo -e "\nFor coverage results see htmlcov/index.html\n"

pylint.rc

@@ -28,7 +28,7 @@ unsafe-load-any-extension=no
 # A comma-separated list of package or module names from where C extensions may
 # be loaded. Extensions are loading into the active Python interpreter and may
 # run arbitrary code
-extension-pkg-whitelist=ubjson.compat,ubjson.encoder,ubjson.decoder,ubjson.markers
+extension-pkg-whitelist=_ubjson
 
 # Allow optimization of some AST trees. This will activate a peephole AST
 # optimizer, which will apply various small optimizations. For instance, it can

setup.py

@@ -12,12 +12,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# pylint: disable=import-error,wrong-import-order
+# pylint: disable=import-error,wrong-import-order,ungrouped-imports
+
+from __future__ import print_function
 
 import sys
 import os
 import warnings
-from glob import iglob
+from glob import glob
 
 # Allow for environments without setuptools
 try:
@@ -39,10 +41,10 @@
 try:
     from pypandoc import convert
 except ImportError:
-    READ_MD = lambda f: open(f, 'r').read()
+    READ_MD = lambda f: open(f, 'r').read()  # noqa: E731
     print('Warning: pypandoc module not found, will not convert Markdown to RST')
 else:
-    READ_MD = lambda f: convert(f, 'rst')
+    READ_MD = lambda f: convert(f, 'rst')  # noqa: E731
 
 
 # Loosely based on https://github.com/mongodb/mongo-python-driver/blob/master/setup.py
@@ -67,9 +69,13 @@ def build_extension(self, ext):
                           % ext.name)
 
 
-EXTENSION = '.py3.c' if sys.version_info[0] >= 3 else '.py2.c'
 BUILD_EXTENSIONS = 'PYUBJSON_NO_EXTENSION' not in os.environ
 
+COMPILE_ARGS = ['-std=c99']
+# For testing/debug only - some of these are GCC-specific
+COMPILE_ARGS += ['-Wall', '-Wextra', '-Wundef', '-Wshadow', '-Wcast-align', '-Wcast-qual', '-Wstrict-prototypes',
+                 '-pedantic']
+
 setup(
     name='py-ubjson',
     version=version,
@@ -83,7 +89,7 @@ def build_extension(self, ext):
     license='Apache License 2.0',
     packages=['ubjson'],
     zip_safe=False,
-    ext_modules=([Extension(name[:-len(EXTENSION)], [name]) for name in iglob('ubjson/*' + EXTENSION)]
+    ext_modules=([Extension('_ubjson', glob('src/*.c'), extra_compile_args=COMPILE_ARGS)]
                  if BUILD_EXTENSIONS else []),
     cmdclass={"build_ext": BuildExtWarnOnFail},
     keywords=['ubjson', 'ubj'],

src/_ubjson.c

@@ -0,0 +1,229 @@
+/*
+ * Copyright (c) 2017 Iotic Labs Ltd. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://github.com/Iotic-Labs/py-ubjson/blob/master/LICENSE
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <Python.h>
+
+#include "common.h"
+#include "encoder.h"
+#include "decoder.h"
+
+/******************************************************************************/
+
+static _ubjson_encoder_prefs_t _ubjson_encoder_prefs_defaults = {
+    .container_count = 0,
+    .sort_keys = 0,
+    .no_float32 = 1
+};
+
+static _ubjson_decoder_prefs_t _ubjson_decoder_prefs_defaults = {
+    .no_bytes = 0,
+    .object_pairs_hook = NULL
+};
+
+/******************************************************************************/
+
+PyDoc_STRVAR(_ubjson_dump__doc__, "See pure Python version (encoder.dump) for documentation.");
+#define FUNC_DEF_DUMP {"dump", (PyCFunction)_ubjson_dump, METH_VARARGS | METH_KEYWORDS, _ubjson_dump__doc__}
+static PyObject*
+_ubjson_dump(PyObject *self, PyObject *args, PyObject *kwargs) {
+    static const char *format = "OO|iii:dump";
+    static char *keywords[] = {"obj", "fp", "container_count", "sort_keys", "no_float32", NULL};
+
+    _ubjson_encoder_buffer_t *buffer = NULL;
+    _ubjson_encoder_prefs_t prefs = _ubjson_encoder_prefs_defaults;
+    PyObject *obj;
+    PyObject *fp;
+    PyObject *fp_write = NULL;
+    UNUSED(self);
+
+    if (!PyArg_ParseTupleAndKeywords(args, kwargs, format, keywords, &obj, &fp, &prefs.container_count,
+                                     &prefs.sort_keys, &prefs.no_float32)) {
+        goto bail;
+    }
+    BAIL_ON_NULL(fp_write = PyObject_GetAttrString(fp, "write"));
+    BAIL_ON_NULL(buffer = _ubjson_encoder_buffer_create(&prefs, fp_write));
+    // buffer creation has added reference
+    Py_CLEAR(fp_write);
+
+    BAIL_ON_NONZERO(_ubjson_encode_value(obj, buffer));
+    BAIL_ON_NULL(obj = _ubjson_encoder_buffer_finalise(buffer));
+    Py_DECREF(obj);
+    _ubjson_encoder_buffer_free(buffer);
+    Py_RETURN_NONE;
+
+bail:
+    Py_XDECREF(fp_write);
+    _ubjson_encoder_buffer_free(buffer);
+    return NULL;
+}
+
+PyDoc_STRVAR(_ubjson_dumpb__doc__, "See pure Python version (encoder.dumpb) for documentation.");
+#define FUNC_DEF_DUMPB {"dumpb", (PyCFunction)_ubjson_dumpb, METH_VARARGS | METH_KEYWORDS, _ubjson_dumpb__doc__}
+static PyObject*
+_ubjson_dumpb(PyObject *self, PyObject *args, PyObject *kwargs) {
+    static const char *format = "O|iii:dumpb";
+    static char *keywords[] = {"obj", "container_count", "sort_keys", "no_float32", NULL};
+
+    _ubjson_encoder_buffer_t *buffer = NULL;
+    _ubjson_encoder_prefs_t prefs = _ubjson_encoder_prefs_defaults;
+    PyObject *obj;
+    UNUSED(self);
+
+    if (!PyArg_ParseTupleAndKeywords(args, kwargs, format, keywords, &obj, &prefs.container_count, &prefs.sort_keys,
+                                     &prefs.no_float32)) {
+        goto bail;
+    }
+
+    BAIL_ON_NULL(buffer = _ubjson_encoder_buffer_create(&prefs, NULL));
+    BAIL_ON_NONZERO(_ubjson_encode_value(obj, buffer));
+    BAIL_ON_NULL(obj = _ubjson_encoder_buffer_finalise(buffer));
+    _ubjson_encoder_buffer_free(buffer);
+    return obj;
+
+bail:
+    _ubjson_encoder_buffer_free(buffer);
+    return NULL;
+}
+
+/******************************************************************************/
+
+PyDoc_STRVAR(_ubjson_load__doc__, "See pure Python version (encoder.load) for documentation.");
+#define FUNC_DEF_LOAD {"load", (PyCFunction)_ubjson_load, METH_VARARGS | METH_KEYWORDS, _ubjson_load__doc__}
+static PyObject*
+_ubjson_load(PyObject *self, PyObject *args, PyObject *kwargs) {
+    static const char *format = "O|iO:load";
+    static char *keywords[] = {"fp", "no_bytes", "object_pairs_hook", NULL};
+
+    _ubjson_decoder_buffer_t *buffer = NULL;
+    _ubjson_decoder_prefs_t prefs = _ubjson_decoder_prefs_defaults;
+    PyObject *fp;
+    PyObject *fp_read = NULL;
+    PyObject *obj = NULL;
+    UNUSED(self);
+
+    if (!PyArg_ParseTupleAndKeywords(args, kwargs, format, keywords, &fp, &prefs.no_bytes, &prefs.object_pairs_hook)) {
+        goto bail;
+    }
+    BAIL_ON_NULL(fp_read = PyObject_GetAttrString(fp, "read"));
+    if (!PyCallable_Check(fp_read)) {
+        PyErr_SetString(PyExc_TypeError, "fp.read not callable");
+        goto bail;
+    }
+
+    BAIL_ON_NULL(buffer = _ubjson_decoder_buffer_create(&prefs, fp_read));
+    // buffer creation has added reference
+    Py_CLEAR(fp_read);
+
+    BAIL_ON_NULL(obj = _ubjson_decode_value(buffer, NULL));
+    _ubjson_decoder_buffer_free(buffer);
+    return obj;
+
+bail:
+    Py_XDECREF(fp_read);
+    _ubjson_decoder_buffer_free(buffer);
+    return NULL;
+}
+
+PyDoc_STRVAR(_ubjson_loadb__doc__, "See pure Python version (encoder.loadb) for documentation.");
+#define FUNC_DEF_LOADB {"loadb", (PyCFunction)_ubjson_loadb, METH_VARARGS | METH_KEYWORDS, _ubjson_loadb__doc__}
+static PyObject*
+_ubjson_loadb(PyObject *self, PyObject *args, PyObject *kwargs) {
+    static const char *format = "O|iO:loadb";
+    static char *keywords[] = {"chars", "no_bytes", "object_pairs_hook", NULL};
+
+    _ubjson_decoder_buffer_t *buffer = NULL;
+    _ubjson_decoder_prefs_t prefs = _ubjson_decoder_prefs_defaults;
+    PyObject *chars;
+    PyObject *obj = NULL;
+    UNUSED(self);
+
+    if (!PyArg_ParseTupleAndKeywords(args, kwargs, format, keywords, &chars, &prefs.no_bytes,
+                                     &prefs.object_pairs_hook)) {
+        goto bail;
+    }
+    if (PyUnicode_Check(chars)) {
+        PyErr_SetString(PyExc_TypeError, "chars must be a bytes-like object, not str");
+        goto bail;
+    }
+    if (!PyObject_CheckBuffer(chars)) {
+        PyErr_SetString(PyExc_TypeError, "chars does not support buffer interface");
+        goto bail;
+    }
+
+    BAIL_ON_NULL(buffer = _ubjson_decoder_buffer_create(&prefs, chars));
+
+    BAIL_ON_NULL(obj = _ubjson_decode_value(buffer, NULL));
+    _ubjson_decoder_buffer_free(buffer);
+    return obj;
+
+bail:
+    _ubjson_decoder_buffer_free(buffer);
+    return NULL;
+}
+
+/******************************************************************************/
+
+static PyMethodDef UbjsonMethods[] = {
+    FUNC_DEF_DUMP, FUNC_DEF_DUMPB,
+    FUNC_DEF_LOAD, FUNC_DEF_LOADB,
+    {NULL, NULL, 0, NULL}
+};
+
+#if PY_MAJOR_VERSION >= 3
+static struct PyModuleDef moduledef = {
+        PyModuleDef_HEAD_INIT,
+        "_ubjson",
+        NULL,
+        -1,
+        UbjsonMethods,
+        NULL,
+        NULL,
+        NULL,
+        NULL
+};
+
+#define INITERROR return NULL
+PyObject*
+PyInit__ubjson(void)
+
+#else
+#define INITERROR return
+
+void
+init_ubjson(void)
+#endif
+{
+#if PY_MAJOR_VERSION >= 3
+    PyObject *module = PyModule_Create(&moduledef);
+#else
+    PyObject *module = Py_InitModule("_ubjson", UbjsonMethods);
+#endif
+
+    BAIL_ON_NONZERO(_ubjson_encoder_init());
+    BAIL_ON_NONZERO(_ubjson_decoder_init());
+
+#if PY_MAJOR_VERSION >= 3
+    return module;
+#else
+    return;
+#endif
+
+bail:
+    _ubjson_encoder_cleanup();
+    _ubjson_decoder_cleanup();
+    Py_XDECREF(module);
+    INITERROR;
+}

src/common.h

@@ -0,0 +1,47 @@
+/*
+ * Copyright (c) 2017 Iotic Labs Ltd. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://github.com/Iotic-Labs/py-ubjson/blob/master/LICENSE
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+/******************************************************************************/
+
+#define MIN(x, y) (x) <= (y) ? (x) : (y)
+#define MAX(x, y) (x) >= (y) ? (x) : (y)
+
+#define UNUSED(x) (void)(x)
+
+#define BAIL_ON_NULL(result)\
+if (NULL == (result)) {\
+    goto bail;\
+}
+
+#define BAIL_ON_NONZERO(result)\
+if (result) {\
+    goto bail;\
+}
+
+#define BAIL_ON_NEGATIVE(result)\
+if ((result) < 0) {\
+    goto bail;\
+}
+
+#if defined (__cplusplus)
+}
+#endif