prelim gil impl for Int64Hashtable

Author: jreback

pandas/hashtable.pyx

@@ -49,18 +49,11 @@ def list_to_object_array(list obj):
 
 cdef size_t _INIT_VEC_CAP = 32
 
-cdef class ObjectVector:
+cdef class Vector:
 
     cdef:
         size_t n, m
         ndarray ao
-        PyObject **data
-
-    def __cinit__(self):
-        self.n = 0
-        self.m = _INIT_VEC_CAP
-        self.ao = np.empty(_INIT_VEC_CAP, dtype=object)
-        self.data = <PyObject**> self.ao.data
 
     def __len__(self):
         return self.n
@@ -70,6 +63,18 @@ cdef class ObjectVector:
         self.m = self.n
         return self.ao
 
+
+cdef class ObjectVector(Vector):
+
+    cdef:
+        PyObject **data
+
+    def __cinit__(self):
+        self.n = 0
+        self.m = _INIT_VEC_CAP
+        self.ao = np.empty(_INIT_VEC_CAP, dtype=object)
+        self.data = <PyObject**> self.ao.data
+
     cdef inline append(self, object o):
         if self.n == self.m:
             self.m = max(self.m * 2, _INIT_VEC_CAP)
@@ -81,11 +86,9 @@ cdef class ObjectVector:
         self.n += 1
 
 
-cdef class Int64Vector:
+cdef class Int64Vector(Vector):
 
     cdef:
-        size_t n, m
-        ndarray ao
         int64_t *data
 
     def __cinit__(self):
@@ -94,28 +97,29 @@ cdef class Int64Vector:
         self.ao = np.empty(_INIT_VEC_CAP, dtype=np.int64)
         self.data = <int64_t*> self.ao.data
 
-    def __len__(self):
-        return self.n
+    cdef inline uint8_t needs_resize(self) nogil:
+        # if we need to resize
+        return self.n == self.m
 
-    def to_array(self):
-        self.ao.resize(self.n)
-        self.m = self.n
-        return self.ao
+    cdef resize(self):
+        self.m = max(self.m * 2, _INIT_VEC_CAP)
+        self.ao.resize(self.m)
+        self.data = <int64_t*> self.ao.data
 
-    cdef inline append(self, int64_t x):
-        if self.n == self.m:
-            self.m = max(self.m * 2, _INIT_VEC_CAP)
-            self.ao.resize(self.m)
-            self.data = <int64_t*> self.ao.data
+    cdef inline void append(self, int64_t x) nogil:
 
-        self.data[self.n] = x
-        self.n += 1
+        with nogil:
+
+            if self.needs_resize():
+                with gil:
+                    self.resize()
 
-cdef class Float64Vector:
+            self.data[self.n] = x
+            self.n += 1
+
+cdef class Float64Vector(Vector):
 
     cdef:
-        size_t n, m
-        ndarray ao
         float64_t *data
 
     def __cinit__(self):
@@ -124,14 +128,6 @@ cdef class Float64Vector:
         self.ao = np.empty(_INIT_VEC_CAP, dtype=np.float64)
         self.data = <float64_t*> self.ao.data
 
-    def __len__(self):
-        return self.n
-
-    def to_array(self):
-        self.ao.resize(self.n)
-        self.m = self.n
-        return self.ao
-
     cdef inline append(self, float64_t x):
         if self.n == self.m:
             self.m = max(self.m * 2, _INIT_VEC_CAP)
@@ -142,18 +138,17 @@ cdef class Float64Vector:
         self.n += 1
 
 
-cdef class HashTable:
-    pass
-
-
-cdef class StringHashTable(HashTable):
+cdef class StringHashTable:
     cdef kh_str_t *table
 
     def __cinit__(self, int size_hint=1):
         self.table = kh_init_str()
         if size_hint is not None:
             kh_resize_str(self.table, size_hint)
 
+    def __len__(self):
+        return self.table.size
+
     def __dealloc__(self):
         kh_destroy_str(self.table)
 
@@ -256,7 +251,7 @@ cdef class StringHashTable(HashTable):
 
         return reverse, labels
 
-cdef class Int32HashTable(HashTable):
+cdef class Int32HashTable:
     cdef kh_int32_t *table
 
     def __init__(self, size_hint=1):
@@ -266,6 +261,9 @@ cdef class Int32HashTable(HashTable):
     def __cinit__(self):
         self.table = kh_init_int32()
 
+    def __len__(self):
+        return self.table.size
+
     def __dealloc__(self):
         kh_destroy_int32(self.table)
 
@@ -353,14 +351,16 @@ cdef class Int32HashTable(HashTable):
 
         return reverse, labels
 
-cdef class Int64HashTable: #(HashTable):
-    # cdef kh_int64_t *table
+cdef class Int64HashTable:
 
     def __cinit__(self, size_hint=1):
         self.table = kh_init_int64()
         if size_hint is not None:
             kh_resize_int64(self.table, size_hint)
 
+    def __len__(self):
+        return self.table.size
+
     def __dealloc__(self):
         kh_destroy_int64(self.table)
 
@@ -369,9 +369,6 @@ cdef class Int64HashTable: #(HashTable):
         k = kh_get_int64(self.table, key)
         return k != self.table.n_buckets
 
-    def __len__(self):
-        return self.table.size
-
     cpdef get_item(self, int64_t val):
         cdef khiter_t k
         k = kh_get_int64(self.table, val)
@@ -446,6 +443,7 @@ cdef class Int64HashTable: #(HashTable):
         labels = self.get_labels(values, reverse, 0)
         return reverse, labels
 
+    @cython.boundscheck(False)
     def get_labels(self, ndarray[int64_t] values, Int64Vector uniques,
                    Py_ssize_t count_prior, Py_ssize_t na_sentinel):
         cdef:
@@ -458,21 +456,23 @@ cdef class Int64HashTable: #(HashTable):
 
         labels = np.empty(n, dtype=np.int64)
 
-        for i in range(n):
-            val = values[i]
-            k = kh_get_int64(self.table, val)
-            if k != self.table.n_buckets:
-                idx = self.table.vals[k]
-                labels[i] = idx
-            else:
-                k = kh_put_int64(self.table, val, &ret)
-                self.table.vals[k] = count
-                uniques.append(val)
-                labels[i] = count
-                count += 1
+        with nogil:
+            for i in range(n):
+                val = values[i]
+                k = kh_get_int64(self.table, val)
+                if k != self.table.n_buckets:
+                    idx = self.table.vals[k]
+                    labels[i] = idx
+                else:
+                    k = kh_put_int64(self.table, val, &ret)
+                    self.table.vals[k] = count
+                    uniques.append(val)
+                    labels[i] = count
+                    count += 1
 
         return labels
 
+    @cython.boundscheck(False)
     def get_labels_groupby(self, ndarray[int64_t] values):
         cdef:
             Py_ssize_t i, n = len(values)
@@ -485,24 +485,25 @@ cdef class Int64HashTable: #(HashTable):
 
         labels = np.empty(n, dtype=np.int64)
 
-        for i in range(n):
-            val = values[i]
-
-            # specific for groupby
-            if val < 0:
-                labels[i] = -1
-                continue
-
-            k = kh_get_int64(self.table, val)
-            if k != self.table.n_buckets:
-                idx = self.table.vals[k]
-                labels[i] = idx
-            else:
-                k = kh_put_int64(self.table, val, &ret)
-                self.table.vals[k] = count
-                uniques.append(val)
-                labels[i] = count
-                count += 1
+        with nogil:
+            for i in range(n):
+                val = values[i]
+
+                # specific for groupby
+                if val < 0:
+                    labels[i] = -1
+                    continue
+
+                k = kh_get_int64(self.table, val)
+                if k != self.table.n_buckets:
+                    idx = self.table.vals[k]
+                    labels[i] = idx
+                else:
+                    k = kh_put_int64(self.table, val, &ret)
+                    self.table.vals[k] = count
+                    uniques.append(val)
+                    labels[i] = count
+                    count += 1
 
         arr_uniques = uniques.to_array()
 
@@ -530,6 +531,7 @@ cdef class Int64HashTable: #(HashTable):
 
 
 cdef class Float64HashTable(HashTable):
+
     def __cinit__(self, size_hint=1):
         self.table = kh_init_float64()
         if size_hint is not None:
@@ -658,7 +660,6 @@ cdef class Float64HashTable(HashTable):
 na_sentinel = object
 
 cdef class PyObjectHashTable(HashTable):
-    # cdef kh_pymap_t *table
 
     def __init__(self, size_hint=1):
         self.table = kh_init_pymap()

pandas/src/khash.pxd

@@ -62,15 +62,15 @@ cdef extern from "khash_python.h":
         int64_t *keys
         size_t *vals
 
-    inline kh_int64_t* kh_init_int64()
-    inline void kh_destroy_int64(kh_int64_t*)
-    inline void kh_clear_int64(kh_int64_t*)
-    inline khint_t kh_get_int64(kh_int64_t*, int64_t)
-    inline void kh_resize_int64(kh_int64_t*, khint_t)
-    inline khint_t kh_put_int64(kh_int64_t*, int64_t, int*)
-    inline void kh_del_int64(kh_int64_t*, khint_t)
+    inline kh_int64_t* kh_init_int64() nogil
+    inline void kh_destroy_int64(kh_int64_t*) nogil
+    inline void kh_clear_int64(kh_int64_t*) nogil
+    inline khint_t kh_get_int64(kh_int64_t*, int64_t) nogil
+    inline void kh_resize_int64(kh_int64_t*, khint_t) nogil
+    inline khint_t kh_put_int64(kh_int64_t*, int64_t, int*) nogil
+    inline void kh_del_int64(kh_int64_t*, khint_t) nogil
 
-    bint kh_exist_int64(kh_int64_t*, khiter_t)
+    bint kh_exist_int64(kh_int64_t*, khiter_t) nogil
 
     ctypedef struct kh_float64_t:
         khint_t n_buckets, size, n_occupied, upper_bound
@@ -121,4 +121,3 @@ cdef extern from "khash_python.h":
     inline void kh_del_strbox(kh_strbox_t*, khint_t)
 
     bint kh_exist_strbox(kh_strbox_t*, khiter_t)
-