added Hyperscan 5.2.0 support and literal API (#16)

.semaphore/semaphore.yml

@@ -9,7 +9,7 @@ blocks:
     task:
       env_vars:
         - name: MANYLINUX_DOCKER_IMAGE
-          value: darvid/manylinux-hyperscan:latest
+          value: docker.pkg.github.com/darvid/manylinux-hyperscan/manylinux-hyperscan:v5.2.1
       prologue:
         commands:
           - checkout

build.py

@@ -1,54 +1,55 @@
-import os
-import subprocess
-import sys
-
-from distutils.core import Extension
-from distutils.errors import (
-    CCompilerError,
-    DistutilsExecError,
-    DistutilsPlatformError,
-)
-from distutils.command.build_ext import build_ext
-
-
-# http://code.activestate.com/recipes/502261-python-distutils-pkg-config/
-def pkgconfig(libs, optional=''):
-    flag_map = {
-        'include_dirs': ('--cflags-only-I', 2),
-        'library_dirs': ('--libs-only-L', 2),
-        'libraries': ('--libs-only-l', 2),
-        'extra_compile_args': ('--cflags-only-other', 0),
-        'extra_link_args': ('--libs-only-other', 0),
-    }
-    ext_kwargs = {}
-    for lib in libs:
-        for distutils_kwarg, (pkg_option, trim_offset) in flag_map.items():
-            try:
-                options = (
-                    subprocess.check_output(
-                        ['pkg-config', optional, pkg_option, lib]
-                    )
-                    .decode()
-                    .split()
-                )
-            except subprocess.CalledProcessError:
-                continue
-            ext_kwargs.setdefault(distutils_kwarg, []).extend(
-                [opt[trim_offset:] for opt in options]
-            )
-    return ext_kwargs
-
-
-def build(setup_kwargs):
-    setup_kwargs.update(
-        {
-            'ext_modules': [
-                Extension(
-                    'hyperscan._hyperscan',
-                    ['hyperscan/hyperscanmodule.c'],
-                    **pkgconfig(['libhs'])
-                )
-            ],
-            'cmdclass': {'build_ext': build_ext},
-        }
-    )
+import os
+import subprocess
+import sys
+
+from distutils.core import Extension
+from distutils.errors import (
+    CCompilerError,
+    DistutilsExecError,
+    DistutilsPlatformError,
+)
+from distutils.command.build_ext import build_ext
+
+
+# http://code.activestate.com/recipes/502261-python-distutils-pkg-config/
+def pkgconfig(libs, optional=''):
+    flag_map = {
+        'include_dirs': ('--cflags-only-I', 2),
+        'library_dirs': ('--libs-only-L', 2),
+        'libraries': ('--libs-only-l', 2),
+        'extra_compile_args': ('--cflags-only-other -O0', 0),
+        'extra_link_args': ('--libs-only-other', 0),
+    }
+    ext_kwargs = {}
+    for lib in libs:
+        for distutils_kwarg, (pkg_option, trim_offset) in flag_map.items():
+            try:
+                options = (
+                    subprocess.check_output(
+                        ['pkg-config', optional, pkg_option, lib]
+                    )
+                    .decode()
+                    .split()
+                )
+            except subprocess.CalledProcessError:
+                continue
+            ext_kwargs.setdefault(distutils_kwarg, []).extend(
+                [opt[trim_offset:] for opt in options]
+            )
+    print(ext_kwargs)
+    return ext_kwargs
+
+
+def build(setup_kwargs):
+    setup_kwargs.update(
+        {
+            'ext_modules': [
+                Extension(
+                    'hyperscan._hyperscan',
+                    ['hyperscan/hyperscanmodule.c'],
+                    **pkgconfig(['libhs']),
+                )
+            ],
+            'cmdclass': {'build_ext': build_ext},
+        }
+    )

hyperscan/hyperscanmodule.c

@@ -1,3 +1,4 @@
+#define PY_SSIZE_T_CLEAN
 #include <Python.h>
 #include <bytesobject.h>
 #include <structmember.h>
@@ -122,14 +123,14 @@ static PyObject* Database_compile(Database *self, PyObject *args,
            *oflags       = Py_None,
            *oflag        = Py_None,
            *oids         = Py_None,
-           *ostrict      = Py_False;
+           *oliteral     = Py_False;
   int elements = -1;
 
   static char *kwlist[] = {"expressions", "ids", "elements", "flags",
-                           "strict", NULL};
+                           "literal", NULL};
   if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|OIOO", kwlist,
                                    &oexpressions, &oids, &elements,
-                                   &oflags, &ostrict))
+                                   &oflags, &oliteral))
     return NULL;
 
   if (elements == -1) {
@@ -150,8 +151,7 @@ static PyObject* Database_compile(Database *self, PyObject *args,
 
   PyErr_Clear();
 
-  int i;
-  for (i = 0; i < elements; i++) {
+  for (int i = 0; i < elements; i++) {
     oexpr = PySequence_ITEM(oexpressions, i);
     expressions[i] = PyBytes_AsString(oexpr);
     if (PyErr_Occurred())
@@ -189,8 +189,17 @@ static PyObject* Database_compile(Database *self, PyObject *args,
   hs_compile_error_t *compile_err;
 
   Py_BEGIN_ALLOW_THREADS
-  err = hs_compile_ext_multi(expressions, flags, ids, NULL, elements,
-                             self->mode, NULL, &self->db, &compile_err);
+  if (PyObject_IsTrue(oliteral)) {
+    size_t lens[elements];
+    for (int i = 0; i < elements; i++) {
+      lens[i] = PySequence_Length(PySequence_ITEM(oexpressions, i)) - 1;
+    }
+    err = hs_compile_lit_multi(expressions, flags, ids, lens, elements,
+                               self->mode, NULL, &self->db, &compile_err);
+  } else {
+    err = hs_compile_ext_multi(expressions, flags, ids, NULL, elements,
+                               self->mode, NULL, &self->db, &compile_err);
+  }
   Py_END_ALLOW_THREADS
 
   if (err != HS_SUCCESS) {
@@ -289,7 +298,7 @@ static PyMemberDef Database_members[] = {
 
 static PyMethodDef Database_methods[] = {
   {"compile", (PyCFunction)Database_compile, METH_VARARGS|METH_KEYWORDS,
-   "compile(expressions, ids=None, elements=None, flags=0)\n\n"
+   "compile(expressions, ids=None, elements=None, flags=0, literal=False)\n\n"
    "    Compiles regular expressions.\n\n"
    "    Args:\n"
    "        expressions (sequence of :obj:`str`): A sequence of regular\n"
@@ -300,7 +309,9 @@ static PyMethodDef Database_methods[] = {
    "            sequence.\n"
    "        flags (sequence of :obj:`int` or :obj:`int`, optional): Sequence\n"
    "            of flags associated with each expression, or a single value\n"
-   "            which is applied to all expressions.\n\n"},
+   "            which is applied to all expressions.\n"
+   "        literal (bool, optional): If True, uses the pure literal\n"
+   "            expression compiler introduced in Hyperscan 5.2.0.\n\n"},
   {"info", (PyCFunction)Database_info, METH_VARARGS,
    "info()\n\n"
    "    Returns database information.\n\n"

pyproject.toml

@@ -14,7 +14,7 @@ skip-string-normalization = true
 [tool.poetry]
 build = "build.py"
 name = "hyperscan"
-version = "0.1.3"
+version = "0.1.4"
 description = "Python bindings for Hyperscan."
 authors = ["David Gidwani <david.gidwani@gmail.com>"]
 license = "MIT"

tests/test_hyperscan.py

@@ -94,8 +94,23 @@ def test_database_deserialize(database_stream):
     db = hyperscan.loads(bytearray(serialized))
     assert id(db) != id(database_stream)
 
+
 def test_database_exception_in_callback(database_block, mocker):
     callback = mocker.Mock(side_effect=RuntimeError('oops'))
 
     with pytest.raises(RuntimeError, match=r'^oops$'):
         database_block.scan(b'foobar', match_event_handler=callback)
+
+
+def test_literal_expressions(mocker):
+    db = hyperscan.Database()
+    expressions, ids, _ = zip(*patterns)
+    expressions = [e + b'\0' for e in expressions]
+    db.compile(expressions=expressions, ids=ids, literal=True)
+    callback = mocker.Mock(return_value=None)
+    expected = []
+    for i, expression in enumerate(expressions):
+        expression = expression[:-1]
+        db.scan(expression, match_event_handler=callback, context=expression)
+        expected.append(mocker.call(ids[i], 0, len(expression), 0, expression))
+    assert callback.mock_calls == expected