Merge branch develop into master

CHANGELOG.md

@@ -2,6 +2,17 @@
 
 This is a list of notable changes to Hyperscan, in reverse chronological order.
 
+## [4.6.0] 2017-09-22
+- New API feature: stream state compression. This allows the user to compress
+  and restore state for streams to reduce memory usage.
+- Many improvements to literal matching performance, including more support
+  for Intel(R) Advanced Vector Extensions 512 (Intel(R) AVX-512).
+- Compile time improvements, mainly reducing compiler memory allocation.
+  Also results in reduced compile time for some pattern sets.
+- Bugfix for issue #62: fix error building Hyperscan using older versions of
+  Boost.
+- Small updates to fix warnings identified by Coverity.
+
 ## [4.5.2] 2017-07-26
 - Bugfix for issue #57: Treat characters between `\Q.\E` as codepoints in
   UTF8 mode.

CMakeLists.txt

@@ -2,8 +2,8 @@ cmake_minimum_required (VERSION 2.8.11)
 project (hyperscan C CXX)
 
 set (HS_MAJOR_VERSION 4)
-set (HS_MINOR_VERSION 5)
-set (HS_PATCH_VERSION 2)
+set (HS_MINOR_VERSION 6)
+set (HS_PATCH_VERSION 0)
 set (HS_VERSION ${HS_MAJOR_VERSION}.${HS_MINOR_VERSION}.${HS_PATCH_VERSION})
 
 set(CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake)
@@ -38,6 +38,7 @@ endif()
 
 set(BINDIR "${PROJECT_BINARY_DIR}/bin")
 set(LIBDIR "${PROJECT_BINARY_DIR}/lib")
+
 set(INCLUDE_INSTALL_DIR ${CMAKE_INSTALL_INCLUDEDIR})
 
 # First for the generic no-config case
@@ -57,6 +58,11 @@ if(CMAKE_GENERATOR STREQUAL Xcode)
     set(XCODE TRUE)
 endif()
 
+# older versions of cmake don't know things support isystem
+if (XCODE OR CMAKE_CXX_COMPILER_ID MATCHES "Intel")
+    set(CMAKE_INCLUDE_SYSTEM_FLAG_CXX "-isystem")
+endif ()
+
 set(CMAKE_INCLUDE_CURRENT_DIR 1)
 include_directories(${PROJECT_SOURCE_DIR}/src)
 include_directories(${PROJECT_BINARY_DIR})
@@ -148,8 +154,9 @@ if(MSVC OR MSVC_IDE)
             # todo: change these as required
             set(ARCH_C_FLAGS "/arch:AVX2")
             set(ARCH_CXX_FLAGS "/arch:AVX2")
-            set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS}  /O2 /wd4244 /wd4267")
-            set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /O2 /wd4244 /wd4267 /wd4800 -DBOOST_DETAIL_NO_CONTAINER_FWD -D_SCL_SECURE_NO_WARNINGS")
+            set(MSVC_WARNS "/wd4101 /wd4146 /wd4172 /wd4200 /wd4244 /wd4267 /wd4307 /wd4334 /wd4805 -D_CRT_SECURE_NO_WARNINGS")
+            set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS}  /O2 ${MSVC_WARNS}")
+            set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /O2 ${MSVC_WARNS} /wd4800 -DBOOST_DETAIL_NO_CONTAINER_FWD")
         endif()
         string(REPLACE "/RTC1" "" CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG}")
         string(REPLACE "/RTC1" "" CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG}")
@@ -248,19 +255,20 @@ else()
     endif()
 
     if(CMAKE_COMPILER_IS_GNUCXX)
-        set(EXTRA_CXX_FLAGS "${EXTRA_CXX_FLAGS} -fabi-version=0 -Wno-unused-local-typedefs -Wno-maybe-uninitialized")
+        set(EXTRA_CXX_FLAGS "${EXTRA_CXX_FLAGS} -Wno-maybe-uninitialized")
+        if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 5.0)
+            set(EXTRA_CXX_FLAGS "${EXTRA_CXX_FLAGS} -fabi-version=0")
+        endif ()
+        # don't complain about abi
+        set(EXTRA_C_FLAGS "${EXTRA_C_FLAGS} -Wno-abi")
+        set(EXTRA_CXX_FLAGS "${EXTRA_CXX_FLAGS} -Wno-abi")
     endif()
 
     if (NOT(ARCH_IA32 AND RELEASE_BUILD))
         set(EXTRA_C_FLAGS "${EXTRA_C_FLAGS} -fno-omit-frame-pointer")
         set(EXTRA_CXX_FLAGS "${EXTRA_CXX_FLAGS} -fno-omit-frame-pointer")
     endif()
 
-    if (RELEASE_BUILD)
-        # we don't need the noise of ABI warnings in a release build
-        set(EXTRA_C_FLAGS "${EXTRA_C_FLAGS} -Wno-abi")
-        set(EXTRA_CXX_FLAGS "${EXTRA_CXX_FLAGS} -Wno-abi")
-    endif ()
 
     if (CMAKE_C_COMPILER_ID MATCHES "Intel")
         set(SKYLAKE_FLAG "-xCORE-AVX512")
@@ -396,18 +404,14 @@ if (CXX_MISSING_DECLARATIONS)
     set(EXTRA_CXX_FLAGS "${EXTRA_CXX_FLAGS} -Wmissing-declarations")
 endif()
 
+CHECK_CXX_COMPILER_FLAG("-Wunused-local-typedefs" CXX_UNUSED_LOCAL_TYPEDEFS)
+
 # gcc5 complains about this
 CHECK_CXX_COMPILER_FLAG("-Wunused-variable" CXX_WUNUSED_VARIABLE)
 
 endif()
 
-if (NOT XCODE)
-    include_directories(SYSTEM ${Boost_INCLUDE_DIRS})
-else()
-    # cmake doesn't think Xcode supports isystem
-    set(EXTRA_CXX_FLAGS "${EXTRA_CXX_FLAGS} -isystem ${Boost_INCLUDE_DIRS}")
-endif()
-
+include_directories(SYSTEM ${Boost_INCLUDE_DIRS})
 
 if(CMAKE_SYSTEM_NAME MATCHES "Linux")
     set(LINUX TRUE)
@@ -419,10 +423,10 @@ endif(CMAKE_SYSTEM_NAME MATCHES "FreeBSD")
 
 if(NOT WIN32)
 if(CMAKE_C_COMPILER_ID MATCHES "Intel")
-    SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -diag-error 10006 -diag-disable 177 -diag-disable 2304 -diag-disable 2305 -diag-disable 2338 -diag-disable 1418 -diag-disable=remark")
+    SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -diag-error 10006 -diag-disable 68 -diag-disable 177 -diag-disable 186 -diag-disable 2304 -diag-disable 2305 -diag-disable 2338 -diag-disable 1418 -diag-disable=remark")
 endif()
 if(CMAKE_CXX_COMPILER_ID MATCHES "Intel")
-    SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -diag-error 10006 -diag-disable 177 -diag-disable 2304 -diag-disable 2305 -diag-disable 2338 -diag-disable 1418 -diag-disable 1170 -diag-disable 3373 -diag-disable=remark")
+    SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -diag-error 10006 -diag-disable 68 -diag-disable 177 -diag-disable 186 -diag-disable 2304 -diag-disable 2305 -diag-disable 2338 -diag-disable 1418 -diag-disable 1170 -diag-disable 3373 -diag-disable=remark")
 endif()
 endif()
 
@@ -513,6 +517,9 @@ set (hs_exec_SRCS
     src/crc32.h
     src/report.h
     src/runtime.c
+    src/stream_compress.c
+    src/stream_compress.h
+    src/stream_compress_impl.h
     src/fdr/fdr.c
     src/fdr/fdr.h
     src/fdr/fdr_internal.h
@@ -629,6 +636,7 @@ set (hs_exec_SRCS
     src/util/masked_move.h
     src/util/multibit.h
     src/util/multibit.c
+    src/util/multibit_compress.h
     src/util/multibit_internal.h
     src/util/pack_bits.h
     src/util/popcount.h
@@ -651,15 +659,14 @@ set (hs_exec_avx2_SRCS
 )
 
 
-SET (hs_SRCS
+SET (hs_compile_SRCS
     ${hs_HEADERS}
     src/crc32.h
     src/database.h
     src/grey.cpp
     src/grey.h
     src/hs.cpp
     src/hs_internal.h
-    src/hs_version.c
     src/hs_version.h
     src/scratch.h
     src/state.h
@@ -735,6 +742,7 @@ SET (hs_SRCS
     src/nfa/nfa_build_util.h
     src/nfa/nfa_internal.h
     src/nfa/nfa_kind.h
+    src/nfa/rdfa.cpp
     src/nfa/rdfa.h
     src/nfa/rdfa_graph.cpp
     src/nfa/rdfa_graph.h
@@ -960,6 +968,7 @@ SET (hs_SRCS
     src/rose/rose_build_merge.cpp
     src/rose/rose_build_merge.h
     src/rose/rose_build_misc.cpp
+    src/rose/rose_build_misc.h
     src/rose/rose_build_program.cpp
     src/rose/rose_build_program.h
     src/rose/rose_build_resources.h
@@ -996,9 +1005,13 @@ SET (hs_SRCS
     src/util/dump_mask.h
     src/util/fatbit_build.cpp
     src/util/fatbit_build.h
+    src/util/flat_containers.h
     src/util/graph.h
+    src/util/graph_range.h
+    src/util/graph_small_color_map.h
     src/util/hash.h
     src/util/hash_dynamic_bitset.h
+    src/util/insertion_ordered.h
     src/util/math.h
     src/util/multibit_build.cpp
     src/util/multibit_build.h
@@ -1016,14 +1029,14 @@ SET (hs_SRCS
     src/util/small_vector.h
     src/util/target_info.cpp
     src/util/target_info.h
-    src/util/ue2_containers.h
     src/util/ue2_graph.h
     src/util/ue2string.cpp
     src/util/ue2string.h
     src/util/unaligned.h
     src/util/unicode_def.h
     src/util/unicode_set.h
     src/util/uniform_ops.h
+    src/util/unordered.h
     src/util/verify_types.h
 )
 
@@ -1076,7 +1089,7 @@ set(hs_dump_SRCS
 )
 
 if (DUMP_SUPPORT)
-    set(hs_SRCS ${hs_SRCS} ${hs_dump_SRCS})
+    set(hs_compile_SRCS ${hs_compile_SRCS} ${hs_dump_SRCS})
 endif()
 
 # we group things by sublibraries, specifying shared and static and then
@@ -1099,12 +1112,20 @@ if (NOT FAT_RUNTIME)
         add_library(hs_runtime STATIC src/hs_version.c src/hs_valid_platform.c $<TARGET_OBJECTS:hs_exec>)
         set_target_properties(hs_runtime PROPERTIES LINKER_LANGUAGE C)
 
-        add_library(hs STATIC ${hs_SRCS} src/hs_valid_platform.c $<TARGET_OBJECTS:hs_exec>)
+        add_library(hs_compile OBJECT ${hs_compile_SRCS})
+
+        add_library(hs STATIC
+            src/hs_version.c
+            src/hs_valid_platform.c
+            $<TARGET_OBJECTS:hs_exec>
+            $<TARGET_OBJECTS:hs_compile>)
     endif (BUILD_STATIC_LIBS)
 
     if (BUILD_STATIC_AND_SHARED OR BUILD_SHARED_LIBS)
         add_library(hs_exec_shared OBJECT ${hs_exec_SRCS})
         set_target_properties(hs_exec_shared PROPERTIES POSITION_INDEPENDENT_CODE TRUE)
+        add_library(hs_compile_shared OBJECT ${hs_compile_SRCS})
+        set_target_properties(hs_compile_shared PROPERTIES POSITION_INDEPENDENT_CODE TRUE)
     endif()
 
 else (FAT_RUNTIME)
@@ -1158,17 +1179,20 @@ else (FAT_RUNTIME)
            $<TARGET_OBJECTS:hs_exec_common>
            ${RUNTIME_LIBS})
        set_target_properties(hs_runtime PROPERTIES LINKER_LANGUAGE C)
+        add_library(hs_compile OBJECT ${hs_compile_SRCS})
 
        # we want the static lib for testing
        add_library(hs STATIC src/hs_version.c src/hs_valid_platform.c
-           ${hs_SRCS}
+           $<TARGET_OBJECTS:hs_compile>
            $<TARGET_OBJECTS:hs_exec_common>
            ${RUNTIME_LIBS})
 
     endif (BUILD_STATIC_LIBS)
 
     if (BUILD_STATIC_AND_SHARED OR BUILD_SHARED_LIBS)
         # build shared libs
+        add_library(hs_compile_shared OBJECT ${hs_compile_SRCS})
+        set_target_properties(hs_compile_shared PROPERTIES POSITION_INDEPENDENT_CODE TRUE)
         add_library(hs_exec_shared_core2 OBJECT ${hs_exec_SRCS})
         list(APPEND RUNTIME_SHLIBS $<TARGET_OBJECTS:hs_exec_shared_core2>)
         set_target_properties(hs_exec_shared_core2 PROPERTIES
@@ -1249,10 +1273,10 @@ endif()
 if (BUILD_STATIC_AND_SHARED OR BUILD_SHARED_LIBS)
     if (NOT FAT_RUNTIME)
         add_library(hs_shared SHARED src/hs_version.c src/hs_valid_platform.c
-            ${hs_SRCS} $<TARGET_OBJECTS:hs_exec_shared>)
+            $<TARGET_OBJECTS:hs_compile_shared> $<TARGET_OBJECTS:hs_exec_shared>)
     else()
         add_library(hs_shared SHARED src/hs_version.c src/hs_valid_platform.c
-            ${hs_SRCS} $<TARGET_OBJECTS:hs_exec_common_shared>
+            $<TARGET_OBJECTS:hs_compile_shared> $<TARGET_OBJECTS:hs_exec_common_shared>
             ${RUNTIME_SHLIBS})
 
     endif()

doc/dev-reference/compilation.rst

@@ -64,7 +64,7 @@ libpcre are supported. The use of unsupported constructs will result in
 compilation errors.
 
 The version of PCRE used to validate Hyperscan's interpretation of this syntax
-is 8.40.
+is 8.41.
 
 ====================
 Supported Constructs

doc/dev-reference/runtime.rst

@@ -80,6 +80,42 @@ functions for the management of streams:
   another, resetting the destination stream first. This call avoids the
   allocation done by :c:func:`hs_copy_stream`.
 
+==================
+Stream Compression
+==================
+
+A stream object is allocated as a fixed size region of memory which has been
+sized to ensure that no memory allocations are required during scan
+operations. When the system is under memory pressure, it may be useful to reduce
+the memory consumed by streams that are not expected to be used soon. The
+Hyperscan API provides calls for translating a stream to and from a compressed
+representation for this purpose. The compressed representation differs from the
+full stream object as it does not reserve space for components which are not
+required given the current stream state. The Hyperscan API functions for this
+functionality are:
+
+* :c:func:`hs_compress_stream`: fills the provided buffer with a compressed
+  representation of the stream and returns the number of bytes consumed by the
+  compressed representation. If the buffer is not large enough to hold the
+  compressed representation, :c:member:`HS_INSUFFICIENT_SPACE` is returned along
+  with the required size. This call does not modify the original stream in any
+  way: it may still be written to with :c:func:`hs_scan_stream`, used as part of
+  the various reset calls to reinitialise its state, or
+  :c:func:`hs_close_stream` may be called to free its resources.
+
+* :c:func:`hs_expand_stream`: creates a new stream based on a buffer containing
+  a compressed representation.
+
+* :c:func:`hs_reset_and_expand_stream`: constructs a stream based on a buffer
+  containing a compressed representation on top of an existing stream, resetting
+  the existing stream first. This call avoids the allocation done by
+  :c:func:`hs_expand_stream`.
+
+Note: it is not recommended to use stream compression between every call to scan
+for performance reasons as it takes time to convert between the compressed
+representation and a standard stream.
+
+
 **********
 Block Mode
 **********

examples/patbench.cc

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015-2016, Intel Corporation
+ * Copyright (c) 2015-2017, Intel Corporation
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
@@ -165,6 +165,7 @@ static bool higher_is_better(Criterion c) {
 }
 
 static void print_criterion(Criterion c, double val) {
+    std::ios::fmtflags f(cout.flags());
     switch (c) {
     case CRITERION_THROUGHPUT:
         cout << std::fixed << std::setprecision(3) << val << " Megabits/s";
@@ -179,6 +180,7 @@ static void print_criterion(Criterion c, double val) {
         cout << static_cast<size_t>(val) << " bytes";
         break;
     }
+    cout.flags(f);
 }
 
 // Key for identifying a stream in our pcap input data, using data from its IP
@@ -596,11 +598,13 @@ double eval_set(Benchmark &bench, Sigdata &sigs, unsigned int mode,
     size_t bytes = bench.bytes();
     size_t matches = bench.matches();
     if (diagnose) {
+        std::ios::fmtflags f(cout.flags());
         cout << "Scan time " << std::fixed << std::setprecision(3) << scan_time
              << " sec, Scanned " << bytes * repeatCount << " bytes, Throughput "
              << std::fixed << std::setprecision(3)
              << (bytes * 8 * repeatCount) / (scan_time * 1000000)
              << " Mbps, Matches " << matches << endl;
+        cout.flags(f);
     }
     return (bytes * 8 * repeatCount) / (scan_time * 1000000);
 }
@@ -755,10 +759,12 @@ int main(int argc, char **argv) {
         for (unsigned i = count; i < 16; i++) {
             cout << " ";
         }
+        std::ios::fmtflags out_f(cout.flags());
         cout << "Performance: ";
         print_criterion(criterion, best);
         cout << " (" << std::fixed << std::setprecision(3) << (best / score_base)
              << "x) after cutting:" << endl;
+        cout.flags(out_f);
 
         // s now has factor_max signatures
         for (const auto &found : s) {

src/dispatcher.c

@@ -127,6 +127,16 @@ CREATE_DISPATCH(hs_error_t, hs_serialized_database_info, const char *bytes,
 CREATE_DISPATCH(hs_error_t, hs_serialized_database_size, const char *bytes,
                 const size_t length, size_t *deserialized_size);
 
+CREATE_DISPATCH(hs_error_t, hs_compress_stream, const hs_stream_t *stream,
+                char *buf, size_t buf_space, size_t *used_space);
+
+CREATE_DISPATCH(hs_error_t, hs_expand_stream, const hs_database_t *db,
+                hs_stream_t **stream, const char *buf,size_t buf_size);
+
+CREATE_DISPATCH(hs_error_t, hs_reset_and_expand_stream, hs_stream_t *to_stream,
+                const char *buf, size_t buf_size, hs_scratch_t *scratch,
+                match_event_handler onEvent, void *context);
+
 /** INTERNALS **/
 
 CREATE_DISPATCH(u32, Crc32c_ComputeBuf, u32 inCrc32, const void *buf, size_t bufLen);