Add support for LZ4

cmake/modules/FindLZ4.cmake

@@ -0,0 +1,41 @@
+# Find the LZ4 includes and library.
+#
+# This module defines
+# LZ4_INCLUDE_DIR, where to locate LZ4 header files
+# LZ4_LIBRARIES, the libraries to link against to use LZ4
+# LZ4_FOUND.  If false, you cannot build anything that requires LZ4.
+
+if(LZ4_CONFIG_EXECUTABLE)
+  set(LZ4_FIND_QUIETLY 1)
+endif()
+set(LZ4_FOUND 0)
+
+find_path(LZ4_INCLUDE_DIR lz4.h
+  $ENV{LZ4_DIR}/include
+  /usr/local/include
+  /opt/lz4/include
+  DOC "Specify the directory containing lz4.h"
+)
+
+find_library(LZ4_LIBRARY NAMES lz4 PATHS
+  $ENV{LZ4_DIR}/lib
+  /usr/local/lz4/lib
+  /usr/local/lib
+  /usr/lib/lz4
+  /usr/local/lib/lz4
+  /usr/lz4/lib /usr/lib
+  /usr/lz4 /usr/local/lz4
+  /opt/lz4 /opt/lz4/lib
+  DOC "Specify the lz4 library here."
+)
+
+if(LZ4_INCLUDE_DIR AND LZ4_LIBRARY)
+  set(LZ4_FOUND 1)
+  if(NOT LZ4_FIND_QUIETLY)
+     message(STATUS "Found LZ4 includes at ${LZ4_INCLUDE_DIR}")
+     message(STATUS "Found LZ4 library at ${LZ4_LIBRARY}")
+  endif()
+endif()
+
+set(LZ4_LIBRARIES ${LZ4_LIBRARY})
+mark_as_advanced(LZ4_FOUND LZ4_LIBRARY LZ4_INCLUDE_DIR)

cmake/modules/RootBuildOptions.cmake

@@ -77,6 +77,7 @@ ROOT_BUILD_OPTION(builtin_glew ON "Build included libGLEW, or use system libGLEW
 ROOT_BUILD_OPTION(builtin_gsl OFF "Build the GSL library internally (downloading tarfile from the Web)")
 ROOT_BUILD_OPTION(builtin_llvm ON "Build the LLVM internally")
 ROOT_BUILD_OPTION(builtin_lzma OFF "Build included liblzma, or use system liblzma")
+ROOT_BUILD_OPTION(builtin_lz4 OFF "Built included liblz4, or use system liblz4")
 ROOT_BUILD_OPTION(builtin_openssl OFF "Build OpenSSL internally, or use system OpenSSL")
 ROOT_BUILD_OPTION(builtin_pcre OFF "Build included libpcre, or use system libpcre")
 ROOT_BUILD_OPTION(builtin_tbb OFF "Build the TBB internally")

cmake/modules/RootConfiguration.cmake

@@ -457,6 +457,11 @@ if(lzma)
 else()
   set(haslzmacompression undef)
 endif()
+if(lz4)
+  set(haslz4compression define)
+else()
+  set(haslz4compression undef)
+endif()
 if(cocoa)
   set(hascocoa define)
 else()

cmake/modules/RootNewMacros.cmake

@@ -276,7 +276,7 @@ function(ROOT_GENERATE_DICTIONARY dictionary)
   # Replace the non-standard folder layout of Core.
   if (ARG_STAGE1 AND ARG_MODULE STREQUAL "Core")
     # FIXME: Glob these folders.
-    set(core_folders "base|clib|clingutils|cont|dictgen|doc|foundation|lzma|macosx|meta|metacling|multiproc|newdelete|pcre|rint|rootcling_stage1|textinput|thread|unix|winnt|zip")
+    set(core_folders "base|clib|clingutils|cont|dictgen|doc|foundation|lzma|lz4|macosx|meta|metacling|multiproc|newdelete|pcre|rint|rootcling_stage1|textinput|thread|unix|winnt|zip")
     string(REGEX REPLACE "${CMAKE_SOURCE_DIR}/core/(${core_folders})/inc/" ""  headerfiles "${headerfiles}")
   endif()
 

cmake/modules/SearchInstalledSoftware.cmake

@@ -231,6 +231,40 @@ if(builtin_lzma)
 endif()
 
 
+#---Check for LZ4--------------------------------------------------------------------
+if(NOT builtin_lz4)
+  message(STATUS "Looking for LZ4")
+  find_package(LZ4)
+  if(LZ4_FOUND)
+  else()
+    message(STATUS "LZ4 not found. Switching on builtin_lz4 option")
+    set(builtin_lz4 ON CACHE BOOL "" FORCE)
+  endif()
+endif()
+# Note: the above if-statement may change the value of builtin_lz4 to ON.
+if(builtin_lz4)
+  set(lz4_version 1.7.5)
+  message(STATUS "Building LZ4 version ${lz4_version} included in ROOT itself")
+  if(CMAKE_CXX_COMPILER_ID STREQUAL Clang)
+    set(LZ4_CFLAGS "-Wno-format-nonliteral")
+  elseif( CMAKE_CXX_COMPILER_ID STREQUAL Intel)
+    set(LZ4_CFLAGS "-wd188 -wd181 -wd1292 -wd10006 -wd10156 -wd2259 -wd981 -wd128 -wd3179")
+  endif()
+  set(LZ4_LIBRARIES ${CMAKE_BINARY_DIR}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}lz4${CMAKE_STATIC_LIBRARY_SUFFIX})
+  ExternalProject_Add(
+    LZ4
+    URL https://github.com/lz4/lz4/archive/v1.7.5.tar.gz
+    URL_MD5 c9610c5ce97eb431dddddf0073d919b9
+    INSTALL_DIR ${CMAKE_BINARY_DIR}
+    CONFIGURE_COMMAND  /bin/sh -c "PREFIX=<INSTALL_DIR> make cmake"
+    BUILD_COMMAND /bin/sh -c "PREFIX=<INSTALL_DIR> MOREFLAGS=-fPIC make"
+    INSTALL_COMMAND /bin/sh -c "PREFIX=<INSTALL_DIR> make install"
+    LOG_DOWNLOAD 1 LOG_CONFIGURE 1 LOG_BUILD 1 LOG_INSTALL 1 BUILD_IN_SOURCE 1
+    BUILD_BYPRODUCTS ${LZ4_LIBRARIES})
+  set(LZ4_INCLUDE_DIR ${CMAKE_BINARY_DIR}/include)
+endif()
+
+
 #---Check for X11 which is mandatory lib on Unix--------------------------------------
 if(x11)
   message(STATUS "Looking for X11")

core/CMakeLists.txt

@@ -16,6 +16,7 @@ add_subdirectory(thread)
 add_subdirectory(imt)
 add_subdirectory(zip)
 add_subdirectory(lzma)
+add_subdirectory(lz4)
 
 if(NOT WIN32)
   add_subdirectory(newdelete)
@@ -47,6 +48,7 @@ set(objectlibs $<TARGET_OBJECTS:Base>
                $<TARGET_OBJECTS:Cont>
                $<TARGET_OBJECTS:Foundation>
                $<TARGET_OBJECTS:Lzma>
+               $<TARGET_OBJECTS:Lz4>
                $<TARGET_OBJECTS:Zip>
                $<TARGET_OBJECTS:Meta>
                $<TARGET_OBJECTS:TextInput>
@@ -110,9 +112,9 @@ add_subdirectory(rootcling_stage1)
 ROOT_LINKER_LIBRARY(Core
                     $<TARGET_OBJECTS:BaseTROOT>
                     ${objectlibs}
-                    LIBRARIES ${PCRE_LIBRARIES} ${LZMA_LIBRARIES} ${ZLIB_LIBRARIES}
+                    LIBRARIES ${PCRE_LIBRARIES} ${LZMA_LIBRARIES} ${LZ4_LIBRARIES} ${ZLIB_LIBRARIES}
                               ${CMAKE_DL_LIBS} ${CMAKE_THREAD_LIBS_INIT} ${corelinklibs}
-                    BUILTINS PCRE LZMA)
+                    BUILTINS PCRE LZMA LZ4)
 
 if(cling)
   add_dependencies(Core CLING)

core/README

@@ -27,6 +27,7 @@ and depending on the platform
   - winnt
 and optionaly
   - lzma
+  - lz4
   - zip
 
 libCling depends on libCore and libRIO and contains:

core/lz4/CMakeLists.txt

@@ -0,0 +1,22 @@
+############################################################################
+# CMakeLists.txt file for building ROOT core/lz4 package
+############################################################################
+
+
+#---The builtin LZ4 library is built using the CMake ExternalProject standard module
+#   in cmake/modules/SearchInstalledSoftare.cmake
+
+#---Declare ZipLZ4 sources as part of libCore-------------------------------
+set(headers ${CMAKE_CURRENT_SOURCE_DIR}/inc/ZipLZ4.h)
+set(sources ${CMAKE_CURRENT_SOURCE_DIR}/src/ZipLZ4.c)
+
+
+include_directories(${LZ4_INCLUDE_DIR})
+ROOT_OBJECT_LIBRARY(Lz4 ${sources})
+
+if(builtin_lz4)
+  add_dependencies(Lz4 LZ4)
+endif()
+
+ROOT_INSTALL_HEADERS()
+install(FILES ${LZ4_headers} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})

core/lz4/inc/ZipLZ4.h

@@ -0,0 +1,13 @@
+// Author: Brian Bockelman March 2015
+
+/*************************************************************************
+ * Copyright (C) 1995-2017, Rene Brun and Fons Rademakers.               *
+ * All rights reserved.                                                  *
+ *                                                                       *
+ * For the licensing terms see $ROOTSYS/LICENSE.                         *
+ * For the list of contributors see $ROOTSYS/README/CREDITS.             *
+ *************************************************************************/
+
+void R__zipLZ4(int cxlevel, int *srcsize, char *src, int *tgtsize, char *tgt, int *irep);
+
+void R__unzipLZ4(int *srcsize, unsigned char *src, int *tgtsize, unsigned char *tgt, int *irep);

core/lz4/src/ZipLZ4.c

@@ -0,0 +1,92 @@
+// Original Author: Brian Bockelman
+
+/*************************************************************************
+ * Copyright (C) 1995-2017, Rene Brun and Fons Rademakers.               *
+ * All rights reserved.                                                  *
+ *                                                                       *
+ * For the licensing terms see $ROOTSYS/LICENSE.                         *
+ * For the list of contributors see $ROOTSYS/README/CREDITS.             *
+ *************************************************************************/
+
+#include "ZipLZ4.h"
+#include "lz4.h"
+#include "lz4hc.h"
+#include <stdio.h>
+#include <stdint.h>
+
+#include "RConfig.h"
+
+static const int kHeaderSize = 9;
+
+void R__zipLZ4(int cxlevel, int *srcsize, char *src, int *tgtsize, char *tgt, int *irep)
+{
+   int LZ4_version = LZ4_versionNumber();
+   uint64_t out_size; /* compressed size */
+   uint64_t in_size = (unsigned)(*srcsize);
+
+   *irep = 0;
+
+   if (*tgtsize <= 0) {
+      return;
+   }
+
+   if (*srcsize > 0xffffff || *srcsize < 0) {
+      return;
+   }
+
+   int returnStatus;
+   if (cxlevel > 9) {
+      cxlevel = 9;
+   }
+   if (cxlevel >= 4) {
+      returnStatus = LZ4_compress_HC(src, &tgt[kHeaderSize], *srcsize, *tgtsize - kHeaderSize, cxlevel);
+   } else {
+      returnStatus = LZ4_compress_default(src, &tgt[kHeaderSize], *srcsize, *tgtsize - kHeaderSize);
+   }
+
+   if (R__unlikely(returnStatus == 0)) { /* LZ4 compression failed */
+      return;
+   }
+
+   tgt[0] = 'L';
+   tgt[1] = '4';
+   tgt[2] = (LZ4_version / (100 * 100));
+
+   out_size = returnStatus; /* compressed size */
+
+   tgt[3] = (char)(out_size & 0xff);
+   tgt[4] = (char)((out_size >> 8) & 0xff);
+   tgt[5] = (char)((out_size >> 16) & 0xff);
+
+   tgt[6] = (char)(in_size & 0xff); /* decompressed size */
+   tgt[7] = (char)((in_size >> 8) & 0xff);
+   tgt[8] = (char)((in_size >> 16) & 0xff);
+
+   *irep = (int)returnStatus + kHeaderSize;
+}
+
+void R__unzipLZ4(int *srcsize, unsigned char *src, int *tgtsize, unsigned char *tgt, int *irep)
+{
+   int LZ4_version = LZ4_versionNumber() / (100 * 100);
+   *irep = 0;
+   if (R__unlikely(src[0] != 'L' || src[1] != '4')) {
+      fprintf(stderr, "R__unzipLZ4: algorithm run against buffer with incorrect header (got %d%d; expected %d%d).\n",
+              src[0], src[1], 'L', '4');
+      return;
+   }
+   if (R__unlikely(src[2] != LZ4_version)) {
+      fprintf(stderr,
+              "R__unzipLZ4: This version of LZ4 is incompatible with the on-disk version (got %d; expected %d).\n",
+              src[2], LZ4_version);
+      return;
+   }
+
+   int returnStatus = LZ4_decompress_safe((char *)(&src[kHeaderSize]), (char *)(tgt), *srcsize - kHeaderSize, *tgtsize);
+   if (R__unlikely(returnStatus < 0)) {
+      fprintf(stderr, "R__unzipLZ4: error in decompression around byte %d out of maximum %d.\n", -returnStatus,
+              *tgtsize);
+      return;
+   }
+
+   *irep = returnStatus;
+}

core/zip/inc/Compression.h

@@ -16,33 +16,38 @@
 namespace ROOT {
 #endif
 
-   // The global settings depend on a global variable named
-   // R__ZipMode which can be modified by a global function
-   // named R__SetZipMode.  Both are defined in Bits.h.
-   // The default is to use the global setting
-   // and the default of the global setting is to use the
-   // ZLIB compression algorithm.  The LZMA algorithm
-   // (from the XZ package) is also available.
-   // The LZMA compression usually results
-   // in greater compression factors, but takes more CPU time
-   // and memory when compressing.  LZMA memory usage is particularly
-   // high for compression levels 8 and 9.
-   //
-   // The current algorithms support level 1 to 9. The higher
-   // the level the greater the compression and more CPU time
-   // and memory resources used during compression. Level 0
-   // means no compression.
-   enum ECompressionAlgorithm { kUseGlobalCompressionSetting,
-                                kZLIB,
-                                kLZMA,
-                                kOldCompressionAlgo,
-                                // if adding new algorithm types,
-                                // keep this enum value last
-                                kUndefinedCompressionAlgorithm
-   };
-
-   // Deprecated name, do *not* use:
-   static const enum ECompressionAlgorithm kUseGlobalSetting = kUseGlobalCompressionSetting;
+// The global settings depend on a global variable named
+// R__ZipMode which can be modified by a global function
+// named R__SetZipMode.  Both are defined in Bits.h.
+// The default is to use the global setting
+// and the default of the global setting is to use the
+// ZLIB compression algorithm.  The LZMA algorithm
+// (from the XZ package) is also available.
+// The LZMA compression usually results
+// in greater compression factors, but takes more CPU time
+// and memory when compressing.  LZMA memory usage is particularly
+// high for compression levels 8 and 9.
+//
+// Finally, the LZ4 package results in worse compression ratios
+// than ZLIB but achieves much faster decompression rates.
+//
+// The current algorithms support level 1 to 9. The higher
+// the level the greater the compression and more CPU time
+// and memory resources used during compression. Level 0
+// means no compression.
+enum ECompressionAlgorithm {
+   kUseGlobalCompressionSetting,
+   kZLIB,
+   kLZMA,
+   kOldCompressionAlgo,
+   kLZ4,
+   // if adding new algorithm types,
+   // keep this enum value last
+   kUndefinedCompressionAlgorithm
+};
+
+// Deprecated name, do *not* use:
+static const enum ECompressionAlgorithm kUseGlobalSetting = kUseGlobalCompressionSetting;
 
 #if defined(__cplusplus)
 

core/zip/src/Bits.h

@@ -16,6 +16,7 @@
 #include "Compression.h"
 #include "RConfigure.h"
 #include "ZipLZMA.h"
+#include "ZipLZ4.h"
 
 #include <stdio.h>
 #include <assert.h>
@@ -282,17 +283,21 @@ struct bits_internal_state {
   } \
 }
 
-
 /* ===========================================================================
    R__ZipMode is used to select the compression algorithm when R__zip is called
    and when R__zipMultipleAlgorithm is called with its last argument set to 0.
    R__ZipMode = 1 : ZLIB compression algorithm is used (default)
    R__ZipMode = 2 : LZMA compression algorithm is used
+   R__ZipMode = 4 : LZ4  compression algorithm is used
    R__ZipMode = 0 or 3 : a very old compression algorithm is used
    (the very old algorithm is supported for backward compatibility)
    The LZMA algorithm requires the external XZ package be installed when linking
    is done. LZMA typically has significantly higher compression factors, but takes
    more CPU time and memory resources while compressing.
+
+  The LZ4 algorithm requires the external LZ4 package to be installed when linking
+  is done.  LZ4 typically has the worst compression ratios, but much faster decompression
+  speeds - sometimes by an order of magnitude.
 */
 enum ECompressionAlgorithm R__ZipMode = 1;
 
@@ -587,8 +592,11 @@ void R__zipMultipleAlgorithm(int cxlevel, int *srcsize, char *src, int *tgtsize,
 
   // The LZMA compression algorithm from the XZ package
   if (compressionAlgorithm == kLZMA) {
-    R__zipLZMA(cxlevel, srcsize, src, tgtsize, tgt, irep);
-    return;
+     R__zipLZMA(cxlevel, srcsize, src, tgtsize, tgt, irep);
+     return;
+  } else if (compressionAlgorithm == kLZ4) {
+     R__zipLZ4(cxlevel, srcsize, src, tgtsize, tgt, irep);
+     return;
   }
 
   // The very old algorithm for backward compatibility