Add support for LZ4 as a compression format

cmake/modules/FindLZ4.cmake

@@ -0,0 +1,41 @@
+# Find the LZ4 includes and library.
+#
+# This module defines
+# LZ4_INCLUDE_DIR, where to locate LZ4 header files
+# LZ4_LIBRARIES, the libraries to link against to use LZ4
+# LZ4_FOUND.  If false, you cannot build anything that requires LZ4.
+
+if(LZ4_CONFIG_EXECUTABLE)
+  set(LZ4_FIND_QUIETLY 1)
+endif()
+set(LZ4_FOUND 0)
+
+find_path(LZ4_INCLUDE_DIR lz4.h
+  $ENV{LZ4_DIR}/include
+  /usr/local/include
+  /opt/lzma/include
+  DOC "Specify the directory containing lz4.h"
+)
+
+find_library(LZ4_LIBRARY NAMES lz4 PATHS
+  $ENV{LZ4_DIR}/lib
+  /usr/local/lz4/lib
+  /usr/local/lib
+  /usr/lib/lz4
+  /usr/local/lib/lz4
+  /usr/lz4/lib /usr/lib
+  /usr/lz4 /usr/local/lz4
+  /opt/lz4 /opt/lz4/lib
+  DOC "Specify the lz4 library here."
+)
+
+if(LZ4_INCLUDE_DIR AND LZ4_LIBRARY)
+  set(LZ4_FOUND 1)
+  if(NOT LZ4_FIND_QUIETLY)
+     message(STATUS "Found LZ4 includes at ${LZ4_INCLUDE_DIR}")
+     message(STATUS "Found LZ4 library at ${LZ4_LIBRARY}")
+  endif()
+endif()
+
+set(LZ4_LIBRARIES ${LZ4_LIBRARY})
+mark_as_advanced(LZ4_FOUND LZ4_LIBRARY LZ4_INCLUDE_DIR)

cmake/modules/FindLZMA.cmake

@@ -2,8 +2,8 @@
 #
 # This module defines
 # LZMA_INCLUDE_DIR, where to locate LZMA header files
-# LZMA_LIBRARIES, the libraries to link against to use Pythia6
-# LZMA_FOUND.  If false, you cannot build anything that requires Pythia6.
+# LZMA_LIBRARIES, the libraries to link against to use LZMA
+# LZMA_FOUND.  If false, you cannot build anything that requires LZMA.
 
 if(LZMA_CONFIG_EXECUTABLE)
   set(LZMA_FIND_QUIETLY 1)

cmake/modules/RootBuildOptions.cmake

@@ -75,6 +75,7 @@ ROOT_BUILD_OPTION(builtin_openssl OFF "Build OpenSSL internally, or use system O
 ROOT_BUILD_OPTION(builtin_pcre OFF "Built included libpcre, or use system libpcre")
 ROOT_BUILD_OPTION(builtin_zlib OFF "Built included libz, or use system libz")
 ROOT_BUILD_OPTION(builtin_lzma OFF "Built included liblzma, or use system liblzma")
+ROOT_BUILD_OPTION(builtin_lz4 OFF "Built included liblz4, or use system liblz4")
 ROOT_BUILD_OPTION(builtin_davix OFF "Built the Davix library internally (downloading tarfile from the Web)")
 ROOT_BUILD_OPTION(builtin_gsl OFF "Built the GSL library internally (downloading tarfile from the Web)")
 ROOT_BUILD_OPTION(builtin_cfitsio OFF "Built the FITSIO library internally (downloading tarfile from the Web)")

cmake/modules/RootConfiguration.cmake

@@ -435,6 +435,11 @@ if(lzma)
 else()
   set(haslzmacompression undef)
 endif()
+if(lz4)
+  set(haslz4compression define)
+else()
+  set(haslz4compression undef)
+endif()
 if(cocoa)
   set(hascocoa define)
 else()

cmake/modules/SearchInstalledSoftware.cmake

@@ -180,6 +180,38 @@ if(builtin_lzma)
 endif()
 
 
+#---Check for LZ4--------------------------------------------------------------------
+if(NOT builtin_lz4)
+  message(STATUS "Looking for LZ4")
+  find_package(LZ4)
+  if(LZ4_FOUND)
+  else()
+    message(STATUS "LZ4 not found. Switching on builtin_lz4 option")
+    set(builtin_lz4 ON CACHE BOOL "" FORCE)
+  endif()
+endif()
+if(builtin_lz4)
+  set(lz4_version r127)
+  message(STATUS "Building LZ4 version ${lz4_version} included in ROOT itself")
+  if(CMAKE_CXX_COMPILER_ID STREQUAL Clang)
+    set(LZ4_CFLAGS "-Wno-format-nonliteral")
+  elseif( CMAKE_CXX_COMPILER_ID STREQUAL Intel)
+    set(LZ4_CFLAGS "-wd188 -wd181 -wd1292 -wd10006 -wd10156 -wd2259 -wd981 -wd128 -wd3179")
+  endif()
+  ExternalProject_Add(
+    LZ4
+    URL ${CMAKE_SOURCE_DIR}/core/lz4/src/lz4-${lz4_version}.tar.gz
+    URL_MD5 0601f6b8477209d07db33d504feb6ac4
+    INSTALL_DIR ${CMAKE_BINARY_DIR}
+    CONFIGURE_COMMAND  PREFIX=<INSTALL_DIR> make cmake
+    BUILD_COMMAND PREFIX=<INSTALL_DIR> MOREFLAGS=-fPIC make
+    INSTALL_COMMAND PREFIX=<INSTALL_DIR> make install
+    BUILD_IN_SOURCE 1)
+  set(LZ4_LIBRARIES ${CMAKE_BINARY_DIR}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}lz4${CMAKE_STATIC_LIBRARY_SUFFIX})
+  set(LZ4_INCLUDE_DIR ${CMAKE_BINARY_DIR}/include)
+endif()
+
+
 #---Check for X11 which is mandatory lib on Unix--------------------------------------
 if(x11)
   message(STATUS "Looking for X11")

core/CMakeLists.txt

@@ -1,7 +1,6 @@
 ############################################################################
 # CMakeLists.txt file for building ROOT (global) core package
 ############################################################################
-
 add_subdirectory(rint)
 add_subdirectory(thread)
 add_subdirectory(multiproc)
@@ -31,13 +30,15 @@ if(cocoa)
 endif()
 add_subdirectory(zip)
 add_subdirectory(lzma)
+add_subdirectory(lz4)
 add_subdirectory(base)
 
 set(objectlibs $<TARGET_OBJECTS:Base>
                $<TARGET_OBJECTS:Clib>
                $<TARGET_OBJECTS:Cont>
-               $<TARGET_OBJECTS:Lzma>
                $<TARGET_OBJECTS:Zip>
+               $<TARGET_OBJECTS:Lzma>
+               $<TARGET_OBJECTS:Lz4>
                $<TARGET_OBJECTS:MetaUtils>
                $<TARGET_OBJECTS:Meta>
                $<TARGET_OBJECTS:TextInput>
@@ -96,7 +97,7 @@ add_subdirectory(utils)
 ROOT_LINKER_LIBRARY(Core
                     $<TARGET_OBJECTS:BaseTROOT>
                     ${objectlibs}
-                    LIBRARIES ${PCRE_LIBRARIES} ${LZMA_LIBRARIES} ${ZLIB_LIBRARY}
+                    LIBRARIES ${PCRE_LIBRARIES} ${LZMA_LIBRARIES} ${LZ4_LIBRARIES} ${ZLIB_LIBRARY}
                               ${CMAKE_DL_LIBS} ${CMAKE_THREAD_LIBS_INIT} ${corelinklibs} )
 
 if(cling)
@@ -108,5 +109,8 @@ endif()
 if(builtin_lzma)
   ROOT_ADD_BUILTIN_DEPENDENCIES(Core LZMA)
 endif()
+if(builtin_lz4)
+  ROOT_ADD_BUILTIN_DEPENDENCIES(Core LZ4)
+endif()
 
 #----------------------------------------------------------------------------------------

core/lz4/CMakeLists.txt

@@ -0,0 +1,22 @@
+############################################################################
+# CMakeLists.txt file for building ROOT core/lz4 package
+############################################################################
+
+
+#---The builtin LZ4 library is built using the CMake ExternalProject standard module
+#   in cmake/modules/SearchInstalledSoftare.cmake
+
+#---Declare ZipLZ4 sources as part of libCore-------------------------------
+set(headers ${CMAKE_CURRENT_SOURCE_DIR}/inc/ZipLZ4.h)
+set(sources ${CMAKE_CURRENT_SOURCE_DIR}/src/ZipLZ4.c)
+
+
+include_directories(${LZ4_INCLUDE_DIR})
+ROOT_OBJECT_LIBRARY(Lz4 ${sources})
+
+if(builtin_lz4)
+  add_dependencies(Lz4 LZ4)
+endif()
+
+ROOT_INSTALL_HEADERS()
+install(FILES ${LZ4_headers} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})

core/lz4/inc/ZipLZ4.h

@@ -0,0 +1,13 @@
+// Author: Brian Bockelman March 2015
+
+/*************************************************************************
+ * Copyright (C) 1995-2015, Rene Brun and Fons Rademakers.               *
+ * All rights reserved.                                                  *
+ *                                                                       *
+ * For the licensing terms see $ROOTSYS/LICENSE.                         *
+ * For the list of contributors see $ROOTSYS/README/CREDITS.             *
+ *************************************************************************/
+
+void R__zipLZ4(int cxlevel, int *srcsize, char *src, int *tgtsize, char *tgt, int *irep);
+
+void R__unzipLZ4(int *srcsize, unsigned char *src, int *tgtsize, unsigned char *tgt, int *irep);

core/lz4/src/ZipLZ4.c

@@ -0,0 +1,99 @@
+// Original Author: Brian Bockelman
+
+/*************************************************************************
+ * Copyright (C) 1995-2015, Rene Brun and Fons Rademakers.               *
+ * All rights reserved.                                                  *
+ *                                                                       *
+ * For the licensing terms see $ROOTSYS/LICENSE.                         *
+ * For the list of contributors see $ROOTSYS/README/CREDITS.             *
+ *************************************************************************/
+
+#include "ZipLZ4.h"
+#include "lz4.h"
+#include "lz4hc.h"
+#include <stdio.h>
+#include <stdint.h>
+
+#if (__GNUC__ >= 3) || defined(__INTEL_COMPILER)
+#if !defined(R__unlikely)
+  #define R__unlikely(expr) __builtin_expect(!!(expr), 0)
+#endif
+#if !defined(R__likely)
+  #define R__likely(expr) __builtin_expect(!!(expr), 1)
+#endif
+#else
+  #define R__unlikely(expr) expr
+  #define R__likely(expr) expr
+#endif
+
+static const int kHeaderSize = 9;
+
+void R__zipLZ4(int cxlevel, int *srcsize, char *src, int *tgtsize, char *tgt, int *irep)
+{
+   int LZ4_version = LZ4_versionNumber();
+   uint64_t out_size; /* compressed size */
+   uint64_t in_size = (unsigned) (*srcsize);
+
+   *irep = 0;
+
+   if (*tgtsize <= 0) {
+      return;
+   }
+
+   if (*srcsize > 0xffffff || *srcsize < 0) {
+      return;
+   }
+
+   int returnStatus;
+   if (cxlevel > 9) {
+      cxlevel = 9;
+   }
+   if (cxlevel >= 4) {
+      returnStatus = LZ4_compress_HC(src, &tgt[kHeaderSize], *srcsize, *tgtsize - kHeaderSize, cxlevel);
+   } else {
+      returnStatus = LZ4_compress_default(src, &tgt[kHeaderSize], *srcsize, *tgtsize - kHeaderSize);
+   }
+
+   if (R__unlikely(returnStatus == 0)) { /* LZ4 compression failed */
+      return;
+   }
+
+   tgt[0] = 'L';
+   tgt[1] = '4';
+   tgt[2] = (LZ4_version / (100*100));
+
+   out_size  = returnStatus;             /* compressed size */
+
+   tgt[3] = (char)(out_size & 0xff);
+   tgt[4] = (char)((out_size >> 8) & 0xff);
+   tgt[5] = (char)((out_size >> 16) & 0xff);
+
+   tgt[6] = (char)(in_size & 0xff);         /* decompressed size */
+   tgt[7] = (char)((in_size >> 8) & 0xff);
+   tgt[8] = (char)((in_size >> 16) & 0xff);
+
+   *irep = (int)returnStatus + kHeaderSize;
+}
+
+void R__unzipLZ4(int *srcsize, unsigned char *src, int *tgtsize, unsigned char *tgt, int *irep)
+{
+   int LZ4_version = LZ4_versionNumber()/(100*100);
+   *irep = 0;
+   if (R__unlikely(src[0] != 'L' || src[1] != '4')) {
+      fprintf(stderr, "R__unzipLZ4: algorithm run against buffer with incorrect header (got %d%d; expected %d%d).\n", src[0], src[1], 'L', '4');
+      return;
+   }
+   if (R__unlikely(src[2] != LZ4_version)) {
+      fprintf(stderr, "R__unzipLZ4: This version of LZ4 is incompatible with the on-disk version (got %d; expected %d).\n", src[2], LZ4_version);
+      return;
+   }
+
+   int returnStatus = LZ4_decompress_safe((char *)(&src[kHeaderSize]), (char *)(tgt), *srcsize-kHeaderSize, *tgtsize);
+   if (R__unlikely(returnStatus < 0)) {
+      fprintf(stderr, "R__unzipLZ4: error in decompression around byte %d out of maximum %d.\n", -returnStatus, *tgtsize);
+      return;
+   }
+
+   *irep = returnStatus;
+}
+