[wip] more and new compression algorithms

cmake/modules/FindLZ4.cmake

@@ -0,0 +1,62 @@
+# Find the LZ4 includes and library.
+#
+# This module defines
+# LZ4_INCLUDE_DIR, where to locate LZ4 header files
+# LZ4_LIBRARIES, the libraries to link against to use LZ4
+# LZ4_FOUND.  If false, you cannot build anything that requires LZ4.
+#
+# inspired by @zzxuanyuan's pull request https://github.com/root-mirror/root/pull/81
+
+if(LZ4_CONFIG_EXECUTABLE)
+  set(LZ4_FIND_QUIETLY 1)
+endif()
+set(LZ4_FOUND 0)
+
+if(NOT LZ4_DIR)
+  set(LZ4_DIR $ENV{LZ4_DIR})
+endif()
+
+find_path(LZ4_INCLUDE_DIR lz4.h PATHS
+  ${LZ4_DIR}/include
+  /usr/include
+  /usr/local/include
+  /opt/lz4/include
+  NO_DEFAULT_PATH
+  NO_SYSTEM_ENVIRONMENT_PATH
+  DOC "Specify the directory containing lz4.h"
+)
+
+find_library(LZ4_LIBRARY NAMES lz4 PATHS
+  ${LZ4_DIR}/lib
+  /usr/local/lz4/lib
+  /usr/local/lib
+  /usr/lib/lz4
+  /usr/local/lib/lz4
+  /usr/lz4/lib /usr/lib
+  /usr/lz4 /usr/local/lz4
+  /opt/lz4 /opt/lz4/lib
+  NO_DEFAULT_PATH
+  NO_SYSTEM_ENVIRONMENT_PATH
+  DOC "Specify the lz4 library here."
+)
+
+if(LZ4_INCLUDE_DIR)
+  message(STATUS "Found LZ4 includes at ${LZ4_INCLUDE_DIR}")
+else()
+  message(STATUS "LZ4 includes not found")
+  message(STATUS "not even in $ENV{LZ4_DIR}")
+endif()
+
+if(LZ4_LIBRARY)
+  message(STATUS "Found LZ4 library at ${LZ4_LIBRARY}")
+else()
+  message(STATUS "LZ4 library not found")
+endif()
+
+
+if(LZ4_INCLUDE_DIR AND LZ4_LIBRARY)
+  set(LZ4_FOUND 1)
+endif()
+
+set(LZ4_LIBRARIES ${LZ4_LIBRARY})
+mark_as_advanced(LZ4_FOUND LZ4_LIBRARY LZ4_INCLUDE_DIR)

cmake/modules/FindLZMA.cmake

@@ -2,8 +2,8 @@
 #
 # This module defines
 # LZMA_INCLUDE_DIR, where to locate LZMA header files
-# LZMA_LIBRARIES, the libraries to link against to use Pythia6
-# LZMA_FOUND.  If false, you cannot build anything that requires Pythia6.
+# LZMA_LIBRARIES, the libraries to link against to use LZMA
+# LZMA_FOUND.  If false, you cannot build anything that requires LZMA
 
 if(LZMA_CONFIG_EXECUTABLE)
   set(LZMA_FIND_QUIETLY 1)

cmake/modules/FindLZO.cmake

@@ -0,0 +1,48 @@
+# Find the LZO includes and library.
+#
+# This module defines
+# LZO_INCLUDE_DIR, where to locate LZO header files
+# LZO_LIBRARIES, the libraries to link against to use LZO
+# LZO_FOUND.  If false, you cannot build anything that requires LZO.
+#
+# inspired by @zzxuanyuan's pull request https://github.com/root-mirror/root/pull/81
+
+if(LZO_CONFIG_EXECUTABLE)
+  set(LZO_FIND_QUIETLY 1)
+endif()
+set(LZO_FOUND 0)
+
+if(NOT LZO_DIR)
+  set(LZO_DIR $ENV{LZO_DIR})
+endif()
+
+find_path(LZO_INCLUDE_DIR lzo/lzoutil.h
+  ${LZO_DIR}/include
+  /usr/include
+  /usr/local/include
+  /opt/lzo/include
+  DOC "Specify the directory containing lzoutil.h"
+)
+
+find_library(LZO_LIBRARY NAMES lzo2 PATHS
+  ${LZO_DIR}/lib
+  /usr/local/lzo/lib
+  /usr/local/lib
+  /usr/lib/lzo
+  /usr/local/lib/lzo
+  /usr/lzo/lib /usr/lib
+  /usr/lzo /usr/local/lzo
+  /opt/lzo /opt/lzo/lib
+  DOC "Specify the lzo2 library here."
+)
+
+if(LZO_INCLUDE_DIR AND LZO_LIBRARY)
+  set(LZO_FOUND 1)
+  if(NOT LZO_FIND_QUIETLY)
+     message(STATUS "Found LZO includes at ${LZO_INCLUDE_DIR}")
+     message(STATUS "Found LZO library at ${LZO_LIBRARY}")
+  endif()
+endif()
+
+set(LZO_LIBRARIES ${LZO_LIBRARY})
+mark_as_advanced(LZO_FOUND LZO_LIBRARY LZO_INCLUDE_DIR)

cmake/modules/FindZOPFLI.cmake

@@ -0,0 +1,61 @@
+# Find the ZOPFLI includes and library.
+#
+# This module defines
+# ZOPFLI_INCLUDE_DIR, where to locate ZOPFLI header files
+# ZOPFLI_LIBRARIES, the libraries to link against to use ZOPFLI
+# ZOPFLI_FOUND.  If false, you cannot build anything that requires ZOPFLI.
+#
+# inspired by @zzxuanyuan's pull request https://github.com/root-mirror/root/pull/81
+
+if(ZOPFLI_CONFIG_EXECUTABLE)
+  set(ZOPFLI_FIND_QUIETLY 1)
+endif()
+set(ZOPFLI_FOUND 0)
+
+if(NOT ZOPFLI_DIR)
+  set(ZOPFLI_DIR $ENV{ZOPFLI_DIR})
+endif()
+
+find_path(ZOPFLI_INCLUDE_DIR zopfli/zopfli.h PATHS
+  ${ZOPFLI_DIR}/include
+  /usr/include
+  /usr/local/include
+  /opt/zopfli/include
+  NO_DEFAULT_PATH
+  NO_SYSTEM_ENVIRONMENT_PATH
+  DOC "Specify the directory containing zopfli.h"
+)
+
+find_library(ZOPFLI_LIBRARY NAMES zopfli PATHS
+  ${ZOPFLI_DIR}/lib
+  /usr/local/zopfli/lib
+  /usr/local/lib
+  /usr/lib/zopfli
+  /usr/local/lib/zopfli
+  /usr/zopfli/lib /usr/lib
+  /usr/zopfli /usr/local/zopfli
+  /opt/zopfli /opt/zopfli/lib
+  NO_DEFAULT_PATH
+  NO_SYSTEM_ENVIRONMENT_PATH
+  DOC "Specify the zopfli library here."
+)
+
+if(ZOPFLI_INCLUDE_DIR)
+  message(STATUS "Found ZOPFLI includes at ${ZOPFLI_INCLUDE_DIR}")
+else()
+  message(STATUS "ZOPFLI includes not found")
+endif()
+
+if(ZOPFLI_LIBRARY)
+  message(STATUS "Found ZOPFLI library at ${ZOPFLI_LIBRARY}")
+else()
+  message(STATUS "ZOPFLI library not found")
+endif()
+
+
+if(ZOPFLI_INCLUDE_DIR AND ZOPFLI_LIBRARY)
+  set(ZOPFLI_FOUND 1)
+endif()
+
+set(ZOPFLI_LIBRARIES ${ZOPFLI_LIBRARY})
+mark_as_advanced(ZOPFLI_FOUND ZOPFLI_LIBRARY ZOPFLI_INCLUDE_DIR)

cmake/modules/RootBuildOptions.cmake

@@ -75,6 +75,11 @@ ROOT_BUILD_OPTION(builtin_openssl OFF "Build OpenSSL internally, or use system O
 ROOT_BUILD_OPTION(builtin_pcre OFF "Build included libpcre, or use system libpcre")
 ROOT_BUILD_OPTION(builtin_zlib OFF "Build included libz, or use system libz")
 ROOT_BUILD_OPTION(builtin_lzma OFF "Build included liblzma, or use system liblzma")
+ROOT_BUILD_OPTION(lzo OFF "LZO support, requires liblzo2")
+ROOT_BUILD_OPTION(zopfli OFF "ZOPFLI support, requires libzopfli")
+ROOT_BUILD_OPTION(brotli OFF "BROTLI support (built in)")
+ROOT_BUILD_OPTION(lz4 OFF "LZ4 support, (either included lz4 or system lz4)")
+ROOT_BUILD_OPTION(builtin_lz4 OFF "Build included lz4, or use system liblz4")
 ROOT_BUILD_OPTION(builtin_davix OFF "Build the Davix library internally (downloading tarfile from the Web)")
 ROOT_BUILD_OPTION(builtin_gsl OFF "Build the GSL library internally (downloading tarfile from the Web)")
 ROOT_BUILD_OPTION(builtin_cfitsio OFF "Build the FITSIO library internally (downloading tarfile from the Web)")

cmake/modules/RootConfiguration.cmake

@@ -439,6 +439,26 @@ if(lzma)
 else()
   set(haslzmacompression undef)
 endif()
+if(brotli)
+  set(hasbrotlicompression define)
+else()
+  set(hasbrotlicompression undef)
+endif()
+if(zopfli)
+  set(haszopflicompression define)
+else()
+  set(haszopflicompression undef)
+endif()
+if(lz4)
+  set(haslz4compression define)
+else()
+  set(haslz4compression undef)
+endif()
+if(lzo)
+  set(haslzocompression define)
+else()
+  set(haslzocompression undef)
+endif()
 if(cocoa)
   set(hascocoa define)
 else()

cmake/modules/RootNewMacros.cmake

@@ -331,8 +331,12 @@ function(ROOT_GENERATE_DICTIONARY dictionary)
   endif()
 
   #---call rootcint------------------------------------------
+  if(zopfli)
+    set(zopfli_library ${ZOPFLI_LIBRARIES})
+  endif()
   add_custom_command(OUTPUT ${dictionary}.cxx ${pcm_name} ${rootmap_name}
-                     COMMAND ${command} -f  ${dictionary}.cxx ${newargs} ${rootmapargs}
+    #COMMAND ${ld_library_path}=$ENV{${ld_library_path}}:${zopfli_library} ${command} -f  ${dictionary}.cxx ${newargs} ${rootmapargs}
+    COMMAND ${command} -f  ${dictionary}.cxx ${newargs} ${rootmapargs}
                                         ${ARG_OPTIONS} ${definitions} ${includedirs} ${rheaderfiles} ${_linkdef}
                      IMPLICIT_DEPENDS CXX ${_linkdef} ${headerfiles}
                      DEPENDS ${headerfiles} ${_linkdef} ${ROOTCINTDEP})

cmake/modules/SearchInstalledSoftware.cmake

@@ -523,6 +523,58 @@ if(xml)
   endif()
 endif()
 
+#---Check for LZ4------------------------------------------------------------------
+if(lz4)
+  if(NOT builtin_lz4)
+    message(STATUS "Looking for LZ4")
+    find_package(LZ4)
+    if(NOT LZ4_FOUND)
+      if(fail-on-missing)
+        message(FATAL_ERROR "LZ4 not found and is required ('fail-on-missing' enabled)."
+          "Alternatively, you can enable the option 'builtin_lz4' to build the internal LZ4.")
+      else()
+        message(STATUS "LZ4 not found. Switching on builtin_lz4 option")
+        set(builtin_lz4 ON CACHE BOOL "" FORCE)
+      endif()
+    endif()
+  else()
+    set(lz4_version 0.0) #FIXME look up which version this is
+    message(STATUS "Building LZ4 version ${lz4_version} included in ROOT itself")
+  endif()
+else()
+  if(builtin_lz4)
+    message(FATAL_ERROR "'builtin_lz4' is enabled, but 'lz4' is disabled'.")
+  endif()
+endif()
+
+#---Check for LZO------------------------------------------------------------------
+if(lzo)
+  message(STATUS "Looking for LZO")
+  find_package(LZO)
+  if(NOT LZO_FOUND)
+    if(fail-on-missing)
+      message(FATAL_ERROR "lzo not found but required (lzo option enabled)")
+    else()
+      message(STATUS "lzo not found. Switching off lzo option")
+      set(lzo OFF CACHE BOOL "" FORCE)
+    endif()
+  endif()
+endif()
+
+#---Check for ZOPFLI------------------------------------------------------------------
+if(zopfli)
+  message(STATUS "Looking for ZOPFLI")
+  find_package(ZOPFLI)
+  if(NOT ZOPFLI_FOUND)
+    if(fail-on-missing)
+      message(FATAL_ERROR "zopfli not found but required (zopfli option enabled)")
+    else()
+      message(STATUS "zopfli not found. Switching off zopfli option")
+      set(zopfli OFF CACHE BOOL "" FORCE)
+    endif()
+  endif()
+endif()
+
 #---Check for OpenSSL------------------------------------------------------------------
 if(ssl OR builtin_openssl)
   if(builtin_openssl)

core/CMakeLists.txt

@@ -31,6 +31,28 @@ if(cocoa)
 endif()
 add_subdirectory(zip)
 add_subdirectory(lzma)
+if(lzo)
+  add_subdirectory(lzo)
+  set(lzo_objects $<TARGET_OBJECTS:Lzo>)
+  set(lzo_library ${LZO_LIBRARIES})
+endif()
+if(brotli)
+  add_subdirectory(brotli)
+  set(brotli_objects $<TARGET_OBJECTS:Brotli>)
+endif()
+if(zopfli)
+  add_subdirectory(zopfli)
+  set(zopfli_objects $<TARGET_OBJECTS:Zopfli>)
+  set(zopfli_library ${ZOPFLI_LIBRARIES})
+endif()
+if(lz4)
+  add_subdirectory(lz4)
+  set(lz4_objects $<TARGET_OBJECTS:Lz4>)
+  if (NOT builtin_lz4)
+    # builtin lz4 is statically linked
+    set(lz4_library ${LZ4_LIBRARIES})
+  endif()
+endif()
 add_subdirectory(base)
 
 set(objectlibs $<TARGET_OBJECTS:Base>
@@ -43,6 +65,10 @@ set(objectlibs $<TARGET_OBJECTS:Base>
                $<TARGET_OBJECTS:TextInput>
                ${macosx_objects}
                ${unix_objects}
+               ${lzo_objects}
+               ${zopfli_objects}
+               ${brotli_objects}
+               ${lz4_objects}
                ${winnt_objects})
 
 #---Generation of RGitCommit.h-----------------------------------------------------------
@@ -101,7 +127,7 @@ add_subdirectory(utils)
 ROOT_LINKER_LIBRARY(Core
                     $<TARGET_OBJECTS:BaseTROOT>
                     ${objectlibs}
-                    LIBRARIES ${PCRE_LIBRARIES} ${LZMA_LIBRARIES} ${ZLIB_LIBRARY}
+                    LIBRARIES ${PCRE_LIBRARIES} ${lzo_library} ${zopfli_library} ${lz4_library} ${LZMA_LIBRARIES} ${ZLIB_LIBRARY}
                               ${CMAKE_DL_LIBS} ${CMAKE_THREAD_LIBS_INIT} ${corelinklibs} )
 
 if(cling)

core/brotli/CMakeLists.txt

@@ -0,0 +1,10 @@
+############################################################################
+# CMakeLists.txt file for building ROOT core/brotli package
+############################################################################
+#
+# inspired by @zzxuanyuan's pull request https://github.com/root-mirror/root/pull/81
+
+#---Declare ZipBROTLI sources as part of libCore-------------------------------
+set(sources ${CMAKE_CURRENT_SOURCE_DIR}/src/ZipBROTLI.cc ${CMAKE_CURRENT_SOURCE_DIR}/src/enc/backward_references.cc ${CMAKE_CURRENT_SOURCE_DIR}/src/enc/block_splitter.cc ${CMAKE_CURRENT_SOURCE_DIR}/src/enc/brotli_bit_stream.cc ${CMAKE_CURRENT_SOURCE_DIR}/src/enc/compress_fragment.cc ${CMAKE_CURRENT_SOURCE_DIR}/src/enc/compress_fragment_two_pass.cc ${CMAKE_CURRENT_SOURCE_DIR}/src/enc/encode.cc ${CMAKE_CURRENT_SOURCE_DIR}/src/enc/encode_parallel.cc ${CMAKE_CURRENT_SOURCE_DIR}/src/enc/entropy_encode.cc ${CMAKE_CURRENT_SOURCE_DIR}/src/enc/histogram.cc ${CMAKE_CURRENT_SOURCE_DIR}/src/enc/literal_cost.cc ${CMAKE_CURRENT_SOURCE_DIR}/src/enc/metablock.cc ${CMAKE_CURRENT_SOURCE_DIR}/src/enc/static_dict.cc ${CMAKE_CURRENT_SOURCE_DIR}/src/enc/streams.cc ${CMAKE_CURRENT_SOURCE_DIR}/src/enc/utf8_util.cc ${CMAKE_CURRENT_SOURCE_DIR}/src/dec/bit_reader.c ${CMAKE_CURRENT_SOURCE_DIR}/src/dec/decode.c ${CMAKE_CURRENT_SOURCE_DIR}/src/dec/dictionary.c ${CMAKE_CURRENT_SOURCE_DIR}/src/dec/huffman.c ${CMAKE_CURRENT_SOURCE_DIR}/src/dec/state.c)
+
+ROOT_OBJECT_LIBRARY(Brotli ${sources})

core/brotli/doc/CONTRIBUTING.md

@@ -0,0 +1,27 @@
+Want to contribute? Great! First, read this page (including the small print at
+the end).
+
+### Before you contribute
+Before we can use your code, you must sign the
+[Google Individual Contributor License Agreement]
+(https://cla.developers.google.com/about/google-individual)
+(CLA), which you can do online. The CLA is necessary mainly because you own the
+copyright to your changes, even after your contribution becomes part of our
+codebase, so we need your permission to use and distribute your code. We also
+need to be sure of various other things—for instance that you'll tell us if you
+know that your code infringes on other people's patents. You don't have to sign
+the CLA until after you've submitted your code for review and a member has
+approved it, but you must do it before we can put your code into our codebase.
+Before you start working on a larger contribution, you should get in touch with
+us first through the issue tracker with your idea so that we can help out and
+possibly guide you. Coordinating up front makes it much easier to avoid
+frustration later on.
+
+### Code reviews
+All submissions, including submissions by project members, require review. We
+use Github pull requests for this purpose.
+
+### The small print
+Contributions made by corporations are covered by a different agreement than
+the one above, the [Software Grant and Corporate Contributor License Agreement]
+(https://cla.developers.google.com/about/google-corporate).