blake3 portable + upstream-support

this is a significantly smaller alternative to the PR at #6358 , now blake3 "portable c" version is bundled (specifically version 0.3.7 plus a few patches that will be part of the 0.3.8 release..), and ./configure supports a new optional --with-blake3-upstream-c-source-dir=DIR argument for specifying the location of the upstream BLAKE3 C implementation, if invoked, the SSE2/SSE4.1/AVX2/AVX512 optimized versions of BLAKE3 will be compiled in when applicable (this has not been added to MSVC, i don't know how to do it on MSVC, and i don't have a MSVC system to figure it out out on, if someone think getting those optimizations available on MSVC is important, take it up with the windows php mailing list.. just getting the portable version to compile on MSVC was good enough for me.) also userland scripts can detect at runtime if the portable version or the upstream version, of BLAKE was compiled by consulting phpinfo(), it will either say "blake3 implementation: portable 0.3.7" or "blake3 implementation: upstream X.X.X"
php · divinity76 · Oct 23, 2020 · Oct 28, 2020 · Oct 30, 2020 · Oct 30, 2020
commit 7b51423f0162823afa6f99f6a85241d7083e8111
diff --git a/build/php.m4 b/build/php.m4
@@ -2703,3 +2703,83 @@ AC_DEFUN([PHP_PATCH_CONFIG_HEADERS], [
   $SED -e 's/^#undef PACKAGE_[^ ]*/\/\* & \*\//g' < $srcdir/$1 \
     > $srcdir/$1.tmp && mv $srcdir/$1.tmp $srcdir/$1
 ])
+
+
+
+
+
+dnl
+dnl PHP_CHECK_X86_TARGET
+dnl
+dnl check if we're compiling for x86/x86_64
+dnl
+AC_DEFUN([PHP_CHECK_X86_TARGET], [
+  AC_CACHE_CHECK([for x86 target],ac_cv_target_x86,[
+  AC_RUN_IFELSE([AC_LANG_SOURCE([[
+int main(void) {
+#if defined(__x86_64__) || defined(__i386__)
+    return 0;
+#else
+    return 1;
+#endif
+}
+]])],[
+  ac_cv_target_x86=yes
+],[
+  ac_cv_target_x86=no
+],[
+  ac_cv_target_x86=no
+])])
+
+])
+
+
+dnl
+dnl PHP_CHECK_WINDOWS_TARGET
+dnl
+dnl check if we're compiling for windows
+dnl
+AC_DEFUN([PHP_CHECK_WINDOWS_TARGET], [
+  AC_CACHE_CHECK([for windows target],ac_cv_target_windows,[
+  AC_RUN_IFELSE([AC_LANG_SOURCE([[
+int main(void) {
+#if defined(_WIN32)
+    return 0;
+#else
+    return 1;
+#endif
+}
+]])],[
+  ac_cv_target_windows=yes
+],[
+  ac_cv_target_windows=no
+],[
+  ac_cv_target_windows=no
+])])
+
+])
+
+dnl
+dnl PHP_CHECK_UNIX_TARGET
+dnl
+dnl check if we're compiling for a unix-ish target
+dnl
+AC_DEFUN([PHP_CHECK_UNIX_TARGET], [
+  AC_CACHE_CHECK([for unix-ish target],ac_cv_target_unix,[
+  AC_RUN_IFELSE([AC_LANG_SOURCE([[
+int main(void) {
+#if defined(unix) || defined(__unix) || defined(__unix__)
+    return 0;
+#else
+    return 1;
+#endif
+}
+]])],[
+  ac_cv_target_unix=yes
+],[
+  ac_cv_target_unix=no
+],[
+  ac_cv_target_unix=no
+])])
+
+])
diff --git a/configure.ac b/configure.ac
@@ -324,6 +324,15 @@ PHP_EBCDIC
 dnl Check whether the system byte ordering is bigendian.
 PHP_C_BIGENDIAN
 
+dnl Check if we're targeting x86 / x86_64
+PHP_CHECK_X86_TARGET
+
+dnl Check if we're targeting Windows
+PHP_CHECK_WINDOWS_TARGET
+
+dnl Check whether we're targeting a unix-ish system
+PHP_CHECK_UNIX_TARGET
+
 dnl Check whether writing to stdout works.
 PHP_TEST_WRITE_STDOUT
 

diff --git a/ext/hash/blake3/blake3.c b/ext/hash/blake3/blake3.c
@@ -5,6 +5,10 @@
 #include "blake3.h"
 #include "blake3_impl.h"
 
+const char * blake3_version(void) {
+  return BLAKE3_VERSION_STRING;
+}
+
 INLINE void chunk_state_init(blake3_chunk_state *self, const uint32_t key[8],
                              uint8_t flags) {
   memcpy(self->cv, key, BLAKE3_KEY_LEN);

diff --git a/ext/hash/blake3/blake3.h b/ext/hash/blake3/blake3.h
@@ -8,6 +8,7 @@
 extern "C" {
 #endif
 
+#define BLAKE3_VERSION_STRING "0.3.7"
 #define BLAKE3_KEY_LEN 32
 #define BLAKE3_OUT_LEN 32
 #define BLAKE3_BLOCK_LEN 64
@@ -38,6 +39,7 @@ typedef struct {
   uint8_t cv_stack[(BLAKE3_MAX_DEPTH + 1) * BLAKE3_OUT_LEN];
 } blake3_hasher;
 
+const char * blake3_version(void);
 void blake3_hasher_init(blake3_hasher *self);
 void blake3_hasher_init_keyed(blake3_hasher *self,
                               const uint8_t key[BLAKE3_KEY_LEN]);

diff --git a/ext/hash/config.m4 b/ext/hash/config.m4
@@ -11,7 +11,28 @@ if test "$PHP_MHASH" != "no"; then
   AC_DEFINE(PHP_MHASH_BC, 1, [ ])
 fi
 
+
+
+PHP_ARG_WITH([blake3-upstream-c-source-dir],
+  [for upstream blake3 implementation, github.com/blake3-team/BLAKE3/tree/0.3.7/c ],
+  [AS_HELP_STRING([[--with-blake3-upstream-c-source-dir=DIR]],
+    [Include upstream blake3 source code, github.com/blake3-team/BLAKE3/tree/0.3.7/c])],
+    "", "")
+
+if test "$PHP_BLAKE3_UPSTREAM_C_SOURCE_DIR" = ""; then
+  EXT_HASH_BLAKE3_SOURCES="hash_blake3.c blake3/blake3.c blake3/blake3_dispatch.c blake3/blake3_portable.c"
+  EXT_HASH_BLAKE3_CFLAGS="-I@ext_srcdir@/blake3"
+  PHP_ADD_BUILD_DIR(ext/hash/blake3, 1)
+else
+  AC_DEFINE(PHP_BLAKE3_UPSTREAM_C_SOURCE_DIR, $PHP_BLAKE3_UPSTREAM_C_SOURCE_DIR, [ ])
+  EXT_HASH_BLAKE3_SOURCES="hash_blake3.c $PHP_BLAKE3_UPSTREAM_C_SOURCE_DIR/blake3.c $PHP_BLAKE3_UPSTREAM_C_SOURCE_DIR/blake3_dispatch.c $PHP_BLAKE3_UPSTREAM_C_SOURCE_DIR/blake3_portable.c"
+  PHP_ADD_BUILD_DIR($PHP_BLAKE3_UPSTREAM_C_SOURCE_DIR, 1)
+  EXT_HASH_BLAKE3_CFLAGS="-I$PHP_BLAKE3_UPSTREAM_C_SOURCE_DIR"
+fi
+
+
 if test $ac_cv_c_bigendian_php = yes; then
+  dnl todo: check if $PHP_BLAKE3_UPSTREAM_C_SOURCE_DIR/blake3_neon.c is applicable ?
   EXT_HASH_SHA3_SOURCES="hash_sha3.c"
   AC_DEFINE(HAVE_SLOW_HASH3, 1, [Define is hash3 algo is available])
   AC_MSG_WARN("Use SHA3 slow implementation on bigendian")
@@ -20,26 +41,53 @@ else
   AC_MSG_CHECKING([if we're at 64-bit platform])
   AS_IF([test "$ac_cv_sizeof_long" -eq 4],[
     AC_MSG_RESULT([no])
+    if test $ac_cv_target_x86 = yes; then 
+      dnl i think there are some 32bit x86 cpus with SSE2 but.. cba
+      EXT_HASH_BLAKE3_CFLAGS="$EXT_HASH_BLAKE3_CFLAGS -DBLAKE3_NO_SSE2 -DBLAKE3_NO_SSE41 -DBLAKE3_NO_AVX2 -DBLAKE3_NO_AVX512"
+    fi
     SHA3_DIR="sha3/generic32lc"
     SHA3_OPT_SRC="$SHA3_DIR/KeccakP-1600-inplace32BI.c"
   ],[
     AC_MSG_RESULT([yes])
     SHA3_DIR="sha3/generic64lc"
     SHA3_OPT_SRC="$SHA3_DIR/KeccakP-1600-opt64.c"
+    if test $ac_cv_target_x86 = yes; then 
+      if test "$PHP_BLAKE3_UPSTREAM_C_SOURCE_DIR" = ""; then
+        dnl AC_MSG_WARN("Use BLAKE3 slow/portable implementation")
+        EXT_HASH_BLAKE3_CFLAGS="$EXT_HASH_BLAKE3_CFLAGS -DBLAKE3_NO_SSE2 -DBLAKE3_NO_SSE41 -DBLAKE3_NO_AVX2 -DBLAKE3_NO_AVX512"
+      else
+        if test $ac_cv_target_windows = yes; then
+          dnl x86_64 windows gnuc
+          EXT_HASH_BLAKE3_SOURCES="$EXT_HASH_BLAKE3_SOURCES $PHP_BLAKE3_UPSTREAM_C_SOURCE_DIR/blake3_avx512_x86-64_windows_gnu.S $PHP_BLAKE3_UPSTREAM_C_SOURCE_DIR/blake3_avx2_x86-64_windows_gnu.S $PHP_BLAKE3_UPSTREAM_C_SOURCE_DIR/blake3_sse41_x86-64_windows_gnu.S $PHP_BLAKE3_UPSTREAM_C_SOURCE_DIR/blake3_sse2_x86-64_windows_gnu.S"
+        else
+          if test $ac_cv_target_unix = yes; then 
+            dnl x86_64 unix gnuc
+            EXT_HASH_BLAKE3_SOURCES="$EXT_HASH_BLAKE3_SOURCES $PHP_BLAKE3_UPSTREAM_C_SOURCE_DIR/blake3_avx512_x86-64_unix.S $PHP_BLAKE3_UPSTREAM_C_SOURCE_DIR/blake3_avx2_x86-64_unix.S $PHP_BLAKE3_UPSTREAM_C_SOURCE_DIR/blake3_sse41_x86-64_unix.S $PHP_BLAKE3_UPSTREAM_C_SOURCE_DIR/blake3_sse2_x86-64_unix.S"
+          else
+            dnl blake still has C instrictics versions (eg $PHP_BLAKE3_UPSTREAM_C_SOURCE_DIR/blake3_avx512.c  ) which COULD be used in this situation, 
+            dnl we're targeting x86_64 but neither windows nor unix-ish.. 
+            dnl i have nothing to test this on, so i'm just disabling it for now, feel free to fix it.
+            EXT_HASH_BLAKE3_CFLAGS="$EXT_HASH_BLAKE3_CFLAGS -DBLAKE3_NO_SSE2 -DBLAKE3_NO_SSE41 -DBLAKE3_NO_AVX2 -DBLAKE3_NO_AVX512"
+          fi
+        fi
+      fi
+    fi
   ])
   EXT_HASH_SHA3_SOURCES="$SHA3_OPT_SRC $SHA3_DIR/KeccakHash.c $SHA3_DIR/KeccakSponge.c hash_sha3.c"
   PHP_HASH_CFLAGS="-I@ext_srcdir@/$SHA3_DIR -DKeccakP200_excluded -DKeccakP400_excluded -DKeccakP800_excluded -DZEND_ENABLE_STATIC_TSRMLS_CACHE=1"
 
   PHP_ADD_BUILD_DIR(ext/hash/$SHA3_DIR, 1)
 fi
+PHP_HASH_CFLAGS="$PHP_HASH_CFLAGS $EXT_HASH_BLAKE3_CFLAGS"
 
 EXT_HASH_SOURCES="hash.c hash_md.c hash_sha.c hash_ripemd.c hash_haval.c \
   hash_tiger.c hash_gost.c hash_snefru.c hash_whirlpool.c hash_adler32.c \
-  hash_crc32.c hash_fnv.c hash_joaat.c $EXT_HASH_SHA3_SOURCES"
+  hash_crc32.c hash_fnv.c hash_joaat.c $EXT_HASH_SHA3_SOURCES \
+  $EXT_HASH_BLAKE3_SOURCES"
 EXT_HASH_HEADERS="php_hash.h php_hash_md.h php_hash_sha.h php_hash_ripemd.h \
   php_hash_haval.h php_hash_tiger.h php_hash_gost.h php_hash_snefru.h \
   php_hash_whirlpool.h php_hash_adler32.h php_hash_crc32.h \
-  php_hash_fnv.h php_hash_joaat.h php_hash_sha3.h"
+  php_hash_fnv.h php_hash_joaat.h php_hash_sha3.h php_hash_blake3.h"
 
 PHP_NEW_EXTENSION(hash, $EXT_HASH_SOURCES, 0,,$PHP_HASH_CFLAGS)
 PHP_INSTALL_HEADERS(ext/hash, $EXT_HASH_HEADERS)
diff --git a/ext/hash/config.w32 b/ext/hash/config.w32
@@ -11,7 +11,9 @@ PHP_HASH = 'yes';
 EXTENSION('hash',	'hash.c hash_md.c hash_sha.c hash_ripemd.c hash_haval.c ' +
 					'hash_tiger.c hash_gost.c hash_snefru.c hash_whirlpool.c ' +
 					'hash_adler32.c hash_crc32.c hash_joaat.c hash_fnv.c ' +
-					'hash_sha3.c', false);
+					'hash_sha3.c hash_blake3.c', false);
+
+ADD_SOURCES('ext/hash/blake3', 'blake3.c blake3_dispatch.c blake3_portable.c');
 
 var hash_sha3_dir = 'ext/hash/sha3/generic' + (X64 ? '64' : '32') + 'lc';
 
@@ -26,9 +28,11 @@ if (!CHECK_HEADER_ADD_INCLUDE('KeccakHash.h', 'CFLAGS_HASH', hash_sha3_dir)) {
 	ERROR('Unable to locate SHA3 headers');
 }
 
-ADD_FLAG('CFLAGS_HASH', '/DKeccakP200_excluded /DKeccakP400_excluded /DKeccakP800_excluded /DZEND_ENABLE_STATIC_TSRMLS_CACHE=1');
+ADD_FLAG('CFLAGS_HASH', '/DKeccakP200_excluded /DKeccakP400_excluded /DKeccakP800_excluded /DZEND_ENABLE_STATIC_TSRMLS_CACHE=1 ' +
+                        '/DBLAKE3_NO_SSE2 /DBLAKE3_NO_SSE41 /DBLAKE3_NO_AVX2 /DBLAKE3_NO_AVX512');
 
 PHP_INSTALL_HEADERS('ext/hash/',	'php_hash.h php_hash_md.h php_hash_sha.h ' +
 									'php_hash_ripemd.h php_hash_haval.h php_hash_tiger.h ' +
 									'php_hash_gost.h php_hash_snefru.h php_hash_whirlpool.h ' +
-									'php_hash_adler32.h php_hash_crc32.h php_hash_sha3.h');
+									'php_hash_adler32.h php_hash_crc32.h php_hash_sha3.h ' +
+									'php_hash_blake3.h');
diff --git a/ext/hash/hash.c b/ext/hash/hash.c
@@ -19,6 +19,8 @@
 #include "config.h"
 #endif
 
+#include "blake3.h"
+
 #include <math.h>
 #include "php_hash.h"
 #include "ext/standard/info.h"
@@ -1586,6 +1588,7 @@ PHP_MINIT_FUNCTION(hash)
 	php_hash_register_algo("fnv164",		&php_hash_fnv164_ops);
 	php_hash_register_algo("fnv1a64",		&php_hash_fnv1a64_ops);
 	php_hash_register_algo("joaat",			&php_hash_joaat_ops);
+	php_hash_register_algo("blake3",		&php_hash_blake3_ops);
 
 	PHP_HASH_HAVAL_REGISTER(3,128);
 	PHP_HASH_HAVAL_REGISTER(3,160);
@@ -1650,6 +1653,11 @@ PHP_MINFO_FUNCTION(hash)
 	php_info_print_table_start();
 	php_info_print_table_row(2, "hash support", "enabled");
 	php_info_print_table_row(2, "Hashing Engines", buffer);
+#ifdef PHP_BLAKE3_UPSTREAM_C_SOURCE_DIR
+	php_info_print_table_row(2, "blake3 implementation", "upstream " BLAKE3_VERSION_STRING);
+#else
+	php_info_print_table_row(2, "blake3 implementation", "portable " BLAKE3_VERSION_STRING);
+#endif
 	php_info_print_table_end();
 
 #ifdef PHP_MHASH_BC

diff --git a/ext/hash/hash_blake3.c b/ext/hash/hash_blake3.c
@@ -0,0 +1,27 @@
+#include "php_hash.h"
+#include "php_hash_blake3.h"
+#include "blake3.h"
+
+
+#include <string.h> // memcpy
+
+PHP_HASH_API void PHP_BLAKE3Init(PHP_BLAKE3_CTX *context)
+{
+	blake3_hasher_init(context);
+}
+
+PHP_HASH_API void PHP_BLAKE3Update(PHP_BLAKE3_CTX *context, const unsigned char *input, size_t len)
+{
+	blake3_hasher_update(context, input, len);
+}
+
+PHP_HASH_API void PHP_BLAKE3Final(unsigned char digest[BLAKE3_OUT_LEN/*32*/], PHP_BLAKE3_CTX *context)
+{
+	blake3_hasher_finalize(context, digest, BLAKE3_OUT_LEN);
+}
+
+PHP_HASH_API int PHP_BLAKE3Copy(const php_hash_ops *ops, PHP_BLAKE3_CTX *orig_context, PHP_BLAKE3_CTX *copy_context)
+{
+	memcpy(copy_context, orig_context, sizeof(*orig_context));
+	return SUCCESS;
+}
diff --git a/ext/hash/php_hash.h b/ext/hash/php_hash.h
@@ -105,6 +105,7 @@ extern const php_hash_ops php_hash_fnv1a32_ops;
 extern const php_hash_ops php_hash_fnv164_ops;
 extern const php_hash_ops php_hash_fnv1a64_ops;
 extern const php_hash_ops php_hash_joaat_ops;
+extern const php_hash_ops php_hash_blake3_ops;
 
 #define PHP_HASH_HAVAL_OPS(p,b)	extern const php_hash_ops php_hash_##p##haval##b##_ops;
 

diff --git a/ext/hash/php_hash_blake3.h b/ext/hash/php_hash_blake3.h
@@ -0,0 +1,35 @@
+#ifndef PHP_HASH_BLAKE3_H
+#define PHP_HASH_BLAKE3_H
+
+#include "ext/standard/basic_functions.h"
+#include "php_hash.h"
+#include "blake3.h"
+
+
+// typedef struct blake3_hasher PHP_BLAKE3_CTX;
+#define PHP_BLAKE3_CTX blake3_hasher
+// help: is V correct?
+#define PHP_BLAKE3_SPEC "b8b8qb64bbbbb1760"
+
+PHP_HASH_API void PHP_BLAKE3Init(PHP_BLAKE3_CTX *context);
+PHP_HASH_API void PHP_BLAKE3Update(PHP_BLAKE3_CTX *context, const unsigned char *input, size_t len);
+PHP_HASH_API void PHP_BLAKE3Final(unsigned char digest[BLAKE3_OUT_LEN/*32*/], PHP_BLAKE3_CTX *context);
+PHP_HASH_API int PHP_BLAKE3Copy(const php_hash_ops *ops, PHP_BLAKE3_CTX *orig_context, PHP_BLAKE3_CTX *copy_context);
+
+const php_hash_ops php_hash_blake3_ops = {
+	"blake3",
+	(php_hash_init_func_t) PHP_BLAKE3Init,
+	(php_hash_update_func_t) PHP_BLAKE3Update,
+	(php_hash_final_func_t) PHP_BLAKE3Final,
+	(php_hash_copy_func_t) PHP_BLAKE3Copy,
+	php_hash_serialize,
+	php_hash_unserialize,
+	PHP_BLAKE3_SPEC, // << don't know what this should be, hopefully a dev that knows can remove this comment
+	BLAKE3_OUT_LEN /*32*/,
+	BLAKE3_CHUNK_LEN /*1024*/,
+	sizeof(PHP_BLAKE3_CTX),
+	1
+};
+
+#endif
+
diff --git a/ext/hash/tests/hash-clone.phpt b/ext/hash/tests/hash-clone.phpt
@@ -143,6 +143,9 @@ string(16) "bebc746a33b6ab62"
 string(5) "joaat"
 string(8) "aaebf370"
 string(8) "aaebf370"
+string(6) "blake3"
+string(64) "e232493e3d416e24ffc588b24a1772ccc6f80290f1920cf15f21313bc3543a51"
+string(64) "e232493e3d416e24ffc588b24a1772ccc6f80290f1920cf15f21313bc3543a51"
 string(10) "haval128,3"
 string(32) "86362472c8895e68e223ef8b3711d8d9"
 string(32) "86362472c8895e68e223ef8b3711d8d9"
@@ -302,6 +305,9 @@ string(16) "893899e4415a920f"
 string(5) "joaat"
 string(8) "aaebf370"
 string(8) "836fb0e5"
+string(6) "blake3"
+string(64) "e232493e3d416e24ffc588b24a1772ccc6f80290f1920cf15f21313bc3543a51"
+string(64) "dbdea45e5a6c3bad18a4f96d9d4b9105e4cceaa4fc06568f69829435c47587fb"
 string(10) "haval128,3"
 string(32) "86362472c8895e68e223ef8b3711d8d9"
 string(32) "ebeeeb05c18af1e53d2d127b561d5e0d"

diff --git a/ext/hash/tests/hash_algos.phpt b/ext/hash/tests/hash_algos.phpt
@@ -9,7 +9,7 @@ var_dump(hash_algos());
 ?>
 --EXPECTF--
 *** Testing hash_algos() : basic functionality ***
-array(53) {
+array(54) {
   [%d]=>
   string(3) "md2"
   [%d]=>
@@ -87,6 +87,8 @@ array(53) {
   [%d]=>
   string(5) "joaat"
   [%d]=>
+  string(6) "blake3"
+  [%d]=>
   string(10) "haval128,3"
   [%d]=>
   string(10) "haval160,3"

diff --git a/ext/hash/tests/hash_copy_001.phpt b/ext/hash/tests/hash_copy_001.phpt
@@ -143,6 +143,9 @@ string(16) "bebc746a33b6ab62"
 string(5) "joaat"
 string(8) "aaebf370"
 string(8) "aaebf370"
+string(6) "blake3"
+string(64) "e232493e3d416e24ffc588b24a1772ccc6f80290f1920cf15f21313bc3543a51"
+string(64) "e232493e3d416e24ffc588b24a1772ccc6f80290f1920cf15f21313bc3543a51"
 string(10) "haval128,3"
 string(32) "86362472c8895e68e223ef8b3711d8d9"
 string(32) "86362472c8895e68e223ef8b3711d8d9"
@@ -302,6 +305,9 @@ string(16) "893899e4415a920f"
 string(5) "joaat"
 string(8) "aaebf370"
 string(8) "836fb0e5"
+string(6) "blake3"
+string(64) "e232493e3d416e24ffc588b24a1772ccc6f80290f1920cf15f21313bc3543a51"
+string(64) "dbdea45e5a6c3bad18a4f96d9d4b9105e4cceaa4fc06568f69829435c47587fb"
 string(10) "haval128,3"
 string(32) "86362472c8895e68e223ef8b3711d8d9"
 string(32) "ebeeeb05c18af1e53d2d127b561d5e0d"