Restore ASMABI and other Unify work

Make sure all SHA2 transform function has wrappers

For ASMABI to work, it is required the calling convention
is consistent.

Signed-off-by: Joergen Lundman <lundman@lundman.net>

Author: lundman

module/icp/algs/blake3/blake3_impl.c

@@ -34,15 +34,15 @@
 	(defined(__x86_64) && defined(HAVE_SSE2)) || \
 	(defined(__PPC64__) && defined(__LITTLE_ENDIAN__))
 
-extern void zfs_blake3_compress_in_place_sse2(uint32_t cv[8],
+extern void ASMABI zfs_blake3_compress_in_place_sse2(uint32_t cv[8],
     const uint8_t block[BLAKE3_BLOCK_LEN], uint8_t block_len,
     uint64_t counter, uint8_t flags);
 
-extern void zfs_blake3_compress_xof_sse2(const uint32_t cv[8],
+extern void ASMABI zfs_blake3_compress_xof_sse2(const uint32_t cv[8],
     const uint8_t block[BLAKE3_BLOCK_LEN], uint8_t block_len,
     uint64_t counter, uint8_t flags, uint8_t out[64]);
 
-extern void zfs_blake3_hash_many_sse2(const uint8_t * const *inputs,
+extern void ASMABI zfs_blake3_hash_many_sse2(const uint8_t * const *inputs,
     size_t num_inputs, size_t blocks, const uint32_t key[8],
     uint64_t counter, boolean_t increment_counter, uint8_t flags,
     uint8_t flags_start, uint8_t flags_end, uint8_t *out);
@@ -100,15 +100,15 @@ const blake3_ops_t blake3_sse2_impl = {
 	(defined(__x86_64) && defined(HAVE_SSE2)) || \
 	(defined(__PPC64__) && defined(__LITTLE_ENDIAN__))
 
-extern void zfs_blake3_compress_in_place_sse41(uint32_t cv[8],
+extern void ASMABI zfs_blake3_compress_in_place_sse41(uint32_t cv[8],
     const uint8_t block[BLAKE3_BLOCK_LEN], uint8_t block_len,
     uint64_t counter, uint8_t flags);
 
-extern void zfs_blake3_compress_xof_sse41(const uint32_t cv[8],
+extern void ASMABI zfs_blake3_compress_xof_sse41(const uint32_t cv[8],
     const uint8_t block[BLAKE3_BLOCK_LEN], uint8_t block_len,
     uint64_t counter, uint8_t flags, uint8_t out[64]);
 
-extern void zfs_blake3_hash_many_sse41(const uint8_t * const *inputs,
+extern void ASMABI zfs_blake3_hash_many_sse41(const uint8_t * const *inputs,
     size_t num_inputs, size_t blocks, const uint32_t key[8],
     uint64_t counter, boolean_t increment_counter, uint8_t flags,
     uint8_t flags_start, uint8_t flags_end, uint8_t *out);
@@ -163,7 +163,7 @@ const blake3_ops_t blake3_sse41_impl = {
 #endif
 
 #if defined(__x86_64) && defined(HAVE_SSE4_1) && defined(HAVE_AVX2)
-extern void zfs_blake3_hash_many_avx2(const uint8_t * const *inputs,
+extern void ASMABI zfs_blake3_hash_many_avx2(const uint8_t * const *inputs,
     size_t num_inputs, size_t blocks, const uint32_t key[8],
     uint64_t counter, boolean_t increment_counter, uint8_t flags,
     uint8_t flags_start, uint8_t flags_end, uint8_t *out);
@@ -196,15 +196,15 @@ blake3_avx2_impl = {
 #endif
 
 #if defined(__x86_64) && defined(HAVE_AVX512F) && defined(HAVE_AVX512VL)
-extern void zfs_blake3_compress_in_place_avx512(uint32_t cv[8],
+extern void ASMABI zfs_blake3_compress_in_place_avx512(uint32_t cv[8],
     const uint8_t block[BLAKE3_BLOCK_LEN], uint8_t block_len,
     uint64_t counter, uint8_t flags);
 
-extern void zfs_blake3_compress_xof_avx512(const uint32_t cv[8],
+extern void ASMABI zfs_blake3_compress_xof_avx512(const uint32_t cv[8],
     const uint8_t block[BLAKE3_BLOCK_LEN], uint8_t block_len,
     uint64_t counter, uint8_t flags, uint8_t out[64]);
 
-extern void zfs_blake3_hash_many_avx512(const uint8_t * const *inputs,
+extern void ASMABI zfs_blake3_hash_many_avx512(const uint8_t * const *inputs,
     size_t num_inputs, size_t blocks, const uint32_t key[8],
     uint64_t counter, boolean_t increment_counter, uint8_t flags,
     uint8_t flags_start, uint8_t flags_end, uint8_t *out);

module/icp/algs/sha2/sha256_impl.c

@@ -29,9 +29,10 @@
 #include <sys/simd.h>
 
 #include <sha2/sha2_impl.h>
+#include <sys/asm_linkage.h>
 
 #define	TF(E, N) \
-	extern void E(uint32_t s[8], const void *, size_t); \
+	extern void ASMABI E(uint32_t s[8], const void *, size_t); \
 	static inline void N(uint32_t s[8], const void *d, size_t b) { \
 	kfpu_begin(); E(s, d, b); kfpu_end(); \
 }
@@ -44,10 +45,19 @@ static inline boolean_t sha2_is_supported(void)
 
 #if defined(__x86_64)
 
-extern void zfs_sha256_transform_x64(uint32_t s[8], const void *, size_t);
+/* Users of ASMABI requires all calls to be from wrappers */
+extern void ASMABI
+zfs_sha256_transform_x64(uint32_t s[8], const void *, size_t);
+
+static inline void
+tf_sha256_transform_x64(uint32_t s[8], const void *d, size_t b)
+{
+	zfs_sha256_transform_x64(s, d, b);
+}
+
 const sha256_ops_t sha256_x64_impl = {
 	.is_supported = sha2_is_supported,
-	.transform = zfs_sha256_transform_x64,
+	.transform = tf_sha256_transform_x64,
 	.name = "x64"
 };
 

module/icp/algs/sha2/sha512_impl.c

@@ -29,9 +29,10 @@
 #include <sys/simd.h>
 
 #include <sha2/sha2_impl.h>
+#include <sys/asm_linkage.h>
 
 #define	TF(E, N) \
-	extern void E(uint64_t s[8], const void *, size_t); \
+	extern void ASMABI E(uint64_t s[8], const void *, size_t); \
 	static inline void N(uint64_t s[8], const void *d, size_t b) { \
 	kfpu_begin(); E(s, d, b); kfpu_end(); \
 }
@@ -44,10 +45,18 @@ static inline boolean_t sha2_is_supported(void)
 
 #if defined(__x86_64)
 
-extern void zfs_sha512_transform_x64(uint64_t s[8], const void *, size_t);
+/* Users of ASMABI requires all calls to be from wrappers */
+extern void ASMABI
+zfs_sha512_transform_x64(uint32_t s[8], const void *, size_t);
+
+static inline void
+tf_sha512_transform_x64(uint32_t s[8], const void *d, size_t b)
+{
+	zfs_sha512_transform_x64(s, d, b);
+}
 const sha512_ops_t sha512_x64_impl = {
 	.is_supported = sha2_is_supported,
-	.transform = zfs_sha512_transform_x64,
+	.transform = tf_sha512_transform_x64,
 	.name = "x64"
 };
 

module/icp/asm-x86_64/sha2/sha256-x86_64.S

@@ -27,7 +27,7 @@
 SECTION_STATIC
 
 .align	64
-.type	K256,@object
+SET_OBJ(K256)
 K256:
 .long	0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5
 .long	0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5

module/icp/asm-x86_64/sha2/sha512-x86_64.S

@@ -27,7 +27,7 @@
 SECTION_STATIC
 
 .align	64
-.type	K512,@object
+SET_OBJ(K512)
 K512:
 .quad	0x428a2f98d728ae22,0x7137449123ef65cd
 .quad	0x428a2f98d728ae22,0x7137449123ef65cd