Skip to content

Address HACL SIMD compilation in a better way #645

Open
@geofft

Description

@geofft

HACL has some optimized SIMD implementations that are conditionally compiled in by CPython's build system. Because we rework the build process for extension modules, we ended up with a build failing because these implementations were referenced but not compiled. The simplest thing to do for #610 was to change the build system to unconditionally choose not to use the SIMD implementations (patch-python-configure-hacl-no-simd.patch).

Also, I'm a little unsure whether this properly handles the case where the target machine doesn't actually have SIMD (x86_64-v2 for SIMD128, x86_64-v3 for SIMD256), so it seemed safer to turn it off until confirming that. Also, there are some comments about the fact that the performance is worse on arm64.

We should investigate this in more detail. I imagine that we'll probably want to fix (or ideally get rid of) our custom build for extension modules. We may also need to submit something to upstream CPython to properly handle the case where we're building with a toolchain that supports SSE/AVX but we want to produce binaries that work without it.

For reference, here's the relevant code in configure.ac (as of HEAD right now, i.e. 3.15.0a):

dnl Check if universal2 HACL* implementation should be used.
if test "$UNIVERSAL_ARCHS" = "universal2" -o \
   \( "$build_cpu" = "aarch64" -a "$build_vendor" = "apple" \)
then
  use_hacl_universal2_impl=yes
else
  use_hacl_universal2_impl=no
fi

# The SIMD files use aligned_alloc, which is not available on older versions of
# Android.
# The *mmintrin.h headers are x86-family-specific, so can't be used on WASI.
if test "$ac_sys_system" != "Linux-android" -a "$ac_sys_system" != "WASI" || \
   { test -n "$ANDROID_API_LEVEL" && test "$ANDROID_API_LEVEL" -ge 28; }
then
  dnl This can be extended here to detect e.g. Power8, which HACL* should also support.
  AX_CHECK_COMPILE_FLAG([-msse -msse2 -msse3 -msse4.1 -msse4.2],[
    [LIBHACL_SIMD128_FLAGS="-msse -msse2 -msse3 -msse4.1 -msse4.2"]

    AC_DEFINE([HACL_CAN_COMPILE_SIMD128], [1], [HACL* library can compile SIMD128 implementations])

    # macOS universal2 builds *support* the -msse etc flags because they're
    # available on x86_64. However, performance of the HACL SIMD128 implementation
    # isn't great, so it's disabled on ARM64.
    AC_MSG_CHECKING([for HACL* SIMD128 implementation])
    if test "$use_hacl_universal2_impl" = "yes"; then
      [LIBHACL_BLAKE2_SIMD128_OBJS="Modules/_hacl/Hacl_Hash_Blake2s_Simd128_universal2.o"]
      AC_MSG_RESULT([universal2])
    else
      [LIBHACL_BLAKE2_SIMD128_OBJS="Modules/_hacl/Hacl_Hash_Blake2s_Simd128.o"]
      AC_MSG_RESULT([standard])
    fi

  ], [], [-Werror])
fi
AC_SUBST([LIBHACL_SIMD128_FLAGS])
AC_SUBST([LIBHACL_BLAKE2_SIMD128_OBJS])

# The SIMD files use aligned_alloc, which is not available on older versions of
# Android.
# The *mmintrin.h headers are x86-family-specific, so can't be used on WASI.
#
# Although AVX support is not guaranteed on Android
# (https://developer.android.com/ndk/guides/abis#86-64), this is safe because we do a
# runtime CPUID check.
if test "$ac_sys_system" != "Linux-android" -a "$ac_sys_system" != "WASI" || \
   { test -n "$ANDROID_API_LEVEL" && test "$ANDROID_API_LEVEL" -ge 28; }
then
  AX_CHECK_COMPILE_FLAG([-mavx2],[
    [LIBHACL_SIMD256_FLAGS="-mavx2"]
    AC_DEFINE([HACL_CAN_COMPILE_SIMD256], [1], [HACL* library can compile SIMD256 implementations])

    # macOS universal2 builds *support* the -mavx2 compiler flag because it's
    # available on x86_64; but the HACL SIMD256 build then fails because the
    # implementation requires symbols that aren't available on ARM64. Use a
    # wrapped implementation if we're building for universal2.
    AC_MSG_CHECKING([for HACL* SIMD256 implementation])
    if test "$use_hacl_universal2_impl" = "yes"; then
      [LIBHACL_BLAKE2_SIMD256_OBJS="Modules/_hacl/Hacl_Hash_Blake2b_Simd256_universal2.o"]
      AC_MSG_RESULT([universal2])
    else
      [LIBHACL_BLAKE2_SIMD256_OBJS="Modules/_hacl/Hacl_Hash_Blake2b_Simd256.o"]
      AC_MSG_RESULT([standard])
    fi
  ], [], [-Werror])
fi
AC_SUBST([LIBHACL_SIMD256_FLAGS])
AC_SUBST([LIBHACL_BLAKE2_SIMD256_OBJS])
### end(HACL* configuration)

and Makefile.pre.in does

LIBHACL_BLAKE2_SIMD128_CFLAGS=@LIBHACL_SIMD128_FLAGS@ -DHACL_CAN_COMPILE_VEC128
LIBHACL_BLAKE2_SIMD256_CFLAGS=@LIBHACL_SIMD256_FLAGS@ -DHACL_CAN_COMPILE_VEC256
LIBHACL_BLAKE2_SIMD128_OBJS=@LIBHACL_BLAKE2_SIMD128_OBJS@
LIBHACL_BLAKE2_SIMD256_OBJS=@LIBHACL_BLAKE2_SIMD256_OBJS@
LIBHACL_BLAKE2_OBJS= \
                Modules/_hacl/Hacl_Hash_Blake2s.o \
                Modules/_hacl/Hacl_Hash_Blake2b.o \
                Modules/_hacl/Lib_Memzero0.o \
                $(LIBHACL_BLAKE2_SIMD128_OBJS) \
                $(LIBHACL_BLAKE2_SIMD256_OBJS)
Modules/_hacl/Hacl_Hash_Blake2s.o: $(srcdir)/Modules/_hacl/Hacl_Hash_Blake2s.c $(LIBHACL_BLAKE2_HEADERS)
        $(CC) -c $(LIBHACL_CFLAGS) -o $@ $(srcdir)/Modules/_hacl/Hacl_Hash_Blake2s.c
Modules/_hacl/Hacl_Hash_Blake2b.o: $(srcdir)/Modules/_hacl/Hacl_Hash_Blake2b.c $(LIBHACL_BLAKE2_HEADERS)
        $(CC) -c $(LIBHACL_CFLAGS) -o $@ $(srcdir)/Modules/_hacl/Hacl_Hash_Blake2b.c
Modules/_hacl/Hacl_Hash_Blake2s_Simd128.o: $(srcdir)/Modules/_hacl/Hacl_Hash_Blake2s_Simd128.c $(LIBHACL_BLAKE2_HEADERS)
        $(CC) -c $(LIBHACL_CFLAGS) $(LIBHACL_BLAKE2_SIMD128_CFLAGS) -o $@ $(srcdir)/Modules/_hacl/Hacl_Hash_Blake2s_Simd128.c
Modules/_hacl/Hacl_Hash_Blake2s_Simd128_universal2.o: $(srcdir)/Modules/_hacl/Hacl_Hash_Blake2s_Simd128_universal2.c $(LIBHACL_BLAKE2_HEADERS)
        $(CC) -c $(LIBHACL_CFLAGS) $(LIBHACL_BLAKE2_SIMD128_CFLAGS) -o $@ $(srcdir)/Modules/_hacl/Hacl_Hash_Blake2s_Simd128_universal2.c
Modules/_hacl/Hacl_Hash_Blake2b_Simd256.o: $(srcdir)/Modules/_hacl/Hacl_Hash_Blake2b_Simd256.c $(LIBHACL_BLAKE2_HEADERS)
        $(CC) -c $(LIBHACL_CFLAGS) $(LIBHACL_BLAKE2_SIMD256_CFLAGS) -o $@ $(srcdir)/Modules/_hacl/Hacl_Hash_Blake2b_Simd256.c
Modules/_hacl/Hacl_Hash_Blake2b_Simd256_universal2.o: $(srcdir)/Modules/_hacl/Hacl_Hash_Blake2b_Simd256_universal2.c $(LIBHACL_BLAKE2_HEADERS)
        $(CC) -c $(LIBHACL_CFLAGS) $(LIBHACL_BLAKE2_SIMD256_CFLAGS) -o $@ $(srcdir)/Modules/_hacl/Hacl_Hash_Blake2b_Simd256_universal2.c

Metadata

Metadata

Assignees

No one assigned

    Labels

    No labels
    No labels

    Type

    No type

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions