WL#14633: Add support for jemalloc on Windows

Use jemalloc.dll (if present) for my_malloc in my_sys.lib and for
ut::malloc etc in innobase.lib via function pointers.

The function pointers are initialized to std::malloc, std::free etc
so if jemalloc.dll is not present, the standard memory management
functions are used.

The use of the jemalloc functions is rendered "off by default" by the
presence of a use_jemalloc_allocations function in mysys.lib that
returns false.

Clients of mysys.lib and innobase.lib wishing to use jemalloc (currently
only the MySQL server executable and unit tests) enable it by providing an
"overriding" definition of the use_jemalloc_allocations function that
returns true.

RB: 26872
Change-Id: If2e12d2e8504cb63c6ea0147832e2bb07ed13cb4

include/jemalloc_win.h

@@ -0,0 +1,69 @@
+/*****************************************************************************
+
+Copyright (c) 2021, Oracle and/or its affiliates.
+
+This program is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License, version 2.0, as published by the
+Free Software Foundation.
+
+This program is also distributed with certain software (including but not
+limited to OpenSSL) that is licensed under separate terms, as designated in a
+particular file or component or in included license documentation. The authors
+of MySQL hereby grant you an additional permission to link the program and
+your derivative works with the separately licensed software that they have
+included with MySQL.
+
+This program is distributed in the hope that it will be useful, but WITHOUT
+ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+FOR A PARTICULAR PURPOSE. See the GNU General Public License, version 2.0,
+for more details.
+
+You should have received a copy of the GNU General Public License along with
+this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin St, Fifth Floor, Boston, MA 02110-1301  USA
+
+*****************************************************************************/
+#ifndef JEMALLOC_WIN_INCLUDED
+#define JEMALLOC_WIN_INCLUDED
+
+/** @file jemalloc_win.h
+ Details for dynamically loading and using jemalloc.dll on Windows */
+
+#ifdef _WIN32
+#include <mutex>
+#include <string>
+#include <vector>
+#include "my_loglevel.h"
+const constexpr char *jemalloc_dll_name = "jemalloc.dll";
+const constexpr char *jemalloc_malloc_function_name = "je_malloc";
+const constexpr char *jemalloc_calloc_function_name = "je_calloc";
+const constexpr char *jemalloc_realloc_function_name = "je_realloc";
+const constexpr char *jemalloc_free_function_name = "je_free";
+
+namespace mysys {
+extern bool is_my_malloc_using_jemalloc();
+struct LogMessageInfo {
+  loglevel m_severity;
+  int64_t m_ecode;
+  std::string m_message;
+};
+extern std::vector<LogMessageInfo> fetch_jemalloc_initialization_messages();
+
+const int64_t MY_MALLOC_USING_JEMALLOC_ER = 0;
+const int64_t MY_MALLOC_USING_STD_MALLOC_ER = 1;
+const int64_t MY_MALLOC_LOADLIBRARY_FAILED_ER = 2;
+const int64_t MY_MALLOC_GETPROCADDRESS_FAILED_ER = 3;
+
+namespace detail {
+extern void *(*pfn_malloc)(size_t size);
+extern void *(*pfn_calloc)(size_t number, size_t size);
+extern void *(*pfn_realloc)(void *ptr, size_t size);
+extern void (*pfn_free)(void *ptr);
+extern std::once_flag init_malloc_pointers_flag;
+void init_malloc_pointers();
+}  // namespace detail
+}  // namespace mysys
+
+#endif  // _WIN32
+
+#endif  // JEMALLOC_WIN_INCLUDED

mysys/CMakeLists.txt

@@ -150,6 +150,7 @@ IF (WIN32)
     my_winerr.cc
     my_winfile.cc
     win_timers.cc
+    my_nojemalloc.cc
   )
 ENDIF()
 

mysys/my_malloc.cc

@@ -34,6 +34,10 @@
 #include <string.h>
 #include <sys/types.h>
 
+#ifdef _WIN32
+#include "jemalloc_win.h"
+#endif  // _WIN32
+
 #include "memory_debugging.h"
 #include "my_compiler.h"
 #include "my_dbug.h"
@@ -56,6 +60,192 @@ static void *my_raw_malloc(size_t size, myf my_flags);
 static void my_raw_free(void *ptr);
 extern void my_free(void *ptr);
 
+#ifdef _WIN32
+#include <memory>
+#include <mutex>
+#include <sstream>
+#include <string>
+
+extern "C" bool use_jemalloc_allocations();
+namespace mysys {
+namespace detail {
+void *(*pfn_malloc)(size_t size);
+void *(*pfn_calloc)(size_t number, size_t size);
+void *(*pfn_realloc)(void *ptr, size_t size);
+void (*pfn_free)(void *ptr);
+}  // namespace detail
+}  // namespace mysys
+
+static std::string win_error_to_string(DWORD err) {
+  struct local_freer {
+    void operator()(LPTSTR ptr) { LocalFree(ptr); }
+  };
+  LPTSTR pmsg = nullptr;
+  if (!FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM |
+                         FORMAT_MESSAGE_IGNORE_INSERTS |
+                         FORMAT_MESSAGE_ALLOCATE_BUFFER,
+                     nullptr, err, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),
+                     (LPTSTR)&pmsg, 0, nullptr)) {
+    return "";
+  }
+  std::unique_ptr<TCHAR, local_freer> scope_guard{pmsg};
+  std::string message(pmsg);
+  // Remove any trailing CRLF
+  if (message.rfind("\r\n") == (message.length() - 2)) {
+    message.resize(message.length() - 2);
+  }
+  return message;
+}
+
+namespace mysys {
+static bool using_jemalloc = false;
+bool is_my_malloc_using_jemalloc() { return using_jemalloc; }
+// This unwieldy add_or_return_messages function is a workaround for the fact
+// that we can't control the order of global initializers, and the server
+// logger is not available at the time the jemalloc allocation
+// functions are loaded. We can't split the add_or_return_messages function
+// into separate add_messages and return_messages functions with the
+// messages stored in a file scoped static vector, as messages stored in
+// that vector during the initialization of _another_ file scoped static
+// object could be lost when a file scoped static vector is itself
+// subsequently initialized.
+// So message codes and strings are stored in a static function scoped vector
+// for subsequent retrieval when the server logger IS available. This static
+// function scoped vector is immune to the order of initialization of file
+// scoped variables.
+// Note also that we can't use error codes from mysqld_error.h
+// here, as the mysys library is used in the generation of those
+// error codes.  To avoid a circular dependency, the codes stored
+// in the LogMessageInfo::m_ecode are offsets to be applied to
+// the base error code ER_MY_MALLOC_USING_JEMALLOC to find the
+// "real" error code.
+static void add_or_return_messages(
+    const LogMessageInfo &msg_info,
+    std::vector<LogMessageInfo> *pvec = nullptr) {
+  static std::vector<LogMessageInfo> vec_messages;
+
+  if (pvec) {
+    *pvec = vec_messages;
+    return;
+  }
+  vec_messages.emplace_back(msg_info);
+}
+
+static void log_err_deferred(loglevel severity, int64_t ecode,
+                             const std::string &message) {
+  add_or_return_messages({severity, ecode, message});
+}
+
+std::vector<LogMessageInfo> fetch_jemalloc_initialization_messages() {
+  std::vector<LogMessageInfo> jemalloc_init_messages;
+  LogMessageInfo dummy;
+  add_or_return_messages(dummy, &jemalloc_init_messages);
+  return jemalloc_init_messages;
+}
+template <class T>
+static T get_proc_address(const char *function_name, HMODULE hlib,
+                          const char *dll_path) {
+  auto tmp_pfn = reinterpret_cast<T>(GetProcAddress(hlib, function_name));
+  if (!tmp_pfn) {
+    DWORD err = GetLastError();
+    std::ostringstream os;
+    os << "GetProcAddress failed for \"" << function_name << "\" from \""
+       << dll_path << "\" with error code " << err << ": \""
+       << win_error_to_string(err) << "\" Falling back to default malloc";
+
+    log_err_deferred(ERROR_LEVEL, MY_MALLOC_GETPROCADDRESS_FAILED_ER, os.str());
+  }
+  return tmp_pfn;
+}
+namespace detail {
+std::once_flag init_malloc_pointers_flag;
+/**
+  Initialize (Windows only) function pointers used for some memory allocations,
+  including the my_malloc family.  Note that these memory allocations can occur
+  early in the process lifecycle (during construction of global objects) and
+  thus init_malloc_pointers is called using std::call_once from the functions
+  that wrap memory allocations rather than from my_win_init, where it would be
+  too late.
+  Note that as this function is invoked very early in the lifetime of the
+  process (typically before main), the logging system is not yet initialized
+  so calls to my_message_local send their output to stderr.
+  Also note that when used from the MySQL server on Windows, there are
+  typically two instances of the MySQL server process launched (see the
+  --no-monitor option for launching a single instance) and thus the information
+  message emitted on not finding the jemalloc dll will typically appear on
+  stderr twice.
+*/
+void init_malloc_pointers() {
+  // Set up to use the MSVC library routines by default.
+  pfn_malloc = std::malloc;
+  pfn_calloc = std::calloc;
+  pfn_realloc = std::realloc;
+  pfn_free = std::free;
+  if (!use_jemalloc_allocations()) {
+    return;
+  }
+  // If we can find the jemalloc library in the same directory as this
+  // executable, and load all the required functions successfully, use the
+  // jemalloc routines instead.
+  // Note that the handle returned by LoadLibraryEx is deliberately NOT
+  // released with a FreeLibrary call, as the functions used from this
+  // library can be called from global object destructors late in the
+  // process lifecycle. Not calling FreeLibrary is benign is this case.
+  HMODULE hlib = LoadLibraryEx(jemalloc_dll_name, NULL,
+                               LOAD_LIBRARY_SEARCH_APPLICATION_DIR);
+  if (NULL == hlib) {
+    DWORD err = GetLastError();
+    if (ERROR_MOD_NOT_FOUND == err) {
+      // Normal behaviour: not finding the jemalloc dll means
+      // we don't want to use it.
+      std::ostringstream os;
+      os << jemalloc_dll_name << " not found. Falling back to default malloc";
+      log_err_deferred(INFORMATION_LEVEL, MY_MALLOC_USING_STD_MALLOC_ER,
+                       os.str());
+      return;
+    }
+
+    std::ostringstream os;
+    os << "LoadLibraryEx(\"" << jemalloc_dll_name
+       << "\", NULL, LOAD_LIBRARY_SEARCH_APPLICATION_DIR) failed with error "
+          "code "
+       << err << ": \"" << win_error_to_string(err)
+       << "\", using default malloc";
+
+    log_err_deferred(ERROR_LEVEL, MY_MALLOC_LOADLIBRARY_FAILED_ER, os.str());
+    return;
+  }
+
+  auto tmp_je_malloc = get_proc_address<decltype(pfn_malloc)>(
+      jemalloc_malloc_function_name, hlib, jemalloc_dll_name);
+  auto tmp_je_calloc = get_proc_address<decltype(pfn_calloc)>(
+      jemalloc_calloc_function_name, hlib, jemalloc_dll_name);
+  auto tmp_je_realloc = get_proc_address<decltype(pfn_realloc)>(
+      jemalloc_realloc_function_name, hlib, jemalloc_dll_name);
+  auto tmp_je_free = get_proc_address<decltype(pfn_free)>(
+      jemalloc_free_function_name, hlib, jemalloc_dll_name);
+
+  if (!tmp_je_malloc || !tmp_je_calloc || !tmp_je_realloc || !tmp_je_free) {
+    return;
+  }
+
+  pfn_malloc = tmp_je_malloc;
+  pfn_calloc = tmp_je_calloc;
+  pfn_realloc = tmp_je_realloc;
+  pfn_free = tmp_je_free;
+  using_jemalloc = true;
+  return;
+}
+}  // namespace detail
+}  // namespace mysys
+
+#define malloc(size) mysys::detail::pfn_malloc((size))
+#define calloc(count, size) mysys::detail::pfn_calloc((count), (size))
+#define realloc(p, size) mysys::detail::pfn_realloc((p), (size))
+#define free(p) mysys::detail::pfn_free((p))
+
+#endif  // _WIN32
+
 #ifdef USE_MALLOC_WRAPPER
 
 void *my_malloc(PSI_memory_key key, size_t size, myf flags) {
@@ -182,11 +372,15 @@ static void *my_raw_malloc(size_t size, myf my_flags) {
   else
     point = _malloc_dbg(size, _CLIENT_BLOCK, __FILE__, __LINE__);
 #else
+#ifdef _WIN32
+  std::call_once(mysys::detail::init_malloc_pointers_flag,
+                 mysys::detail::init_malloc_pointers);
+#endif  // _WIN32
   if (my_flags & MY_ZEROFILL)
     point = calloc(size, 1);
   else
     point = malloc(size);
-#endif
+#endif  // MY_MSCRT_DEBUG
 
   DBUG_EXECUTE_IF("simulate_out_of_memory", {
     free(point);
@@ -236,8 +430,12 @@ static void *my_raw_realloc(void *oldpoint, size_t size, myf my_flags) {
 #if defined(MY_MSCRT_DEBUG)
   point = _realloc_dbg(oldpoint, size, _CLIENT_BLOCK, __FILE__, __LINE__);
 #else
+#ifdef _WIN32
+  std::call_once(mysys::detail::init_malloc_pointers_flag,
+                 mysys::detail::init_malloc_pointers);
+#endif  // _WIN32
   point = realloc(oldpoint, size);
-#endif
+#endif  // MY_MSCRT_DEBUG
 #ifndef NDEBUG
 end:
 #endif
@@ -253,7 +451,7 @@ static void *my_raw_realloc(void *oldpoint, size_t size, myf my_flags) {
   DBUG_PRINT("exit", ("ptr: %p", point));
   return point;
 }
-#endif
+#endif  // !USE_MALLOC_WRAPPER
 
 /**
   Free memory allocated with my_raw_malloc.
@@ -266,8 +464,12 @@ static void my_raw_free(void *ptr) {
 #if defined(MY_MSCRT_DEBUG)
   _free_dbg(ptr, _CLIENT_BLOCK);
 #else
+#ifdef _WIN32
+  std::call_once(mysys::detail::init_malloc_pointers_flag,
+                 mysys::detail::init_malloc_pointers);
+#endif  // _WIN32
   free(ptr);
-#endif
+#endif  // MY_MSCRT_DEBUG
 }
 
 void *my_memdup(PSI_memory_key key, const void *from, size_t length,

mysys/my_nojemalloc.cc

@@ -0,0 +1,41 @@
+/*****************************************************************************
+
+Copyright (c) 2021, Oracle and/or its affiliates.
+
+This program is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License, version 2.0, as published by the
+Free Software Foundation.
+
+This program is also distributed with certain software (including but not
+limited to OpenSSL) that is licensed under separate terms, as designated in a
+particular file or component or in included license documentation. The authors
+of MySQL hereby grant you an additional permission to link the program and
+your derivative works with the separately licensed software that they have
+included with MySQL.
+
+This program is distributed in the hope that it will be useful, but WITHOUT
+ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+FOR A PARTICULAR PURPOSE. See the GNU General Public License, version 2.0,
+for more details.
+
+You should have received a copy of the GNU General Public License along with
+this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin St, Fifth Floor, Boston, MA 02110-1301  USA
+
+*****************************************************************************/
+
+#ifdef _WIN32
+/* In order to use jemalloc in the mysys library, the use_jemalloc_allocations
+function must return true.
+
+This definition of the function (in the mysys library) returns false, so by
+default, the mysys library will not use jemalloc for memory allocation.
+
+Note that the linker always uses a symbol defined in an .obj file in preference
+to the same symbol defined in a lib. Thus a user of the mysys library who wants
+the mysys library to use jemalloc must provide an overriding definition (i.e.
+link with an obj file definition) of the use_jemalloc_allocations function that
+returns true.
+*/
+extern "C" bool use_jemalloc_allocations() { return false; }
+#endif

share/messages_to_error_log.txt

@@ -11617,6 +11617,18 @@ ER_PERSISTED_VARIABLES_DECRYPTION_FAILED
 ER_PERSISTED_VARIABLES_LACK_KEYRING_SUPPORT
   eng "Persisting SENSITIVE variables in encrypted form requires keyring component loaded through manifest file."
 
+ER_MY_MALLOC_USING_JEMALLOC
+  eng "Using jemalloc.dll for my_malloc and ut::malloc etc."
+
+ER_MY_MALLOC_USING_STD_MALLOC
+  eng "%s."
+
+ER_MY_MALLOC_LOADLIBRARY_FAILED
+  eng "%s."
+
+ER_MY_MALLOC_GETPROCADDRESS_FAILED
+  eng "%s."
+
 # DO NOT add server-to-client messages here;
 # they go in messages_to_clients.txt
 # in the same directory as this file.