jemalloc improvements

.gitattributes

@@ -7,4 +7,4 @@
 src/etc/pkg/rust-logo.ico binary
 src/etc/pkg/rust-logo.png binary
 src/rt/msvc/* -whitespace
-src/rt/vg/* -whitespace
+src/rt/valgrind/* -whitespace

mk/dist.mk

@@ -36,7 +36,7 @@ LICENSE.txt: $(S)COPYRIGHT $(S)LICENSE-APACHE $(S)LICENSE-MIT
 PKG_TAR = dist/$(PKG_NAME).tar.gz
 
 PKG_GITMODULES := $(S)src/libuv $(S)src/llvm $(S)src/gyp $(S)src/compiler-rt \
-		  $(S)src/rt/hoedown
+		  $(S)src/rt/hoedown $(S)src/jemalloc
 PKG_FILES := \
     $(S)COPYRIGHT                              \
     $(S)LICENSE-APACHE                         \

mk/rt.mk

@@ -260,6 +260,7 @@ $$(JEMALLOC_LIB_$(1)): $$(JEMALLOC_DEPS) $$(MKFILE_DEPS)
 		CC="$$(CC_$(1))" \
 		AR="$$(AR_$(1))" \
 		RANLIB="$$(AR_$(1)) s" \
+		CPPFLAGS="-I $(S)src/rt/" \
 		EXTRA_CFLAGS="$$(CFG_CFLAGS_$(1))"
 	$$(Q)$$(MAKE) -C "$$(JEMALLOC_BUILD_DIR_$(1))" build_lib_static
 	$$(Q)cp $$(JEMALLOC_BUILD_DIR_$(1))/lib/$$(JEMALLOC_REAL_NAME_$(1)) $$(JEMALLOC_LIB_$(1))

mk/tests.mk

@@ -227,8 +227,8 @@ ALL_HS := $(wildcard $(S)src/rt/*.h \
                      $(S)src/rt/*/*.h \
                      $(S)src/rt/*/*/*.h \
                      $(S)src/rustllvm/*.h)
-ALL_HS := $(filter-out $(S)src/rt/vg/valgrind.h \
-                       $(S)src/rt/vg/memcheck.h \
+ALL_HS := $(filter-out $(S)src/rt/valgrind/valgrind.h \
+                       $(S)src/rt/valgrind/memcheck.h \
                        $(S)src/rt/msvc/typeof.h \
                        $(S)src/rt/msvc/stdint.h \
                        $(S)src/rt/msvc/inttypes.h \

src/libstd/rc.rs

@@ -33,7 +33,7 @@ use option::{Option, Some, None};
 use ptr;
 use ptr::RawPtr;
 use mem::{min_align_of, size_of};
-use rt::heap::exchange_free;
+use rt::heap::deallocate;
 
 struct RcBox<T> {
     value: T,
@@ -105,8 +105,8 @@ impl<T> Drop for Rc<T> {
                     self.dec_weak();
 
                     if self.weak() == 0 {
-                        exchange_free(self.ptr as *mut u8, size_of::<RcBox<T>>(),
-                                      min_align_of::<RcBox<T>>())
+                        deallocate(self.ptr as *mut u8, size_of::<RcBox<T>>(),
+                                   min_align_of::<RcBox<T>>())
                     }
                 }
             }
@@ -179,8 +179,8 @@ impl<T> Drop for Weak<T> {
                 // the weak count starts at 1, and will only go to
                 // zero if all the strong pointers have disappeared.
                 if self.weak() == 0 {
-                    exchange_free(self.ptr as *mut u8, size_of::<RcBox<T>>(),
-                                  min_align_of::<RcBox<T>>())
+                    deallocate(self.ptr as *mut u8, size_of::<RcBox<T>>(),
+                               min_align_of::<RcBox<T>>())
                 }
             }
         }

src/libstd/rt/heap.rs

@@ -12,8 +12,9 @@
 // FIXME: #13996: need a way to mark the `allocate` and `reallocate` return values as `noalias`
 
 use intrinsics::{abort, cttz32};
-use libc::{c_int, c_void, size_t};
-use ptr::RawPtr;
+use libc::{c_char, c_int, c_void, size_t};
+use ptr::{RawPtr, mut_null, null};
+use option::{None, Option};
 
 #[link(name = "jemalloc", kind = "static")]
 extern {
@@ -22,6 +23,9 @@ extern {
     fn je_xallocx(ptr: *mut c_void, size: size_t, extra: size_t, flags: c_int) -> size_t;
     fn je_dallocx(ptr: *mut c_void, flags: c_int);
     fn je_nallocx(size: size_t, flags: c_int) -> size_t;
+    fn je_malloc_stats_print(write_cb: Option<extern "C" fn(cbopaque: *mut c_void, *c_char)>,
+                             cbopaque: *mut c_void,
+                             opts: *c_char);
 }
 
 // -lpthread needs to occur after -ljemalloc, the earlier argument isn't enough
@@ -99,6 +103,16 @@ pub fn usable_size(size: uint, align: uint) -> uint {
     unsafe { je_nallocx(size as size_t, mallocx_align(align)) as uint }
 }
 
+/// Print implementation-defined allocator statistics.
+///
+/// These statistics may be inconsistent if other threads use the allocator during the call.
+#[unstable]
+pub fn stats_print() {
+    unsafe {
+        je_malloc_stats_print(None, mut_null(), null())
+    }
+}
+
 /// The allocator for unique pointers.
 #[cfg(stage0)]
 #[lang="exchange_malloc"]
@@ -151,13 +165,8 @@ pub unsafe fn exchange_malloc(size: uint, align: uint) -> *mut u8 {
 #[lang="exchange_free"]
 #[inline]
 // FIXME: #13994 (rustc should pass align and size here)
-pub unsafe fn exchange_free_(ptr: *mut u8) {
-    exchange_free(ptr, 0, 8)
-}
-
-#[inline]
-pub unsafe fn exchange_free(ptr: *mut u8, size: uint, align: uint) {
-    deallocate(ptr, size, align);
+unsafe fn exchange_free(ptr: *mut u8) {
+    deallocate(ptr, 0, 8);
 }
 
 // FIXME: #7496
@@ -179,26 +188,26 @@ unsafe fn closure_exchange_malloc(drop_glue: fn(*mut u8), size: uint, align: uin
 #[doc(hidden)]
 #[deprecated]
 #[cfg(stage0, not(test))]
-pub extern "C" fn rust_malloc(size: uint) -> *mut u8 {
-    unsafe { exchange_malloc(size) }
+pub unsafe extern "C" fn rust_malloc(size: uint) -> *mut u8 {
+    exchange_malloc(size)
 }
 
 // hack for libcore
 #[no_mangle]
 #[doc(hidden)]
 #[deprecated]
 #[cfg(not(stage0), not(test))]
-pub extern "C" fn rust_malloc(size: uint, align: uint) -> *mut u8 {
-    unsafe { exchange_malloc(size, align) }
+pub unsafe extern "C" fn rust_malloc(size: uint, align: uint) -> *mut u8 {
+    exchange_malloc(size, align)
 }
 
 // hack for libcore
 #[no_mangle]
 #[doc(hidden)]
 #[deprecated]
 #[cfg(not(test))]
-pub extern "C" fn rust_free(ptr: *mut u8, size: uint, align: uint) {
-    unsafe { exchange_free(ptr, size, align) }
+pub unsafe extern "C" fn rust_free(ptr: *mut u8, size: uint, align: uint) {
+    deallocate(ptr, size, align)
 }
 
 #[cfg(test)]

src/libstd/slice.rs

@@ -109,7 +109,7 @@ use ops::Drop;
 use option::{None, Option, Some};
 use ptr::RawPtr;
 use ptr;
-use rt::heap::{exchange_malloc, exchange_free};
+use rt::heap::{exchange_malloc, deallocate};
 use unstable::finally::try_finally;
 use vec::Vec;
 
@@ -330,7 +330,7 @@ impl<'a, T: Clone> CloneableVector<T> for &'a [T] {
                         ptr::read(&*p.offset(j));
                     }
                     // FIXME: #13994 (should pass align and size here)
-                    exchange_free(ret as *mut u8, 0, 8);
+                    deallocate(ret as *mut u8, 0, 8);
                 });
             mem::transmute(ret)
         }
@@ -377,7 +377,7 @@ impl<'a, T: Clone> CloneableVector<T> for &'a [T] {
                         ptr::read(&*p.offset(j));
                     }
                     // FIXME: #13994 (should pass align and size here)
-                    exchange_free(ret as *mut u8, 0, 8);
+                    deallocate(ret as *mut u8, 0, 8);
                 });
             mem::transmute(ret)
         }
@@ -817,7 +817,7 @@ impl<T> Drop for MoveItems<T> {
         for _x in *self {}
         unsafe {
             // FIXME: #13994 (should pass align and size here)
-            exchange_free(self.allocation, 0, 8)
+            deallocate(self.allocation, 0, 8)
         }
     }
 }

src/libsync/arc.rs

@@ -15,7 +15,7 @@
 
 use std::mem;
 use std::ptr;
-use std::rt::heap::exchange_free;
+use std::rt::heap::deallocate;
 use std::sync::atomics;
 use std::mem::{min_align_of, size_of};
 
@@ -191,8 +191,8 @@ impl<T: Share + Send> Drop for Arc<T> {
 
         if self.inner().weak.fetch_sub(1, atomics::Release) == 1 {
             atomics::fence(atomics::Acquire);
-            unsafe { exchange_free(self.x as *mut u8, size_of::<ArcInner<T>>(),
-                                   min_align_of::<ArcInner<T>>()) }
+            unsafe { deallocate(self.x as *mut u8, size_of::<ArcInner<T>>(),
+                                min_align_of::<ArcInner<T>>()) }
         }
     }
 }
@@ -242,8 +242,8 @@ impl<T: Share + Send> Drop for Weak<T> {
         // the memory orderings
         if self.inner().weak.fetch_sub(1, atomics::Release) == 1 {
             atomics::fence(atomics::Acquire);
-            unsafe { exchange_free(self.x as *mut u8, size_of::<ArcInner<T>>(),
-                                   min_align_of::<ArcInner<T>>()) }
+            unsafe { deallocate(self.x as *mut u8, size_of::<ArcInner<T>>(),
+                                min_align_of::<ArcInner<T>>()) }
         }
     }
 }

src/rt/rust_builtin.c

@@ -10,7 +10,7 @@
 
 /* Foreign builtins. */
 
-#include "vg/valgrind.h"
+#include "valgrind/valgrind.h"
 
 #include <stdint.h>
 #include <time.h>