Correct nits from Niko. Add tests for cdylib and staticlib.

Author: Firstyear

src/librustc_metadata/creader.rs

@@ -860,10 +860,11 @@ impl<'a> CrateLoader<'a> {
                     config::CrateTypeExecutable => true,
                     // This crate will be compiled with the required
                     // instrumentation pass
-                    config::CrateTypeRlib => false,
-                    config::CrateTypeDylib => false,
-                    config::CrateTypeCdylib => false,
-                    config::CrateTypeStaticlib => false,
+                    config::CrateTypeRlib |
+                    config::CrateTypeDylib |
+                    config::CrateTypeCdylib |
+                    config::CrateTypeStaticlib =>
+                        false,
                     _ => {
                         self.sess.err(&format!("Only executables, dylibs and rlibs can be \
                                                 compiled with `-Z sanitizer`"));

src/test/run-make/sanitizer-cdylib-link/Makefile

@@ -0,0 +1,26 @@
+-include ../tools.mk
+
+# This test builds a shared object, then an executable that links it as a native
+# rust library (constrast to an rlib). The shared library and executable both
+# are compiled with address sanitizer, and we assert that a fault in the cdylib
+# is correctly detected.
+
+# NOTE the address sanitizer only supports x86_64 linux and macOS
+
+ifeq ($(TARGET),x86_64-apple-darwin)
+ASAN_SUPPORT=$(SANITIZER_SUPPORT)
+EXTRA_RUSTFLAG=-C rpath
+else
+ifeq ($(TARGET),x86_64-unknown-linux-gnu)
+ASAN_SUPPORT=$(SANITIZER_SUPPORT)
+EXTRA_RUSTFLAG=
+endif
+endif
+
+all:
+ifeq ($(ASAN_SUPPORT),1)
+	$(RUSTC) -g -Z sanitizer=address --crate-type cdylib --target $(TARGET) library.rs
+	$(RUSTC) -g -Z sanitizer=address --crate-type bin --target $(TARGET) -llibrary program.rs
+	LD_LIBRARY_PATH=$(TMPDIR) $(TMPDIR)/program 2>&1 | grep -q stack-buffer-overflow
+endif
+

src/test/run-make/sanitizer-cdylib-link/library.rs

@@ -0,0 +1,15 @@
+// Copyright 2017 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+#[no_mangle]
+pub extern fn overflow() {
+    let xs = [0, 1, 2, 3];
+    let y = unsafe { *xs.as_ptr().offset(4) };
+}

src/test/run-make/sanitizer-cdylib-link/program.rs

@@ -0,0 +1,17 @@
+// Copyright 2017 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+extern {
+    fn overflow();
+}
+
+fn main() {
+    unsafe { overflow() }
+}

src/test/run-make/sanitizer-staticlib-link/Makefile

@@ -0,0 +1,25 @@
+-include ../tools.mk
+
+# This test builds a staticlib, then an executable that links to it.
+# The staticlib and executable both  are compiled with address sanitizer, 
+# and we assert that a fault in the staticlib is correctly detected.
+
+# NOTE the address sanitizer only supports x86_64 linux and macOS
+
+ifeq ($(TARGET),x86_64-apple-darwin)
+ASAN_SUPPORT=$(SANITIZER_SUPPORT)
+EXTRA_RUSTFLAG=-C rpath
+else
+ifeq ($(TARGET),x86_64-unknown-linux-gnu)
+ASAN_SUPPORT=$(SANITIZER_SUPPORT)
+EXTRA_RUSTFLAG=
+endif
+endif
+
+all:
+ifeq ($(ASAN_SUPPORT),1)
+	$(RUSTC) -g -Z sanitizer=address --crate-type staticlib --target $(TARGET) library.rs
+	$(CC) program.c $(call STATICLIB,library) $(call OUT_EXE,program) -lasan $(EXTRACFLAGS) $(EXTRACXXFLAGS)
+	LD_LIBRARY_PATH=$(TMPDIR) $(TMPDIR)/program 2>&1 | grep -q stack-buffer-overflow
+endif
+

src/test/run-make/sanitizer-staticlib-link/library.rs

@@ -0,0 +1,15 @@
+// Copyright 2017 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+#[no_mangle]
+pub extern fn overflow() {
+    let xs = [0, 1, 2, 3];
+    let y = unsafe { *xs.as_ptr().offset(4) };
+}

src/test/run-make/sanitizer-staticlib-link/program.c

@@ -0,0 +1,8 @@
+// ignore-license
+void overflow();
+
+int main() {
+    overflow();
+    return 0;
+}
+