From 304b2c2a45213bb31f610d1c9d139ccb289bbe30 Mon Sep 17 00:00:00 2001 From: Kuba Brecka Date: Mon, 6 Jun 2016 18:18:47 +0000 Subject: [PATCH] [tsan] Switch to InternalAlloc everywhere __libc_malloc is currently used MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This patch replaces all uses of __libc_malloc and friends with the internal allocator. It seems that the only reason why we have calls to __libc_malloc in the first place was the lack of the internal allocator at the time. Using the internal allocator will also make sure that the system allocator is never used (this is the same behavior as ASan), and we don’t have to worry about working with unknown pointers coming from the system allocator. Differential Revision: http://reviews.llvm.org/D21025 llvm-svn: 271916 --- .../sanitizer_common/sanitizer_allocator.cc | 82 +++++++++++++++---- .../sanitizer_allocator_internal.h | 7 +- compiler-rt/lib/tsan/dd/dd_interceptors.cc | 3 - compiler-rt/lib/tsan/rtl/tsan_interceptors.cc | 36 +++----- compiler-rt/lib/tsan/rtl/tsan_interceptors.h | 8 -- compiler-rt/lib/tsan/rtl/tsan_malloc_mac.cc | 39 ++++----- compiler-rt/lib/tsan/rtl/tsan_new_delete.cc | 8 +- 7 files changed, 102 insertions(+), 81 deletions(-) diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_allocator.cc b/compiler-rt/lib/sanitizer_common/sanitizer_allocator.cc index 538e2db95d4e..df298c62271d 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_allocator.cc +++ b/compiler-rt/lib/sanitizer_common/sanitizer_allocator.cc @@ -22,30 +22,47 @@ namespace __sanitizer { #if defined(SANITIZER_GO) || defined(SANITIZER_USE_MALLOC) # if SANITIZER_LINUX && !SANITIZER_ANDROID extern "C" void *__libc_malloc(uptr size); +extern "C" void *__libc_memalign(uptr alignment, uptr size); +extern "C" void *__libc_realloc(void *ptr, uptr size); extern "C" void __libc_free(void *ptr); -# define LIBC_MALLOC __libc_malloc -# define LIBC_FREE __libc_free # else # include -# define LIBC_MALLOC malloc -# define LIBC_FREE free +# define __libc_malloc malloc +static void *__libc_memalign(uptr alignment, uptr size) { + void *p; + uptr error = posix_memalign(&p, alignment, size); + if (error) return nullptr; + return p; +} +# define __libc_realloc realloc +# define __libc_free free # endif -static void *RawInternalAlloc(uptr size, InternalAllocatorCache *cache) { +static void *RawInternalAlloc(uptr size, InternalAllocatorCache *cache, + uptr alignment) { (void)cache; - return LIBC_MALLOC(size); + if (alignment == 0) + return __libc_malloc(size); + else + return __libc_memalign(alignment, size); +} + +static void *RawInternalRealloc(void *ptr, uptr size, + InternalAllocatorCache *cache) { + (void)cache; + return __libc_realloc(ptr, size); } static void RawInternalFree(void *ptr, InternalAllocatorCache *cache) { (void)cache; - LIBC_FREE(ptr); + __libc_free(ptr); } InternalAllocator *internal_allocator() { return 0; } -#else // SANITIZER_GO +#else // defined(SANITIZER_GO) || defined(SANITIZER_USE_MALLOC) static ALIGNED(64) char internal_alloc_placeholder[sizeof(InternalAllocator)]; static atomic_uint8_t internal_allocator_initialized; @@ -68,13 +85,26 @@ InternalAllocator *internal_allocator() { return internal_allocator_instance; } -static void *RawInternalAlloc(uptr size, InternalAllocatorCache *cache) { +static void *RawInternalAlloc(uptr size, InternalAllocatorCache *cache, + uptr alignment) { + if (alignment == 0) alignment = 8; if (cache == 0) { SpinMutexLock l(&internal_allocator_cache_mu); - return internal_allocator()->Allocate(&internal_allocator_cache, size, 8, - false); + return internal_allocator()->Allocate(&internal_allocator_cache, size, + alignment, false); } - return internal_allocator()->Allocate(cache, size, 8, false); + return internal_allocator()->Allocate(cache, size, alignment, false); +} + +static void *RawInternalRealloc(void *ptr, uptr size, + InternalAllocatorCache *cache) { + uptr alignment = 8; + if (cache == 0) { + SpinMutexLock l(&internal_allocator_cache_mu); + return internal_allocator()->Reallocate(&internal_allocator_cache, ptr, + size, alignment); + } + return internal_allocator()->Reallocate(cache, ptr, size, alignment); } static void RawInternalFree(void *ptr, InternalAllocatorCache *cache) { @@ -85,20 +115,42 @@ static void RawInternalFree(void *ptr, InternalAllocatorCache *cache) { internal_allocator()->Deallocate(cache, ptr); } -#endif // SANITIZER_GO +#endif // defined(SANITIZER_GO) || defined(SANITIZER_USE_MALLOC) const u64 kBlockMagic = 0x6A6CB03ABCEBC041ull; -void *InternalAlloc(uptr size, InternalAllocatorCache *cache) { +void *InternalAlloc(uptr size, InternalAllocatorCache *cache, uptr alignment) { if (size + sizeof(u64) < size) return nullptr; - void *p = RawInternalAlloc(size + sizeof(u64), cache); + void *p = RawInternalAlloc(size + sizeof(u64), cache, alignment); if (!p) return nullptr; ((u64*)p)[0] = kBlockMagic; return (char*)p + sizeof(u64); } +void *InternalRealloc(void *addr, uptr size, InternalAllocatorCache *cache) { + if (!addr) + return InternalAlloc(size, cache); + if (size + sizeof(u64) < size) + return nullptr; + addr = (char*)addr - sizeof(u64); + size = size + sizeof(u64); + CHECK_EQ(kBlockMagic, ((u64*)addr)[0]); + void *p = RawInternalRealloc(addr, size, cache); + if (!p) + return nullptr; + return (char*)p + sizeof(u64); +} + +void *InternalCalloc(uptr count, uptr size, InternalAllocatorCache *cache) { + if (CallocShouldReturnNullDueToOverflow(count, size)) + return internal_allocator()->ReturnNullOrDie(); + void *p = InternalAlloc(count * size, cache); + if (p) internal_memset(p, 0, count * size); + return p; +} + void InternalFree(void *addr, InternalAllocatorCache *cache) { if (!addr) return; diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_allocator_internal.h b/compiler-rt/lib/sanitizer_common/sanitizer_allocator_internal.h index 3dcfccd7cba3..a7ea454ff17b 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_allocator_internal.h +++ b/compiler-rt/lib/sanitizer_common/sanitizer_allocator_internal.h @@ -45,7 +45,12 @@ typedef SizeClassAllocatorLocalCache typedef CombinedAllocator > InternalAllocator; -void *InternalAlloc(uptr size, InternalAllocatorCache *cache = nullptr); +void *InternalAlloc(uptr size, InternalAllocatorCache *cache = nullptr, + uptr alignment = 0); +void *InternalRealloc(void *p, uptr size, + InternalAllocatorCache *cache = nullptr); +void *InternalCalloc(uptr countr, uptr size, + InternalAllocatorCache *cache = nullptr); void InternalFree(void *p, InternalAllocatorCache *cache = nullptr); InternalAllocator *internal_allocator(); diff --git a/compiler-rt/lib/tsan/dd/dd_interceptors.cc b/compiler-rt/lib/tsan/dd/dd_interceptors.cc index 8151f7fd3c67..97c72dd2b7fd 100644 --- a/compiler-rt/lib/tsan/dd/dd_interceptors.cc +++ b/compiler-rt/lib/tsan/dd/dd_interceptors.cc @@ -15,9 +15,6 @@ using namespace __dsan; -extern "C" void *__libc_malloc(uptr size); -extern "C" void __libc_free(void *ptr); - __attribute__((tls_model("initial-exec"))) static __thread Thread *thr; __attribute__((tls_model("initial-exec"))) diff --git a/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc b/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc index 4d42cf0decc7..4a1821e69dbc 100644 --- a/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc +++ b/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc @@ -41,20 +41,8 @@ using namespace __tsan; // NOLINT #define stderr __stderrp #endif -#if SANITIZER_FREEBSD -#define __libc_realloc __realloc -#define __libc_calloc __calloc -#elif SANITIZER_MAC -#define __libc_malloc REAL(malloc) -#define __libc_realloc REAL(realloc) -#define __libc_calloc REAL(calloc) -#define __libc_free REAL(free) -#elif SANITIZER_ANDROID +#if SANITIZER_ANDROID #define __errno_location __errno -#define __libc_malloc REAL(malloc) -#define __libc_realloc REAL(realloc) -#define __libc_calloc REAL(calloc) -#define __libc_free REAL(free) #define mallopt(a, b) #endif @@ -109,10 +97,6 @@ extern "C" void *pthread_self(); extern "C" void _exit(int status); extern "C" int *__errno_location(); extern "C" int fileno_unlocked(void *stream); -#if !SANITIZER_ANDROID -extern "C" void *__libc_calloc(uptr size, uptr n); -extern "C" void *__libc_realloc(void *ptr, uptr size); -#endif extern "C" int dirfd(void *dirp); #if !SANITIZER_FREEBSD && !SANITIZER_ANDROID extern "C" int mallopt(int param, int value); @@ -396,7 +380,7 @@ static void at_exit_wrapper(void *arg) { Acquire(thr, pc, (uptr)arg); AtExitCtx *ctx = (AtExitCtx*)arg; ((void(*)(void *arg))ctx->f)(ctx->arg); - __libc_free(ctx); + InternalFree(ctx); } static int setup_at_exit_wrapper(ThreadState *thr, uptr pc, void(*f)(), @@ -422,7 +406,7 @@ TSAN_INTERCEPTOR(int, __cxa_atexit, void (*f)(void *a), void *arg, void *dso) { static int setup_at_exit_wrapper(ThreadState *thr, uptr pc, void(*f)(), void *arg, void *dso) { - AtExitCtx *ctx = (AtExitCtx*)__libc_malloc(sizeof(AtExitCtx)); + AtExitCtx *ctx = (AtExitCtx*)InternalAlloc(sizeof(AtExitCtx)); ctx->f = f; ctx->arg = arg; Release(thr, pc, (uptr)ctx); @@ -441,14 +425,14 @@ static void on_exit_wrapper(int status, void *arg) { Acquire(thr, pc, (uptr)arg); AtExitCtx *ctx = (AtExitCtx*)arg; ((void(*)(int status, void *arg))ctx->f)(status, ctx->arg); - __libc_free(ctx); + InternalFree(ctx); } TSAN_INTERCEPTOR(int, on_exit, void(*f)(int, void*), void *arg) { if (cur_thread()->in_symbolizer) return 0; SCOPED_TSAN_INTERCEPTOR(on_exit, f, arg); - AtExitCtx *ctx = (AtExitCtx*)__libc_malloc(sizeof(AtExitCtx)); + AtExitCtx *ctx = (AtExitCtx*)InternalAlloc(sizeof(AtExitCtx)); ctx->f = (void(*)())f; ctx->arg = arg; Release(thr, pc, (uptr)ctx); @@ -601,7 +585,7 @@ TSAN_INTERCEPTOR(void, siglongjmp, uptr *env, int val) { #if !SANITIZER_MAC TSAN_INTERCEPTOR(void*, malloc, uptr size) { if (cur_thread()->in_symbolizer) - return __libc_malloc(size); + return InternalAlloc(size); void *p = 0; { SCOPED_INTERCEPTOR_RAW(malloc, size); @@ -618,7 +602,7 @@ TSAN_INTERCEPTOR(void*, __libc_memalign, uptr align, uptr sz) { TSAN_INTERCEPTOR(void*, calloc, uptr size, uptr n) { if (cur_thread()->in_symbolizer) - return __libc_calloc(size, n); + return InternalCalloc(size, n); void *p = 0; { SCOPED_INTERCEPTOR_RAW(calloc, size, n); @@ -630,7 +614,7 @@ TSAN_INTERCEPTOR(void*, calloc, uptr size, uptr n) { TSAN_INTERCEPTOR(void*, realloc, void *p, uptr size) { if (cur_thread()->in_symbolizer) - return __libc_realloc(p, size); + return InternalRealloc(p, size); if (p) invoke_free_hook(p); { @@ -645,7 +629,7 @@ TSAN_INTERCEPTOR(void, free, void *p) { if (p == 0) return; if (cur_thread()->in_symbolizer) - return __libc_free(p); + return InternalFree(p); invoke_free_hook(p); SCOPED_INTERCEPTOR_RAW(free, p); user_free(thr, pc, p); @@ -655,7 +639,7 @@ TSAN_INTERCEPTOR(void, cfree, void *p) { if (p == 0) return; if (cur_thread()->in_symbolizer) - return __libc_free(p); + return InternalFree(p); invoke_free_hook(p); SCOPED_INTERCEPTOR_RAW(cfree, p); user_free(thr, pc, p); diff --git a/compiler-rt/lib/tsan/rtl/tsan_interceptors.h b/compiler-rt/lib/tsan/rtl/tsan_interceptors.h index a27800de54d6..a0f9a0753a63 100644 --- a/compiler-rt/lib/tsan/rtl/tsan_interceptors.h +++ b/compiler-rt/lib/tsan/rtl/tsan_interceptors.h @@ -46,12 +46,4 @@ class ScopedInterceptor { #define TSAN_INTERCEPTOR(ret, func, ...) INTERCEPTOR(ret, func, __VA_ARGS__) -#if SANITIZER_FREEBSD -#define __libc_free __free -#define __libc_malloc __malloc -#endif - -extern "C" void __libc_free(void *ptr); -extern "C" void *__libc_malloc(uptr size); - #endif // TSAN_INTERCEPTORS_H diff --git a/compiler-rt/lib/tsan/rtl/tsan_malloc_mac.cc b/compiler-rt/lib/tsan/rtl/tsan_malloc_mac.cc index 7fd94273c314..8d31ccbcaaca 100644 --- a/compiler-rt/lib/tsan/rtl/tsan_malloc_mac.cc +++ b/compiler-rt/lib/tsan/rtl/tsan_malloc_mac.cc @@ -27,33 +27,28 @@ using namespace __tsan; #define COMMON_MALLOC_MEMALIGN(alignment, size) \ void *p = \ user_alloc(cur_thread(), StackTrace::GetCurrentPc(), size, alignment) -#define COMMON_MALLOC_MALLOC(size) \ - if (cur_thread()->in_symbolizer) \ - return REAL(malloc)(size); \ - SCOPED_INTERCEPTOR_RAW(malloc, size); \ +#define COMMON_MALLOC_MALLOC(size) \ + if (cur_thread()->in_symbolizer) return InternalAlloc(size); \ + SCOPED_INTERCEPTOR_RAW(malloc, size); \ void *p = user_alloc(thr, pc, size) -#define COMMON_MALLOC_REALLOC(ptr, size) \ - if (cur_thread()->in_symbolizer) \ - return REAL(realloc)(ptr, size); \ - SCOPED_INTERCEPTOR_RAW(realloc, ptr, size); \ +#define COMMON_MALLOC_REALLOC(ptr, size) \ + if (cur_thread()->in_symbolizer) return InternalRealloc(ptr, size); \ + SCOPED_INTERCEPTOR_RAW(realloc, ptr, size); \ void *p = user_realloc(thr, pc, ptr, size) -#define COMMON_MALLOC_CALLOC(count, size) \ - if (cur_thread()->in_symbolizer) \ - return REAL(calloc)(count, size); \ - SCOPED_INTERCEPTOR_RAW(calloc, size, count); \ +#define COMMON_MALLOC_CALLOC(count, size) \ + if (cur_thread()->in_symbolizer) return InternalCalloc(count, size); \ + SCOPED_INTERCEPTOR_RAW(calloc, size, count); \ void *p = user_calloc(thr, pc, size, count) -#define COMMON_MALLOC_VALLOC(size) \ - if (cur_thread()->in_symbolizer) \ - return REAL(valloc)(size); \ - SCOPED_INTERCEPTOR_RAW(valloc, size); \ +#define COMMON_MALLOC_VALLOC(size) \ + if (cur_thread()->in_symbolizer) \ + return InternalAlloc(size, nullptr, GetPageSizeCached()); \ + SCOPED_INTERCEPTOR_RAW(valloc, size); \ void *p = user_alloc(thr, pc, size, GetPageSizeCached()) -#define COMMON_MALLOC_FREE(ptr) \ - if (cur_thread()->in_symbolizer) \ - return REAL(free)(ptr); \ - SCOPED_INTERCEPTOR_RAW(free, ptr); \ +#define COMMON_MALLOC_FREE(ptr) \ + if (cur_thread()->in_symbolizer) return InternalFree(ptr); \ + SCOPED_INTERCEPTOR_RAW(free, ptr); \ user_free(thr, pc, ptr) -#define COMMON_MALLOC_SIZE(ptr) \ - uptr size = user_alloc_usable_size(ptr); +#define COMMON_MALLOC_SIZE(ptr) uptr size = user_alloc_usable_size(ptr); #define COMMON_MALLOC_FILL_STATS(zone, stats) #define COMMON_MALLOC_REPORT_UNKNOWN_REALLOC(ptr, zone_ptr, zone_name) \ (void)zone_name; \ diff --git a/compiler-rt/lib/tsan/rtl/tsan_new_delete.cc b/compiler-rt/lib/tsan/rtl/tsan_new_delete.cc index ebb422cf2023..b6478bb08c57 100644 --- a/compiler-rt/lib/tsan/rtl/tsan_new_delete.cc +++ b/compiler-rt/lib/tsan/rtl/tsan_new_delete.cc @@ -23,14 +23,10 @@ struct nothrow_t {}; DECLARE_REAL(void *, malloc, uptr size) DECLARE_REAL(void, free, void *ptr) -#if SANITIZER_MAC || SANITIZER_ANDROID -#define __libc_malloc REAL(malloc) -#define __libc_free REAL(free) -#endif #define OPERATOR_NEW_BODY(mangled_name) \ if (cur_thread()->in_symbolizer) \ - return __libc_malloc(size); \ + return InternalAlloc(size); \ void *p = 0; \ { \ SCOPED_INTERCEPTOR_RAW(mangled_name, size); \ @@ -66,7 +62,7 @@ void *operator new[](__sanitizer::uptr size, std::nothrow_t const&) { #define OPERATOR_DELETE_BODY(mangled_name) \ if (ptr == 0) return; \ if (cur_thread()->in_symbolizer) \ - return __libc_free(ptr); \ + return InternalFree(ptr); \ invoke_free_hook(ptr); \ SCOPED_INTERCEPTOR_RAW(mangled_name, ptr); \ user_free(thr, pc, ptr);