[TSan] Fix madvise(MADV_NOHUGEPAGE) for meta shadow memory

Summary:
Move madvise(MADV_NOHUGEPAGE) for the meta shadow memory after the meta
shadow memory is mapped (currently it silently fails with ENOMEM).

Add a diagnostic message to detect similar problems in the future.

Reviewers: dvyukov

Subscribers: kubamracek, delcypher, #sanitizers, llvm-commits

Differential Revision: https://reviews.llvm.org/D48097

llvm-svn: 334624

compiler-rt/lib/sanitizer_common/sanitizer_common.h

@@ -120,8 +120,8 @@ void ReleaseMemoryPagesToOS(uptr beg, uptr end);
 void IncreaseTotalMmap(uptr size);
 void DecreaseTotalMmap(uptr size);
 uptr GetRSS();
-void NoHugePagesInRegion(uptr addr, uptr length);
-void DontDumpShadowMemory(uptr addr, uptr length);
+bool NoHugePagesInRegion(uptr addr, uptr length);
+bool DontDumpShadowMemory(uptr addr, uptr length);
 // Check if the built VMA size matches the runtime one.
 void CheckVMASize();
 void RunMallocHooks(const void *ptr, uptr size);

compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cc

@@ -69,16 +69,20 @@ void ReleaseMemoryPagesToOS(uptr beg, uptr end) {
             SANITIZER_MADVISE_DONTNEED);
 }
 
-void NoHugePagesInRegion(uptr addr, uptr size) {
+bool NoHugePagesInRegion(uptr addr, uptr size) {
 #ifdef MADV_NOHUGEPAGE  // May not be defined on old systems.
-  madvise((void *)addr, size, MADV_NOHUGEPAGE);
+  return madvise((void *)addr, size, MADV_NOHUGEPAGE) == 0;
+#else
+  return true;
 #endif  // MADV_NOHUGEPAGE
 }
 
-void DontDumpShadowMemory(uptr addr, uptr length) {
+bool DontDumpShadowMemory(uptr addr, uptr length) {
 #ifdef MADV_DONTDUMP
-  madvise((void *)addr, length, MADV_DONTDUMP);
-#endif
+  return madvise((void *)addr, length, MADV_DONTDUMP) == 0;
+#else
+  return true;
+#endif  // MADV_DONTDUMP
 }
 
 static rlim_t getlim(int res) {

compiler-rt/lib/sanitizer_common/sanitizer_win.cc

@@ -311,13 +311,15 @@ void ReleaseMemoryPagesToOS(uptr beg, uptr end) {
   // FIXME: add madvise-analog when we move to 64-bits.
 }
 
-void NoHugePagesInRegion(uptr addr, uptr size) {
+bool NoHugePagesInRegion(uptr addr, uptr size) {
   // FIXME: probably similar to ReleaseMemoryToOS.
+  return true;
 }
 
-void DontDumpShadowMemory(uptr addr, uptr length) {
+bool DontDumpShadowMemory(uptr addr, uptr length) {
   // This is almost useless on 32-bits.
   // FIXME: add madvise-analog when we move to 64-bits.
+  return true;
 }
 
 uptr FindAvailableMemoryRange(uptr size, uptr alignment, uptr left_padding,

compiler-rt/lib/tsan/rtl/tsan_platform_posix.cc

@@ -23,6 +23,27 @@
 
 namespace __tsan {
 
+static const char kShadowMemoryMappingWarning[] =
+    "FATAL: %s can not madvise shadow region [%zx, %zx] with %s\n";
+
+static void NoHugePagesInShadow(uptr addr, uptr size) {
+  if (common_flags()->no_huge_pages_for_shadow)
+    if (!NoHugePagesInRegion(addr, size)) {
+      Printf(kShadowMemoryMappingWarning, SanitizerToolName, addr, addr + size,
+             "MADV_NOHUGEPAGE");
+      Die();
+    }
+}
+
+static void DontDumpShadow(uptr addr, uptr size) {
+  if (common_flags()->use_madv_dontdump)
+    if (!DontDumpShadowMemory(addr, size)) {
+      Printf(kShadowMemoryMappingWarning, SanitizerToolName, addr, addr + size,
+             "MADV_DONTDUMP");
+      Die();
+    }
+}
+
 #if !SANITIZER_GO
 void InitializeShadowMemory() {
   // Map memory shadow.
@@ -74,14 +95,9 @@ void InitializeShadowMemory() {
     DCHECK(0);
   }
 #endif
-  NoHugePagesInRegion(MemToShadow(kMadviseRangeBeg),
+  NoHugePagesInShadow(MemToShadow(kMadviseRangeBeg),
                       kMadviseRangeSize * kShadowMultiplier);
-  // Meta shadow is compressing and we don't flush it,
-  // so it makes sense to mark it as NOHUGEPAGE to not over-allocate memory.
-  // On one program it reduces memory consumption from 5GB to 2.5GB.
-  NoHugePagesInRegion(MetaShadowBeg(), MetaShadowEnd() - MetaShadowBeg());
-  if (common_flags()->use_madv_dontdump)
-    DontDumpShadowMemory(ShadowBeg(), ShadowEnd() - ShadowBeg());
+  DontDumpShadow(ShadowBeg(), ShadowEnd() - ShadowBeg());
   DPrintf("memory shadow: %zx-%zx (%zuGB)\n",
       ShadowBeg(), ShadowEnd(),
       (ShadowEnd() - ShadowBeg()) >> 30);
@@ -96,8 +112,8 @@ void InitializeShadowMemory() {
                "to link with -pie (%p, %p).\n", meta, MetaShadowBeg());
     Die();
   }
-  if (common_flags()->use_madv_dontdump)
-    DontDumpShadowMemory(meta, meta_size);
+  NoHugePagesInShadow(meta, meta_size);
+  DontDumpShadow(meta, meta_size);
   DPrintf("meta shadow: %zx-%zx (%zuGB)\n",
       meta, meta + meta_size, meta_size >> 30);