tsan: fix unbounded memory consumption for large mallocs

This happens only in corner cases, but we observed this on a real app.
See the test for description of the exact scenario that lead to unbounded memory consumption.

llvm-svn: 240535

compiler-rt/lib/tsan/rtl/tsan_mman.cc

@@ -36,6 +36,23 @@ struct MapUnmapCallback {
     // We are about to unmap a chunk of user memory.
     // Mark the corresponding shadow memory as not needed.
     DontNeedShadowFor(p, size);
+    // Mark the corresponding meta shadow memory as not needed.
+    // Note the block does not contain any meta info at this point
+    // (this happens after free).
+    const uptr kMetaRatio = kMetaShadowCell / kMetaShadowSize;
+    const uptr kPageSize = GetPageSizeCached() * kMetaRatio;
+    // Block came from LargeMmapAllocator, so must be large.
+    // We rely on this in the calculations below.
+    CHECK_GE(size, 2 * kPageSize);
+    uptr diff = RoundUp(p, kPageSize) - p;
+    if (diff != 0) {
+      p += diff;
+      size -= diff;
+    }
+    diff = p + size - RoundDown(p + size, kPageSize);
+    if (diff != 0)
+      size -= diff;
+    FlushUnneededShadowMemory((uptr)MemToMeta(p), size / kMetaRatio);
   }
 };
 

compiler-rt/test/tsan/large_malloc_meta.cc

@@ -0,0 +1,28 @@
+// RUN: %clangxx_tsan -O1 %s -o %t && %run %t 2>&1 | FileCheck %s
+#include "test.h"
+#include <sys/mman.h>
+
+// Test for previously unbounded memory consumption for large mallocs.
+// Code allocates a large memory block (that is handled by LargeMmapAllocator),
+// and forces allocation of meta shadow for the block. Then freed the block.
+// But meta shadow was not unmapped. Then code occupies the virtual memory
+// range of the block with something else (that does not need meta shadow).
+// And repeats. As the result meta shadow growed infinitely.
+// This program used to consume >2GB. Now it consumes <50MB.
+
+int main() {
+  for (int i = 0; i < 1000; i++) {
+    const int kSize = 1 << 20;
+    const int kPageSize = 4 << 10;
+    volatile int *p = new int[kSize];
+    for (int j = 0; j < kSize; j += kPageSize / sizeof(*p))
+      __atomic_store_n(&p[i], 1, __ATOMIC_RELEASE);
+    delete[] p;
+    mmap(0, kSize * sizeof(*p) + kPageSize, PROT_NONE, MAP_PRIVATE | MAP_ANON,
+        -1, 0);
+  }
+  fprintf(stderr, "DONE\n");
+  return 0;
+}
+
+// CHECK: DONE