tsan: add memory_limit_mb flag

The flag allows to bound maximum process memory consumption (best effort).
If RSS reaches memory_limit_mb, tsan flushes all shadow memory.

llvm-svn: 191913

compiler-rt/lib/tsan/go/buildgo.sh

@@ -34,6 +34,7 @@ if [ "`uname -a | grep Linux`" != "" ]; then
 		../../sanitizer_common/sanitizer_posix_libcdep.cc
 		../../sanitizer_common/sanitizer_linux.cc
 		../../sanitizer_common/sanitizer_linux_libcdep.cc
+		../../sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cc
 	"
 elif [ "`uname -a | grep Darwin`" != "" ]; then
 	SUFFIX="darwin_amd64"

compiler-rt/lib/tsan/rtl/tsan_flags.cc

@@ -59,6 +59,7 @@ void InitializeFlags(Flags *f, const char *env) {
   f->profile_memory = "";
   f->flush_memory_ms = 0;
   f->flush_symbolizer_ms = 5000;
+  f->memory_limit_mb = 0;
   f->stop_on_start = false;
   f->running_on_valgrind = false;
   f->external_symbolizer_path = "";
@@ -92,6 +93,7 @@ void InitializeFlags(Flags *f, const char *env) {
   ParseFlag(env, &f->profile_memory, "profile_memory");
   ParseFlag(env, &f->flush_memory_ms, "flush_memory_ms");
   ParseFlag(env, &f->flush_symbolizer_ms, "flush_symbolizer_ms");
+  ParseFlag(env, &f->memory_limit_mb, "memory_limit_mb");
   ParseFlag(env, &f->stop_on_start, "stop_on_start");
   ParseFlag(env, &f->external_symbolizer_path, "external_symbolizer_path");
   ParseFlag(env, &f->history_size, "history_size");

compiler-rt/lib/tsan/rtl/tsan_platform.h

@@ -134,6 +134,7 @@ static inline uptr AlternativeAddress(uptr addr) {
 
 void FlushShadowMemory();
 void WriteMemoryProfile(char *buf, uptr buf_size);
+uptr GetRSS();
 
 const char *InitializePlatform();
 void FinalizePlatform();

compiler-rt/lib/tsan/rtl/tsan_platform_linux.cc

@@ -19,6 +19,7 @@
 #include "sanitizer_common/sanitizer_common.h"
 #include "sanitizer_common/sanitizer_libc.h"
 #include "sanitizer_common/sanitizer_procmaps.h"
+#include "sanitizer_common/sanitizer_stoptheworld.h"
 #include "tsan_platform.h"
 #include "tsan_rtl.h"
 #include "tsan_flags.h"
@@ -106,10 +107,23 @@ void WriteMemoryProfile(char *buf, uptr buf_size) {
       mi.arena >> 20, mi.hblkhd >> 20, mi.fordblks >> 20, mi.keepcost >> 20);
 }
 
-void FlushShadowMemory() {
+uptr GetRSS() {
+  uptr mem[7] = {};
+  __sanitizer::GetMemoryProfile(FillProfileCallback, mem, 7);
+  return mem[6];
+}
+
+
+void FlushShadowMemoryCallback(
+    const SuspendedThreadsList &suspended_threads_list,
+    void *argument) {
   FlushUnneededShadowMemory(kLinuxShadowBeg, kLinuxShadowEnd - kLinuxShadowBeg);
 }
 
+void FlushShadowMemory() {
+  StopTheWorld(FlushShadowMemoryCallback, 0);
+}
+
 #ifndef TSAN_GO
 static void ProtectRange(uptr beg, uptr end) {
   ScopedInRtl in_rtl;

compiler-rt/lib/tsan/rtl/tsan_rtl.cc

@@ -130,17 +130,38 @@ static void BackgroundThread(void *arg) {
   }
 
   u64 last_flush = NanoTime();
+  uptr last_rss = 0;
   for (int i = 0; ; i++) {
     SleepForSeconds(1);
     u64 now = NanoTime();
 
     // Flush memory if requested.
-    if (flags()->flush_memory_ms) {
+    if (flags()->flush_memory_ms > 0) {
       if (last_flush + flags()->flush_memory_ms * kMs2Ns < now) {
+        if (flags()->verbosity > 0)
+          Printf("ThreadSanitizer: periodic memory flush\n");
         FlushShadowMemory();
         last_flush = NanoTime();
       }
     }
+    if (flags()->memory_limit_mb > 0) {
+      uptr rss = GetRSS();
+      uptr limit = uptr(flags()->memory_limit_mb) << 20;
+      if (flags()->verbosity > 0) {
+        Printf("ThreadSanitizer: memory flush check"
+               " RSS=%llu LAST=%llu LIMIT=%llu\n",
+               (u64)rss>>20, (u64)last_rss>>20, (u64)limit>>20);
+      }
+      if (2 * rss > limit + last_rss) {
+        if (flags()->verbosity > 0)
+          Printf("ThreadSanitizer: flushing memory due to RSS\n");
+        FlushShadowMemory();
+        rss = GetRSS();
+        if (flags()->verbosity > 0)
+          Printf("ThreadSanitizer: memory flushed RSS=%llu\n", (u64)rss>>20);
+      }
+      last_rss = rss;
+    }
 
     // Write memory profile if requested.
     if (mprof_fd != kInvalidFd)