[libFuzzer] Guard symbolization with try-lock.

Summary:
When out-of-memory or timeout occurs, threads can be stopped during
symbolization, thereby causing a deadlock when the OOM/TO handlers
attempt symbolization.  We avoid this deadlock by skipping symbolization
if another thread is symbolizing.

Reviewers: kcc

Reviewed By: kcc

Subscribers: llvm-commits

Differential Revision: https://reviews.llvm.org/D46605

llvm-svn: 331825

compiler-rt/lib/fuzzer/FuzzerLoop.cpp

@@ -105,7 +105,7 @@ void MallocHook(const volatile void *ptr, size_t size) {
       return;
     Printf("MALLOC[%zd] %p %zd\n", N, ptr, size);
     if (TraceLevel >= 2 && EF)
-      EF->__sanitizer_print_stack_trace();
+      PrintStackTrace();
   }
 }
 
@@ -118,7 +118,7 @@ void FreeHook(const volatile void *ptr) {
       return;
     Printf("FREE[%zd]   %p\n", N, ptr);
     if (TraceLevel >= 2 && EF)
-      EF->__sanitizer_print_stack_trace();
+      PrintStackTrace();
   }
 }
 
@@ -129,8 +129,7 @@ void Fuzzer::HandleMalloc(size_t Size) {
   Printf("==%d== ERROR: libFuzzer: out-of-memory (malloc(%zd))\n", GetPid(),
          Size);
   Printf("   To change the out-of-memory limit use -rss_limit_mb=<N>\n\n");
-  if (EF->__sanitizer_print_stack_trace)
-    EF->__sanitizer_print_stack_trace();
+  PrintStackTrace();
   DumpCurrentUnit("oom-");
   Printf("SUMMARY: libFuzzer: out-of-memory\n");
   PrintFinalStats();
@@ -231,8 +230,7 @@ void Fuzzer::CrashCallback() {
   if (EF->__sanitizer_acquire_crash_state)
     EF->__sanitizer_acquire_crash_state();
   Printf("==%lu== ERROR: libFuzzer: deadly signal\n", GetPid());
-  if (EF->__sanitizer_print_stack_trace)
-    EF->__sanitizer_print_stack_trace();
+  PrintStackTrace();
   Printf("NOTE: libFuzzer has rudimentary signal handlers.\n"
          "      Combine libFuzzer with AddressSanitizer or similar for better "
          "crash reports.\n");
@@ -249,8 +247,7 @@ void Fuzzer::ExitCallback() {
       !EF->__sanitizer_acquire_crash_state())
     return;
   Printf("==%lu== ERROR: libFuzzer: fuzz target exited\n", GetPid());
-  if (EF->__sanitizer_print_stack_trace)
-    EF->__sanitizer_print_stack_trace();
+  PrintStackTrace();
   Printf("SUMMARY: libFuzzer: fuzz target exited\n");
   DumpCurrentUnit("crash-");
   PrintFinalStats();
@@ -296,8 +293,7 @@ void Fuzzer::AlarmCallback() {
     DumpCurrentUnit("timeout-");
     Printf("==%lu== ERROR: libFuzzer: timeout after %d seconds\n", GetPid(),
            Seconds);
-    if (EF->__sanitizer_print_stack_trace)
-      EF->__sanitizer_print_stack_trace();
+    PrintStackTrace();
     Printf("SUMMARY: libFuzzer: timeout\n");
     PrintFinalStats();
     _Exit(Options.TimeoutExitCode); // Stop right now.
@@ -312,8 +308,7 @@ void Fuzzer::RssLimitCallback() {
       "==%lu== ERROR: libFuzzer: out-of-memory (used: %zdMb; limit: %zdMb)\n",
       GetPid(), GetPeakRSSMb(), Options.RssLimitMb);
   Printf("   To change the out-of-memory limit use -rss_limit_mb=<N>\n\n");
-  if (EF->__sanitizer_print_memory_profile)
-    EF->__sanitizer_print_memory_profile(95, 8);
+  PrintMemoryProfile();
   DumpCurrentUnit("oom-");
   Printf("SUMMARY: libFuzzer: out-of-memory\n");
   PrintFinalStats();

compiler-rt/lib/fuzzer/FuzzerUtil.cpp

@@ -16,6 +16,7 @@
 #include <chrono>
 #include <cstring>
 #include <errno.h>
+#include <mutex>
 #include <signal.h>
 #include <sstream>
 #include <stdio.h>
@@ -179,8 +180,12 @@ std::string Base64(const Unit &U) {
   return Res;
 }
 
+static std::mutex SymbolizeMutex;
+
 std::string DescribePC(const char *SymbolizedFMT, uintptr_t PC) {
-  if (!EF->__sanitizer_symbolize_pc) return "<can not symbolize>";
+  std::unique_lock<std::mutex> l(SymbolizeMutex, std::try_to_lock);
+  if (!EF->__sanitizer_symbolize_pc || !l.owns_lock())
+    return "<can not symbolize>";
   char PcDescr[1024] = {};
   EF->__sanitizer_symbolize_pc(reinterpret_cast<void*>(PC),
                                SymbolizedFMT, PcDescr, sizeof(PcDescr));
@@ -195,6 +200,18 @@ void PrintPC(const char *SymbolizedFMT, const char *FallbackFMT, uintptr_t PC) {
     Printf(FallbackFMT, PC);
 }
 
+void PrintStackTrace() {
+  std::unique_lock<std::mutex> l(SymbolizeMutex, std::try_to_lock);
+  if (EF->__sanitizer_print_stack_trace && l.owns_lock())
+    EF->__sanitizer_print_stack_trace();
+}
+
+void PrintMemoryProfile() {
+  std::unique_lock<std::mutex> l(SymbolizeMutex, std::try_to_lock);
+  if (EF->__sanitizer_print_memory_profile && l.owns_lock())
+    EF->__sanitizer_print_memory_profile(95, 8);
+}
+
 unsigned NumberOfCpuCores() {
   unsigned N = std::thread::hardware_concurrency();
   if (!N) {

compiler-rt/lib/fuzzer/FuzzerUtil.h

@@ -40,6 +40,10 @@ void PrintPC(const char *SymbolizedFMT, const char *FallbackFMT, uintptr_t PC);
 
 std::string DescribePC(const char *SymbolizedFMT, uintptr_t PC);
 
+void PrintStackTrace();
+
+void PrintMemoryProfile();
+
 unsigned NumberOfCpuCores();
 
 // Platform specific functions.

compiler-rt/test/fuzzer/SymbolizeDeadlock.cpp

@@ -0,0 +1,35 @@
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+
+// Tests that deadlocks do not occur when an OOM occurs during symbolization.
+
+#include <cassert>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <unistd.h>
+
+#include "Bingo.h"
+
+volatile unsigned Sink = 0;
+
+// Do not inline this function.  We want to trigger NEW_FUNC symbolization when
+// libFuzzer finds this function.  We use a macro to make the name as long
+// possible, hoping to increase the time spent in symbolization and increase the
+// chances of triggering a deadlock.
+__attribute__((noinline)) void BINGO() {
+  // Busy work.  Inserts a delay here so the deadlock is more likely to trigger.
+  for (unsigned i = 0; i < 330000000; i++) Sink += i;
+}
+
+extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
+  assert(Data);
+  if (Size < 3) return 0;
+  if (Data[0] == 'F' &&
+      Data[1] == 'U' &&
+      Data[2] == 'Z')
+    BINGO();
+  return 0;
+}
+

compiler-rt/test/fuzzer/symbolize-deadlock.test

@@ -0,0 +1,2 @@
+RUN: %cpp_compiler %S/SymbolizeDeadlock.cpp -o %t
+RUN: not %t -rss_limit_mb=20 2>&1