[scudo] Adding a public Scudo interface

Summary:
The first and only function to start with allows to set the soft or hard RSS
limit at runtime. Add associated tests.

Reviewers: alekseyshl

Reviewed By: alekseyshl

Subscribers: mgorny, #sanitizers, llvm-commits

Differential Revision: https://reviews.llvm.org/D41128

llvm-svn: 320611

compiler-rt/include/CMakeLists.txt

@@ -10,6 +10,7 @@ if (COMPILER_RT_BUILD_SANITIZERS)
     sanitizer/linux_syscall_hooks.h
     sanitizer/lsan_interface.h
     sanitizer/msan_interface.h
+    sanitizer/scudo_interface.h
     sanitizer/tsan_interface.h
     sanitizer/tsan_interface_atomic.h)
 endif(COMPILER_RT_BUILD_SANITIZERS)

compiler-rt/include/sanitizer/scudo_interface.h

@@ -0,0 +1,34 @@
+//===-- sanitizer/scudo_interface.h -----------------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+/// Public Scudo interface header.
+//
+//===----------------------------------------------------------------------===//
+#ifndef SANITIZER_SCUDO_INTERFACE_H_
+#define SANITIZER_SCUDO_INTERFACE_H_
+
+#include <sanitizer/common_interface_defs.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+  // This function may be optionally provided by a user and should return
+  // a string containing Scudo runtime options. See scudo_flags.h for details.
+  const char* __scudo_default_options();
+
+  // This function allows to set the RSS limit at runtime. This can be either
+  // the hard limit (HardLimit=1) or the soft limit (HardLimit=0). The limit
+  // can be removed by setting LimitMb to 0. This function's parameters should
+  // be fully trusted to avoid security mishaps.
+  void __scudo_set_rss_limit(unsigned long LimitMb, int HardLimit);
+#ifdef __cplusplus
+}  // extern "C"
+#endif
+
+#endif  // SANITIZER_SCUDO_INTERFACE_H_

compiler-rt/lib/scudo/scudo_allocator.cpp

@@ -597,6 +597,14 @@ struct ScudoAllocator {
     initThreadMaybe();
     return FailureHandler::OnBadRequest();
   }
+
+  void setRssLimit(uptr LimitMb, bool HardLimit) {
+    if (HardLimit)
+      HardRssLimitMb = LimitMb;
+    else
+      SoftRssLimitMb = LimitMb;
+    CheckRssLimit = HardRssLimitMb || SoftRssLimitMb;
+  }
 };
 
 static ScudoAllocator Instance(LINKER_INITIALIZED);
@@ -726,3 +734,13 @@ int __sanitizer_get_ownership(const void *Ptr) {
 uptr __sanitizer_get_allocated_size(const void *Ptr) {
   return Instance.getUsableSize(Ptr);
 }
+
+// Interface functions
+
+extern "C" {
+void __scudo_set_rss_limit(unsigned long LimitMb, int HardLimit) {  // NOLINT
+  if (!SCUDO_CAN_USE_PUBLIC_INTERFACE)
+    return;
+  Instance.setRssLimit(LimitMb, !!HardLimit);
+}
+}  // extern "C"

compiler-rt/lib/scudo/scudo_interface_internal.h

@@ -0,0 +1,22 @@
+//===-- scudo_interface_internal.h ------------------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+///
+/// Private Scudo interface header.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef SCUDO_INTERFACE_INTERNAL_H_
+#define SCUDO_INTERFACE_INTERNAL_H_
+
+extern "C" {
+SANITIZER_INTERFACE_ATTRIBUTE
+void __scudo_set_rss_limit(unsigned long LimitMb, int HardLimit);  // NOLINT
+}  // extern "C"
+
+#endif  // SCUDO_INTERFACE_INTERNAL_H_

compiler-rt/lib/scudo/scudo_platform.h

@@ -8,6 +8,7 @@
 //===----------------------------------------------------------------------===//
 ///
 /// Scudo platform specific definitions.
+/// TODO(kostyak): add tests for the compile time defines.
 ///
 //===----------------------------------------------------------------------===//
 
@@ -45,6 +46,11 @@
 # define SCUDO_SHARED_TSD_POOL_SIZE 32U
 #endif  // SCUDO_SHARED_TSD_POOL_SIZE
 
+// The following allows the public interface functions to be disabled.
+#ifndef SCUDO_CAN_USE_PUBLIC_INTERFACE
+# define SCUDO_CAN_USE_PUBLIC_INTERFACE 1
+#endif
+
 namespace __scudo {
 
 #if SANITIZER_CAN_USE_ALLOCATOR64

compiler-rt/test/scudo/interface.cpp

@@ -1,17 +1,21 @@
 // RUN: %clangxx_scudo %s -lstdc++ -o %t
-// RUN: %run %t ownership          2>&1
-// RUN: %run %t ownership-and-size 2>&1
-// RUN: %run %t heap-size          2>&1
+// RUN:                                                   %run %t ownership          2>&1
+// RUN:                                                   %run %t ownership-and-size 2>&1
+// RUN:                                                   %run %t heap-size          2>&1
+// RUN: %env_scudo_opts="allocator_may_return_null=1"     %run %t soft-limit         2>&1
+// RUN: %env_scudo_opts="allocator_may_return_null=1" not %run %t hard-limit         2>&1
 
 // Tests that the sanitizer interface functions behave appropriately.
 
 #include <stdlib.h>
 #include <assert.h>
 #include <string.h>
+#include <unistd.h>
 
 #include <vector>
 
 #include <sanitizer/allocator_interface.h>
+#include <sanitizer/scudo_interface.h>
 
 int main(int argc, char **argv)
 {
@@ -42,6 +46,41 @@ int main(int argc, char **argv)
     // allocator function.
     assert(__sanitizer_get_heap_size() >= 0);
   }
+  if (!strcmp(argv[1], "soft-limit")) {
+    // Verifies that setting the soft RSS limit at runtime works as expected.
+    std::vector<void *> pointers;
+    size_t size = 1 << 19;  // 512Kb
+    for (int i = 0; i < 5; i++)
+      pointers.push_back(malloc(size));
+    // Set the soft RSS limit to 1Mb.
+    __scudo_set_rss_limit(1, 0);
+    usleep(20000);
+    // The following allocation should return NULL.
+    void *p = malloc(size);
+    assert(!p);
+    // Remove the soft RSS limit.
+    __scudo_set_rss_limit(0, 0);
+    // The following allocation should succeed.
+    p = malloc(size);
+    assert(p);
+    free(p);
+    while (!pointers.empty()) {
+      free(pointers.back());
+      pointers.pop_back();
+    }
+  }
+  if (!strcmp(argv[1], "hard-limit")) {
+    // Verifies that setting the hard RSS limit at runtime works as expected.
+    std::vector<void *> pointers;
+    size_t size = 1 << 19;  // 512Kb
+    for (int i = 0; i < 5; i++)
+      pointers.push_back(malloc(size));
+    // Set the hard RSS limit to 1Mb
+    __scudo_set_rss_limit(1, 1);
+    usleep(20000);
+    // The following should trigger our death.
+    void *p = malloc(size);
+  }
 
   return 0;
 }