[xray] Detect MPROTECT and error out when it's enabled (on NetBSD)

Add a CheckMPROTECT() routine to detect when pax MPROTECT is enabled
on NetBSD, and error xray out when it is.  The solution is adapted
from existing CheckASLR().

Differential Revision: https://reviews.llvm.org/D56049

llvm-svn: 350030

compiler-rt/lib/sanitizer_common/sanitizer_common.h

@@ -223,6 +223,7 @@ bool SetEnv(const char *name, const char *value);
 u32 GetUid();
 void ReExec();
 void CheckASLR();
+void CheckMPROTECT();
 char **GetArgv();
 char **GetEnviron();
 void PrintCmdline();

compiler-rt/lib/sanitizer_common/sanitizer_fuchsia.cc

@@ -89,6 +89,7 @@ void GetThreadStackTopAndBottom(bool, uptr *stack_top, uptr *stack_bottom) {
 void InitializePlatformEarly() {}
 void MaybeReexec() {}
 void CheckASLR() {}
+void CheckMPROTECT() {}
 void PlatformPrepareForSandboxing(__sanitizer_sandbox_arguments *args) {}
 void DisableCoreDumperIfNecessary() {}
 void InstallDeadlySignalHandlers(SignalHandlerType handler) {}

compiler-rt/lib/sanitizer_common/sanitizer_linux.cc

@@ -2023,6 +2023,30 @@ void CheckASLR() {
 #endif
 }
 
+void CheckMPROTECT() {
+#if SANITIZER_NETBSD
+  int mib[3];
+  int paxflags;
+  uptr len = sizeof(paxflags);
+
+  mib[0] = CTL_PROC;
+  mib[1] = internal_getpid();
+  mib[2] = PROC_PID_PAXFLAGS;
+
+  if (UNLIKELY(internal_sysctl(mib, 3, &paxflags, &len, NULL, 0) == -1)) {
+    Printf("sysctl failed\n");
+    Die();
+  }
+
+  if (UNLIKELY(paxflags & CTL_PROC_PAXFLAGS_MPROTECT)) {
+    Printf("This sanitizer is not compatible with enabled MPROTECT\n");
+    Die();
+  }
+#else
+  // Do nothing
+#endif
+}
+
 void PrintModuleMap() { }
 
 void CheckNoDeepBind(const char *filename, int flag) {

compiler-rt/lib/sanitizer_common/sanitizer_mac.cc

@@ -377,6 +377,10 @@ void CheckASLR() {
   // Do nothing
 }
 
+void CheckMPROTECT() {
+  // Do nothing
+}
+
 uptr GetPageSize() {
   return sysconf(_SC_PAGESIZE);
 }

compiler-rt/lib/sanitizer_common/sanitizer_rtems.cc

@@ -98,6 +98,7 @@ void GetThreadStackAndTls(bool main, uptr *stk_addr, uptr *stk_size,
 void InitializePlatformEarly() {}
 void MaybeReexec() {}
 void CheckASLR() {}
+void CheckMPROTECT() {}
 void DisableCoreDumperIfNecessary() {}
 void InstallDeadlySignalHandlers(SignalHandlerType handler) {}
 void SetAlternateSignalStack() {}

compiler-rt/lib/sanitizer_common/sanitizer_win.cc

@@ -1016,6 +1016,10 @@ void CheckASLR() {
   // Do nothing
 }
 
+void CheckMPROTECT() {
+  // Do nothing
+}
+
 char **GetArgv() {
   // FIXME: Actually implement this function.
   return 0;

compiler-rt/lib/xray/xray_init.cc

@@ -67,6 +67,9 @@ void __xray_init() XRAY_NEVER_INSTRUMENT {
   if (atomic_load(&XRayInitialized, memory_order_acquire))
     return;
 
+  // XRAY is not compatible with PaX MPROTECT
+  CheckMPROTECT();
+
   if (!atomic_load(&XRayFlagsInitialized, memory_order_acquire)) {
     initializeFlags();
     atomic_store(&XRayFlagsInitialized, true, memory_order_release);