[Demangle] Extract nonMicrosoftDemangle from llvm::demangle

Introduce a new demangling function that supports symbols using Itanium
mangling and Rust v0 mangling, and is expected in the near future to
include support for D mangling as well.

Unlike llvm::demangle, the function does not accept extra underscore
decoration. The callers generally know exactly when symbols should
include the extra decoration and so they should be responsible for
stripping it.

Functionally the only intended change is to allow demangling Rust
symbols with an extra underscore decoration through llvm::demangle,
which matches the existing behaviour for Itanium symbols.

Reviewed By: dblaikie, jhenderson

Part of https://reviews.llvm.org/D110664

llvm/include/llvm/Demangle/Demangle.h

@@ -67,6 +67,8 @@ char *rustDemangle(const char *MangledName, char *Buf, size_t *N, int *Status);
 /// demangling occurred.
 std::string demangle(const std::string &MangledName);
 
+bool nonMicrosoftDemangle(const char *MangledName, std::string &Result);
+
 /// "Partial" demangler. This supports demangling a string into an AST
 /// (typically an intermediate stage in itaniumDemangle) and querying certain
 /// properties or partially printing the demangled name.

llvm/lib/Demangle/Demangle.cpp

@@ -12,32 +12,46 @@
 
 #include "llvm/Demangle/Demangle.h"
 #include <cstdlib>
+#include <cstring>
 
-static bool isItaniumEncoding(const std::string &MangledName) {
-  size_t Pos = MangledName.find_first_not_of('_');
-  // A valid Itanium encoding requires 1-4 leading underscores, followed by 'Z'.
-  return Pos > 0 && Pos <= 4 && MangledName[Pos] == 'Z';
+static bool isItaniumEncoding(const char *S) {
+  // Itanium encoding requires 1 or 3 leading underscores, followed by 'Z'.
+  return std::strncmp(S, "_Z", 2) == 0 || std::strncmp(S, "___Z", 4) == 0;
 }
 
-static bool isRustEncoding(const std::string &MangledName) {
-  return MangledName.size() >= 2 && MangledName[0] == '_' &&
-         MangledName[1] == 'R';
-}
+static bool isRustEncoding(const char *S) { return S[0] == '_' && S[1] == 'R'; }
 
 std::string llvm::demangle(const std::string &MangledName) {
-  char *Demangled;
+  std::string Result;
+  const char *S = MangledName.c_str();
+
+  if (nonMicrosoftDemangle(S, Result))
+    return Result;
+
+  if (S[0] == '_' && nonMicrosoftDemangle(S + 1, Result))
+    return Result;
+
+  if (char *Demangled =
+          microsoftDemangle(S, nullptr, nullptr, nullptr, nullptr)) {
+    Result = Demangled;
+    std::free(Demangled);
+    return Result;
+  }
+
+  return MangledName;
+}
+
+bool llvm::nonMicrosoftDemangle(const char *MangledName, std::string &Result) {
+  char *Demangled = nullptr;
   if (isItaniumEncoding(MangledName))
-    Demangled = itaniumDemangle(MangledName.c_str(), nullptr, nullptr, nullptr);
+    Demangled = itaniumDemangle(MangledName, nullptr, nullptr, nullptr);
   else if (isRustEncoding(MangledName))
-    Demangled = rustDemangle(MangledName.c_str(), nullptr, nullptr, nullptr);
-  else
-    Demangled = microsoftDemangle(MangledName.c_str(), nullptr, nullptr,
-                                  nullptr, nullptr);
+    Demangled = rustDemangle(MangledName, nullptr, nullptr, nullptr);
 
   if (!Demangled)
-    return MangledName;
+    return false;
 
-  std::string Ret = Demangled;
+  Result = Demangled;
   std::free(Demangled);
-  return Ret;
+  return true;
 }

llvm/unittests/Demangle/DemangleTest.cpp

@@ -22,6 +22,7 @@ TEST(Demangle, demangleTest) {
   EXPECT_EQ(demangle("?foo@@YAXH@Z"), "void __cdecl foo(int)");
   EXPECT_EQ(demangle("foo"), "foo");
   EXPECT_EQ(demangle("_RNvC3foo3bar"), "foo::bar");
+  EXPECT_EQ(demangle("__RNvC3foo3bar"), "foo::bar");
 
   // Regression test for demangling of optional template-args for vendor
   // extended type qualifier (https://bugs.llvm.org/show_bug.cgi?id=48009)