[llvm] [Debuginfo] Add llvm-debuginfod-find tool and end-to-end-tests.

This implements the `llvm-debuginfod-find` tool, which wraps the Debuginfod library (D112758) to query debuginfod servers for artifacts according to the [[ https://www.mankier.com/8/debuginfod#Webapi | specification ]].

Reviewed By: labath

Differential Revision: https://reviews.llvm.org/D112759

llvm/test/CMakeLists.txt

@@ -6,6 +6,7 @@ llvm_canonicalize_cmake_booleans(
   LLVM_ENABLE_DIA_SDK
   LLVM_ENABLE_FFI
   LLVM_ENABLE_THREADS
+  LLVM_ENABLE_CURL
   LLVM_ENABLE_ZLIB
   LLVM_ENABLE_LIBXML2
   LLVM_INCLUDE_GO_TESTS
@@ -72,6 +73,7 @@ set(LLVM_TEST_DEPENDS
           llvm-cxxdump
           llvm-cxxfilt
           llvm-cxxmap
+          llvm-debuginfod-find
           llvm-diff
           llvm-dis
           llvm-dlltool

llvm/test/lit.cfg.py

@@ -158,9 +158,9 @@ tools = [
 tools.extend([
     'dsymutil', 'lli', 'lli-child-target', 'llvm-ar', 'llvm-as',
     'llvm-addr2line', 'llvm-bcanalyzer', 'llvm-bitcode-strip', 'llvm-config',
-    'llvm-cov', 'llvm-cxxdump', 'llvm-cvtres', 'llvm-diff', 'llvm-dis',
-    'llvm-dwarfdump', 'llvm-dlltool', 'llvm-exegesis', 'llvm-extract',
-    'llvm-isel-fuzzer', 'llvm-ifs',
+    'llvm-cov', 'llvm-cxxdump', 'llvm-cvtres', 'llvm-debuginfod-find',
+    'llvm-diff', 'llvm-dis', 'llvm-dwarfdump', 'llvm-dlltool', 'llvm-exegesis',
+    'llvm-extract', 'llvm-isel-fuzzer', 'llvm-ifs',
     'llvm-install-name-tool', 'llvm-jitlink', 'llvm-opt-fuzzer', 'llvm-lib',
     'llvm-link', 'llvm-lto', 'llvm-lto2', 'llvm-mc', 'llvm-mca',
     'llvm-modextract', 'llvm-nm', 'llvm-objcopy', 'llvm-objdump', 'llvm-otool',
@@ -394,6 +394,9 @@ if config.enable_threads:
 if config.have_libxml2:
     config.available_features.add('libxml2')
 
+if config.have_curl:
+    config.available_features.add('curl')
+
 if config.have_opt_viewer_modules:
     config.available_features.add('have_opt_viewer_modules')
 

llvm/test/lit.site.cfg.py.in

@@ -39,6 +39,7 @@ config.llvm_use_sanitizer = "@LLVM_USE_SANITIZER@"
 config.have_zlib = @LLVM_ENABLE_ZLIB@
 config.have_libxar = @LLVM_HAVE_LIBXAR@
 config.have_libxml2 = @LLVM_ENABLE_LIBXML2@
+config.have_curl = @LLVM_ENABLE_CURL@
 config.have_dia_sdk = @LLVM_ENABLE_DIA_SDK@
 config.enable_ffi = @LLVM_ENABLE_FFI@
 config.build_examples = @LLVM_BUILD_EXAMPLES@

llvm/test/tools/llvm-debuginfod-find/Inputs/buildid/abcdef/debuginfo

@@ -0,0 +1 @@
+fake_debuginfo

llvm/test/tools/llvm-debuginfod-find/Inputs/buildid/abcdef/executable

@@ -0,0 +1 @@
+fake_executable

llvm/test/tools/llvm-debuginfod-find/Inputs/buildid/abcdef/source/directory/file.c

@@ -0,0 +1 @@
+int foo = 0;

llvm/test/tools/llvm-debuginfod-find/debuginfod.test

@@ -0,0 +1,77 @@
+# REQUIRES: curl
+# RUN: rm -rf %t
+# RUN: mkdir %t
+# # Query the python server for artifacts
+# RUN: DEBUGINFOD_CACHE_PATH=%t python %s --server-path %S/Inputs \
+# RUN:   --tool-cmd 'llvm-debuginfod-find --dump --executable abcdef' | \
+# RUN:   FileCheck %s --check-prefix=EXECUTABLE
+# RUN: DEBUGINFOD_CACHE_PATH=%t python %s --server-path %S/Inputs \
+# RUN:   --tool-cmd 'llvm-debuginfod-find --dump --source=/directory/file.c abcdef' | \
+# RUN:   FileCheck %s --check-prefix=SOURCE
+# RUN: DEBUGINFOD_CACHE_PATH=%t python %s --server-path %S/Inputs \
+# RUN:   --tool-cmd 'llvm-debuginfod-find --dump --debuginfo abcdef' | \
+# RUN:   FileCheck %s --check-prefix=DEBUGINFO
+
+# EXECUTABLE: fake_executable
+# SOURCE: int foo = 0;
+# DEBUGINFO: fake_debuginfo
+
+# # The artifacts should still be present in the cache without needing to query
+# # the server.
+# RUN: DEBUGINFOD_CACHE_PATH=%t llvm-debuginfod-find --dump --executable abcdef | \
+# RUN:   FileCheck %s --check-prefix=EXECUTABLE
+# RUN: DEBUGINFOD_CACHE_PATH=%t llvm-debuginfod-find --dump \
+# RUN:   --source=/directory/file.c abcdef | \
+# RUN:   FileCheck %s --check-prefix=SOURCE
+# RUN: DEBUGINFOD_CACHE_PATH=%t llvm-debuginfod-find --dump --debuginfo abcdef | \
+# RUN:   FileCheck %s --check-prefix=DEBUGINFO
+
+
+# This script is used to test the debuginfod client within a host tool.
+# It first stands up a Python HTTP static file server and then executes the tool.
+# This way the tool can make debuginfod HTTP requests to the static file server.
+import argparse
+import threading
+import http.server
+import functools
+import subprocess
+import sys
+import os
+
+
+# Serves files at the server_path, then runs the tool with specified args.
+# Sets the DEBUGINFOD_CACHE_PATH env var to point at the given cache_directory.
+# Sets the DEBUGINFOD_URLS env var to point at the local server.
+def test_tool(server_path, tool_args):
+    httpd = http.server.ThreadingHTTPServer(
+        ('',0), functools.partial(
+            http.server.SimpleHTTPRequestHandler,
+            directory=server_path))
+    port = httpd.server_port
+    thread = threading.Thread(target=httpd.serve_forever)
+    try:
+        thread.start()
+        env = os.environ
+        env['DEBUGINFOD_URLS'] = f'http://localhost:%s' % port
+        process = subprocess.Popen(
+            tool_args, env=env)
+        code = process.wait()
+        if code != 0:
+            print('nontrivial return code %s' % code)
+            return 1
+    finally:
+        httpd.shutdown()
+        thread.join()
+    return 0
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--server-path', default='./')
+    parser.add_argument('--tool-cmd', required=True, type=str)
+    args = parser.parse_args()
+    result = test_tool(args.server_path,
+        args.tool_cmd.split())
+    sys.exit(result)
+
+if __name__ == '__main__':
+    main()

llvm/tools/llvm-debuginfod-find/CMakeLists.txt

@@ -0,0 +1,10 @@
+set(LLVM_LINK_COMPONENTS
+  Debuginfod
+  Support
+  )
+add_llvm_tool(llvm-debuginfod-find
+  llvm-debuginfod-find.cpp
+  )
+if(LLVM_INSTALL_BINUTILS_SYMLINKS)
+  add_llvm_tool_symlink(debuginfod-find llvm-debuginfod-find)
+endif()

llvm/tools/llvm-debuginfod-find/llvm-debuginfod-find.cpp

@@ -0,0 +1,109 @@
+//===-- llvm-debuginfod-find.cpp - Simple CLI for libdebuginfod-client ----===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file contains the llvm-debuginfod-find tool. This tool
+/// queries the debuginfod servers in the DEBUGINFOD_URLS environment
+/// variable (delimited by space (" ")) for the executable,
+/// debuginfo, or specified source file of the binary matching the
+/// given build-id.
+///
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Debuginfod/Debuginfod.h"
+#include "llvm/Debuginfod/HTTPClient.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/InitLLVM.h"
+
+using namespace llvm;
+
+cl::OptionCategory DebuginfodFindCategory("llvm-debuginfod-find Options");
+
+cl::opt<std::string> InputBuildID(cl::Positional, cl::Required,
+                                  cl::desc("<input build_id>"), cl::init("-"),
+                                  cl::cat(DebuginfodFindCategory));
+
+static cl::opt<bool>
+    FetchExecutable("executable", cl::init(false),
+                    cl::desc("If set, fetch a binary file associated with this "
+                             "build id, containing the executable sections."),
+                    cl::cat(DebuginfodFindCategory));
+
+static cl::opt<bool>
+    FetchDebuginfo("debuginfo", cl::init(false),
+                   cl::desc("If set, fetch a binary file associated with this "
+                            "build id, containing the debuginfo sections."),
+                   cl::cat(DebuginfodFindCategory));
+
+static cl::opt<std::string> FetchSource(
+    "source", cl::init(""),
+    cl::desc("Fetch a source file associated with this build id, which is at "
+             "this relative path relative to the compilation directory."),
+    cl::cat(DebuginfodFindCategory));
+
+static cl::opt<bool>
+    DumpToStdout("dump", cl::init(false),
+                 cl::desc("If set, dumps the contents of the fetched artifact "
+                          "to standard output. Otherwise, dumps the absolute "
+                          "path to the cached artifact on disk."),
+                 cl::cat(DebuginfodFindCategory));
+
+[[noreturn]] static void helpExit() {
+  errs() << "Must specify exactly one of --executable, "
+            "--source=/path/to/file, or --debuginfo.";
+  exit(1);
+}
+
+ExitOnError ExitOnErr;
+
+int main(int argc, char **argv) {
+  InitLLVM X(argc, argv);
+  HTTPClient::initialize();
+
+  cl::HideUnrelatedOptions({&DebuginfodFindCategory});
+  cl::ParseCommandLineOptions(
+      argc, argv,
+      "llvm-debuginfod-find: Fetch debuginfod artifacts\n\n"
+      "This program is a frontend to the debuginfod client library. The cache "
+      "directory, request timeout (in seconds), and debuginfod server urls are "
+      "set by these environment variables:\n"
+      "DEBUGINFOD_CACHE_PATH (default set by sys::path::cache_directory)\n"
+      "DEBUGINFOD_TIMEOUT (defaults to 90s)\n"
+      "DEBUGINFOD_URLS=[comma separated URLs] (defaults to empty)\n");
+
+  if (FetchExecutable + FetchDebuginfo + (FetchSource != "") != 1)
+    helpExit();
+
+  std::string IDString;
+  if (!tryGetFromHex(InputBuildID, IDString)) {
+    errs() << "Build ID " << InputBuildID << " is not a hex string.\n";
+    exit(1);
+  }
+  BuildID ID(IDString.begin(), IDString.end());
+
+  std::string Path;
+  if (FetchSource != "")
+    Path = ExitOnErr(getCachedOrDownloadSource(ID, FetchSource));
+  else if (FetchExecutable)
+    Path = ExitOnErr(getCachedOrDownloadExecutable(ID));
+  else if (FetchDebuginfo)
+    Path = ExitOnErr(getCachedOrDownloadDebuginfo(ID));
+  else
+    llvm_unreachable("We have already checked that exactly one of the above "
+                     "conditions is true.");
+
+  if (DumpToStdout) {
+    // Print the contents of the artifact.
+    ErrorOr<std::unique_ptr<MemoryBuffer>> Buf = MemoryBuffer::getFile(
+        Path, /*IsText=*/false, /*RequiresNullTerminator=*/false);
+    ExitOnErr(errorCodeToError(Buf.getError()));
+    outs() << Buf.get()->getBuffer();
+  } else
+    // Print the path to the cached artifact file.
+    outs() << Path << "\n";
+}