[lld-macho] Add LTO cache support

This adds support for the lld-only `--thinlto-cache-policy` option, as well as
implementations for ld64's `-cache_path_lto`, `-prune_interval_lto`,
`-prune_after_lto`, and `-max_relative_cache_size_lto`.

Test is adapted from lld/test/ELF/lto/cache.ll

Differential Revision: https://reviews.llvm.org/D105922

lld/MachO/Config.h

@@ -14,6 +14,7 @@
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/BinaryFormat/MachO.h"
+#include "llvm/Support/CachePruning.h"
 #include "llvm/Support/GlobPattern.h"
 #include "llvm/Support/VersionTuple.h"
 #include "llvm/TextAPI/Architecture.h"
@@ -139,6 +140,8 @@ struct Configuration {
   llvm::StringRef thinLTOJobs;
   llvm::StringRef umbrella;
   uint32_t ltoo = 2;
+  llvm::CachePruningPolicy thinLTOCachePolicy;
+  llvm::StringRef thinLTOCacheDir;
   bool deadStripDylibs = false;
   bool demangle = false;
   bool deadStrip = false;

lld/MachO/Driver.cpp

@@ -195,6 +195,35 @@ getFrameworkSearchPaths(InputArgList &args,
                         {"/Library/Frameworks", "/System/Library/Frameworks"});
 }
 
+static llvm::CachePruningPolicy getLTOCachePolicy(InputArgList &args) {
+  SmallString<128> ltoPolicy;
+  auto add = [&ltoPolicy](Twine val) {
+    if (!ltoPolicy.empty())
+      ltoPolicy += ":";
+    val.toVector(ltoPolicy);
+  };
+  for (const Arg *arg :
+       args.filtered(OPT_thinlto_cache_policy, OPT_prune_interval_lto,
+                     OPT_prune_after_lto, OPT_max_relative_cache_size_lto)) {
+    switch (arg->getOption().getID()) {
+    case OPT_thinlto_cache_policy: add(arg->getValue()); break;
+    case OPT_prune_interval_lto:
+      if (!strcmp("-1", arg->getValue()))
+        add("prune_interval=87600h"); // 10 years
+      else
+        add(Twine("prune_interval=") + arg->getValue() + "s");
+      break;
+    case OPT_prune_after_lto:
+      add(Twine("prune_after=") + arg->getValue() + "s");
+      break;
+    case OPT_max_relative_cache_size_lto:
+      add(Twine("cache_size=") + arg->getValue() + "%");
+      break;
+    }
+  }
+  return CHECK(parseCachePruningPolicy(ltoPolicy), "invalid LTO cache policy");
+}
+
 namespace {
 struct ArchiveMember {
   MemoryBufferRef mbref;
@@ -1171,6 +1200,8 @@ bool macho::link(ArrayRef<const char *> argsArr, bool canExitEarly,
   config->ltoo = args::getInteger(args, OPT_lto_O, 2);
   if (config->ltoo > 3)
     error("--lto-O: invalid optimization level: " + Twine(config->ltoo));
+  config->thinLTOCacheDir = args.getLastArgValue(OPT_cache_path_lto);
+  config->thinLTOCachePolicy = getLTOCachePolicy(args);
   config->runtimePaths = args::getStrings(args, OPT_rpath);
   config->allLoad = args.hasArg(OPT_all_load);
   config->archMultiple = args.hasArg(OPT_arch_multiple);

lld/MachO/LTO.cpp

@@ -17,6 +17,8 @@
 #include "lld/Common/ErrorHandler.h"
 #include "lld/Common/Strings.h"
 #include "lld/Common/TargetOptionsCommandFlags.h"
+#include "llvm/LTO/Caching.h"
+#include "llvm/LTO/Config.h"
 #include "llvm/LTO/LTO.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Path.h"
@@ -97,11 +99,28 @@ void BitcodeCompiler::add(BitcodeFile &f) {
 std::vector<ObjFile *> BitcodeCompiler::compile() {
   unsigned maxTasks = ltoObj->getMaxTasks();
   buf.resize(maxTasks);
+  files.resize(maxTasks);
 
-  checkError(ltoObj->run([&](size_t task) {
+  // The -cache_path_lto option specifies the path to a directory in which
-    return std::make_unique<lto::NativeObjectStream>(
+  // to cache native object files for ThinLTO incremental builds. If a path was
-        std::make_unique<raw_svector_ostream>(buf[task]));
+  // specified, configure LTO to use it as the cache directory.
-  }));
+  lto::NativeObjectCache cache;
+  if (!config->thinLTOCacheDir.empty())
+    cache = check(
+        lto::localCache(config->thinLTOCacheDir,
+                        [&](size_t task, std::unique_ptr<MemoryBuffer> mb) {
+                          files[task] = std::move(mb);
+                        }));
+
+  checkError(ltoObj->run(
+      [&](size_t task) {
+        return std::make_unique<lto::NativeObjectStream>(
+            std::make_unique<raw_svector_ostream>(buf[task]));
+      },
+      cache));
+
+  if (!config->thinLTOCacheDir.empty())
+    pruneCache(config->thinLTOCacheDir, config->thinLTOCachePolicy);
 
   if (config->saveTemps) {
     if (!buf[0].empty())
@@ -130,6 +149,8 @@ std::vector<ObjFile *> BitcodeCompiler::compile() {
     ret.push_back(make<ObjFile>(
         MemoryBufferRef(buf[i], saver.save(filePath.str())), modTime, ""));
   }
-
+  for (std::unique_ptr<MemoryBuffer> &file : files)
+    if (file)
+      ret.push_back(make<ObjFile>(*file, 0, ""));
   return ret;
 }

lld/MachO/LTO.h

@@ -10,6 +10,7 @@
 #define LLD_MACHO_LTO_H
 
 #include "llvm/ADT/SmallString.h"
+#include "llvm/Support/MemoryBuffer.h"
 #include <memory>
 #include <vector>
 
@@ -35,6 +36,7 @@ public:
 private:
   std::unique_ptr<llvm::lto::LTO> ltoObj;
   std::vector<llvm::SmallString<0>> buf;
+  std::vector<std::unique_ptr<llvm::MemoryBuffer>> files;
 };
 
 } // namespace macho

lld/MachO/Options.td

@@ -68,6 +68,9 @@ def lto_O: Joined<["--"], "lto-O">,
     HelpText<"Set optimization level for LTO (default: 2)">,
     MetaVarName<"<opt-level>">,
     Group<grp_lld>;
+def thinlto_cache_policy: Joined<["--"], "thinlto-cache-policy=">,
+    HelpText<"Pruning policy for the ThinLTO cache">,
+    Group<grp_lld>;
 
 // This is a complete Options.td compiled from Apple's ld(1) manpage
 // dated 2018-03-07 and cross checked with ld64 source code in repo
@@ -915,22 +918,18 @@ def object_path_lto : Separate<["-"], "object_path_lto">,
 def cache_path_lto : Separate<["-"], "cache_path_lto">,
     MetaVarName<"<path>">,
     HelpText<"Use <path> as a directory for the incremental LTO cache">,
-    Flags<[HelpHidden]>,
     Group<grp_rare>;
 def prune_interval_lto : Separate<["-"], "prune_interval_lto">,
     MetaVarName<"<seconds>">,
     HelpText<"Prune the incremental LTO cache after <seconds> (-1 disables pruning)">,
-    Flags<[HelpHidden]>,
     Group<grp_rare>;
 def prune_after_lto : Separate<["-"], "prune_after_lto">,
     MetaVarName<"<seconds>">,
     HelpText<"Remove LTO cache entries after <seconds>">,
-    Flags<[HelpHidden]>,
     Group<grp_rare>;
 def max_relative_cache_size_lto : Separate<["-"], "max_relative_cache_size_lto">,
     MetaVarName<"<percent>">,
     HelpText<"Limit the incremental LTO cache growth to <percent> of free disk, space">,
-    Flags<[HelpHidden]>,
     Group<grp_rare>;
 def page_align_data_atoms : Flag<["-"], "page_align_data_atoms">,
     HelpText<"Distribute global variables on separate pages so page used/dirty status can guide creation of an order file to cluster commonly used/dirty globals">,

lld/test/MachO/lto-cache.ll

@@ -0,0 +1,94 @@
+; REQUIRES: x86
+; NetBSD: noatime mounts currently inhibit 'touch' from updating atime
+; UNSUPPORTED: system-netbsd
+
+; RUN: rm -rf %t; split-file %s %t
+; RUN: opt -module-hash -module-summary %t/foo.ll -o %t/foo.o
+; RUN: opt -module-hash -module-summary %t/bar.ll -o %t/bar.o
+
+; RUN: rm -Rf %t/cache && mkdir %t/cache
+;; Create two files that would be removed by cache pruning due to age.
+;; We should only remove files matching the pattern "llvmcache-*".
+; RUN: touch -t 197001011200 %t/cache/llvmcache-baz %t/cache/baz
+; RUN: %lld -cache_path_lto %t/cache \
+; RUN:   --thinlto-cache-policy=prune_after=1h:prune_interval=0s \
+; RUN:   -o %t/test %t/foo.o %t/bar.o
+
+;; Two cached objects, plus a timestamp file and "baz", minus the file we removed.
+; RUN: ls %t/cache | count 4
+
+;; Same thing, but with `-prune_after_lto`
+; RUN: touch -t 197001011200 %t/cache/llvmcache-baz
+; RUN: %lld -cache_path_lto %t/cache -prune_after_lto 3600 -prune_interval_lto 0 \
+; RUN:   -o %t/test %t/foo.o %t/bar.o
+; RUN: ls %t/cache | count 4
+
+;; Create a file of size 64KB.
+; RUN: %python -c "print(' ' * 65536)" > %t/cache/llvmcache-baz
+
+;; This should leave the file in place.
+; RUN: %lld -cache_path_lto %t/cache \
+; RUN:   --thinlto-cache-policy=cache_size_bytes=128k:prune_interval=0s \
+; RUN:   -o %t/test %t/foo.o %t/bar.o
+; RUN: ls %t/cache | count 5
+
+;; Increase the age of llvmcache-baz, which will give it the oldest time stamp
+;; so that it is processed and removed first.
+; RUN: %python -c 'import os,sys,time; t=time.time()-120; os.utime(sys.argv[1],(t,t))' \
+; RUN:    %t/cache/llvmcache-baz
+
+;; This should remove it.
+; RUN: %lld -cache_path_lto %t/cache \
+; RUN:   --thinlto-cache-policy=cache_size_bytes=32k:prune_interval=0s \
+; RUN:   -o %t/test %t/foo.o %t/bar.o
+; RUN: ls %t/cache | count 4
+
+;; Delete everything except for the timestamp, "baz" and one cache file.
+; RUN: %lld -cache_path_lto %t/cache \
+; RUN:   --thinlto-cache-policy=prune_after=0s:cache_size=0%:cache_size_files=1:prune_interval=0s \
+; RUN:   -o %t/test %t/foo.o %t/bar.o
+; RUN: ls %t/cache | count 3
+
+;; Check that we remove the least recently used file first.
+; RUN: rm -fr %t/cache
+; RUN: mkdir %t/cache
+; RUN: echo xyz > %t/cache/llvmcache-old
+; RUN: touch -t 198002011200 %t/cache/llvmcache-old
+; RUN: echo xyz > %t/cache/llvmcache-newer
+; RUN: touch -t 198002021200 %t/cache/llvmcache-newer
+; RUN: %lld -cache_path_lto %t/cache \
+; RUN:   --thinlto-cache-policy=prune_after=0s:cache_size=0%:cache_size_files=3:prune_interval=0s \
+; RUN:   -o %t/test %t/foo.o %t/bar.o
+; RUN: ls %t/cache | FileCheck %s
+
+;; Check that  `-max_relative_cache_size_lto` is a legal argument.
+; RUN: %lld -cache_path_lto %t/cache -max_relative_cache_size_lto 10 \
+; RUN:   -o %t/test %t/foo.o %t/bar.o
+
+; CHECK-NOT: llvmcache-old
+; CHECK: llvmcache-newer
+; CHECK-NOT: llvmcache-old
+
+;--- foo.ll
+
+target triple = "x86_64-apple-macosx10.15.0"
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+
+define void @globalfunc() #0 {
+entry:
+  ret void
+}
+
+
+;--- bar.ll
+
+target triple = "x86_64-apple-macosx10.15.0"
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+
+define i32 @main() {
+entry:
+  call void (...) @globalfunc()
+  ret i32 0
+}
+
+declare void @globalfunc(...)