replica
/
hanchenye-llvm-project
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

[WebAssembly] Initial support for LTO 

Differential Revision: https://reviews.llvm.org/D47162

llvm-svn: 333570
This commit is contained in:
Sam Clegg 2018-05-30 18:07:52 +00:00
parent dff5b311af
commit c729c1b47d
26 changed files with 674 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
lld/test/lit.cfg.py
View File

@ -40,7 +40,8 @@ llvm_config.use_lld()
tool_patterns = [
'llc', 'llvm-as', 'llvm-mc', 'llvm-nm', 'llvm-objdump', 'llvm-pdbutil',
'llvm-dwarfdump', 'llvm-readelf', 'llvm-readobj', 'obj2yaml', 'yaml2obj']
'llvm-dwarfdump', 'llvm-readelf', 'llvm-readobj', 'obj2yaml', 'yaml2obj',
'opt', 'llvm-dis']
llvm_config.add_tool_substitutions(tool_patterns)

10
lld/test/wasm/lto/Inputs/cache.ll Normal file
View File

@ -0,0 +1,10 @@
target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"
target triple = "wasm32-unknown-unknown"
define i32 @_start() {
entry:
call void (...) @globalfunc()
ret i32 0
}
declare void @globalfunc(...)

6
lld/test/wasm/lto/Inputs/save-temps.ll Normal file
View File

@ -0,0 +1,6 @@
target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"
target triple = "wasm32-unknown-unknown"
define void @bar() {
ret void
}

7
lld/test/wasm/lto/Inputs/thinlto.ll Normal file
View File

@ -0,0 +1,7 @@
target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"
target triple = "wasm32-unknown-unknown"
define void @g() {
entry:
ret void
}

38
lld/test/wasm/lto/cache.ll Normal file
View File

@ -0,0 +1,38 @@
; RUN: opt -module-hash -module-summary %s -o %t.o
; RUN: opt -module-hash -module-summary %p/Inputs/cache.ll -o %t2.o
; RUN: rm -Rf %t.cache && mkdir %t.cache
; Create two files that would be removed by cache pruning due to age.
; We should only remove files matching the pattern "llvmcache-*".
; RUN: touch -t 197001011200 %t.cache/llvmcache-foo %t.cache/foo
; RUN: wasm-ld --thinlto-cache-dir=%t.cache --thinlto-cache-policy prune_after=1h:prune_interval=0s -o %t.wasm %t2.o %t.o
; Two cached objects, plus a timestamp file and "foo", minus the file we removed.
; RUN: ls %t.cache | count 4
; Create a file of size 64KB.
; RUN: "%python" -c "print(' ' * 65536)" > %t.cache/llvmcache-foo
; This should leave the file in place.
; RUN: wasm-ld --thinlto-cache-dir=%t.cache --thinlto-cache-policy cache_size_bytes=128k:prune_interval=0s -o %t.wasm %t2.o %t.o
; RUN: ls %t.cache | count 5
; This should remove it.
; RUN: wasm-ld --thinlto-cache-dir=%t.cache --thinlto-cache-policy cache_size_bytes=32k:prune_interval=0s -o %t.wasm %t2.o %t.o
; RUN: ls %t.cache | count 4
; Setting max number of files to 0 should disable the limit, not delete everything.
; RUN: wasm-ld --thinlto-cache-dir=%t.cache --thinlto-cache-policy prune_after=0s:cache_size=0%:cache_size_files=0:prune_interval=0s -o %t.wasm %t2.o %t.o
; RUN: ls %t.cache | count 4
; Delete everything except for the timestamp, "foo" and one cache file.
; RUN: wasm-ld --thinlto-cache-dir=%t.cache --thinlto-cache-policy prune_after=0s:cache_size=0%:cache_size_files=1:prune_interval=0s -o %t.wasm %t2.o %t.o
; RUN: ls %t.cache | count 3
target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"
target triple = "wasm32-unknown-unknown-wasm"
define void @globalfunc() #0 {
entry:
ret void
}

8
lld/test/wasm/lto/incompatible.ll Normal file
View File

@ -0,0 +1,8 @@
; REQUIRES: x86
; RUN: llvm-as %s -o %t.bc
; RUN: not wasm-ld %t.bc -o out.wasm 2>&1 | FileCheck %s
target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
target triple = "x86_64-unknown-linux-gnu"
; CHECK: {{.*}}incompatible.ll.tmp.bc: machine type must be wasm32

20
lld/test/wasm/lto/internalize-basic.ll Normal file
View File

@ -0,0 +1,20 @@
; RUN: llvm-as %s -o %t.o
; RUN: wasm-ld %t.o -o %t2 -save-temps
; RUN: llvm-dis < %t2.0.2.internalize.bc | FileCheck %s
target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"
target triple = "wasm32-unknown-unknown-wasm"
define void @_start() {
ret void
}
define hidden void @foo() {
ret void
}
; Check that _start is not internalized.
; CHECK: define void @_start()
; Check that foo function is correctly internalized.
; CHECK: define internal void @foo()

18
lld/test/wasm/lto/lto-start.ll Normal file
View File

@ -0,0 +1,18 @@
; RUN: llvm-as %s -o %t.o
; RUN: wasm-ld %t.o -o %t.wasm
; RUN: obj2yaml %t.wasm | FileCheck %s
; CHECK: - Type: CUSTOM
; CHECK-NEXT: Name: name
; CHECK-NEXT: FunctionNames:
; CHECK-NEXT: - Index: 0
; CHECK-NEXT: Name: __wasm_call_ctors
; CHECK-NEXT: - Index: 1
; CHECK-NEXT: Name: _start
target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"
target triple = "wasm32-unknown-unknown-wasm"
define void @_start() {
ret void
}

30
lld/test/wasm/lto/opt-level.ll Normal file
View File

@ -0,0 +1,30 @@
; RUN: llvm-as -o %t.o %s
; RUN: wasm-ld -o %t0 -e main --lto-O0 %t.o
; RUN: obj2yaml %t0 | FileCheck --check-prefix=CHECK-O0 %s
; RUN: wasm-ld -o %t2 -e main --lto-O2 %t.o
; RUN: obj2yaml %t2 | FileCheck --check-prefix=CHECK-O2 %s
; RUN: wasm-ld -o %t2a -e main %t.o
; RUN: obj2yaml %t2a | FileCheck --check-prefix=CHECK-O2 %s
; Reject invalid optimization levels.
; RUN: not ld.lld -o %t3 -e main --lto-O6 %t.o 2>&1 | \
; RUN: FileCheck --check-prefix=INVALID %s
; INVALID: invalid optimization level for LTO: 6
; RUN: not ld.lld -o %t3 -m elf_x86_64 -e main --lto-O-1 %t.o 2>&1 | \
; RUN: FileCheck --check-prefix=INVALIDNEGATIVE %s
; INVALIDNEGATIVE: invalid optimization level for LTO: 4294967295
target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"
target triple = "wasm32-unknown-unknown-wasm"
; CHECK-O0: Name: foo
; CHECK-O2-NOT: Name: foo
define internal void @foo() {
ret void
}
define void @main() {
call void @foo()
ret void
}

24
lld/test/wasm/lto/parallel.ll Normal file
View File

@ -0,0 +1,24 @@
; RUN: llvm-as -o %t.bc %s
; RUN: rm -f %t.lto.o %t1.lto.o
; RUN: wasm-ld --lto-partitions=2 -save-temps -o %t %t.bc -r
; RUN: llvm-nm %t.lto.o | FileCheck --check-prefix=CHECK0 %s
; RUN: llvm-nm %t1.lto.o | FileCheck --check-prefix=CHECK1 %s
target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"
target triple = "wasm32-unknown-unknown-wasm"
; CHECK0-NOT: bar
; CHECK0: T foo
; CHECK0-NOT: bar
define void @foo() {
call void @bar()
ret void
}
; CHECK1-NOT: foo
; CHECK1: T bar
; CHECK1-NOT: foo
define void @bar() {
call void @foo()
ret void
}

19
lld/test/wasm/lto/save-temps.ll Normal file
View File

@ -0,0 +1,19 @@
; RUN: cd %T
; RUN: rm -f a.out a.out.lto.bc a.out.lto.o
; RUN: llvm-as %s -o %t.o
; RUN: llvm-as %p/Inputs/save-temps.ll -o %t2.o
; RUN: wasm-ld -r -o a.out %t.o %t2.o -save-temps
; RUN: llvm-nm a.out | FileCheck %s
; RUN: llvm-nm a.out.0.0.preopt.bc | FileCheck %s
; RUN: llvm-nm a.out.lto.o | FileCheck %s
; RUN: llvm-dis a.out.0.0.preopt.bc
target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"
target triple = "wasm32-unknown-unknown"
define void @foo() {
ret void
}
; CHECK: T bar
; CHECK: T foo

34
lld/test/wasm/lto/thinlto.ll Normal file
View File

@ -0,0 +1,34 @@
; Basic ThinLTO tests.
; RUN: opt -module-summary %s -o %t1.o
; RUN: opt -module-summary %p/Inputs/thinlto.ll -o %t2.o
; First force single-threaded mode
; RUN: rm -f %t31.lto.o %t32.lto.o
; RUN: wasm-ld -r -save-temps --thinlto-jobs=1 %t1.o %t2.o -o %t3
; RUN: llvm-nm %t31.lto.o | FileCheck %s --check-prefix=NM1
; RUN: llvm-nm %t32.lto.o | FileCheck %s --check-prefix=NM2
; Next force multi-threaded mode
; RUN: rm -f %t31.lto.o %t32.lto.o
; RUN: wasm-ld -r -save-temps --thinlto-jobs=2 %t1.o %t2.o -o %t3
; RUN: llvm-nm %t31.lto.o | FileCheck %s --check-prefix=NM1
; RUN: llvm-nm %t32.lto.o | FileCheck %s --check-prefix=NM2
; Check without --thinlto-jobs (which currently default to hardware_concurrency)
; RUN: wasm-ld -r %t1.o %t2.o -o %t3
; RUN: llvm-nm %t31.lto.o | FileCheck %s --check-prefix=NM1
; RUN: llvm-nm %t32.lto.o | FileCheck %s --check-prefix=NM2
; NM1: T f
; NM2: T g
target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"
target triple = "wasm32-unknown-unknown"
declare void @g(...)
define void @f() {
entry:
call void (...) @g()
ret void
}

16
lld/test/wasm/lto/verify-invalid.ll Normal file
View File

@ -0,0 +1,16 @@
; RUN: llvm-as %s -o %t.o
; RUN: wasm-ld %t.o -o %t2 -mllvm -debug-pass=Arguments \
; RUN: 2>&1 | FileCheck -check-prefix=DEFAULT %s
; RUN: wasm-ld %t.o -o %t2 -mllvm -debug-pass=Arguments \
; RUN: -disable-verify 2>&1 | FileCheck -check-prefix=DISABLE %s
target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"
target triple = "wasm32-unknown-unknown"
define void @_start() {
ret void
}
; -disable-verify should disable the verification of bitcode.
; DEFAULT: Pass Arguments: {{.*}} -verify {{.*}} -verify
; DISABLE-NOT: Pass Arguments: {{.*}} -verify {{.*}} -verify

16
lld/test/wasm/lto/weak.ll Normal file
View File

@ -0,0 +1,16 @@
; RUN: llvm-as %s -o %t.o
; RUN: wasm-ld %t.o %t.o -o %t.wasm -r
; RUN: llvm-readobj -t %t.wasm | FileCheck %s
target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"
target triple = "wasm32-unknown-unknown-wasm"
define weak void @f() {
ret void
}
; CHECK: Symbol {
; CHECK-NEXT: Name: f
; CHECK-NEXT: Type: FUNCTION (0x0)
; CHECK-NEXT: Flags: 0x1
; CHECK-NEXT: }

3
lld/wasm/CMakeLists.txt
View File

@ -6,6 +6,7 @@ add_lld_library(lldWasm
Driver.cpp
InputChunks.cpp
InputFiles.cpp
LTO.cpp
MarkLive.cpp
OutputSections.cpp
SymbolTable.cpp
@ -18,6 +19,8 @@ add_lld_library(lldWasm
BinaryFormat
Core
Demangle
LTO
MC
Object
Option
Support

10
lld/wasm/Config.h
View File

@ -13,6 +13,7 @@
#include "llvm/ADT/StringRef.h"
#include "llvm/ADT/StringSet.h"
#include "llvm/BinaryFormat/Wasm.h"
#include "llvm/Support/CachePruning.h"
namespace lld {
namespace wasm {
@ -21,6 +22,7 @@ struct Configuration {
bool AllowUndefined;
bool CompressRelocTargets;
bool Demangle;
bool DisableVerify;
bool ExportTable;
bool GcSections;
bool ImportMemory;
@ -28,19 +30,25 @@ struct Configuration {
bool MergeDataSegments;
bool PrintGcSections;
bool Relocatable;
bool SaveTemps;
bool StripAll;
bool StripDebug;
bool StackFirst;
uint32_t GlobalBase;
uint32_t InitialMemory;
uint32_t MaxMemory;
uint32_t Optimize;
uint32_t ZStackSize;
unsigned LTOPartitions;
unsigned LTOO;
unsigned Optimize;
unsigned ThinLTOJobs;
llvm::StringRef Entry;
llvm::StringRef OutputFile;
llvm::StringRef ThinLTOCacheDir;
llvm::StringSet<> AllowUndefinedSymbols;
std::vector<llvm::StringRef> SearchPaths;
llvm::CachePruningPolicy ThinLTOCachePolicy;
};
// The only instance of Configuration struct.

64
lld/wasm/Driver.cpp
View File

@ -26,6 +26,7 @@
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Path.h"
#include "llvm/Support/Process.h"
#include "llvm/Support/TargetSelect.h"
#define DEBUG_TYPE "lld"
@ -48,6 +49,17 @@ enum {
#undef OPTION
};
// This function is called on startup. We need this for LTO since
// LTO calls LLVM functions to compile bitcode files to native code.
// Technically this can be delayed until we read bitcode files, but
// we don't bother to do lazily because the initialization is fast.
static void initLLVM() {
InitializeAllTargets();
InitializeAllTargetMCs();
InitializeAllAsmPrinters();
InitializeAllAsmParsers();
}
class LinkerDriver {
public:
void link(ArrayRef<const char *> ArgsArr);
@ -72,6 +84,7 @@ bool lld::wasm::link(ArrayRef<const char *> Args, bool CanExitEarly,
Config = make<Configuration>();
Symtab = make<SymbolTable>();
initLLVM();
LinkerDriver().link(Args);
// Exit immediately if we don't need to return to the caller.
@ -173,7 +186,8 @@ void LinkerDriver::addFile(StringRef Path) {
return;
MemoryBufferRef MBRef = *Buffer;
if (identify_magic(MBRef.getBuffer()) == file_magic::archive) {
switch (identify_magic(MBRef.getBuffer())) {
case file_magic::archive: {
SmallString<128> ImportFile = Path;
path::replace_extension(ImportFile, ".imports");
if (fs::exists(ImportFile))
@ -182,8 +196,12 @@ void LinkerDriver::addFile(StringRef Path) {
Files.push_back(make<ArchiveFile>(MBRef));
return;
}
Files.push_back(make<ObjFile>(MBRef));
case file_magic::bitcode:
Files.push_back(make<BitcodeFile>(MBRef));
break;
default:
Files.push_back(make<ObjFile>(MBRef));
}
}
// Add a given library by searching it from input search paths.
@ -261,6 +279,17 @@ static void handleWeakUndefines() {
}
}
// Force Sym to be entered in the output. Used for -u or equivalent.
static Symbol *addUndefined(StringRef Name) {
Symbol *S = Symtab->addUndefinedFunction(Name, 0, nullptr, nullptr);
// Since symbol S may not be used inside the program, LTO may
// eliminate it. Mark the symbol as "used" to prevent it.
S->IsUsedInRegularObj = true;
return S;
}
void LinkerDriver::link(ArrayRef<const char *> ArgsArr) {
WasmOptTable Parser;
opt::InputArgList Args = Parser.parse(ArgsArr.slice(1));
@ -288,12 +317,15 @@ void LinkerDriver::link(ArrayRef<const char *> ArgsArr) {
Config->AllowUndefined = Args.hasArg(OPT_allow_undefined);
Config->Demangle = Args.hasFlag(OPT_demangle, OPT_no_demangle, true);
Config->DisableVerify = Args.hasArg(OPT_disable_verify);
Config->Entry = getEntry(Args, Args.hasArg(OPT_relocatable) ? "" : "_start");
Config->ExportTable = Args.hasArg(OPT_export_table);
errorHandler().FatalWarnings =
Args.hasFlag(OPT_fatal_warnings, OPT_no_fatal_warnings, false);
Config->ImportMemory = Args.hasArg(OPT_import_memory);
Config->ImportTable = Args.hasArg(OPT_import_table);
Config->LTOO = args::getInteger(Args, OPT_lto_O, 2);
Config->LTOPartitions = args::getInteger(Args, OPT_lto_partitions, 1);
Config->Optimize = args::getInteger(Args, OPT_O, 0);
Config->OutputFile = Args.getLastArgValue(OPT_o);
Config->Relocatable = Args.hasArg(OPT_relocatable);
@ -304,10 +336,16 @@ void LinkerDriver::link(ArrayRef<const char *> ArgsArr) {
!Config->Relocatable);
Config->PrintGcSections =
Args.hasFlag(OPT_print_gc_sections, OPT_no_print_gc_sections, false);
Config->SaveTemps = Args.hasArg(OPT_save_temps);
Config->SearchPaths = args::getStrings(Args, OPT_L);
Config->StripAll = Args.hasArg(OPT_strip_all);
Config->StripDebug = Args.hasArg(OPT_strip_debug);
Config->StackFirst = Args.hasArg(OPT_stack_first);
Config->ThinLTOCacheDir = Args.getLastArgValue(OPT_thinlto_cache_dir);
Config->ThinLTOCachePolicy = CHECK(
parseCachePruningPolicy(Args.getLastArgValue(OPT_thinlto_cache_policy)),
"--thinlto-cache-policy: invalid cache policy");
Config->ThinLTOJobs = args::getInteger(Args, OPT_thinlto_jobs, -1u);
errorHandler().Verbose = Args.hasArg(OPT_verbose);
ThreadsEnabled = Args.hasFlag(OPT_threads, OPT_no_threads, true);
@ -319,6 +357,13 @@ void LinkerDriver::link(ArrayRef<const char *> ArgsArr) {
Config->CompressRelocTargets = Config->Optimize > 0 && !Config->Relocatable;
if (Config->LTOO > 3)
error("invalid optimization level for LTO: " + Twine(Config->LTOO));
if (Config->LTOPartitions == 0)
error("--lto-partitions: number of threads must be > 0");
if (Config->ThinLTOJobs == 0)
error("--thinlto-jobs: number of threads must be > 0");
if (auto *Arg = Args.getLastArg(OPT_allow_undefined_file))
readImportFile(Arg->getValue());
@ -372,12 +417,11 @@ void LinkerDriver::link(ArrayRef<const char *> ArgsArr) {
// For now, since we don't actually use the start function as the
// wasm start symbol, we don't need to care about it signature.
if (!Config->Entry.empty())
EntrySym =
Symtab->addUndefinedFunction(Config->Entry, 0, nullptr, nullptr);
EntrySym = addUndefined(Config->Entry);
// Handle the `--undefined <sym>` options.
for (auto *Arg : Args.filtered(OPT_undefined))
Symtab->addUndefinedFunction(Arg->getValue(), 0, nullptr, nullptr);
addUndefined(Arg->getValue());
}
createFiles(Args);
@ -388,11 +432,19 @@ void LinkerDriver::link(ArrayRef<const char *> ArgsArr) {
// symbols that we need to the symbol table.
for (InputFile *F : Files)
Symtab->addFile(F);
if (errorCount())
return;
// Add synthetic dummies for weak undefined functions.
if (!Config->Relocatable)
handleWeakUndefines();
// Do link-time optimization if given files are LLVM bitcode files.
// This compiles bitcode files into real object files.
Symtab->addCombinedLTOObject();
if (errorCount())
return;
// Make sure we have resolved all symbols.
if (!Config->Relocatable && !Config->AllowUndefined) {
Symtab->reportRemainingUndefines();

42
lld/wasm/InputFiles.cpp
View File

@ -370,6 +370,48 @@ void ArchiveFile::addMember(const Archive::Symbol *Sym) {
Symtab->addFile(Obj);
}
static uint8_t mapVisibility(GlobalValue::VisibilityTypes GvVisibility) {
switch (GvVisibility) {
case GlobalValue::DefaultVisibility:
return WASM_SYMBOL_VISIBILITY_DEFAULT;
case GlobalValue::HiddenVisibility:
case GlobalValue::ProtectedVisibility:
return WASM_SYMBOL_VISIBILITY_HIDDEN;
}
llvm_unreachable("unknown visibility");
}
static Symbol *createBitcodeSymbol(const lto::InputFile::Symbol &ObjSym,
BitcodeFile &F) {
StringRef Name = Saver.save(ObjSym.getName());
uint32_t Flags = ObjSym.isWeak() ? WASM_SYMBOL_BINDING_WEAK : 0;
Flags |= mapVisibility(ObjSym.getVisibility());
if (ObjSym.isUndefined()) {
if (ObjSym.isExecutable())
return Symtab->addUndefinedFunction(Name, Flags, &F, nullptr);
return Symtab->addUndefinedData(Name, Flags, &F);
}
if (ObjSym.isExecutable())
return Symtab->addDefinedFunction(Name, Flags, &F, nullptr);
return Symtab->addDefinedData(Name, Flags, &F, nullptr, 0, 0);
}
void BitcodeFile::parse() {
Obj = check(lto::InputFile::create(MemoryBufferRef(
MB.getBuffer(), Saver.save(ParentName + MB.getBufferIdentifier()))));
Triple T(Obj->getTargetTriple());
if (T.getArch() != Triple::wasm32) {
error(toString(MB.getBufferIdentifier()) + ": machine type must be wasm32");
return;
}
for (const lto::InputFile::Symbol &ObjSym : Obj->symbols())
Symbols.push_back(createBitcodeSymbol(ObjSym, *this));
}
// Returns a string in the format of "foo.o" or "foo.a(bar.o)".
std::string lld::toString(const wasm::InputFile *File) {
if (!File)

26
lld/wasm/InputFiles.h
View File

@ -14,6 +14,7 @@
#include "lld/Common/LLVM.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/DenseSet.h"
#include "llvm/LTO/LTO.h"
#include "llvm/Object/Archive.h"
#include "llvm/Object/Wasm.h"
#include "llvm/Support/MemoryBuffer.h"
@ -28,6 +29,12 @@ using llvm::wasm::WasmImport;
using llvm::wasm::WasmRelocation;
using llvm::wasm::WasmSignature;
namespace llvm {
namespace lto {
class InputFile;
}
} // namespace llvm
namespace lld {
namespace wasm {
@ -42,6 +49,7 @@ public:
enum Kind {
ObjectKind,
ArchiveKind,
BitcodeKind,
};
virtual ~InputFile() {}
@ -57,10 +65,15 @@ public:
// An archive file name if this file is created from an archive.
StringRef ParentName;
ArrayRef<Symbol *> getSymbols() const { return Symbols; }
protected:
InputFile(Kind K, MemoryBufferRef M) : MB(M), FileKind(K) {}
MemoryBufferRef MB;
// List of all symbols referenced or defined by this file.
std::vector<Symbol *> Symbols;
private:
const Kind FileKind;
};
@ -113,7 +126,6 @@ public:
std::vector<InputSection *> CustomSections;
llvm::DenseMap<uint32_t, InputSection *> CustomSectionsByIndex;
ArrayRef<Symbol *> getSymbols() const { return Symbols; }
Symbol *getSymbol(uint32_t Index) const { return Symbols[Index]; }
FunctionSymbol *getFunctionSymbol(uint32_t Index) const;
DataSymbol *getDataSymbol(uint32_t Index) const;
@ -126,12 +138,18 @@ private:
bool isExcludedByComdat(InputChunk *Chunk) const;
// List of all symbols referenced or defined by this file.
std::vector<Symbol *> Symbols;
std::unique_ptr<WasmObjectFile> WasmObj;
};
class BitcodeFile : public InputFile {
public:
explicit BitcodeFile(MemoryBufferRef M) : InputFile(BitcodeKind, M) {}
static bool classof(const InputFile *F) { return F->kind() == BitcodeKind; }
void parse() override;
std::unique_ptr<llvm::lto::InputFile> Obj;
};
// Opens a given file.
llvm::Optional<MemoryBufferRef> readFile(StringRef Path);

151
lld/wasm/LTO.cpp Normal file
View File

@ -0,0 +1,151 @@
//===- LTO.cpp ------------------------------------------------------------===//
//
// The LLVM Linker
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
#include "LTO.h"
#include "Config.h"
#include "InputFiles.h"
#include "Symbols.h"
#include "lld/Common/ErrorHandler.h"
#include "lld/Common/Strings.h"
#include "lld/Common/TargetOptionsCommandFlags.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallString.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/ADT/Twine.h"
#include "llvm/IR/DiagnosticPrinter.h"
#include "llvm/LTO/Caching.h"
#include "llvm/LTO/Config.h"
#include "llvm/LTO/LTO.h"
#include "llvm/Object/SymbolicFile.h"
#include "llvm/Support/CodeGen.h"
#include "llvm/Support/Error.h"
#include "llvm/Support/FileSystem.h"
#include "llvm/Support/MemoryBuffer.h"
#include "llvm/Support/raw_ostream.h"
#include <algorithm>
#include <cstddef>
#include <memory>
#include <string>
#include <system_error>
#include <vector>
using namespace llvm;
using namespace llvm::object;
using namespace lld;
using namespace lld::wasm;
static std::unique_ptr<lto::LTO> createLTO() {
lto::Config C;
C.Options = InitTargetOptionsFromCodeGenFlags();
// Always emit a section per function/datum with LTO.
C.Options.FunctionSections = true;
C.Options.DataSections = true;
C.DisableVerify = Config->DisableVerify;
C.DiagHandler = diagnosticHandler;
C.OptLevel = Config->LTOO;
if (Config->SaveTemps)
checkError(C.addSaveTemps(Config->OutputFile.str() + ".",
/*UseInputModulePath*/ true));
lto::ThinBackend Backend;
if (Config->ThinLTOJobs != -1U)
Backend = lto::createInProcessThinBackend(Config->ThinLTOJobs);
return llvm::make_unique<lto::LTO>(std::move(C), Backend,
Config->LTOPartitions);
}
BitcodeCompiler::BitcodeCompiler() : LTOObj(createLTO()) {}
BitcodeCompiler::~BitcodeCompiler() = default;
static void undefine(Symbol *S) {
if (isa<DefinedFunction>(S))
replaceSymbol<UndefinedFunction>(S, S->getName(), 0);
else if (isa<DefinedData>(S))
replaceSymbol<UndefinedData>(S, S->getName(), 0);
else
llvm_unreachable("unexpected symbol kind");
}
void BitcodeCompiler::add(BitcodeFile &F) {
lto::InputFile &Obj = *F.Obj;
unsigned SymNum = 0;
ArrayRef<Symbol *> Syms = F.getSymbols();
std::vector<lto::SymbolResolution> Resols(Syms.size());
// Provide a resolution to the LTO API for each symbol.
for (const lto::InputFile::Symbol &ObjSym : Obj.symbols()) {
Symbol *Sym = Syms[SymNum];
lto::SymbolResolution &R = Resols[SymNum];
++SymNum;
// Ideally we shouldn't check for SF_Undefined but currently IRObjectFile
// reports two symbols for module ASM defined. Without this check, lld
// flags an undefined in IR with a definition in ASM as prevailing.
// Once IRObjectFile is fixed to report only one symbol this hack can
// be removed.
R.Prevailing = !ObjSym.isUndefined() && Sym->getFile() == &F;
R.VisibleToRegularObj = Config->Relocatable || Sym->IsUsedInRegularObj;
if (R.Prevailing)
undefine(Sym);
}
checkError(LTOObj->add(std::move(F.Obj), Resols));
}
// Merge all the bitcode files we have seen, codegen the result
// and return the resulting objects.
std::vector<StringRef> BitcodeCompiler::compile() {
unsigned MaxTasks = LTOObj->getMaxTasks();
Buf.resize(MaxTasks);
Files.resize(MaxTasks);
// The --thinlto-cache-dir option specifies the path to a directory in which
// to cache native object files for ThinLTO incremental builds. If a path was
// specified, configure LTO to use it as the cache directory.
lto::NativeObjectCache Cache;
if (!Config->ThinLTOCacheDir.empty())
Cache = check(
lto::localCache(Config->ThinLTOCacheDir,
[&](size_t Task, std::unique_ptr<MemoryBuffer> MB) {
Files[Task] = std::move(MB);
}));
checkError(LTOObj->run(
[&](size_t Task) {
return llvm::make_unique<lto::NativeObjectStream>(
llvm::make_unique<raw_svector_ostream>(Buf[Task]));
},
Cache));
if (!Config->ThinLTOCacheDir.empty())
pruneCache(Config->ThinLTOCacheDir, Config->ThinLTOCachePolicy);
std::vector<StringRef> Ret;
for (unsigned I = 0; I != MaxTasks; ++I) {
if (Buf[I].empty())
continue;
if (Config->SaveTemps) {
if (I == 0)
saveBuffer(Buf[I], Config->OutputFile + ".lto.o");
else
saveBuffer(Buf[I], Config->OutputFile + Twine(I) + ".lto.o");
}
Ret.emplace_back(Buf[I].data(), Buf[I].size());
}
for (std::unique_ptr<MemoryBuffer> &File : Files)
if (File)
Ret.push_back(File->getBuffer());
return Ret;
}

57
lld/wasm/LTO.h Normal file
View File

@ -0,0 +1,57 @@
//===- LTO.h ----------------------------------------------------*- C++ -*-===//
//
// The LLVM Linker
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
//
// This file provides a way to combine bitcode files into one wasm
// file by compiling them using LLVM.
//
// If LTO is in use, your input files are not in regular wasm files
// but instead LLVM bitcode files. In that case, the linker has to
// convert bitcode files into the native format so that we can create
// a wasm file that contains native code. This file provides that
// functionality.
//
//===----------------------------------------------------------------------===//
#ifndef LLD_WASM_LTO_H
#define LLD_WASM_LTO_H
#include "lld/Common/LLVM.h"
#include "llvm/ADT/SmallString.h"
#include <memory>
#include <vector>
namespace llvm {
namespace lto {
class LTO;
}
} // namespace llvm
namespace lld {
namespace wasm {
class BitcodeFile;
class InputFile;
class BitcodeCompiler {
public:
BitcodeCompiler();
~BitcodeCompiler();
void add(BitcodeFile &F);
std::vector<StringRef> compile();
private:
std::unique_ptr<llvm::lto::LTO> LTOObj;
std::vector<SmallString<0>> Buf;
std::vector<std::unique_ptr<MemoryBuffer>> Files;
};
} // namespace wasm
} // namespace lld
#endif

13
lld/wasm/Options.td
View File

@ -136,3 +136,16 @@ def alias_initial_memory_i: Flag<["-"], "i">, Alias<initial_memory>;
def alias_max_memory_m: Flag<["-"], "m">, Alias<max_memory>;
def alias_relocatable_r: Flag<["-"], "r">, Alias<relocatable>;
def alias_undefined_u: JoinedOrSeparate<["-"], "u">, Alias<undefined>;
// LTO-related options.
def lto_O: J<"lto-O">, MetaVarName<"<opt-level>">,
HelpText<"Optimization level for LTO">;
def lto_partitions: J<"lto-partitions=">,
HelpText<"Number of LTO codegen partitions">;
def disable_verify: F<"disable-verify">;
def save_temps: F<"save-temps">;
def thinlto_cache_dir: J<"thinlto-cache-dir=">,
HelpText<"Path to ThinLTO cached object file directory">;
defm thinlto_cache_policy: Eq<"thinlto-cache-policy">,
HelpText<"Pruning policy for the ThinLTO cache">;
def thinlto_jobs: J<"thinlto-jobs=">, HelpText<"Number of ThinLTO jobs">;

58
lld/wasm/SymbolTable.cpp
View File

@ -29,17 +29,46 @@ void SymbolTable::addFile(InputFile *File) {
log("Processing: " + toString(File));
File->parse();
if (auto *F = dyn_cast<ObjFile>(File))
// LLVM bitcode file
if (auto *F = dyn_cast<BitcodeFile>(File))
BitcodeFiles.push_back(F);
else if (auto *F = dyn_cast<ObjFile>(File))
ObjectFiles.push_back(F);
}
// This function is where all the optimizations of link-time
// optimization happens. When LTO is in use, some input files are
// not in native object file format but in the LLVM bitcode format.
// This function compiles bitcode files into a few big native files
// using LLVM functions and replaces bitcode symbols with the results.
// Because all bitcode files that the program consists of are passed
// to the compiler at once, it can do whole-program optimization.
void SymbolTable::addCombinedLTOObject() {
if (BitcodeFiles.empty())
return;
// Compile bitcode files and replace bitcode symbols.
LTO.reset(new BitcodeCompiler);
for (BitcodeFile *F : BitcodeFiles)
LTO->add(*F);
for (StringRef Filename : LTO->compile()) {
auto *Obj = make<ObjFile>(MemoryBufferRef(Filename, "lto.tmp"));
Obj->parse();
ObjectFiles.push_back(Obj);
}
}
void SymbolTable::reportRemainingUndefines() {
SetVector<Symbol *> Undefs;
for (Symbol *Sym : SymVector) {
if (Sym->isUndefined() && !Sym->isWeak() &&
Config->AllowUndefinedSymbols.count(Sym->getName()) == 0) {
Undefs.insert(Sym);
}
if (!Sym->isUndefined() || Sym->isWeak())
continue;
if (Config->AllowUndefinedSymbols.count(Sym->getName()) != 0)
continue;
if (!Sym->IsUsedInRegularObj)
continue;
Undefs.insert(Sym);
}
if (Undefs.empty())
@ -64,6 +93,7 @@ std::pair<Symbol *, bool> SymbolTable::insert(StringRef Name) {
if (Sym)
return {Sym, false};
Sym = reinterpret_cast<Symbol *>(make<SymbolUnion>());
Sym->IsUsedInRegularObj = false;
SymVector.emplace_back(Sym);
return {Sym, true};
}
@ -178,12 +208,16 @@ Symbol *SymbolTable::addDefinedFunction(StringRef Name, uint32_t Flags,
bool WasInserted;
std::tie(S, WasInserted) = insert(Name);
if (!File || File->kind() == InputFile::ObjectKind)
S->IsUsedInRegularObj = true;
if (WasInserted || S->isLazy()) {
replaceSymbol<DefinedFunction>(S, Name, Flags, File, Function);
return S;
}
checkFunctionType(S, File, &Function->Signature);
if (Function)
checkFunctionType(S, File, &Function->Signature);
if (shouldReplace(S, File, Flags))
replaceSymbol<DefinedFunction>(S, Name, Flags, File, Function);
@ -199,6 +233,9 @@ Symbol *SymbolTable::addDefinedData(StringRef Name, uint32_t Flags,
bool WasInserted;
std::tie(S, WasInserted) = insert(Name);
if (!File || File->kind() == InputFile::ObjectKind)
S->IsUsedInRegularObj = true;
if (WasInserted || S->isLazy()) {
replaceSymbol<DefinedData>(S, Name, Flags, File, Segment, Address, Size);
return S;
@ -218,6 +255,9 @@ Symbol *SymbolTable::addDefinedGlobal(StringRef Name, uint32_t Flags,
bool WasInserted;
std::tie(S, WasInserted) = insert(Name);
if (!File || File->kind() == InputFile::ObjectKind)
S->IsUsedInRegularObj = true;
if (WasInserted || S->isLazy()) {
replaceSymbol<DefinedGlobal>(S, Name, Flags, File, Global);
return S;
@ -239,6 +279,9 @@ Symbol *SymbolTable::addUndefinedFunction(StringRef Name, uint32_t Flags,
bool WasInserted;
std::tie(S, WasInserted) = insert(Name);
if (!File || File->kind() == InputFile::ObjectKind)
S->IsUsedInRegularObj = true;
if (WasInserted)
replaceSymbol<UndefinedFunction>(S, Name, Flags, File, Sig);
else if (auto *Lazy = dyn_cast<LazySymbol>(S))
@ -274,6 +317,9 @@ Symbol *SymbolTable::addUndefinedGlobal(StringRef Name, uint32_t Flags,
bool WasInserted;
std::tie(S, WasInserted) = insert(Name);
if (!File || File->kind() == InputFile::ObjectKind)
S->IsUsedInRegularObj = true;
if (WasInserted)
replaceSymbol<UndefinedGlobal>(S, Name, Flags, File, Type);
else if (auto *Lazy = dyn_cast<LazySymbol>(S))

6
lld/wasm/SymbolTable.h
View File

@ -11,6 +11,7 @@
#define LLD_WASM_SYMBOL_TABLE_H
#include "InputFiles.h"
#include "LTO.h"
#include "Symbols.h"
#include "llvm/ADT/CachedHashString.h"
#include "llvm/ADT/DenseSet.h"
@ -39,8 +40,10 @@ class InputSegment;
class SymbolTable {
public:
void addFile(InputFile *File);
void addCombinedLTOObject();
std::vector<ObjFile *> ObjectFiles;
std::vector<BitcodeFile *> BitcodeFiles;
std::vector<InputFunction *> SyntheticFunctions;
std::vector<InputGlobal *> SyntheticGlobals;
@ -80,6 +83,9 @@ private:
std::vector<Symbol *> SymVector;
llvm::DenseSet<llvm::CachedHashStringRef> Comdats;
// For LTO.
std::unique_ptr<BitcodeCompiler> LTO;
};
extern SymbolTable *Symtab;

14
lld/wasm/Symbols.h
View File

@ -91,10 +91,13 @@ public:
WasmSymbolType getWasmType() const;
// True if this symbol was referenced by a regular (non-bitcode) object.
unsigned IsUsedInRegularObj : 1;
protected:
Symbol(StringRef Name, Kind K, uint32_t Flags, InputFile *F)
: Name(Name), SymbolKind(K), Flags(Flags), File(F),
Referenced(!Config->GcSections) {}
: IsUsedInRegularObj(false), Name(Name), SymbolKind(K), Flags(Flags),
File(F), Referenced(!Config->GcSections) {}
StringRef Name;
Kind SymbolKind;
@ -332,7 +335,12 @@ T *replaceSymbol(Symbol *S, ArgT &&... Arg) {
"SymbolUnion not aligned enough");
assert(static_cast<Symbol *>(static_cast<T *>(nullptr)) == nullptr &&
"Not a Symbol");
return new (S) T(std::forward<ArgT>(Arg)...);
Symbol SymCopy = *S;
T *S2 = new (S) T(std::forward<ArgT>(Arg)...);
S2->IsUsedInRegularObj = SymCopy.IsUsedInRegularObj;
return S2;
}
} // namespace wasm

2
lld/wasm/Writer.cpp
View File

@ -715,6 +715,8 @@ void Writer::calculateImports() {
continue;
if (!Sym->isLive())
continue;
if (!Sym->IsUsedInRegularObj)
continue;
LLVM_DEBUG(dbgs() << "import: " << Sym->getName() << "\n");
ImportedSymbols.emplace_back(Sym);