replica
/
hanchenye-llvm-project
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

[CallSite removal] Migrate the statepoint GC infrastructure to use the 

`CallBase` class rather than `CallSite` wrappers.

I pushed this change down through most of the statepoint infrastructure,
completely removing the use of CallSite where I could reasonably do so.
I ended up making a couple of cut-points: generic call handling
(instcombine, TLI, SDAG). As soon as it hit truly generic handling with
users outside the immediate code, I simply transitioned into or out of
a `CallSite` to make this a reasonable sized chunk.

Differential Revision: https://reviews.llvm.org/D56122

llvm-svn: 353660
This commit is contained in:
Chandler Carruth 2019-02-11 07:42:30 +00:00
parent 5b1beda001
commit 3160734af1
11 changed files with 227 additions and 260 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
llvm/include/llvm/CodeGen/TargetLowering.h
View File

@ -194,7 +194,11 @@ public:
IsNest(false), IsByVal(false), IsInAlloca(false), IsReturned(false),
IsSwiftSelf(false), IsSwiftError(false) {}
void setAttributes(ImmutableCallSite *CS, unsigned ArgIdx);
void setAttributes(const CallBase *Call, unsigned ArgIdx);
void setAttributes(ImmutableCallSite *CS, unsigned ArgIdx) {
return setAttributes(cast<CallBase>(CS->getInstruction()), ArgIdx);
}
};
using ArgListTy = std::vector<ArgListEntry>;

141
llvm/include/llvm/IR/Statepoint.h
View File

@ -7,7 +7,7 @@
//===----------------------------------------------------------------------===//
//
// This file contains utility functions and a wrapper class analogous to
// CallSite for accessing the fields of gc.statepoint, gc.relocate,
// CallBase for accessing the fields of gc.statepoint, gc.relocate,
// gc.result intrinsics; and some general utilities helpful when dealing with
// gc.statepoint.
//
@ -20,7 +20,6 @@
#include "llvm/ADT/iterator_range.h"
#include "llvm/IR/Attributes.h"
#include "llvm/IR/BasicBlock.h"
#include "llvm/IR/CallSite.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/Instruction.h"
@ -56,42 +55,39 @@ enum class StatepointFlags {
class GCRelocateInst;
class GCResultInst;
bool isStatepoint(ImmutableCallSite CS);
bool isStatepoint(const CallBase *Call);
bool isStatepoint(const Value *V);
bool isStatepoint(const Value &V);
bool isGCRelocate(ImmutableCallSite CS);
bool isGCRelocate(const CallBase *Call);
bool isGCRelocate(const Value *V);
bool isGCResult(ImmutableCallSite CS);
bool isGCResult(const CallBase *Call);
bool isGCResult(const Value *V);
/// Analogous to CallSiteBase, this provides most of the actual
/// A wrapper around a GC intrinsic call, this provides most of the actual
/// functionality for Statepoint and ImmutableStatepoint. It is
/// templatized to allow easily specializing of const and non-const
/// concrete subtypes. This is structured analogous to CallSite
/// rather than the IntrinsicInst.h helpers since we need to support
/// invokable statepoints.
/// concrete subtypes.
template <typename FunTy, typename InstructionTy, typename ValueTy,
typename CallSiteTy>
typename CallBaseTy>
class StatepointBase {
CallSiteTy StatepointCS;
CallBaseTy *StatepointCall;
protected:
explicit StatepointBase(InstructionTy *I) {
if (isStatepoint(I)) {
StatepointCS = CallSiteTy(I);
assert(StatepointCS && "isStatepoint implies CallSite");
StatepointCall = cast<CallBaseTy>(I);
}
}
explicit StatepointBase(CallSiteTy CS) {
if (isStatepoint(CS))
StatepointCS = CS;
explicit StatepointBase(CallBaseTy *Call) {
if (isStatepoint(Call))
StatepointCall = Call;
}
public:
using arg_iterator = typename CallSiteTy::arg_iterator;
using arg_iterator = typename CallBaseTy::const_op_iterator;
enum {
IDPos = 0,
@ -106,30 +102,30 @@ public:
void *operator new(size_t s) = delete;
explicit operator bool() const {
// We do not assign non-statepoint CallSites to StatepointCS.
return (bool)StatepointCS;
// We do not assign non-statepoint call instructions to StatepointCall.
return (bool)StatepointCall;
}
/// Return the underlying CallSite.
CallSiteTy getCallSite() const {
/// Return the underlying call instruction.
CallBaseTy *getCall() const {
assert(*this && "check validity first!");
return StatepointCS;
return StatepointCall;
}
uint64_t getFlags() const {
return cast<ConstantInt>(getCallSite().getArgument(FlagsPos))
return cast<ConstantInt>(getCall()->getArgOperand(FlagsPos))
->getZExtValue();
}
/// Return the ID associated with this statepoint.
uint64_t getID() const {
const Value *IDVal = getCallSite().getArgument(IDPos);
const Value *IDVal = getCall()->getArgOperand(IDPos);
return cast<ConstantInt>(IDVal)->getZExtValue();
}
/// Return the number of patchable bytes associated with this statepoint.
uint32_t getNumPatchBytes() const {
const Value *NumPatchBytesVal = getCallSite().getArgument(NumPatchBytesPos);
const Value *NumPatchBytesVal = getCall()->getArgOperand(NumPatchBytesPos);
uint64_t NumPatchBytes =
cast<ConstantInt>(NumPatchBytesVal)->getZExtValue();
assert(isInt<32>(NumPatchBytes) && "should fit in 32 bits!");
@ -138,12 +134,11 @@ public:
/// Return the value actually being called or invoked.
ValueTy *getCalledValue() const {
return getCallSite().getArgument(CalledFunctionPos);
return getCall()->getArgOperand(CalledFunctionPos);
}
InstructionTy *getInstruction() const {
return getCallSite().getInstruction();
}
// FIXME: Migrate users of this to `getCall` and remove it.
InstructionTy *getInstruction() const { return getCall(); }
/// Return the function being called if this is a direct call, otherwise
/// return null (if it's an indirect call).
@ -152,12 +147,12 @@ public:
}
/// Return the caller function for this statepoint.
FunTy *getCaller() const { return getCallSite().getCaller(); }
FunTy *getCaller() const { return getCall()->getCaller(); }
/// Determine if the statepoint cannot unwind.
bool doesNotThrow() const {
Function *F = getCalledFunction();
return getCallSite().doesNotThrow() || (F ? F->doesNotThrow() : false);
return getCall()->doesNotThrow() || (F ? F->doesNotThrow() : false);
}
/// Return the type of the value returned by the call underlying the
@ -170,18 +165,18 @@ public:
/// Number of arguments to be passed to the actual callee.
int getNumCallArgs() const {
const Value *NumCallArgsVal = getCallSite().getArgument(NumCallArgsPos);
const Value *NumCallArgsVal = getCall()->getArgOperand(NumCallArgsPos);
return cast<ConstantInt>(NumCallArgsVal)->getZExtValue();
}
size_t arg_size() const { return getNumCallArgs(); }
typename CallSiteTy::arg_iterator arg_begin() const {
assert(CallArgsBeginPos <= (int)getCallSite().arg_size());
return getCallSite().arg_begin() + CallArgsBeginPos;
arg_iterator arg_begin() const {
assert(CallArgsBeginPos <= (int)getCall()->arg_size());
return getCall()->arg_begin() + CallArgsBeginPos;
}
typename CallSiteTy::arg_iterator arg_end() const {
arg_iterator arg_end() const {
auto I = arg_begin() + arg_size();
assert((getCallSite().arg_end() - I) >= 0);
assert((getCall()->arg_end() - I) >= 0);
return I;
}
@ -198,8 +193,8 @@ public:
/// Return true if the call or the callee has the given attribute.
bool paramHasAttr(unsigned i, Attribute::AttrKind A) const {
Function *F = getCalledFunction();
return getCallSite().paramHasAttr(i + CallArgsBeginPos, A) ||
(F ? F->getAttributes().hasAttribute(i, A) : false);
return getCall()->paramHasAttr(i + CallArgsBeginPos, A) ||
(F ? F->getAttributes().hasAttribute(i, A) : false);
}
/// Number of GC transition args.
@ -207,14 +202,14 @@ public:
const Value *NumGCTransitionArgs = *arg_end();
return cast<ConstantInt>(NumGCTransitionArgs)->getZExtValue();
}
typename CallSiteTy::arg_iterator gc_transition_args_begin() const {
arg_iterator gc_transition_args_begin() const {
auto I = arg_end() + 1;
assert((getCallSite().arg_end() - I) >= 0);
assert((getCall()->arg_end() - I) >= 0);
return I;
}
typename CallSiteTy::arg_iterator gc_transition_args_end() const {
arg_iterator gc_transition_args_end() const {
auto I = gc_transition_args_begin() + getNumTotalGCTransitionArgs();
assert((getCallSite().arg_end() - I) >= 0);
assert((getCall()->arg_end() - I) >= 0);
return I;
}
@ -230,14 +225,14 @@ public:
return cast<ConstantInt>(NumVMSArgs)->getZExtValue();
}
typename CallSiteTy::arg_iterator deopt_begin() const {
arg_iterator deopt_begin() const {
auto I = gc_transition_args_end() + 1;
assert((getCallSite().arg_end() - I) >= 0);
assert((getCall()->arg_end() - I) >= 0);
return I;
}
typename CallSiteTy::arg_iterator deopt_end() const {
arg_iterator deopt_end() const {
auto I = deopt_begin() + getNumTotalVMSArgs();
assert((getCallSite().arg_end() - I) >= 0);
assert((getCall()->arg_end() - I) >= 0);
return I;
}
@ -246,15 +241,11 @@ public:
return make_range(deopt_begin(), deopt_end());
}
typename CallSiteTy::arg_iterator gc_args_begin() const {
return deopt_end();
}
typename CallSiteTy::arg_iterator gc_args_end() const {
return getCallSite().arg_end();
}
arg_iterator gc_args_begin() const { return deopt_end(); }
arg_iterator gc_args_end() const { return getCall()->arg_end(); }
unsigned gcArgsStartIdx() const {
return gc_args_begin() - getInstruction()->op_begin();
return gc_args_begin() - getCall()->op_begin();
}
/// range adapter for gc arguments
@ -303,25 +294,24 @@ public:
/// to a gc.statepoint.
class ImmutableStatepoint
: public StatepointBase<const Function, const Instruction, const Value,
ImmutableCallSite> {
using Base =
StatepointBase<const Function, const Instruction, const Value,
ImmutableCallSite>;
const CallBase> {
using Base = StatepointBase<const Function, const Instruction, const Value,
const CallBase>;
public:
explicit ImmutableStatepoint(const Instruction *I) : Base(I) {}
explicit ImmutableStatepoint(ImmutableCallSite CS) : Base(CS) {}
explicit ImmutableStatepoint(const CallBase *Call) : Base(Call) {}
};
/// A specialization of it's base class for read-write access
/// to a gc.statepoint.
class Statepoint
: public StatepointBase<Function, Instruction, Value, CallSite> {
using Base = StatepointBase<Function, Instruction, Value, CallSite>;
: public StatepointBase<Function, Instruction, Value, CallBase> {
using Base = StatepointBase<Function, Instruction, Value, CallBase>;
public:
explicit Statepoint(Instruction *I) : Base(I) {}
explicit Statepoint(CallSite CS) : Base(CS) {}
explicit Statepoint(CallBase *Call) : Base(Call) {}
};
/// Common base class for representing values projected from a statepoint.
@ -346,14 +336,14 @@ public:
}
/// The statepoint with which this gc.relocate is associated.
const Instruction *getStatepoint() const {
const CallBase *getStatepoint() const {
const Value *Token = getArgOperand(0);
// This takes care both of relocates for call statepoints and relocates
// on normal path of invoke statepoint.
if (!isa<LandingPadInst>(Token)) {
assert(isStatepoint(Token));
return cast<Instruction>(Token);
return cast<CallBase>(Token);
}
// This relocate is on exceptional path of an invoke statepoint
@ -365,7 +355,7 @@ public:
"safepoint block should be well formed");
assert(isStatepoint(InvokeBB->getTerminator()));
return InvokeBB->getTerminator();
return cast<CallBase>(InvokeBB->getTerminator());
}
};
@ -394,13 +384,11 @@ public:
}
Value *getBasePtr() const {
ImmutableCallSite CS(getStatepoint());
return *(CS.arg_begin() + getBasePtrIndex());
return *(getStatepoint()->arg_begin() + getBasePtrIndex());
}
Value *getDerivedPtr() const {
ImmutableCallSite CS(getStatepoint());
return *(CS.arg_begin() + getDerivedPtrIndex());
return *(getStatepoint()->arg_begin() + getDerivedPtrIndex());
}
};
@ -417,28 +405,25 @@ public:
};
template <typename FunTy, typename InstructionTy, typename ValueTy,
typename CallSiteTy>
typename CallBaseTy>
std::vector<const GCRelocateInst *>
StatepointBase<FunTy, InstructionTy, ValueTy, CallSiteTy>::getRelocates()
StatepointBase<FunTy, InstructionTy, ValueTy, CallBaseTy>::getRelocates()
const {
std::vector<const GCRelocateInst *> Result;
CallSiteTy StatepointCS = getCallSite();
// Search for relocated pointers. Note that working backwards from the
// gc_relocates ensures that we only get pairs which are actually relocated
// and used after the statepoint.
for (const User *U : getInstruction()->users())
for (const User *U : StatepointCall->users())
if (auto *Relocate = dyn_cast<GCRelocateInst>(U))
Result.push_back(Relocate);
if (!StatepointCS.isInvoke())
auto *StatepointInvoke = dyn_cast<InvokeInst>(StatepointCall);
if (!StatepointInvoke)
return Result;
// We need to scan thorough exceptional relocations if it is invoke statepoint
LandingPadInst *LandingPad =
cast<InvokeInst>(getInstruction())->getLandingPadInst();
LandingPadInst *LandingPad = StatepointInvoke->getLandingPadInst();
// Search for gc relocates that are attached to this landingpad.
for (const User *LandingPadUser : LandingPad->users()) {

5
llvm/include/llvm/Transforms/Utils/Local.h
View File

@ -22,7 +22,6 @@
#include "llvm/Analysis/AliasAnalysis.h"
#include "llvm/Analysis/DomTreeUpdater.h"
#include "llvm/Analysis/Utils/Local.h"
#include "llvm/IR/CallSite.h"
#include "llvm/IR/Constant.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DataLayout.h"
@ -437,7 +436,7 @@ unsigned replaceDominatedUsesWith(Value *From, Value *To, DominatorTree &DT,
unsigned replaceDominatedUsesWith(Value *From, Value *To, DominatorTree &DT,
const BasicBlock *BB);
/// Return true if the CallSite CS calls a gc leaf function.
/// Return true if this call calls a gc leaf function.
///
/// A leaf function is a function that does not safepoint the thread during its
/// execution. During a call or invoke to such a function, the callers stack
@ -445,7 +444,7 @@ unsigned replaceDominatedUsesWith(Value *From, Value *To, DominatorTree &DT,
///
/// Most passes can and should ignore this information, and it is only used
/// during lowering by the GC infrastructure.
bool callsGCLeafFunction(ImmutableCallSite CS, const TargetLibraryInfo &TLI);
bool callsGCLeafFunction(const CallBase *Call, const TargetLibraryInfo &TLI);
/// Copy a nonnull metadata node to a new load instruction.
///

14
llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
View File

@ -8132,7 +8132,7 @@ SDValue SelectionDAGBuilder::lowerRangeToAssertZExt(SelectionDAG &DAG,
/// convention or require stack pointer adjustment. Only a subset of the
/// intrinsic's operands need to participate in the calling convention.
void SelectionDAGBuilder::populateCallLoweringInfo(
TargetLowering::CallLoweringInfo &CLI, ImmutableCallSite CS,
TargetLowering::CallLoweringInfo &CLI, const CallBase *Call,
unsigned ArgIdx, unsigned NumArgs, SDValue Callee, Type *ReturnTy,
bool IsPatchPoint) {
TargetLowering::ArgListTy Args;
@ -8142,21 +8142,21 @@ void SelectionDAGBuilder::populateCallLoweringInfo(
// Attributes for args start at offset 1, after the return attribute.
for (unsigned ArgI = ArgIdx, ArgE = ArgIdx + NumArgs;
ArgI != ArgE; ++ArgI) {
const Value *V = CS->getOperand(ArgI);
const Value *V = Call->getOperand(ArgI);
assert(!V->getType()->isEmptyTy() && "Empty type passed to intrinsic.");
TargetLowering::ArgListEntry Entry;
Entry.Node = getValue(V);
Entry.Ty = V->getType();
Entry.setAttributes(&CS, ArgI);
Entry.setAttributes(Call, ArgI);
Args.push_back(Entry);
}
CLI.setDebugLoc(getCurSDLoc())
.setChain(getRoot())
.setCallee(CS.getCallingConv(), ReturnTy, Callee, std::move(Args))
.setDiscardResult(CS->use_empty())
.setCallee(Call->getCallingConv(), ReturnTy, Callee, std::move(Args))
.setDiscardResult(Call->use_empty())
.setIsPatchPoint(IsPatchPoint);
}
@ -8300,8 +8300,8 @@ void SelectionDAGBuilder::visitPatchpoint(ImmutableCallSite CS,
IsAnyRegCC ? Type::getVoidTy(*DAG.getContext()) : CS->getType();
TargetLowering::CallLoweringInfo CLI(DAG);
populateCallLoweringInfo(CLI, CS, NumMetaOpers, NumCallArgs, Callee, ReturnTy,
true);
populateCallLoweringInfo(CLI, cast<CallBase>(CS.getInstruction()),
NumMetaOpers, NumCallArgs, Callee, ReturnTy, true);
std::pair<SDValue, SDValue> Result = lowerInvokable(CLI, EHPadBB);
SDNode *CallEnd = Result.second.getNode();

6
llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
View File

@ -733,7 +733,7 @@ public:
SDValue Op);
void populateCallLoweringInfo(TargetLowering::CallLoweringInfo &CLI,
ImmutableCallSite CS, unsigned ArgIdx,
const CallBase *Call, unsigned ArgIdx,
unsigned NumArgs, SDValue Callee,
Type *ReturnTy, bool IsPatchPoint);
@ -797,13 +797,13 @@ public:
void LowerStatepoint(ImmutableStatepoint ISP,
const BasicBlock *EHPadBB = nullptr);
void LowerCallSiteWithDeoptBundle(ImmutableCallSite CS, SDValue Callee,
void LowerCallSiteWithDeoptBundle(const CallBase *Call, SDValue Callee,
const BasicBlock *EHPadBB);
void LowerDeoptimizeCall(const CallInst *CI);
void LowerDeoptimizingReturn();
void LowerCallSiteWithDeoptBundleImpl(ImmutableCallSite CS, SDValue Callee,
void LowerCallSiteWithDeoptBundleImpl(const CallBase *Call, SDValue Callee,
const BasicBlock *EHPadBB,
bool VarArgDisallowed,
bool ForceVoidReturnTy);

31
llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp
View File

@ -798,7 +798,7 @@ SDValue SelectionDAGBuilder::LowerAsSTATEPOINT(
void
SelectionDAGBuilder::LowerStatepoint(ImmutableStatepoint ISP,
const BasicBlock *EHPadBB /*= nullptr*/) {
assert(ISP.getCallSite().getCallingConv() != CallingConv::AnyReg &&
assert(ISP.getCall()->getCallingConv() != CallingConv::AnyReg &&
"anyregcc is not supported on statepoints!");
#ifndef NDEBUG
@ -831,7 +831,7 @@ SelectionDAGBuilder::LowerStatepoint(ImmutableStatepoint ISP,
}
StatepointLoweringInfo SI(DAG);
populateCallLoweringInfo(SI.CLI, ISP.getCallSite(),
populateCallLoweringInfo(SI.CLI, ISP.getCall(),
ImmutableStatepoint::CallArgsBeginPos,
ISP.getNumCallArgs(), ActualCallee,
ISP.getActualReturnType(), false /* IsPatchPoint */);
@ -858,7 +858,7 @@ SelectionDAGBuilder::LowerStatepoint(ImmutableStatepoint ISP,
const GCResultInst *GCResult = ISP.getGCResult();
Type *RetTy = ISP.getActualReturnType();
if (!RetTy->isVoidTy() && GCResult) {
if (GCResult->getParent() != ISP.getCallSite().getParent()) {
if (GCResult->getParent() != ISP.getCall()->getParent()) {
// Result value will be used in a different basic block so we need to
// export it now. Default exporting mechanism will not work here because
// statepoint call has a different type than the actual call. It means
@ -870,7 +870,7 @@ SelectionDAGBuilder::LowerStatepoint(ImmutableStatepoint ISP,
unsigned Reg = FuncInfo.CreateRegs(RetTy);
RegsForValue RFV(*DAG.getContext(), DAG.getTargetLoweringInfo(),
DAG.getDataLayout(), Reg, RetTy,
ISP.getCallSite().getCallingConv());
ISP.getCall()->getCallingConv());
SDValue Chain = DAG.getEntryNode();
RFV.getCopyToRegs(ReturnValue, DAG, getCurSDLoc(), Chain, nullptr);
@ -890,22 +890,22 @@ SelectionDAGBuilder::LowerStatepoint(ImmutableStatepoint ISP,
}
void SelectionDAGBuilder::LowerCallSiteWithDeoptBundleImpl(
ImmutableCallSite CS, SDValue Callee, const BasicBlock *EHPadBB,
const CallBase *Call, SDValue Callee, const BasicBlock *EHPadBB,
bool VarArgDisallowed, bool ForceVoidReturnTy) {
StatepointLoweringInfo SI(DAG);
unsigned ArgBeginIndex = CS.arg_begin() - CS.getInstruction()->op_begin();
unsigned ArgBeginIndex = Call->arg_begin() - Call->op_begin();
populateCallLoweringInfo(
SI.CLI, CS, ArgBeginIndex, CS.getNumArgOperands(), Callee,
ForceVoidReturnTy ? Type::getVoidTy(*DAG.getContext()) : CS.getType(),
SI.CLI, Call, ArgBeginIndex, Call->getNumArgOperands(), Callee,
ForceVoidReturnTy ? Type::getVoidTy(*DAG.getContext()) : Call->getType(),
false);
if (!VarArgDisallowed)
SI.CLI.IsVarArg = CS.getFunctionType()->isVarArg();
SI.CLI.IsVarArg = Call->getFunctionType()->isVarArg();
auto DeoptBundle = *CS.getOperandBundle(LLVMContext::OB_deopt);
auto DeoptBundle = *Call->getOperandBundle(LLVMContext::OB_deopt);
unsigned DefaultID = StatepointDirectives::DeoptBundleStatepointID;
auto SD = parseStatepointDirectivesFromAttrs(CS.getAttributes());
auto SD = parseStatepointDirectivesFromAttrs(Call->getAttributes());
SI.ID = SD.StatepointID.getValueOr(DefaultID);
SI.NumPatchBytes = SD.NumPatchBytes.getValueOr(0);
@ -917,15 +917,14 @@ void SelectionDAGBuilder::LowerCallSiteWithDeoptBundleImpl(
// NB! The GC arguments are deliberately left empty.
if (SDValue ReturnVal = LowerAsSTATEPOINT(SI)) {
const Instruction *Inst = CS.getInstruction();
ReturnVal = lowerRangeToAssertZExt(DAG, *Inst, ReturnVal);
setValue(Inst, ReturnVal);
ReturnVal = lowerRangeToAssertZExt(DAG, *Call, ReturnVal);
setValue(Call, ReturnVal);
}
}
void SelectionDAGBuilder::LowerCallSiteWithDeoptBundle(
ImmutableCallSite CS, SDValue Callee, const BasicBlock *EHPadBB) {
LowerCallSiteWithDeoptBundleImpl(CS, Callee, EHPadBB,
const CallBase *Call, SDValue Callee, const BasicBlock *EHPadBB) {
LowerCallSiteWithDeoptBundleImpl(Call, Callee, EHPadBB,
/* VarArgDisallowed = */ false,
/* ForceVoidReturnTy = */ false);
}

24
llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
View File

@ -99,19 +99,19 @@ bool TargetLowering::parametersInCSRMatch(const MachineRegisterInfo &MRI,
/// Set CallLoweringInfo attribute flags based on a call instruction
/// and called function attributes.
void TargetLoweringBase::ArgListEntry::setAttributes(ImmutableCallSite *CS,
void TargetLoweringBase::ArgListEntry::setAttributes(const CallBase *Call,
unsigned ArgIdx) {
IsSExt = CS->paramHasAttr(ArgIdx, Attribute::SExt);
IsZExt = CS->paramHasAttr(ArgIdx, Attribute::ZExt);
IsInReg = CS->paramHasAttr(ArgIdx, Attribute::InReg);
IsSRet = CS->paramHasAttr(ArgIdx, Attribute::StructRet);
IsNest = CS->paramHasAttr(ArgIdx, Attribute::Nest);
IsByVal = CS->paramHasAttr(ArgIdx, Attribute::ByVal);
IsInAlloca = CS->paramHasAttr(ArgIdx, Attribute::InAlloca);
IsReturned = CS->paramHasAttr(ArgIdx, Attribute::Returned);
IsSwiftSelf = CS->paramHasAttr(ArgIdx, Attribute::SwiftSelf);
IsSwiftError = CS->paramHasAttr(ArgIdx, Attribute::SwiftError);
Alignment = CS->getParamAlignment(ArgIdx);
IsSExt = Call->paramHasAttr(ArgIdx, Attribute::SExt);
IsZExt = Call->paramHasAttr(ArgIdx, Attribute::ZExt);
IsInReg = Call->paramHasAttr(ArgIdx, Attribute::InReg);
IsSRet = Call->paramHasAttr(ArgIdx, Attribute::StructRet);
IsNest = Call->paramHasAttr(ArgIdx, Attribute::Nest);
IsByVal = Call->paramHasAttr(ArgIdx, Attribute::ByVal);
IsInAlloca = Call->paramHasAttr(ArgIdx, Attribute::InAlloca);
IsReturned = Call->paramHasAttr(ArgIdx, Attribute::Returned);
IsSwiftSelf = Call->paramHasAttr(ArgIdx, Attribute::SwiftSelf);
IsSwiftError = Call->paramHasAttr(ArgIdx, Attribute::SwiftError);
Alignment = Call->getParamAlignment(ArgIdx);
}
/// Generate a libcall taking the given operands as arguments and returning a

30
llvm/lib/IR/Statepoint.cpp
View File

@ -17,21 +17,15 @@
using namespace llvm;
static const Function *getCalledFunction(ImmutableCallSite CS) {
if (!CS.getInstruction())
return nullptr;
return CS.getCalledFunction();
}
bool llvm::isStatepoint(ImmutableCallSite CS) {
if (auto *F = getCalledFunction(CS))
bool llvm::isStatepoint(const CallBase *Call) {
if (auto *F = Call->getCalledFunction())
return F->getIntrinsicID() == Intrinsic::experimental_gc_statepoint;
return false;
}
bool llvm::isStatepoint(const Value *V) {
if (auto CS = ImmutableCallSite(V))
return isStatepoint(CS);
if (auto *Call = dyn_cast<CallBase>(V))
return isStatepoint(Call);
return false;
}
@ -39,23 +33,21 @@ bool llvm::isStatepoint(const Value &V) {
return isStatepoint(&V);
}
bool llvm::isGCRelocate(ImmutableCallSite CS) {
return CS.getInstruction() && isa<GCRelocateInst>(CS.getInstruction());
bool llvm::isGCRelocate(const CallBase *Call) {
return isa<GCRelocateInst>(Call);
}
bool llvm::isGCRelocate(const Value *V) {
if (auto CS = ImmutableCallSite(V))
return isGCRelocate(CS);
if (auto *Call = dyn_cast<CallBase>(V))
return isGCRelocate(Call);
return false;
}
bool llvm::isGCResult(ImmutableCallSite CS) {
return CS.getInstruction() && isa<GCResultInst>(CS.getInstruction());
}
bool llvm::isGCResult(const CallBase *Call) { return isa<GCResultInst>(Call); }
bool llvm::isGCResult(const Value *V) {
if (auto CS = ImmutableCallSite(V))
return isGCResult(CS);
if (auto *Call = dyn_cast<CallBase>(V))
return isGCResult(Call);
return false;
}

35
llvm/lib/Transforms/Scalar/PlaceSafepoints.cpp
View File

@ -55,7 +55,6 @@
#include "llvm/Analysis/ScalarEvolution.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
#include "llvm/Transforms/Utils/Local.h"
#include "llvm/IR/CallSite.h"
#include "llvm/IR/Dominators.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/LegacyPassManager.h"
@ -178,19 +177,18 @@ struct PlaceSafepoints : public FunctionPass {
// callers job.
static void
InsertSafepointPoll(Instruction *InsertBefore,
std::vector<CallSite> &ParsePointsNeeded /*rval*/,
std::vector<CallBase *> &ParsePointsNeeded /*rval*/,
const TargetLibraryInfo &TLI);
static bool needsStatepoint(const CallSite &CS, const TargetLibraryInfo &TLI) {
if (callsGCLeafFunction(CS, TLI))
static bool needsStatepoint(CallBase *Call, const TargetLibraryInfo &TLI) {
if (callsGCLeafFunction(Call, TLI))
return false;
if (CS.isCall()) {
CallInst *call = cast<CallInst>(CS.getInstruction());
if (call->isInlineAsm())
if (auto *CI = dyn_cast<CallInst>(Call)) {
if (CI->isInlineAsm())
return false;
}
return !(isStatepoint(CS) || isGCRelocate(CS) || isGCResult(CS));
return !(isStatepoint(Call) || isGCRelocate(Call) || isGCResult(Call));
}
/// Returns true if this loop is known to contain a call safepoint which
@ -216,14 +214,14 @@ static bool containsUnconditionalCallSafepoint(Loop *L, BasicBlock *Header,
BasicBlock *Current = Pred;
while (true) {
for (Instruction &I : *Current) {
if (auto CS = CallSite(&I))
if (auto *Call = dyn_cast<CallBase>(&I))
// Note: Technically, needing a safepoint isn't quite the right
// condition here. We should instead be checking if the target method
// has an
// unconditional poll. In practice, this is only a theoretical concern
// since we don't have any methods with conditional-only safepoint
// polls.
if (needsStatepoint(CS, TLI))
if (needsStatepoint(Call, TLI))
return true;
}
@ -359,9 +357,8 @@ bool PlaceBackedgeSafepointsImpl::runOnLoop(Loop *L) {
/// Returns true if an entry safepoint is not required before this callsite in
/// the caller function.
static bool doesNotRequireEntrySafepointBefore(const CallSite &CS) {
Instruction *Inst = CS.getInstruction();
if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Inst)) {
static bool doesNotRequireEntrySafepointBefore(CallBase *Call) {
if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Call)) {
switch (II->getIntrinsicID()) {
case Intrinsic::experimental_gc_statepoint:
case Intrinsic::experimental_patchpoint_void:
@ -423,8 +420,8 @@ static Instruction *findLocationForEntrySafepoint(Function &F,
// which can grow the stack by an unbounded amount. This isn't required
// for GC semantics per se, but is a common requirement for languages
// which detect stack overflow via guard pages and then throw exceptions.
if (auto CS = CallSite(Cursor)) {
if (doesNotRequireEntrySafepointBefore(CS))
if (auto *Call = dyn_cast<CallBase>(Cursor)) {
if (doesNotRequireEntrySafepointBefore(Call))
continue;
break;
}
@ -499,7 +496,7 @@ bool PlaceSafepoints::runOnFunction(Function &F) {
DT.recalculate(F);
SmallVector<Instruction *, 16> PollsNeeded;
std::vector<CallSite> ParsePointNeeded;
std::vector<CallBase *> ParsePointNeeded;
if (enableBackedgeSafepoints(F)) {
// Construct a pass manager to run the LoopPass backedge logic. We
@ -588,7 +585,7 @@ bool PlaceSafepoints::runOnFunction(Function &F) {
// Now that we've identified all the needed safepoint poll locations, insert
// safepoint polls themselves.
for (Instruction *PollLocation : PollsNeeded) {
std::vector<CallSite> RuntimeCalls;
std::vector<CallBase *> RuntimeCalls;
InsertSafepointPoll(PollLocation, RuntimeCalls, TLI);
ParsePointNeeded.insert(ParsePointNeeded.end(), RuntimeCalls.begin(),
RuntimeCalls.end());
@ -621,7 +618,7 @@ INITIALIZE_PASS_END(PlaceSafepoints, "place-safepoints", "Place Safepoints",
static void
InsertSafepointPoll(Instruction *InsertBefore,
std::vector<CallSite> &ParsePointsNeeded /*rval*/,
std::vector<CallBase *> &ParsePointsNeeded /*rval*/,
const TargetLibraryInfo &TLI) {
BasicBlock *OrigBB = InsertBefore->getParent();
Module *M = InsertBefore->getModule();
@ -686,7 +683,7 @@ InsertSafepointPoll(Instruction *InsertBefore,
// These are likely runtime calls. Should we assert that via calling
// convention or something?
ParsePointsNeeded.push_back(CallSite(CI));
ParsePointsNeeded.push_back(CI);
}
assert(ParsePointsNeeded.size() <= Calls.size());
}

187
llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp
View File

@ -31,7 +31,6 @@
#include "llvm/IR/Argument.h"
#include "llvm/IR/Attributes.h"
#include "llvm/IR/BasicBlock.h"
#include "llvm/IR/CallSite.h"
#include "llvm/IR/CallingConv.h"
#include "llvm/IR/Constant.h"
#include "llvm/IR/Constants.h"
@ -285,9 +284,9 @@ struct PartiallyConstructedSafepointRecord {
} // end anonymous namespace
static ArrayRef<Use> GetDeoptBundleOperands(ImmutableCallSite CS) {
static ArrayRef<Use> GetDeoptBundleOperands(const CallBase *Call) {
Optional<OperandBundleUse> DeoptBundle =
CS.getOperandBundle(LLVMContext::OB_deopt);
Call->getOperandBundle(LLVMContext::OB_deopt);
if (!DeoptBundle.hasValue()) {
assert(AllowStatepointWithNoDeoptInfo &&
@ -369,14 +368,11 @@ static std::string suffixed_name_or(Value *V, StringRef Suffix,
// given instruction. The analysis is performed immediately before the
// given instruction. Values defined by that instruction are not considered
// live. Values used by that instruction are considered live.
static void
analyzeParsePointLiveness(DominatorTree &DT,
GCPtrLivenessData &OriginalLivenessData, CallSite CS,
PartiallyConstructedSafepointRecord &Result) {
Instruction *Inst = CS.getInstruction();
static void analyzeParsePointLiveness(
DominatorTree &DT, GCPtrLivenessData &OriginalLivenessData, CallBase *Call,
PartiallyConstructedSafepointRecord &Result) {
StatepointLiveSetTy LiveSet;
findLiveSetAtInst(Inst, OriginalLivenessData, LiveSet);
findLiveSetAtInst(Call, OriginalLivenessData, LiveSet);
if (PrintLiveSet) {
dbgs() << "Live Variables:\n";
@ -384,7 +380,7 @@ analyzeParsePointLiveness(DominatorTree &DT,
dbgs() << " " << V->getName() << " " << *V << "\n";
}
if (PrintLiveSetSize) {
dbgs() << "Safepoint For: " << CS.getCalledValue()->getName() << "\n";
dbgs() << "Safepoint For: " << Call->getCalledValue()->getName() << "\n";
dbgs() << "Number live values: " << LiveSet.size() << "\n";
}
Result.LiveSet = LiveSet;
@ -1177,7 +1173,7 @@ findBasePointers(const StatepointLiveSetTy &live,
/// Find the required based pointers (and adjust the live set) for the given
/// parse point.
static void findBasePointers(DominatorTree &DT, DefiningValueMapTy &DVCache,
CallSite CS,
CallBase *Call,
PartiallyConstructedSafepointRecord &result) {
MapVector<Value *, Value *> PointerToBase;
findBasePointers(result.LiveSet, PointerToBase, &DT, DVCache);
@ -1199,11 +1195,11 @@ static void findBasePointers(DominatorTree &DT, DefiningValueMapTy &DVCache,
/// Given an updated version of the dataflow liveness results, update the
/// liveset and base pointer maps for the call site CS.
static void recomputeLiveInValues(GCPtrLivenessData &RevisedLivenessData,
CallSite CS,
CallBase *Call,
PartiallyConstructedSafepointRecord &result);
static void recomputeLiveInValues(
Function &F, DominatorTree &DT, ArrayRef<CallSite> toUpdate,
Function &F, DominatorTree &DT, ArrayRef<CallBase *> toUpdate,
MutableArrayRef<struct PartiallyConstructedSafepointRecord> records) {
// TODO-PERF: reuse the original liveness, then simply run the dataflow
// again. The old values are still live and will help it stabilize quickly.
@ -1398,16 +1394,16 @@ public:
} // end anonymous namespace
static StringRef getDeoptLowering(CallSite CS) {
static StringRef getDeoptLowering(CallBase *Call) {
const char *DeoptLowering = "deopt-lowering";
if (CS.hasFnAttr(DeoptLowering)) {
// FIXME: CallSite has a *really* confusing interface around attributes
if (Call->hasFnAttr(DeoptLowering)) {
// FIXME: Calls have a *really* confusing interface around attributes
// with values.
const AttributeList &CSAS = CS.getAttributes();
const AttributeList &CSAS = Call->getAttributes();
if (CSAS.hasAttribute(AttributeList::FunctionIndex, DeoptLowering))
return CSAS.getAttribute(AttributeList::FunctionIndex, DeoptLowering)
.getValueAsString();
Function *F = CS.getCalledFunction();
Function *F = Call->getCalledFunction();
assert(F && F->hasFnAttribute(DeoptLowering));
return F->getFnAttribute(DeoptLowering).getValueAsString();
}
@ -1415,7 +1411,7 @@ static StringRef getDeoptLowering(CallSite CS) {
}
static void
makeStatepointExplicitImpl(const CallSite CS, /* to replace */
makeStatepointExplicitImpl(CallBase *Call, /* to replace */
const SmallVectorImpl<Value *> &BasePtrs,
const SmallVectorImpl<Value *> &LiveVariables,
PartiallyConstructedSafepointRecord &Result,
@ -1426,19 +1422,18 @@ makeStatepointExplicitImpl(const CallSite CS, /* to replace */
// immediately before the previous instruction under the assumption that all
// arguments will be available here. We can't insert afterwards since we may
// be replacing a terminator.
Instruction *InsertBefore = CS.getInstruction();
IRBuilder<> Builder(InsertBefore);
IRBuilder<> Builder(Call);
ArrayRef<Value *> GCArgs(LiveVariables);
uint64_t StatepointID = StatepointDirectives::DefaultStatepointID;
uint32_t NumPatchBytes = 0;
uint32_t Flags = uint32_t(StatepointFlags::None);
ArrayRef<Use> CallArgs(CS.arg_begin(), CS.arg_end());
ArrayRef<Use> DeoptArgs = GetDeoptBundleOperands(CS);
ArrayRef<Use> CallArgs(Call->arg_begin(), Call->arg_end());
ArrayRef<Use> DeoptArgs = GetDeoptBundleOperands(Call);
ArrayRef<Use> TransitionArgs;
if (auto TransitionBundle =
CS.getOperandBundle(LLVMContext::OB_gc_transition)) {
Call->getOperandBundle(LLVMContext::OB_gc_transition)) {
Flags |= uint32_t(StatepointFlags::GCTransition);
TransitionArgs = TransitionBundle->Inputs;
}
@ -1449,21 +1444,21 @@ makeStatepointExplicitImpl(const CallSite CS, /* to replace */
bool IsDeoptimize = false;
StatepointDirectives SD =
parseStatepointDirectivesFromAttrs(CS.getAttributes());
parseStatepointDirectivesFromAttrs(Call->getAttributes());
if (SD.NumPatchBytes)
NumPatchBytes = *SD.NumPatchBytes;
if (SD.StatepointID)
StatepointID = *SD.StatepointID;
// Pass through the requested lowering if any. The default is live-through.
StringRef DeoptLowering = getDeoptLowering(CS);
StringRef DeoptLowering = getDeoptLowering(Call);
if (DeoptLowering.equals("live-in"))
Flags |= uint32_t(StatepointFlags::DeoptLiveIn);
else {
assert(DeoptLowering.equals("live-through") && "Unsupported value!");
}
Value *CallTarget = CS.getCalledValue();
Value *CallTarget = Call->getCalledValue();
if (Function *F = dyn_cast<Function>(CallTarget)) {
if (F->getIntrinsicID() == Intrinsic::experimental_deoptimize) {
// Calls to llvm.experimental.deoptimize are lowered to calls to the
@ -1490,57 +1485,56 @@ makeStatepointExplicitImpl(const CallSite CS, /* to replace */
// Create the statepoint given all the arguments
Instruction *Token = nullptr;
if (CS.isCall()) {
CallInst *ToReplace = cast<CallInst>(CS.getInstruction());
CallInst *Call = Builder.CreateGCStatepointCall(
if (auto *CI = dyn_cast<CallInst>(Call)) {
CallInst *SPCall = Builder.CreateGCStatepointCall(
StatepointID, NumPatchBytes, CallTarget, Flags, CallArgs,
TransitionArgs, DeoptArgs, GCArgs, "safepoint_token");
Call->setTailCallKind(ToReplace->getTailCallKind());
Call->setCallingConv(ToReplace->getCallingConv());
SPCall->setTailCallKind(CI->getTailCallKind());
SPCall->setCallingConv(CI->getCallingConv());
// Currently we will fail on parameter attributes and on certain
// function attributes. In case if we can handle this set of attributes -
// set up function attrs directly on statepoint and return attrs later for
// gc_result intrinsic.
Call->setAttributes(legalizeCallAttributes(ToReplace->getAttributes()));
SPCall->setAttributes(legalizeCallAttributes(CI->getAttributes()));
Token = Call;
Token = SPCall;
// Put the following gc_result and gc_relocate calls immediately after the
// the old call (which we're about to delete)
assert(ToReplace->getNextNode() && "Not a terminator, must have next!");
Builder.SetInsertPoint(ToReplace->getNextNode());
Builder.SetCurrentDebugLocation(ToReplace->getNextNode()->getDebugLoc());
assert(CI->getNextNode() && "Not a terminator, must have next!");
Builder.SetInsertPoint(CI->getNextNode());
Builder.SetCurrentDebugLocation(CI->getNextNode()->getDebugLoc());
} else {
InvokeInst *ToReplace = cast<InvokeInst>(CS.getInstruction());
auto *II = cast<InvokeInst>(Call);
// Insert the new invoke into the old block. We'll remove the old one in a
// moment at which point this will become the new terminator for the
// original block.
InvokeInst *Invoke = Builder.CreateGCStatepointInvoke(
StatepointID, NumPatchBytes, CallTarget, ToReplace->getNormalDest(),
ToReplace->getUnwindDest(), Flags, CallArgs, TransitionArgs, DeoptArgs,
GCArgs, "statepoint_token");
InvokeInst *SPInvoke = Builder.CreateGCStatepointInvoke(
StatepointID, NumPatchBytes, CallTarget, II->getNormalDest(),
II->getUnwindDest(), Flags, CallArgs, TransitionArgs, DeoptArgs, GCArgs,
"statepoint_token");
Invoke->setCallingConv(ToReplace->getCallingConv());
SPInvoke->setCallingConv(II->getCallingConv());
// Currently we will fail on parameter attributes and on certain
// function attributes. In case if we can handle this set of attributes -
// set up function attrs directly on statepoint and return attrs later for
// gc_result intrinsic.
Invoke->setAttributes(legalizeCallAttributes(ToReplace->getAttributes()));
SPInvoke->setAttributes(legalizeCallAttributes(II->getAttributes()));
Token = Invoke;
Token = SPInvoke;
// Generate gc relocates in exceptional path
BasicBlock *UnwindBlock = ToReplace->getUnwindDest();
BasicBlock *UnwindBlock = II->getUnwindDest();
assert(!isa<PHINode>(UnwindBlock->begin()) &&
UnwindBlock->getUniquePredecessor() &&
"can't safely insert in this block!");
Builder.SetInsertPoint(&*UnwindBlock->getFirstInsertionPt());
Builder.SetCurrentDebugLocation(ToReplace->getDebugLoc());
Builder.SetCurrentDebugLocation(II->getDebugLoc());
// Attach exceptional gc relocates to the landingpad.
Instruction *ExceptionalToken = UnwindBlock->getLandingPadInst();
@ -1551,7 +1545,7 @@ makeStatepointExplicitImpl(const CallSite CS, /* to replace */
Builder);
// Generate gc relocates and returns for normal block
BasicBlock *NormalDest = ToReplace->getNormalDest();
BasicBlock *NormalDest = II->getNormalDest();
assert(!isa<PHINode>(NormalDest->begin()) &&
NormalDest->getUniquePredecessor() &&
"can't safely insert in this block!");
@ -1568,16 +1562,15 @@ makeStatepointExplicitImpl(const CallSite CS, /* to replace */
// transform the tail-call like structure to a call to a void function
// followed by unreachable to get better codegen.
Replacements.push_back(
DeferredReplacement::createDeoptimizeReplacement(CS.getInstruction()));
DeferredReplacement::createDeoptimizeReplacement(Call));
} else {
Token->setName("statepoint_token");
if (!CS.getType()->isVoidTy() && !CS.getInstruction()->use_empty()) {
StringRef Name =
CS.getInstruction()->hasName() ? CS.getInstruction()->getName() : "";
CallInst *GCResult = Builder.CreateGCResult(Token, CS.getType(), Name);
if (!Call->getType()->isVoidTy() && !Call->use_empty()) {
StringRef Name = Call->hasName() ? Call->getName() : "";
CallInst *GCResult = Builder.CreateGCResult(Token, Call->getType(), Name);
GCResult->setAttributes(
AttributeList::get(GCResult->getContext(), AttributeList::ReturnIndex,
CS.getAttributes().getRetAttributes()));
Call->getAttributes().getRetAttributes()));
// We cannot RAUW or delete CS.getInstruction() because it could be in the
// live set of some other safepoint, in which case that safepoint's
@ -1586,10 +1579,9 @@ makeStatepointExplicitImpl(const CallSite CS, /* to replace */
// after the live sets have been made explicit in the IR, and we no longer
// have raw pointers to worry about.
Replacements.emplace_back(
DeferredReplacement::createRAUW(CS.getInstruction(), GCResult));
DeferredReplacement::createRAUW(Call, GCResult));
} else {
Replacements.emplace_back(
DeferredReplacement::createDelete(CS.getInstruction()));
Replacements.emplace_back(DeferredReplacement::createDelete(Call));
}
}
@ -1606,7 +1598,7 @@ makeStatepointExplicitImpl(const CallSite CS, /* to replace */
// WARNING: Does not do any fixup to adjust users of the original live
// values. That's the callers responsibility.
static void
makeStatepointExplicit(DominatorTree &DT, CallSite CS,
makeStatepointExplicit(DominatorTree &DT, CallBase *Call,
PartiallyConstructedSafepointRecord &Result,
std::vector<DeferredReplacement> &Replacements) {
const auto &LiveSet = Result.LiveSet;
@ -1625,7 +1617,7 @@ makeStatepointExplicit(DominatorTree &DT, CallSite CS,
assert(LiveVec.size() == BaseVec.size());
// Do the actual rewriting and delete the old statepoint
makeStatepointExplicitImpl(CS, BaseVec, LiveVec, Result, Replacements);
makeStatepointExplicitImpl(Call, BaseVec, LiveVec, Result, Replacements);
}
// Helper function for the relocationViaAlloca.
@ -1895,25 +1887,25 @@ template <typename T> static void unique_unsorted(SmallVectorImpl<T> &Vec) {
/// Insert holders so that each Value is obviously live through the entire
/// lifetime of the call.
static void insertUseHolderAfter(CallSite &CS, const ArrayRef<Value *> Values,
static void insertUseHolderAfter(CallBase *Call, const ArrayRef<Value *> Values,
SmallVectorImpl<CallInst *> &Holders) {
if (Values.empty())
// No values to hold live, might as well not insert the empty holder
return;
Module *M = CS.getInstruction()->getModule();
Module *M = Call->getModule();
// Use a dummy vararg function to actually hold the values live
FunctionCallee Func = M->getOrInsertFunction(
"__tmp_use", FunctionType::get(Type::getVoidTy(M->getContext()), true));
if (CS.isCall()) {
if (isa<CallInst>(Call)) {
// For call safepoints insert dummy calls right after safepoint
Holders.push_back(CallInst::Create(Func, Values, "",
&*++CS.getInstruction()->getIterator()));
Holders.push_back(
CallInst::Create(Func, Values, "", &*++Call->getIterator()));
return;
}
// For invoke safepooints insert dummy calls both in normal and
// exceptional destination blocks
auto *II = cast<InvokeInst>(CS.getInstruction());
auto *II = cast<InvokeInst>(Call);
Holders.push_back(CallInst::Create(
Func, Values, "", &*II->getNormalDest()->getFirstInsertionPt()));
Holders.push_back(CallInst::Create(
@ -1921,7 +1913,7 @@ static void insertUseHolderAfter(CallSite &CS, const ArrayRef<Value *> Values,
}
static void findLiveReferences(
Function &F, DominatorTree &DT, ArrayRef<CallSite> toUpdate,
Function &F, DominatorTree &DT, ArrayRef<CallBase *> toUpdate,
MutableArrayRef<struct PartiallyConstructedSafepointRecord> records) {
GCPtrLivenessData OriginalLivenessData;
computeLiveInValues(DT, F, OriginalLivenessData);
@ -2024,7 +2016,7 @@ static bool AreEquivalentPhiNodes(PHINode &OrigRootPhi, PHINode &AlternateRootPh
// to relocate. Remove this values from the live set, rematerialize them after
// statepoint and record them in "Info" structure. Note that similar to
// relocated values we don't do any user adjustments here.
static void rematerializeLiveValues(CallSite CS,
static void rematerializeLiveValues(CallBase *Call,
PartiallyConstructedSafepointRecord &Info,
TargetTransformInfo &TTI) {
const unsigned int ChainLengthThreshold = 10;
@ -2078,7 +2070,7 @@ static void rematerializeLiveValues(CallSite CS,
// For invokes we need to rematerialize each chain twice - for normal and
// for unwind basic blocks. Model this by multiplying cost by two.
if (CS.isInvoke()) {
if (isa<InvokeInst>(Call)) {
Cost *= 2;
}
// If it's too expensive - skip it
@ -2146,14 +2138,14 @@ static void rematerializeLiveValues(CallSite CS,
// Different cases for calls and invokes. For invokes we need to clone
// instructions both on normal and unwind path.
if (CS.isCall()) {
Instruction *InsertBefore = CS.getInstruction()->getNextNode();
if (isa<CallInst>(Call)) {
Instruction *InsertBefore = Call->getNextNode();
assert(InsertBefore);
Instruction *RematerializedValue = rematerializeChain(
InsertBefore, RootOfChain, Info.PointerToBase[LiveValue]);
Info.RematerializedValues[RematerializedValue] = LiveValue;
} else {
InvokeInst *Invoke = cast<InvokeInst>(CS.getInstruction());
auto *Invoke = cast<InvokeInst>(Call);
Instruction *NormalInsertBefore =
&*Invoke->getNormalDest()->getFirstInsertionPt();
@ -2178,25 +2170,25 @@ static void rematerializeLiveValues(CallSite CS,
static bool insertParsePoints(Function &F, DominatorTree &DT,
TargetTransformInfo &TTI,
SmallVectorImpl<CallSite> &ToUpdate) {
SmallVectorImpl<CallBase *> &ToUpdate) {
#ifndef NDEBUG
// sanity check the input
std::set<CallSite> Uniqued;
std::set<CallBase *> Uniqued;
Uniqued.insert(ToUpdate.begin(), ToUpdate.end());
assert(Uniqued.size() == ToUpdate.size() && "no duplicates please!");
for (CallSite CS : ToUpdate)
assert(CS.getInstruction()->getFunction() == &F);
for (CallBase *Call : ToUpdate)
assert(Call->getFunction() == &F);
#endif
// When inserting gc.relocates for invokes, we need to be able to insert at
// the top of the successor blocks. See the comment on
// normalForInvokeSafepoint on exactly what is needed. Note that this step
// may restructure the CFG.
for (CallSite CS : ToUpdate) {
if (!CS.isInvoke())
for (CallBase *Call : ToUpdate) {
auto *II = dyn_cast<InvokeInst>(Call);
if (!II)
continue;
auto *II = cast<InvokeInst>(CS.getInstruction());
normalizeForInvokeSafepoint(II->getNormalDest(), II->getParent(), DT);
normalizeForInvokeSafepoint(II->getUnwindDest(), II->getParent(), DT);
}
@ -2209,17 +2201,17 @@ static bool insertParsePoints(Function &F, DominatorTree &DT,
// actual safepoint insertion as arguments. This ensures reference operands
// in the deopt argument list are considered live through the safepoint (and
// thus makes sure they get relocated.)
for (CallSite CS : ToUpdate) {
for (CallBase *Call : ToUpdate) {
SmallVector<Value *, 64> DeoptValues;
for (Value *Arg : GetDeoptBundleOperands(CS)) {
for (Value *Arg : GetDeoptBundleOperands(Call)) {
assert(!isUnhandledGCPointerType(Arg->getType()) &&
"support for FCA unimplemented");
if (isHandledGCPointerType(Arg->getType()))
DeoptValues.push_back(Arg);
}
insertUseHolderAfter(CS, DeoptValues, Holders);
insertUseHolderAfter(Call, DeoptValues, Holders);
}
SmallVector<PartiallyConstructedSafepointRecord, 64> Records(ToUpdate.size());
@ -2321,7 +2313,7 @@ static bool insertParsePoints(Function &F, DominatorTree &DT,
for (size_t i = 0; i < Records.size(); i++)
makeStatepointExplicit(DT, ToUpdate[i], Records[i], Replacements);
ToUpdate.clear(); // prevent accident use of invalid CallSites
ToUpdate.clear(); // prevent accident use of invalid calls.
for (auto &PR : Replacements)
PR.doReplacement();
@ -2386,7 +2378,7 @@ static bool insertParsePoints(Function &F, DominatorTree &DT,
return !Records.empty();
}
// Handles both return values and arguments for Functions and CallSites.
// Handles both return values and arguments for Functions and calls.
template <typename AttrHolder>
static void RemoveNonValidAttrAtIndex(LLVMContext &Ctx, AttrHolder &AH,
unsigned Index) {
@ -2478,12 +2470,13 @@ static void stripNonValidDataFromBody(Function &F) {
stripInvalidMetadataFromInstruction(I);
if (CallSite CS = CallSite(&I)) {
for (int i = 0, e = CS.arg_size(); i != e; i++)
if (isa<PointerType>(CS.getArgument(i)->getType()))
RemoveNonValidAttrAtIndex(Ctx, CS, i + AttributeList::FirstArgIndex);
if (isa<PointerType>(CS.getType()))
RemoveNonValidAttrAtIndex(Ctx, CS, AttributeList::ReturnIndex);
if (auto *Call = dyn_cast<CallBase>(&I)) {
for (int i = 0, e = Call->arg_size(); i != e; i++)
if (isa<PointerType>(Call->getArgOperand(i)->getType()))
RemoveNonValidAttrAtIndex(Ctx, *Call,
i + AttributeList::FirstArgIndex);
if (isa<PointerType>(Call->getType()))
RemoveNonValidAttrAtIndex(Ctx, *Call, AttributeList::ReturnIndex);
}
}
@ -2528,12 +2521,11 @@ bool RewriteStatepointsForGC::runOnFunction(Function &F, DominatorTree &DT,
assert(shouldRewriteStatepointsIn(F) && "mismatch in rewrite decision");
auto NeedsRewrite = [&TLI](Instruction &I) {
if (ImmutableCallSite CS = ImmutableCallSite(&I))
return !callsGCLeafFunction(CS, TLI) && !isStatepoint(CS);
if (const auto *Call = dyn_cast<CallBase>(&I))
return !callsGCLeafFunction(Call, TLI) && !isStatepoint(Call);
return false;
};
// Delete any unreachable statepoints so that we don't have unrewritten
// statepoints surviving this pass. This makes testing easier and the
// resulting IR less confusing to human readers.
@ -2545,7 +2537,7 @@ bool RewriteStatepointsForGC::runOnFunction(Function &F, DominatorTree &DT,
// Gather all the statepoints which need rewritten. Be careful to only
// consider those in reachable code since we need to ask dominance queries
// when rewriting. We'll delete the unreachable ones in a moment.
SmallVector<CallSite, 64> ParsePointNeeded;
SmallVector<CallBase *, 64> ParsePointNeeded;
for (Instruction &I : instructions(F)) {
// TODO: only the ones with the flag set!
if (NeedsRewrite(I)) {
@ -2555,7 +2547,7 @@ bool RewriteStatepointsForGC::runOnFunction(Function &F, DominatorTree &DT,
// isReachableFromEntry() returns true.
assert(DT.isReachableFromEntry(I.getParent()) &&
"no unreachable blocks expected");
ParsePointNeeded.push_back(CallSite(&I));
ParsePointNeeded.push_back(cast<CallBase>(&I));
}
}
@ -2815,11 +2807,10 @@ static void findLiveSetAtInst(Instruction *Inst, GCPtrLivenessData &Data,
}
static void recomputeLiveInValues(GCPtrLivenessData &RevisedLivenessData,
CallSite CS,
CallBase *Call,
PartiallyConstructedSafepointRecord &Info) {
Instruction *Inst = CS.getInstruction();
StatepointLiveSetTy Updated;
findLiveSetAtInst(Inst, RevisedLivenessData, Updated);
findLiveSetAtInst(Call, RevisedLivenessData, Updated);
// We may have base pointers which are now live that weren't before. We need
// to update the PointerToBase structure to reflect this.

8
llvm/lib/Transforms/Utils/Local.cpp
View File

@ -2462,12 +2462,12 @@ unsigned llvm::replaceDominatedUsesWith(Value *From, Value *To,
return ::replaceDominatedUsesWith(From, To, BB, ProperlyDominates);
}
bool llvm::callsGCLeafFunction(ImmutableCallSite CS,
bool llvm::callsGCLeafFunction(const CallBase *Call,
const TargetLibraryInfo &TLI) {
// Check if the function is specifically marked as a gc leaf function.
if (CS.hasFnAttr("gc-leaf-function"))
if (Call->hasFnAttr("gc-leaf-function"))
return true;
if (const Function *F = CS.getCalledFunction()) {
if (const Function *F = Call->getCalledFunction()) {
if (F->hasFnAttribute("gc-leaf-function"))
return true;
@ -2481,7 +2481,7 @@ bool llvm::callsGCLeafFunction(ImmutableCallSite CS,
// marked as 'gc-leaf-function.' All available Libcalls are
// GC-leaf.
LibFunc LF;
if (TLI.getLibFunc(CS, LF)) {
if (TLI.getLibFunc(ImmutableCallSite(Call), LF)) {
return TLI.has(LF);
}