From 3160734af13ba9797c82ce98be75b63db0c37c50 Mon Sep 17 00:00:00 2001 From: Chandler Carruth Date: Mon, 11 Feb 2019 07:42:30 +0000 Subject: [PATCH] [CallSite removal] Migrate the statepoint GC infrastructure to use the `CallBase` class rather than `CallSite` wrappers. I pushed this change down through most of the statepoint infrastructure, completely removing the use of CallSite where I could reasonably do so. I ended up making a couple of cut-points: generic call handling (instcombine, TLI, SDAG). As soon as it hit truly generic handling with users outside the immediate code, I simply transitioned into or out of a `CallSite` to make this a reasonable sized chunk. Differential Revision: https://reviews.llvm.org/D56122 llvm-svn: 353660 --- llvm/include/llvm/CodeGen/TargetLowering.h | 6 +- llvm/include/llvm/IR/Statepoint.h | 141 ++++++------- llvm/include/llvm/Transforms/Utils/Local.h | 5 +- .../SelectionDAG/SelectionDAGBuilder.cpp | 14 +- .../SelectionDAG/SelectionDAGBuilder.h | 6 +- .../SelectionDAG/StatepointLowering.cpp | 31 ++- .../CodeGen/SelectionDAG/TargetLowering.cpp | 24 +-- llvm/lib/IR/Statepoint.cpp | 30 ++- .../lib/Transforms/Scalar/PlaceSafepoints.cpp | 35 ++-- .../Scalar/RewriteStatepointsForGC.cpp | 187 +++++++++--------- llvm/lib/Transforms/Utils/Local.cpp | 8 +- 11 files changed, 227 insertions(+), 260 deletions(-) diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h index 7bb5944d99a5..fa71024a25f3 100644 --- a/llvm/include/llvm/CodeGen/TargetLowering.h +++ b/llvm/include/llvm/CodeGen/TargetLowering.h @@ -194,7 +194,11 @@ public: IsNest(false), IsByVal(false), IsInAlloca(false), IsReturned(false), IsSwiftSelf(false), IsSwiftError(false) {} - void setAttributes(ImmutableCallSite *CS, unsigned ArgIdx); + void setAttributes(const CallBase *Call, unsigned ArgIdx); + + void setAttributes(ImmutableCallSite *CS, unsigned ArgIdx) { + return setAttributes(cast(CS->getInstruction()), ArgIdx); + } }; using ArgListTy = std::vector; diff --git a/llvm/include/llvm/IR/Statepoint.h b/llvm/include/llvm/IR/Statepoint.h index 995c70f47b2a..aec0e9b6d87b 100644 --- a/llvm/include/llvm/IR/Statepoint.h +++ b/llvm/include/llvm/IR/Statepoint.h @@ -7,7 +7,7 @@ //===----------------------------------------------------------------------===// // // This file contains utility functions and a wrapper class analogous to -// CallSite for accessing the fields of gc.statepoint, gc.relocate, +// CallBase for accessing the fields of gc.statepoint, gc.relocate, // gc.result intrinsics; and some general utilities helpful when dealing with // gc.statepoint. // @@ -20,7 +20,6 @@ #include "llvm/ADT/iterator_range.h" #include "llvm/IR/Attributes.h" #include "llvm/IR/BasicBlock.h" -#include "llvm/IR/CallSite.h" #include "llvm/IR/Constants.h" #include "llvm/IR/Function.h" #include "llvm/IR/Instruction.h" @@ -56,42 +55,39 @@ enum class StatepointFlags { class GCRelocateInst; class GCResultInst; -bool isStatepoint(ImmutableCallSite CS); +bool isStatepoint(const CallBase *Call); bool isStatepoint(const Value *V); bool isStatepoint(const Value &V); -bool isGCRelocate(ImmutableCallSite CS); +bool isGCRelocate(const CallBase *Call); bool isGCRelocate(const Value *V); -bool isGCResult(ImmutableCallSite CS); +bool isGCResult(const CallBase *Call); bool isGCResult(const Value *V); -/// Analogous to CallSiteBase, this provides most of the actual +/// A wrapper around a GC intrinsic call, this provides most of the actual /// functionality for Statepoint and ImmutableStatepoint. It is /// templatized to allow easily specializing of const and non-const -/// concrete subtypes. This is structured analogous to CallSite -/// rather than the IntrinsicInst.h helpers since we need to support -/// invokable statepoints. +/// concrete subtypes. template + typename CallBaseTy> class StatepointBase { - CallSiteTy StatepointCS; + CallBaseTy *StatepointCall; protected: explicit StatepointBase(InstructionTy *I) { if (isStatepoint(I)) { - StatepointCS = CallSiteTy(I); - assert(StatepointCS && "isStatepoint implies CallSite"); + StatepointCall = cast(I); } } - explicit StatepointBase(CallSiteTy CS) { - if (isStatepoint(CS)) - StatepointCS = CS; + explicit StatepointBase(CallBaseTy *Call) { + if (isStatepoint(Call)) + StatepointCall = Call; } public: - using arg_iterator = typename CallSiteTy::arg_iterator; + using arg_iterator = typename CallBaseTy::const_op_iterator; enum { IDPos = 0, @@ -106,30 +102,30 @@ public: void *operator new(size_t s) = delete; explicit operator bool() const { - // We do not assign non-statepoint CallSites to StatepointCS. - return (bool)StatepointCS; + // We do not assign non-statepoint call instructions to StatepointCall. + return (bool)StatepointCall; } - /// Return the underlying CallSite. - CallSiteTy getCallSite() const { + /// Return the underlying call instruction. + CallBaseTy *getCall() const { assert(*this && "check validity first!"); - return StatepointCS; + return StatepointCall; } uint64_t getFlags() const { - return cast(getCallSite().getArgument(FlagsPos)) + return cast(getCall()->getArgOperand(FlagsPos)) ->getZExtValue(); } /// Return the ID associated with this statepoint. uint64_t getID() const { - const Value *IDVal = getCallSite().getArgument(IDPos); + const Value *IDVal = getCall()->getArgOperand(IDPos); return cast(IDVal)->getZExtValue(); } /// Return the number of patchable bytes associated with this statepoint. uint32_t getNumPatchBytes() const { - const Value *NumPatchBytesVal = getCallSite().getArgument(NumPatchBytesPos); + const Value *NumPatchBytesVal = getCall()->getArgOperand(NumPatchBytesPos); uint64_t NumPatchBytes = cast(NumPatchBytesVal)->getZExtValue(); assert(isInt<32>(NumPatchBytes) && "should fit in 32 bits!"); @@ -138,12 +134,11 @@ public: /// Return the value actually being called or invoked. ValueTy *getCalledValue() const { - return getCallSite().getArgument(CalledFunctionPos); + return getCall()->getArgOperand(CalledFunctionPos); } - InstructionTy *getInstruction() const { - return getCallSite().getInstruction(); - } + // FIXME: Migrate users of this to `getCall` and remove it. + InstructionTy *getInstruction() const { return getCall(); } /// Return the function being called if this is a direct call, otherwise /// return null (if it's an indirect call). @@ -152,12 +147,12 @@ public: } /// Return the caller function for this statepoint. - FunTy *getCaller() const { return getCallSite().getCaller(); } + FunTy *getCaller() const { return getCall()->getCaller(); } /// Determine if the statepoint cannot unwind. bool doesNotThrow() const { Function *F = getCalledFunction(); - return getCallSite().doesNotThrow() || (F ? F->doesNotThrow() : false); + return getCall()->doesNotThrow() || (F ? F->doesNotThrow() : false); } /// Return the type of the value returned by the call underlying the @@ -170,18 +165,18 @@ public: /// Number of arguments to be passed to the actual callee. int getNumCallArgs() const { - const Value *NumCallArgsVal = getCallSite().getArgument(NumCallArgsPos); + const Value *NumCallArgsVal = getCall()->getArgOperand(NumCallArgsPos); return cast(NumCallArgsVal)->getZExtValue(); } size_t arg_size() const { return getNumCallArgs(); } - typename CallSiteTy::arg_iterator arg_begin() const { - assert(CallArgsBeginPos <= (int)getCallSite().arg_size()); - return getCallSite().arg_begin() + CallArgsBeginPos; + arg_iterator arg_begin() const { + assert(CallArgsBeginPos <= (int)getCall()->arg_size()); + return getCall()->arg_begin() + CallArgsBeginPos; } - typename CallSiteTy::arg_iterator arg_end() const { + arg_iterator arg_end() const { auto I = arg_begin() + arg_size(); - assert((getCallSite().arg_end() - I) >= 0); + assert((getCall()->arg_end() - I) >= 0); return I; } @@ -198,8 +193,8 @@ public: /// Return true if the call or the callee has the given attribute. bool paramHasAttr(unsigned i, Attribute::AttrKind A) const { Function *F = getCalledFunction(); - return getCallSite().paramHasAttr(i + CallArgsBeginPos, A) || - (F ? F->getAttributes().hasAttribute(i, A) : false); + return getCall()->paramHasAttr(i + CallArgsBeginPos, A) || + (F ? F->getAttributes().hasAttribute(i, A) : false); } /// Number of GC transition args. @@ -207,14 +202,14 @@ public: const Value *NumGCTransitionArgs = *arg_end(); return cast(NumGCTransitionArgs)->getZExtValue(); } - typename CallSiteTy::arg_iterator gc_transition_args_begin() const { + arg_iterator gc_transition_args_begin() const { auto I = arg_end() + 1; - assert((getCallSite().arg_end() - I) >= 0); + assert((getCall()->arg_end() - I) >= 0); return I; } - typename CallSiteTy::arg_iterator gc_transition_args_end() const { + arg_iterator gc_transition_args_end() const { auto I = gc_transition_args_begin() + getNumTotalGCTransitionArgs(); - assert((getCallSite().arg_end() - I) >= 0); + assert((getCall()->arg_end() - I) >= 0); return I; } @@ -230,14 +225,14 @@ public: return cast(NumVMSArgs)->getZExtValue(); } - typename CallSiteTy::arg_iterator deopt_begin() const { + arg_iterator deopt_begin() const { auto I = gc_transition_args_end() + 1; - assert((getCallSite().arg_end() - I) >= 0); + assert((getCall()->arg_end() - I) >= 0); return I; } - typename CallSiteTy::arg_iterator deopt_end() const { + arg_iterator deopt_end() const { auto I = deopt_begin() + getNumTotalVMSArgs(); - assert((getCallSite().arg_end() - I) >= 0); + assert((getCall()->arg_end() - I) >= 0); return I; } @@ -246,15 +241,11 @@ public: return make_range(deopt_begin(), deopt_end()); } - typename CallSiteTy::arg_iterator gc_args_begin() const { - return deopt_end(); - } - typename CallSiteTy::arg_iterator gc_args_end() const { - return getCallSite().arg_end(); - } + arg_iterator gc_args_begin() const { return deopt_end(); } + arg_iterator gc_args_end() const { return getCall()->arg_end(); } unsigned gcArgsStartIdx() const { - return gc_args_begin() - getInstruction()->op_begin(); + return gc_args_begin() - getCall()->op_begin(); } /// range adapter for gc arguments @@ -303,25 +294,24 @@ public: /// to a gc.statepoint. class ImmutableStatepoint : public StatepointBase { - using Base = - StatepointBase; + const CallBase> { + using Base = StatepointBase; public: explicit ImmutableStatepoint(const Instruction *I) : Base(I) {} - explicit ImmutableStatepoint(ImmutableCallSite CS) : Base(CS) {} + explicit ImmutableStatepoint(const CallBase *Call) : Base(Call) {} }; /// A specialization of it's base class for read-write access /// to a gc.statepoint. class Statepoint - : public StatepointBase { - using Base = StatepointBase; + : public StatepointBase { + using Base = StatepointBase; public: explicit Statepoint(Instruction *I) : Base(I) {} - explicit Statepoint(CallSite CS) : Base(CS) {} + explicit Statepoint(CallBase *Call) : Base(Call) {} }; /// Common base class for representing values projected from a statepoint. @@ -346,14 +336,14 @@ public: } /// The statepoint with which this gc.relocate is associated. - const Instruction *getStatepoint() const { + const CallBase *getStatepoint() const { const Value *Token = getArgOperand(0); // This takes care both of relocates for call statepoints and relocates // on normal path of invoke statepoint. if (!isa(Token)) { assert(isStatepoint(Token)); - return cast(Token); + return cast(Token); } // This relocate is on exceptional path of an invoke statepoint @@ -365,7 +355,7 @@ public: "safepoint block should be well formed"); assert(isStatepoint(InvokeBB->getTerminator())); - return InvokeBB->getTerminator(); + return cast(InvokeBB->getTerminator()); } }; @@ -394,13 +384,11 @@ public: } Value *getBasePtr() const { - ImmutableCallSite CS(getStatepoint()); - return *(CS.arg_begin() + getBasePtrIndex()); + return *(getStatepoint()->arg_begin() + getBasePtrIndex()); } Value *getDerivedPtr() const { - ImmutableCallSite CS(getStatepoint()); - return *(CS.arg_begin() + getDerivedPtrIndex()); + return *(getStatepoint()->arg_begin() + getDerivedPtrIndex()); } }; @@ -417,28 +405,25 @@ public: }; template + typename CallBaseTy> std::vector -StatepointBase::getRelocates() +StatepointBase::getRelocates() const { - std::vector Result; - CallSiteTy StatepointCS = getCallSite(); - // Search for relocated pointers. Note that working backwards from the // gc_relocates ensures that we only get pairs which are actually relocated // and used after the statepoint. - for (const User *U : getInstruction()->users()) + for (const User *U : StatepointCall->users()) if (auto *Relocate = dyn_cast(U)) Result.push_back(Relocate); - if (!StatepointCS.isInvoke()) + auto *StatepointInvoke = dyn_cast(StatepointCall); + if (!StatepointInvoke) return Result; // We need to scan thorough exceptional relocations if it is invoke statepoint - LandingPadInst *LandingPad = - cast(getInstruction())->getLandingPadInst(); + LandingPadInst *LandingPad = StatepointInvoke->getLandingPadInst(); // Search for gc relocates that are attached to this landingpad. for (const User *LandingPadUser : LandingPad->users()) { diff --git a/llvm/include/llvm/Transforms/Utils/Local.h b/llvm/include/llvm/Transforms/Utils/Local.h index 6032f6588459..641b645825fc 100644 --- a/llvm/include/llvm/Transforms/Utils/Local.h +++ b/llvm/include/llvm/Transforms/Utils/Local.h @@ -22,7 +22,6 @@ #include "llvm/Analysis/AliasAnalysis.h" #include "llvm/Analysis/DomTreeUpdater.h" #include "llvm/Analysis/Utils/Local.h" -#include "llvm/IR/CallSite.h" #include "llvm/IR/Constant.h" #include "llvm/IR/Constants.h" #include "llvm/IR/DataLayout.h" @@ -437,7 +436,7 @@ unsigned replaceDominatedUsesWith(Value *From, Value *To, DominatorTree &DT, unsigned replaceDominatedUsesWith(Value *From, Value *To, DominatorTree &DT, const BasicBlock *BB); -/// Return true if the CallSite CS calls a gc leaf function. +/// Return true if this call calls a gc leaf function. /// /// A leaf function is a function that does not safepoint the thread during its /// execution. During a call or invoke to such a function, the callers stack @@ -445,7 +444,7 @@ unsigned replaceDominatedUsesWith(Value *From, Value *To, DominatorTree &DT, /// /// Most passes can and should ignore this information, and it is only used /// during lowering by the GC infrastructure. -bool callsGCLeafFunction(ImmutableCallSite CS, const TargetLibraryInfo &TLI); +bool callsGCLeafFunction(const CallBase *Call, const TargetLibraryInfo &TLI); /// Copy a nonnull metadata node to a new load instruction. /// diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp index d41158134b2c..e3c6ddefc3ca 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -8132,7 +8132,7 @@ SDValue SelectionDAGBuilder::lowerRangeToAssertZExt(SelectionDAG &DAG, /// convention or require stack pointer adjustment. Only a subset of the /// intrinsic's operands need to participate in the calling convention. void SelectionDAGBuilder::populateCallLoweringInfo( - TargetLowering::CallLoweringInfo &CLI, ImmutableCallSite CS, + TargetLowering::CallLoweringInfo &CLI, const CallBase *Call, unsigned ArgIdx, unsigned NumArgs, SDValue Callee, Type *ReturnTy, bool IsPatchPoint) { TargetLowering::ArgListTy Args; @@ -8142,21 +8142,21 @@ void SelectionDAGBuilder::populateCallLoweringInfo( // Attributes for args start at offset 1, after the return attribute. for (unsigned ArgI = ArgIdx, ArgE = ArgIdx + NumArgs; ArgI != ArgE; ++ArgI) { - const Value *V = CS->getOperand(ArgI); + const Value *V = Call->getOperand(ArgI); assert(!V->getType()->isEmptyTy() && "Empty type passed to intrinsic."); TargetLowering::ArgListEntry Entry; Entry.Node = getValue(V); Entry.Ty = V->getType(); - Entry.setAttributes(&CS, ArgI); + Entry.setAttributes(Call, ArgI); Args.push_back(Entry); } CLI.setDebugLoc(getCurSDLoc()) .setChain(getRoot()) - .setCallee(CS.getCallingConv(), ReturnTy, Callee, std::move(Args)) - .setDiscardResult(CS->use_empty()) + .setCallee(Call->getCallingConv(), ReturnTy, Callee, std::move(Args)) + .setDiscardResult(Call->use_empty()) .setIsPatchPoint(IsPatchPoint); } @@ -8300,8 +8300,8 @@ void SelectionDAGBuilder::visitPatchpoint(ImmutableCallSite CS, IsAnyRegCC ? Type::getVoidTy(*DAG.getContext()) : CS->getType(); TargetLowering::CallLoweringInfo CLI(DAG); - populateCallLoweringInfo(CLI, CS, NumMetaOpers, NumCallArgs, Callee, ReturnTy, - true); + populateCallLoweringInfo(CLI, cast(CS.getInstruction()), + NumMetaOpers, NumCallArgs, Callee, ReturnTy, true); std::pair Result = lowerInvokable(CLI, EHPadBB); SDNode *CallEnd = Result.second.getNode(); diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h index 37b04e99ccfd..c354a16b6cd2 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h @@ -733,7 +733,7 @@ public: SDValue Op); void populateCallLoweringInfo(TargetLowering::CallLoweringInfo &CLI, - ImmutableCallSite CS, unsigned ArgIdx, + const CallBase *Call, unsigned ArgIdx, unsigned NumArgs, SDValue Callee, Type *ReturnTy, bool IsPatchPoint); @@ -797,13 +797,13 @@ public: void LowerStatepoint(ImmutableStatepoint ISP, const BasicBlock *EHPadBB = nullptr); - void LowerCallSiteWithDeoptBundle(ImmutableCallSite CS, SDValue Callee, + void LowerCallSiteWithDeoptBundle(const CallBase *Call, SDValue Callee, const BasicBlock *EHPadBB); void LowerDeoptimizeCall(const CallInst *CI); void LowerDeoptimizingReturn(); - void LowerCallSiteWithDeoptBundleImpl(ImmutableCallSite CS, SDValue Callee, + void LowerCallSiteWithDeoptBundleImpl(const CallBase *Call, SDValue Callee, const BasicBlock *EHPadBB, bool VarArgDisallowed, bool ForceVoidReturnTy); diff --git a/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp index dcb141b38ec5..a1555376d00f 100644 --- a/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp @@ -798,7 +798,7 @@ SDValue SelectionDAGBuilder::LowerAsSTATEPOINT( void SelectionDAGBuilder::LowerStatepoint(ImmutableStatepoint ISP, const BasicBlock *EHPadBB /*= nullptr*/) { - assert(ISP.getCallSite().getCallingConv() != CallingConv::AnyReg && + assert(ISP.getCall()->getCallingConv() != CallingConv::AnyReg && "anyregcc is not supported on statepoints!"); #ifndef NDEBUG @@ -831,7 +831,7 @@ SelectionDAGBuilder::LowerStatepoint(ImmutableStatepoint ISP, } StatepointLoweringInfo SI(DAG); - populateCallLoweringInfo(SI.CLI, ISP.getCallSite(), + populateCallLoweringInfo(SI.CLI, ISP.getCall(), ImmutableStatepoint::CallArgsBeginPos, ISP.getNumCallArgs(), ActualCallee, ISP.getActualReturnType(), false /* IsPatchPoint */); @@ -858,7 +858,7 @@ SelectionDAGBuilder::LowerStatepoint(ImmutableStatepoint ISP, const GCResultInst *GCResult = ISP.getGCResult(); Type *RetTy = ISP.getActualReturnType(); if (!RetTy->isVoidTy() && GCResult) { - if (GCResult->getParent() != ISP.getCallSite().getParent()) { + if (GCResult->getParent() != ISP.getCall()->getParent()) { // Result value will be used in a different basic block so we need to // export it now. Default exporting mechanism will not work here because // statepoint call has a different type than the actual call. It means @@ -870,7 +870,7 @@ SelectionDAGBuilder::LowerStatepoint(ImmutableStatepoint ISP, unsigned Reg = FuncInfo.CreateRegs(RetTy); RegsForValue RFV(*DAG.getContext(), DAG.getTargetLoweringInfo(), DAG.getDataLayout(), Reg, RetTy, - ISP.getCallSite().getCallingConv()); + ISP.getCall()->getCallingConv()); SDValue Chain = DAG.getEntryNode(); RFV.getCopyToRegs(ReturnValue, DAG, getCurSDLoc(), Chain, nullptr); @@ -890,22 +890,22 @@ SelectionDAGBuilder::LowerStatepoint(ImmutableStatepoint ISP, } void SelectionDAGBuilder::LowerCallSiteWithDeoptBundleImpl( - ImmutableCallSite CS, SDValue Callee, const BasicBlock *EHPadBB, + const CallBase *Call, SDValue Callee, const BasicBlock *EHPadBB, bool VarArgDisallowed, bool ForceVoidReturnTy) { StatepointLoweringInfo SI(DAG); - unsigned ArgBeginIndex = CS.arg_begin() - CS.getInstruction()->op_begin(); + unsigned ArgBeginIndex = Call->arg_begin() - Call->op_begin(); populateCallLoweringInfo( - SI.CLI, CS, ArgBeginIndex, CS.getNumArgOperands(), Callee, - ForceVoidReturnTy ? Type::getVoidTy(*DAG.getContext()) : CS.getType(), + SI.CLI, Call, ArgBeginIndex, Call->getNumArgOperands(), Callee, + ForceVoidReturnTy ? Type::getVoidTy(*DAG.getContext()) : Call->getType(), false); if (!VarArgDisallowed) - SI.CLI.IsVarArg = CS.getFunctionType()->isVarArg(); + SI.CLI.IsVarArg = Call->getFunctionType()->isVarArg(); - auto DeoptBundle = *CS.getOperandBundle(LLVMContext::OB_deopt); + auto DeoptBundle = *Call->getOperandBundle(LLVMContext::OB_deopt); unsigned DefaultID = StatepointDirectives::DeoptBundleStatepointID; - auto SD = parseStatepointDirectivesFromAttrs(CS.getAttributes()); + auto SD = parseStatepointDirectivesFromAttrs(Call->getAttributes()); SI.ID = SD.StatepointID.getValueOr(DefaultID); SI.NumPatchBytes = SD.NumPatchBytes.getValueOr(0); @@ -917,15 +917,14 @@ void SelectionDAGBuilder::LowerCallSiteWithDeoptBundleImpl( // NB! The GC arguments are deliberately left empty. if (SDValue ReturnVal = LowerAsSTATEPOINT(SI)) { - const Instruction *Inst = CS.getInstruction(); - ReturnVal = lowerRangeToAssertZExt(DAG, *Inst, ReturnVal); - setValue(Inst, ReturnVal); + ReturnVal = lowerRangeToAssertZExt(DAG, *Call, ReturnVal); + setValue(Call, ReturnVal); } } void SelectionDAGBuilder::LowerCallSiteWithDeoptBundle( - ImmutableCallSite CS, SDValue Callee, const BasicBlock *EHPadBB) { - LowerCallSiteWithDeoptBundleImpl(CS, Callee, EHPadBB, + const CallBase *Call, SDValue Callee, const BasicBlock *EHPadBB) { + LowerCallSiteWithDeoptBundleImpl(Call, Callee, EHPadBB, /* VarArgDisallowed = */ false, /* ForceVoidReturnTy = */ false); } diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp index 26c495507cf4..405b596bf9d5 100644 --- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp @@ -99,19 +99,19 @@ bool TargetLowering::parametersInCSRMatch(const MachineRegisterInfo &MRI, /// Set CallLoweringInfo attribute flags based on a call instruction /// and called function attributes. -void TargetLoweringBase::ArgListEntry::setAttributes(ImmutableCallSite *CS, +void TargetLoweringBase::ArgListEntry::setAttributes(const CallBase *Call, unsigned ArgIdx) { - IsSExt = CS->paramHasAttr(ArgIdx, Attribute::SExt); - IsZExt = CS->paramHasAttr(ArgIdx, Attribute::ZExt); - IsInReg = CS->paramHasAttr(ArgIdx, Attribute::InReg); - IsSRet = CS->paramHasAttr(ArgIdx, Attribute::StructRet); - IsNest = CS->paramHasAttr(ArgIdx, Attribute::Nest); - IsByVal = CS->paramHasAttr(ArgIdx, Attribute::ByVal); - IsInAlloca = CS->paramHasAttr(ArgIdx, Attribute::InAlloca); - IsReturned = CS->paramHasAttr(ArgIdx, Attribute::Returned); - IsSwiftSelf = CS->paramHasAttr(ArgIdx, Attribute::SwiftSelf); - IsSwiftError = CS->paramHasAttr(ArgIdx, Attribute::SwiftError); - Alignment = CS->getParamAlignment(ArgIdx); + IsSExt = Call->paramHasAttr(ArgIdx, Attribute::SExt); + IsZExt = Call->paramHasAttr(ArgIdx, Attribute::ZExt); + IsInReg = Call->paramHasAttr(ArgIdx, Attribute::InReg); + IsSRet = Call->paramHasAttr(ArgIdx, Attribute::StructRet); + IsNest = Call->paramHasAttr(ArgIdx, Attribute::Nest); + IsByVal = Call->paramHasAttr(ArgIdx, Attribute::ByVal); + IsInAlloca = Call->paramHasAttr(ArgIdx, Attribute::InAlloca); + IsReturned = Call->paramHasAttr(ArgIdx, Attribute::Returned); + IsSwiftSelf = Call->paramHasAttr(ArgIdx, Attribute::SwiftSelf); + IsSwiftError = Call->paramHasAttr(ArgIdx, Attribute::SwiftError); + Alignment = Call->getParamAlignment(ArgIdx); } /// Generate a libcall taking the given operands as arguments and returning a diff --git a/llvm/lib/IR/Statepoint.cpp b/llvm/lib/IR/Statepoint.cpp index baec5ce98cb7..fce89b42e9bf 100644 --- a/llvm/lib/IR/Statepoint.cpp +++ b/llvm/lib/IR/Statepoint.cpp @@ -17,21 +17,15 @@ using namespace llvm; -static const Function *getCalledFunction(ImmutableCallSite CS) { - if (!CS.getInstruction()) - return nullptr; - return CS.getCalledFunction(); -} - -bool llvm::isStatepoint(ImmutableCallSite CS) { - if (auto *F = getCalledFunction(CS)) +bool llvm::isStatepoint(const CallBase *Call) { + if (auto *F = Call->getCalledFunction()) return F->getIntrinsicID() == Intrinsic::experimental_gc_statepoint; return false; } bool llvm::isStatepoint(const Value *V) { - if (auto CS = ImmutableCallSite(V)) - return isStatepoint(CS); + if (auto *Call = dyn_cast(V)) + return isStatepoint(Call); return false; } @@ -39,23 +33,21 @@ bool llvm::isStatepoint(const Value &V) { return isStatepoint(&V); } -bool llvm::isGCRelocate(ImmutableCallSite CS) { - return CS.getInstruction() && isa(CS.getInstruction()); +bool llvm::isGCRelocate(const CallBase *Call) { + return isa(Call); } bool llvm::isGCRelocate(const Value *V) { - if (auto CS = ImmutableCallSite(V)) - return isGCRelocate(CS); + if (auto *Call = dyn_cast(V)) + return isGCRelocate(Call); return false; } -bool llvm::isGCResult(ImmutableCallSite CS) { - return CS.getInstruction() && isa(CS.getInstruction()); -} +bool llvm::isGCResult(const CallBase *Call) { return isa(Call); } bool llvm::isGCResult(const Value *V) { - if (auto CS = ImmutableCallSite(V)) - return isGCResult(CS); + if (auto *Call = dyn_cast(V)) + return isGCResult(Call); return false; } diff --git a/llvm/lib/Transforms/Scalar/PlaceSafepoints.cpp b/llvm/lib/Transforms/Scalar/PlaceSafepoints.cpp index a19a0db0bdd9..b544f0a39ea8 100644 --- a/llvm/lib/Transforms/Scalar/PlaceSafepoints.cpp +++ b/llvm/lib/Transforms/Scalar/PlaceSafepoints.cpp @@ -55,7 +55,6 @@ #include "llvm/Analysis/ScalarEvolution.h" #include "llvm/Analysis/TargetLibraryInfo.h" #include "llvm/Transforms/Utils/Local.h" -#include "llvm/IR/CallSite.h" #include "llvm/IR/Dominators.h" #include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/LegacyPassManager.h" @@ -178,19 +177,18 @@ struct PlaceSafepoints : public FunctionPass { // callers job. static void InsertSafepointPoll(Instruction *InsertBefore, - std::vector &ParsePointsNeeded /*rval*/, + std::vector &ParsePointsNeeded /*rval*/, const TargetLibraryInfo &TLI); -static bool needsStatepoint(const CallSite &CS, const TargetLibraryInfo &TLI) { - if (callsGCLeafFunction(CS, TLI)) +static bool needsStatepoint(CallBase *Call, const TargetLibraryInfo &TLI) { + if (callsGCLeafFunction(Call, TLI)) return false; - if (CS.isCall()) { - CallInst *call = cast(CS.getInstruction()); - if (call->isInlineAsm()) + if (auto *CI = dyn_cast(Call)) { + if (CI->isInlineAsm()) return false; } - return !(isStatepoint(CS) || isGCRelocate(CS) || isGCResult(CS)); + return !(isStatepoint(Call) || isGCRelocate(Call) || isGCResult(Call)); } /// Returns true if this loop is known to contain a call safepoint which @@ -216,14 +214,14 @@ static bool containsUnconditionalCallSafepoint(Loop *L, BasicBlock *Header, BasicBlock *Current = Pred; while (true) { for (Instruction &I : *Current) { - if (auto CS = CallSite(&I)) + if (auto *Call = dyn_cast(&I)) // Note: Technically, needing a safepoint isn't quite the right // condition here. We should instead be checking if the target method // has an // unconditional poll. In practice, this is only a theoretical concern // since we don't have any methods with conditional-only safepoint // polls. - if (needsStatepoint(CS, TLI)) + if (needsStatepoint(Call, TLI)) return true; } @@ -359,9 +357,8 @@ bool PlaceBackedgeSafepointsImpl::runOnLoop(Loop *L) { /// Returns true if an entry safepoint is not required before this callsite in /// the caller function. -static bool doesNotRequireEntrySafepointBefore(const CallSite &CS) { - Instruction *Inst = CS.getInstruction(); - if (IntrinsicInst *II = dyn_cast(Inst)) { +static bool doesNotRequireEntrySafepointBefore(CallBase *Call) { + if (IntrinsicInst *II = dyn_cast(Call)) { switch (II->getIntrinsicID()) { case Intrinsic::experimental_gc_statepoint: case Intrinsic::experimental_patchpoint_void: @@ -423,8 +420,8 @@ static Instruction *findLocationForEntrySafepoint(Function &F, // which can grow the stack by an unbounded amount. This isn't required // for GC semantics per se, but is a common requirement for languages // which detect stack overflow via guard pages and then throw exceptions. - if (auto CS = CallSite(Cursor)) { - if (doesNotRequireEntrySafepointBefore(CS)) + if (auto *Call = dyn_cast(Cursor)) { + if (doesNotRequireEntrySafepointBefore(Call)) continue; break; } @@ -499,7 +496,7 @@ bool PlaceSafepoints::runOnFunction(Function &F) { DT.recalculate(F); SmallVector PollsNeeded; - std::vector ParsePointNeeded; + std::vector ParsePointNeeded; if (enableBackedgeSafepoints(F)) { // Construct a pass manager to run the LoopPass backedge logic. We @@ -588,7 +585,7 @@ bool PlaceSafepoints::runOnFunction(Function &F) { // Now that we've identified all the needed safepoint poll locations, insert // safepoint polls themselves. for (Instruction *PollLocation : PollsNeeded) { - std::vector RuntimeCalls; + std::vector RuntimeCalls; InsertSafepointPoll(PollLocation, RuntimeCalls, TLI); ParsePointNeeded.insert(ParsePointNeeded.end(), RuntimeCalls.begin(), RuntimeCalls.end()); @@ -621,7 +618,7 @@ INITIALIZE_PASS_END(PlaceSafepoints, "place-safepoints", "Place Safepoints", static void InsertSafepointPoll(Instruction *InsertBefore, - std::vector &ParsePointsNeeded /*rval*/, + std::vector &ParsePointsNeeded /*rval*/, const TargetLibraryInfo &TLI) { BasicBlock *OrigBB = InsertBefore->getParent(); Module *M = InsertBefore->getModule(); @@ -686,7 +683,7 @@ InsertSafepointPoll(Instruction *InsertBefore, // These are likely runtime calls. Should we assert that via calling // convention or something? - ParsePointsNeeded.push_back(CallSite(CI)); + ParsePointsNeeded.push_back(CI); } assert(ParsePointsNeeded.size() <= Calls.size()); } diff --git a/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp b/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp index 1654c24291fa..c358258d24cf 100644 --- a/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp +++ b/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp @@ -31,7 +31,6 @@ #include "llvm/IR/Argument.h" #include "llvm/IR/Attributes.h" #include "llvm/IR/BasicBlock.h" -#include "llvm/IR/CallSite.h" #include "llvm/IR/CallingConv.h" #include "llvm/IR/Constant.h" #include "llvm/IR/Constants.h" @@ -285,9 +284,9 @@ struct PartiallyConstructedSafepointRecord { } // end anonymous namespace -static ArrayRef GetDeoptBundleOperands(ImmutableCallSite CS) { +static ArrayRef GetDeoptBundleOperands(const CallBase *Call) { Optional DeoptBundle = - CS.getOperandBundle(LLVMContext::OB_deopt); + Call->getOperandBundle(LLVMContext::OB_deopt); if (!DeoptBundle.hasValue()) { assert(AllowStatepointWithNoDeoptInfo && @@ -369,14 +368,11 @@ static std::string suffixed_name_or(Value *V, StringRef Suffix, // given instruction. The analysis is performed immediately before the // given instruction. Values defined by that instruction are not considered // live. Values used by that instruction are considered live. -static void -analyzeParsePointLiveness(DominatorTree &DT, - GCPtrLivenessData &OriginalLivenessData, CallSite CS, - PartiallyConstructedSafepointRecord &Result) { - Instruction *Inst = CS.getInstruction(); - +static void analyzeParsePointLiveness( + DominatorTree &DT, GCPtrLivenessData &OriginalLivenessData, CallBase *Call, + PartiallyConstructedSafepointRecord &Result) { StatepointLiveSetTy LiveSet; - findLiveSetAtInst(Inst, OriginalLivenessData, LiveSet); + findLiveSetAtInst(Call, OriginalLivenessData, LiveSet); if (PrintLiveSet) { dbgs() << "Live Variables:\n"; @@ -384,7 +380,7 @@ analyzeParsePointLiveness(DominatorTree &DT, dbgs() << " " << V->getName() << " " << *V << "\n"; } if (PrintLiveSetSize) { - dbgs() << "Safepoint For: " << CS.getCalledValue()->getName() << "\n"; + dbgs() << "Safepoint For: " << Call->getCalledValue()->getName() << "\n"; dbgs() << "Number live values: " << LiveSet.size() << "\n"; } Result.LiveSet = LiveSet; @@ -1177,7 +1173,7 @@ findBasePointers(const StatepointLiveSetTy &live, /// Find the required based pointers (and adjust the live set) for the given /// parse point. static void findBasePointers(DominatorTree &DT, DefiningValueMapTy &DVCache, - CallSite CS, + CallBase *Call, PartiallyConstructedSafepointRecord &result) { MapVector PointerToBase; findBasePointers(result.LiveSet, PointerToBase, &DT, DVCache); @@ -1199,11 +1195,11 @@ static void findBasePointers(DominatorTree &DT, DefiningValueMapTy &DVCache, /// Given an updated version of the dataflow liveness results, update the /// liveset and base pointer maps for the call site CS. static void recomputeLiveInValues(GCPtrLivenessData &RevisedLivenessData, - CallSite CS, + CallBase *Call, PartiallyConstructedSafepointRecord &result); static void recomputeLiveInValues( - Function &F, DominatorTree &DT, ArrayRef toUpdate, + Function &F, DominatorTree &DT, ArrayRef toUpdate, MutableArrayRef records) { // TODO-PERF: reuse the original liveness, then simply run the dataflow // again. The old values are still live and will help it stabilize quickly. @@ -1398,16 +1394,16 @@ public: } // end anonymous namespace -static StringRef getDeoptLowering(CallSite CS) { +static StringRef getDeoptLowering(CallBase *Call) { const char *DeoptLowering = "deopt-lowering"; - if (CS.hasFnAttr(DeoptLowering)) { - // FIXME: CallSite has a *really* confusing interface around attributes + if (Call->hasFnAttr(DeoptLowering)) { + // FIXME: Calls have a *really* confusing interface around attributes // with values. - const AttributeList &CSAS = CS.getAttributes(); + const AttributeList &CSAS = Call->getAttributes(); if (CSAS.hasAttribute(AttributeList::FunctionIndex, DeoptLowering)) return CSAS.getAttribute(AttributeList::FunctionIndex, DeoptLowering) .getValueAsString(); - Function *F = CS.getCalledFunction(); + Function *F = Call->getCalledFunction(); assert(F && F->hasFnAttribute(DeoptLowering)); return F->getFnAttribute(DeoptLowering).getValueAsString(); } @@ -1415,7 +1411,7 @@ static StringRef getDeoptLowering(CallSite CS) { } static void -makeStatepointExplicitImpl(const CallSite CS, /* to replace */ +makeStatepointExplicitImpl(CallBase *Call, /* to replace */ const SmallVectorImpl &BasePtrs, const SmallVectorImpl &LiveVariables, PartiallyConstructedSafepointRecord &Result, @@ -1426,19 +1422,18 @@ makeStatepointExplicitImpl(const CallSite CS, /* to replace */ // immediately before the previous instruction under the assumption that all // arguments will be available here. We can't insert afterwards since we may // be replacing a terminator. - Instruction *InsertBefore = CS.getInstruction(); - IRBuilder<> Builder(InsertBefore); + IRBuilder<> Builder(Call); ArrayRef GCArgs(LiveVariables); uint64_t StatepointID = StatepointDirectives::DefaultStatepointID; uint32_t NumPatchBytes = 0; uint32_t Flags = uint32_t(StatepointFlags::None); - ArrayRef CallArgs(CS.arg_begin(), CS.arg_end()); - ArrayRef DeoptArgs = GetDeoptBundleOperands(CS); + ArrayRef CallArgs(Call->arg_begin(), Call->arg_end()); + ArrayRef DeoptArgs = GetDeoptBundleOperands(Call); ArrayRef TransitionArgs; if (auto TransitionBundle = - CS.getOperandBundle(LLVMContext::OB_gc_transition)) { + Call->getOperandBundle(LLVMContext::OB_gc_transition)) { Flags |= uint32_t(StatepointFlags::GCTransition); TransitionArgs = TransitionBundle->Inputs; } @@ -1449,21 +1444,21 @@ makeStatepointExplicitImpl(const CallSite CS, /* to replace */ bool IsDeoptimize = false; StatepointDirectives SD = - parseStatepointDirectivesFromAttrs(CS.getAttributes()); + parseStatepointDirectivesFromAttrs(Call->getAttributes()); if (SD.NumPatchBytes) NumPatchBytes = *SD.NumPatchBytes; if (SD.StatepointID) StatepointID = *SD.StatepointID; // Pass through the requested lowering if any. The default is live-through. - StringRef DeoptLowering = getDeoptLowering(CS); + StringRef DeoptLowering = getDeoptLowering(Call); if (DeoptLowering.equals("live-in")) Flags |= uint32_t(StatepointFlags::DeoptLiveIn); else { assert(DeoptLowering.equals("live-through") && "Unsupported value!"); } - Value *CallTarget = CS.getCalledValue(); + Value *CallTarget = Call->getCalledValue(); if (Function *F = dyn_cast(CallTarget)) { if (F->getIntrinsicID() == Intrinsic::experimental_deoptimize) { // Calls to llvm.experimental.deoptimize are lowered to calls to the @@ -1490,57 +1485,56 @@ makeStatepointExplicitImpl(const CallSite CS, /* to replace */ // Create the statepoint given all the arguments Instruction *Token = nullptr; - if (CS.isCall()) { - CallInst *ToReplace = cast(CS.getInstruction()); - CallInst *Call = Builder.CreateGCStatepointCall( + if (auto *CI = dyn_cast(Call)) { + CallInst *SPCall = Builder.CreateGCStatepointCall( StatepointID, NumPatchBytes, CallTarget, Flags, CallArgs, TransitionArgs, DeoptArgs, GCArgs, "safepoint_token"); - Call->setTailCallKind(ToReplace->getTailCallKind()); - Call->setCallingConv(ToReplace->getCallingConv()); + SPCall->setTailCallKind(CI->getTailCallKind()); + SPCall->setCallingConv(CI->getCallingConv()); // Currently we will fail on parameter attributes and on certain // function attributes. In case if we can handle this set of attributes - // set up function attrs directly on statepoint and return attrs later for // gc_result intrinsic. - Call->setAttributes(legalizeCallAttributes(ToReplace->getAttributes())); + SPCall->setAttributes(legalizeCallAttributes(CI->getAttributes())); - Token = Call; + Token = SPCall; // Put the following gc_result and gc_relocate calls immediately after the // the old call (which we're about to delete) - assert(ToReplace->getNextNode() && "Not a terminator, must have next!"); - Builder.SetInsertPoint(ToReplace->getNextNode()); - Builder.SetCurrentDebugLocation(ToReplace->getNextNode()->getDebugLoc()); + assert(CI->getNextNode() && "Not a terminator, must have next!"); + Builder.SetInsertPoint(CI->getNextNode()); + Builder.SetCurrentDebugLocation(CI->getNextNode()->getDebugLoc()); } else { - InvokeInst *ToReplace = cast(CS.getInstruction()); + auto *II = cast(Call); // Insert the new invoke into the old block. We'll remove the old one in a // moment at which point this will become the new terminator for the // original block. - InvokeInst *Invoke = Builder.CreateGCStatepointInvoke( - StatepointID, NumPatchBytes, CallTarget, ToReplace->getNormalDest(), - ToReplace->getUnwindDest(), Flags, CallArgs, TransitionArgs, DeoptArgs, - GCArgs, "statepoint_token"); + InvokeInst *SPInvoke = Builder.CreateGCStatepointInvoke( + StatepointID, NumPatchBytes, CallTarget, II->getNormalDest(), + II->getUnwindDest(), Flags, CallArgs, TransitionArgs, DeoptArgs, GCArgs, + "statepoint_token"); - Invoke->setCallingConv(ToReplace->getCallingConv()); + SPInvoke->setCallingConv(II->getCallingConv()); // Currently we will fail on parameter attributes and on certain // function attributes. In case if we can handle this set of attributes - // set up function attrs directly on statepoint and return attrs later for // gc_result intrinsic. - Invoke->setAttributes(legalizeCallAttributes(ToReplace->getAttributes())); + SPInvoke->setAttributes(legalizeCallAttributes(II->getAttributes())); - Token = Invoke; + Token = SPInvoke; // Generate gc relocates in exceptional path - BasicBlock *UnwindBlock = ToReplace->getUnwindDest(); + BasicBlock *UnwindBlock = II->getUnwindDest(); assert(!isa(UnwindBlock->begin()) && UnwindBlock->getUniquePredecessor() && "can't safely insert in this block!"); Builder.SetInsertPoint(&*UnwindBlock->getFirstInsertionPt()); - Builder.SetCurrentDebugLocation(ToReplace->getDebugLoc()); + Builder.SetCurrentDebugLocation(II->getDebugLoc()); // Attach exceptional gc relocates to the landingpad. Instruction *ExceptionalToken = UnwindBlock->getLandingPadInst(); @@ -1551,7 +1545,7 @@ makeStatepointExplicitImpl(const CallSite CS, /* to replace */ Builder); // Generate gc relocates and returns for normal block - BasicBlock *NormalDest = ToReplace->getNormalDest(); + BasicBlock *NormalDest = II->getNormalDest(); assert(!isa(NormalDest->begin()) && NormalDest->getUniquePredecessor() && "can't safely insert in this block!"); @@ -1568,16 +1562,15 @@ makeStatepointExplicitImpl(const CallSite CS, /* to replace */ // transform the tail-call like structure to a call to a void function // followed by unreachable to get better codegen. Replacements.push_back( - DeferredReplacement::createDeoptimizeReplacement(CS.getInstruction())); + DeferredReplacement::createDeoptimizeReplacement(Call)); } else { Token->setName("statepoint_token"); - if (!CS.getType()->isVoidTy() && !CS.getInstruction()->use_empty()) { - StringRef Name = - CS.getInstruction()->hasName() ? CS.getInstruction()->getName() : ""; - CallInst *GCResult = Builder.CreateGCResult(Token, CS.getType(), Name); + if (!Call->getType()->isVoidTy() && !Call->use_empty()) { + StringRef Name = Call->hasName() ? Call->getName() : ""; + CallInst *GCResult = Builder.CreateGCResult(Token, Call->getType(), Name); GCResult->setAttributes( AttributeList::get(GCResult->getContext(), AttributeList::ReturnIndex, - CS.getAttributes().getRetAttributes())); + Call->getAttributes().getRetAttributes())); // We cannot RAUW or delete CS.getInstruction() because it could be in the // live set of some other safepoint, in which case that safepoint's @@ -1586,10 +1579,9 @@ makeStatepointExplicitImpl(const CallSite CS, /* to replace */ // after the live sets have been made explicit in the IR, and we no longer // have raw pointers to worry about. Replacements.emplace_back( - DeferredReplacement::createRAUW(CS.getInstruction(), GCResult)); + DeferredReplacement::createRAUW(Call, GCResult)); } else { - Replacements.emplace_back( - DeferredReplacement::createDelete(CS.getInstruction())); + Replacements.emplace_back(DeferredReplacement::createDelete(Call)); } } @@ -1606,7 +1598,7 @@ makeStatepointExplicitImpl(const CallSite CS, /* to replace */ // WARNING: Does not do any fixup to adjust users of the original live // values. That's the callers responsibility. static void -makeStatepointExplicit(DominatorTree &DT, CallSite CS, +makeStatepointExplicit(DominatorTree &DT, CallBase *Call, PartiallyConstructedSafepointRecord &Result, std::vector &Replacements) { const auto &LiveSet = Result.LiveSet; @@ -1625,7 +1617,7 @@ makeStatepointExplicit(DominatorTree &DT, CallSite CS, assert(LiveVec.size() == BaseVec.size()); // Do the actual rewriting and delete the old statepoint - makeStatepointExplicitImpl(CS, BaseVec, LiveVec, Result, Replacements); + makeStatepointExplicitImpl(Call, BaseVec, LiveVec, Result, Replacements); } // Helper function for the relocationViaAlloca. @@ -1895,25 +1887,25 @@ template static void unique_unsorted(SmallVectorImpl &Vec) { /// Insert holders so that each Value is obviously live through the entire /// lifetime of the call. -static void insertUseHolderAfter(CallSite &CS, const ArrayRef Values, +static void insertUseHolderAfter(CallBase *Call, const ArrayRef Values, SmallVectorImpl &Holders) { if (Values.empty()) // No values to hold live, might as well not insert the empty holder return; - Module *M = CS.getInstruction()->getModule(); + Module *M = Call->getModule(); // Use a dummy vararg function to actually hold the values live FunctionCallee Func = M->getOrInsertFunction( "__tmp_use", FunctionType::get(Type::getVoidTy(M->getContext()), true)); - if (CS.isCall()) { + if (isa(Call)) { // For call safepoints insert dummy calls right after safepoint - Holders.push_back(CallInst::Create(Func, Values, "", - &*++CS.getInstruction()->getIterator())); + Holders.push_back( + CallInst::Create(Func, Values, "", &*++Call->getIterator())); return; } // For invoke safepooints insert dummy calls both in normal and // exceptional destination blocks - auto *II = cast(CS.getInstruction()); + auto *II = cast(Call); Holders.push_back(CallInst::Create( Func, Values, "", &*II->getNormalDest()->getFirstInsertionPt())); Holders.push_back(CallInst::Create( @@ -1921,7 +1913,7 @@ static void insertUseHolderAfter(CallSite &CS, const ArrayRef Values, } static void findLiveReferences( - Function &F, DominatorTree &DT, ArrayRef toUpdate, + Function &F, DominatorTree &DT, ArrayRef toUpdate, MutableArrayRef records) { GCPtrLivenessData OriginalLivenessData; computeLiveInValues(DT, F, OriginalLivenessData); @@ -2024,7 +2016,7 @@ static bool AreEquivalentPhiNodes(PHINode &OrigRootPhi, PHINode &AlternateRootPh // to relocate. Remove this values from the live set, rematerialize them after // statepoint and record them in "Info" structure. Note that similar to // relocated values we don't do any user adjustments here. -static void rematerializeLiveValues(CallSite CS, +static void rematerializeLiveValues(CallBase *Call, PartiallyConstructedSafepointRecord &Info, TargetTransformInfo &TTI) { const unsigned int ChainLengthThreshold = 10; @@ -2078,7 +2070,7 @@ static void rematerializeLiveValues(CallSite CS, // For invokes we need to rematerialize each chain twice - for normal and // for unwind basic blocks. Model this by multiplying cost by two. - if (CS.isInvoke()) { + if (isa(Call)) { Cost *= 2; } // If it's too expensive - skip it @@ -2146,14 +2138,14 @@ static void rematerializeLiveValues(CallSite CS, // Different cases for calls and invokes. For invokes we need to clone // instructions both on normal and unwind path. - if (CS.isCall()) { - Instruction *InsertBefore = CS.getInstruction()->getNextNode(); + if (isa(Call)) { + Instruction *InsertBefore = Call->getNextNode(); assert(InsertBefore); Instruction *RematerializedValue = rematerializeChain( InsertBefore, RootOfChain, Info.PointerToBase[LiveValue]); Info.RematerializedValues[RematerializedValue] = LiveValue; } else { - InvokeInst *Invoke = cast(CS.getInstruction()); + auto *Invoke = cast(Call); Instruction *NormalInsertBefore = &*Invoke->getNormalDest()->getFirstInsertionPt(); @@ -2178,25 +2170,25 @@ static void rematerializeLiveValues(CallSite CS, static bool insertParsePoints(Function &F, DominatorTree &DT, TargetTransformInfo &TTI, - SmallVectorImpl &ToUpdate) { + SmallVectorImpl &ToUpdate) { #ifndef NDEBUG // sanity check the input - std::set Uniqued; + std::set Uniqued; Uniqued.insert(ToUpdate.begin(), ToUpdate.end()); assert(Uniqued.size() == ToUpdate.size() && "no duplicates please!"); - for (CallSite CS : ToUpdate) - assert(CS.getInstruction()->getFunction() == &F); + for (CallBase *Call : ToUpdate) + assert(Call->getFunction() == &F); #endif // When inserting gc.relocates for invokes, we need to be able to insert at // the top of the successor blocks. See the comment on // normalForInvokeSafepoint on exactly what is needed. Note that this step // may restructure the CFG. - for (CallSite CS : ToUpdate) { - if (!CS.isInvoke()) + for (CallBase *Call : ToUpdate) { + auto *II = dyn_cast(Call); + if (!II) continue; - auto *II = cast(CS.getInstruction()); normalizeForInvokeSafepoint(II->getNormalDest(), II->getParent(), DT); normalizeForInvokeSafepoint(II->getUnwindDest(), II->getParent(), DT); } @@ -2209,17 +2201,17 @@ static bool insertParsePoints(Function &F, DominatorTree &DT, // actual safepoint insertion as arguments. This ensures reference operands // in the deopt argument list are considered live through the safepoint (and // thus makes sure they get relocated.) - for (CallSite CS : ToUpdate) { + for (CallBase *Call : ToUpdate) { SmallVector DeoptValues; - for (Value *Arg : GetDeoptBundleOperands(CS)) { + for (Value *Arg : GetDeoptBundleOperands(Call)) { assert(!isUnhandledGCPointerType(Arg->getType()) && "support for FCA unimplemented"); if (isHandledGCPointerType(Arg->getType())) DeoptValues.push_back(Arg); } - insertUseHolderAfter(CS, DeoptValues, Holders); + insertUseHolderAfter(Call, DeoptValues, Holders); } SmallVector Records(ToUpdate.size()); @@ -2321,7 +2313,7 @@ static bool insertParsePoints(Function &F, DominatorTree &DT, for (size_t i = 0; i < Records.size(); i++) makeStatepointExplicit(DT, ToUpdate[i], Records[i], Replacements); - ToUpdate.clear(); // prevent accident use of invalid CallSites + ToUpdate.clear(); // prevent accident use of invalid calls. for (auto &PR : Replacements) PR.doReplacement(); @@ -2386,7 +2378,7 @@ static bool insertParsePoints(Function &F, DominatorTree &DT, return !Records.empty(); } -// Handles both return values and arguments for Functions and CallSites. +// Handles both return values and arguments for Functions and calls. template static void RemoveNonValidAttrAtIndex(LLVMContext &Ctx, AttrHolder &AH, unsigned Index) { @@ -2478,12 +2470,13 @@ static void stripNonValidDataFromBody(Function &F) { stripInvalidMetadataFromInstruction(I); - if (CallSite CS = CallSite(&I)) { - for (int i = 0, e = CS.arg_size(); i != e; i++) - if (isa(CS.getArgument(i)->getType())) - RemoveNonValidAttrAtIndex(Ctx, CS, i + AttributeList::FirstArgIndex); - if (isa(CS.getType())) - RemoveNonValidAttrAtIndex(Ctx, CS, AttributeList::ReturnIndex); + if (auto *Call = dyn_cast(&I)) { + for (int i = 0, e = Call->arg_size(); i != e; i++) + if (isa(Call->getArgOperand(i)->getType())) + RemoveNonValidAttrAtIndex(Ctx, *Call, + i + AttributeList::FirstArgIndex); + if (isa(Call->getType())) + RemoveNonValidAttrAtIndex(Ctx, *Call, AttributeList::ReturnIndex); } } @@ -2528,12 +2521,11 @@ bool RewriteStatepointsForGC::runOnFunction(Function &F, DominatorTree &DT, assert(shouldRewriteStatepointsIn(F) && "mismatch in rewrite decision"); auto NeedsRewrite = [&TLI](Instruction &I) { - if (ImmutableCallSite CS = ImmutableCallSite(&I)) - return !callsGCLeafFunction(CS, TLI) && !isStatepoint(CS); + if (const auto *Call = dyn_cast(&I)) + return !callsGCLeafFunction(Call, TLI) && !isStatepoint(Call); return false; }; - // Delete any unreachable statepoints so that we don't have unrewritten // statepoints surviving this pass. This makes testing easier and the // resulting IR less confusing to human readers. @@ -2545,7 +2537,7 @@ bool RewriteStatepointsForGC::runOnFunction(Function &F, DominatorTree &DT, // Gather all the statepoints which need rewritten. Be careful to only // consider those in reachable code since we need to ask dominance queries // when rewriting. We'll delete the unreachable ones in a moment. - SmallVector ParsePointNeeded; + SmallVector ParsePointNeeded; for (Instruction &I : instructions(F)) { // TODO: only the ones with the flag set! if (NeedsRewrite(I)) { @@ -2555,7 +2547,7 @@ bool RewriteStatepointsForGC::runOnFunction(Function &F, DominatorTree &DT, // isReachableFromEntry() returns true. assert(DT.isReachableFromEntry(I.getParent()) && "no unreachable blocks expected"); - ParsePointNeeded.push_back(CallSite(&I)); + ParsePointNeeded.push_back(cast(&I)); } } @@ -2815,11 +2807,10 @@ static void findLiveSetAtInst(Instruction *Inst, GCPtrLivenessData &Data, } static void recomputeLiveInValues(GCPtrLivenessData &RevisedLivenessData, - CallSite CS, + CallBase *Call, PartiallyConstructedSafepointRecord &Info) { - Instruction *Inst = CS.getInstruction(); StatepointLiveSetTy Updated; - findLiveSetAtInst(Inst, RevisedLivenessData, Updated); + findLiveSetAtInst(Call, RevisedLivenessData, Updated); // We may have base pointers which are now live that weren't before. We need // to update the PointerToBase structure to reflect this. diff --git a/llvm/lib/Transforms/Utils/Local.cpp b/llvm/lib/Transforms/Utils/Local.cpp index 933a69caaaa3..b7e696698392 100644 --- a/llvm/lib/Transforms/Utils/Local.cpp +++ b/llvm/lib/Transforms/Utils/Local.cpp @@ -2462,12 +2462,12 @@ unsigned llvm::replaceDominatedUsesWith(Value *From, Value *To, return ::replaceDominatedUsesWith(From, To, BB, ProperlyDominates); } -bool llvm::callsGCLeafFunction(ImmutableCallSite CS, +bool llvm::callsGCLeafFunction(const CallBase *Call, const TargetLibraryInfo &TLI) { // Check if the function is specifically marked as a gc leaf function. - if (CS.hasFnAttr("gc-leaf-function")) + if (Call->hasFnAttr("gc-leaf-function")) return true; - if (const Function *F = CS.getCalledFunction()) { + if (const Function *F = Call->getCalledFunction()) { if (F->hasFnAttribute("gc-leaf-function")) return true; @@ -2481,7 +2481,7 @@ bool llvm::callsGCLeafFunction(ImmutableCallSite CS, // marked as 'gc-leaf-function.' All available Libcalls are // GC-leaf. LibFunc LF; - if (TLI.getLibFunc(CS, LF)) { + if (TLI.getLibFunc(ImmutableCallSite(Call), LF)) { return TLI.has(LF); }