From be9c91173f7b9e2d05418ccd6bcfc92df35db84e Mon Sep 17 00:00:00 2001 From: Johannes Doerfert Date: Fri, 6 Feb 2015 21:39:31 +0000 Subject: [PATCH] [Refactor] Use only one BlockGenerator for a SCoP This change has two main purposes: 1) We do not use a static interface to hide an object we create and destroy for every basic block we copy. 2) We allow the BlockGenerator to store information between calls to the copyBB method. This will ease scalar/phi code generation later on. While a lot of method signatures were changed this should not cause any real behaviour change. Differential Revision: http://reviews.llvm.org/D7467 llvm-svn: 228443 --- polly/include/polly/CodeGen/BlockGenerators.h | 155 +++++++-------- polly/include/polly/ScopInfo.h | 10 + polly/lib/Analysis/ScopInfo.cpp | 2 +- polly/lib/CodeGen/BlockGenerators.cpp | 186 +++++++++--------- polly/lib/CodeGen/IslCodeGeneration.cpp | 12 +- 5 files changed, 185 insertions(+), 180 deletions(-) diff --git a/polly/include/polly/CodeGen/BlockGenerators.h b/polly/include/polly/CodeGen/BlockGenerators.h index 16e60bef76ec..6ccf801772aa 100644 --- a/polly/include/polly/CodeGen/BlockGenerators.h +++ b/polly/include/polly/CodeGen/BlockGenerators.h @@ -67,41 +67,38 @@ bool isIgnoredIntrinsic(const llvm::Value *V); /// The only public function exposed is generate(). class BlockGenerator { public: - /// @brief Generate a new BasicBlock for a ScopStmt. + /// @brief Create a generator for basic blocks. /// - /// @param Builder The LLVM-IR Builder used to generate the statement. The - /// code is generated at the location, the Builder points to. - /// @param Stmt The statement to code generate. - /// @param GlobalMap A map that defines for certain Values referenced from the - /// original code new Values they should be replaced with. - /// @param P A reference to the pass this function is called from. - /// The pass is needed to update other analysis. - /// @param LI The loop info for the current function - /// @param SE The scalar evolution info for the current function - /// @param Build The AST build with the new schedule. + /// @param Builder The LLVM-IR Builder used to generate the statement. The + /// code is generated at the location, the Builder points + /// to. + /// @param P A reference to the pass this function is called from. + /// The pass is needed to update other analysis. + /// @param LI The loop info for the current function + /// @param SE The scalar evolution info for the current function /// @param ExprBuilder An expression builder to generate new access functions. - static void generate(PollyIRBuilder &Builder, ScopStmt &Stmt, - ValueMapT &GlobalMap, LoopToScevMapT <S, Pass *P, - LoopInfo &LI, ScalarEvolution &SE, - __isl_keep isl_ast_build *Build = nullptr, - IslExprBuilder *ExprBuilder = nullptr) { - BlockGenerator Generator(Builder, Stmt, P, LI, SE, Build, ExprBuilder); - Generator.copyBB(GlobalMap, LTS); - } + BlockGenerator(PollyIRBuilder &Builder, Pass *P, LoopInfo &LI, + ScalarEvolution &SE, IslExprBuilder *ExprBuilder = nullptr); + + /// @brief Copy the basic block. + /// + /// This copies the entire basic block and updates references to old values + /// with references to new values, as defined by GlobalMap. + /// + /// @param Stmt The statement to code generate. + /// @param GlobalMap A mapping from old values to their new values + /// (for values recalculated in the new ScoP, but not + /// within this basic block). + /// @param LTS A map from old loops to new induction variables as SCEVs. + void copyBB(ScopStmt &Stmt, ValueMapT &GlobalMap, LoopToScevMapT <S); protected: PollyIRBuilder &Builder; - ScopStmt &Statement; Pass *P; LoopInfo &LI; ScalarEvolution &SE; - isl_ast_build *Build; IslExprBuilder *ExprBuilder; - BlockGenerator(PollyIRBuilder &B, ScopStmt &Stmt, Pass *P, LoopInfo &LI, - ScalarEvolution &SE, __isl_keep isl_ast_build *Build, - IslExprBuilder *ExprBuilder); - /// @brief Get the new version of a value. /// /// Given an old value, we first check if a new version of this value is @@ -111,6 +108,7 @@ protected: /// we return the old value. If the value can still not be derived, this /// function will assert. /// + /// @param Stmt The statement to code generate. /// @param Old The old Value. /// @param BBMap A mapping from old values to their new values /// (for values recalculated within this basic block). @@ -128,10 +126,10 @@ protected: /// @returns o The old value, if it is still valid. /// o The new value, if available. /// o NULL, if no value is found. - Value *getNewValue(const Value *Old, ValueMapT &BBMap, ValueMapT &GlobalMap, - LoopToScevMapT <S, Loop *L) const; + Value *getNewValue(ScopStmt &Stmt, const Value *Old, ValueMapT &BBMap, + ValueMapT &GlobalMap, LoopToScevMapT <S, Loop *L) const; - void copyInstScalar(const Instruction *Inst, ValueMapT &BBMap, + void copyInstScalar(ScopStmt &Stmt, const Instruction *Inst, ValueMapT &BBMap, ValueMapT &GlobalMap, LoopToScevMapT <S); /// @brief Get the innermost loop that surrounds an instruction. @@ -141,41 +139,40 @@ protected: Loop *getLoopForInst(const Instruction *Inst); /// @brief Get the new operand address according to access relation of @p MA. - Value *getNewAccessOperand(const MemoryAccess &MA); + Value *getNewAccessOperand(ScopStmt &Stmt, const MemoryAccess &MA); /// @brief Generate the operand address - Value *generateLocationAccessed(const Instruction *Inst, const Value *Pointer, - ValueMapT &BBMap, ValueMapT &GlobalMap, - LoopToScevMapT <S); + Value *generateLocationAccessed(ScopStmt &Stmt, const Instruction *Inst, + const Value *Pointer, ValueMapT &BBMap, + ValueMapT &GlobalMap, LoopToScevMapT <S); - Value *generateScalarLoad(const LoadInst *load, ValueMapT &BBMap, - ValueMapT &GlobalMap, LoopToScevMapT <S); + Value *generateScalarLoad(ScopStmt &Stmt, const LoadInst *load, + ValueMapT &BBMap, ValueMapT &GlobalMap, + LoopToScevMapT <S); - Value *generateScalarStore(const StoreInst *store, ValueMapT &BBMap, - ValueMapT &GlobalMap, LoopToScevMapT <S); + Value *generateScalarStore(ScopStmt &Stmt, const StoreInst *store, + ValueMapT &BBMap, ValueMapT &GlobalMap, + LoopToScevMapT <S); /// @brief Copy a single Instruction. /// /// This copies a single Instruction and updates references to old values /// with references to new values, as defined by GlobalMap and BBMap. /// + /// @param Stmt The statement to code generate. + /// @param Inst The instruction to copy. /// @param BBMap A mapping from old values to their new values /// (for values recalculated within this basic block). /// @param GlobalMap A mapping from old values to their new values /// (for values recalculated in the new ScoP, but not /// within this basic block). - void copyInstruction(const Instruction *Inst, ValueMapT &BBMap, - ValueMapT &GlobalMap, LoopToScevMapT <S); - - /// @brief Copy the basic block. - /// - /// This copies the entire basic block and updates references to old values - /// with references to new values, as defined by GlobalMap. - /// - /// @param GlobalMap A mapping from old values to their new values + /// @param LTS A mapping from loops virtual canonical induction + /// variable to their new values /// (for values recalculated in the new ScoP, but not - /// within this basic block). - void copyBB(ValueMapT &GlobalMap, LoopToScevMapT <S); + /// within this basic block). + void copyInstruction(ScopStmt &Stmt, const Instruction *Inst, + ValueMapT &BBMap, ValueMapT &GlobalMap, + LoopToScevMapT <S); }; /// @brief Generate a new vector basic block for a polyhedral statement. @@ -191,6 +188,7 @@ public: /// instructions, but e.g. for address calculation instructions we currently /// generate scalar instructions for each vector lane. /// + /// @param BlockGen A block generator object used as parent. /// @param Stmt The statement to code generate. /// @param GlobalMaps A vector of maps that define for certain Values /// referenced from the original code new Values they should @@ -198,25 +196,19 @@ public: /// used for one vector lane. The number of elements in the /// vector defines the width of the generated vector /// instructions. + /// @param VLTS A mapping from loops virtual canonical induction + /// variable to their new values + /// (for values recalculated in the new ScoP, but not + /// within this basic block), one for each lane. /// @param Schedule A map from the statement to a schedule where the /// innermost dimension is the dimension of the innermost /// loop containing the statemenet. - /// @param P A reference to the pass this function is called from. - /// The pass is needed to update other analysis. - /// @param LI The loop info for the current function - /// @param SE The scalar evolution info for the current function - /// @param Build The AST build with the new schedule. - /// @param ExprBuilder An expression builder to generate new access functions. - static void generate(PollyIRBuilder &B, ScopStmt &Stmt, + static void generate(BlockGenerator &BlockGen, ScopStmt &Stmt, VectorValueMapT &GlobalMaps, std::vector &VLTS, - __isl_keep isl_map *Schedule, Pass *P, LoopInfo &LI, - ScalarEvolution &SE, - __isl_keep isl_ast_build *Build = nullptr, - IslExprBuilder *ExprBuilder = nullptr) { - VectorBlockGenerator Generator(B, GlobalMaps, VLTS, Stmt, Schedule, P, LI, - SE, Build, ExprBuilder); - Generator.copyBB(); + __isl_keep isl_map *Schedule) { + VectorBlockGenerator Generator(BlockGen, GlobalMaps, VLTS, Schedule); + Generator.copyBB(Stmt); } private: @@ -249,16 +241,13 @@ private: // dimension of the innermost loop containing the statemenet. isl_map *Schedule; - VectorBlockGenerator(PollyIRBuilder &B, VectorValueMapT &GlobalMaps, - std::vector &VLTS, ScopStmt &Stmt, - __isl_keep isl_map *Schedule, Pass *P, LoopInfo &LI, - ScalarEvolution &SE, - __isl_keep isl_ast_build *Build = nullptr, - IslExprBuilder *ExprBuilder = nullptr); + VectorBlockGenerator(BlockGenerator &BlockGen, VectorValueMapT &GlobalMaps, + std::vector &VLTS, + __isl_keep isl_map *Schedule); int getVectorWidth(); - Value *getVectorValue(const Value *Old, ValueMapT &VectorMap, + Value *getVectorValue(ScopStmt &Stmt, const Value *Old, ValueMapT &VectorMap, VectorValueMapT &ScalarMaps, Loop *L); Type *getVectorPtrTy(const Value *V, int Width); @@ -271,13 +260,14 @@ private: /// %vector_ptr= bitcast double* %p to <4 x double>* /// %vec_full = load <4 x double>* %vector_ptr /// + /// @param Stmt The statement to code generate. /// @param NegativeStride This is used to indicate a -1 stride. In such /// a case we load the end of a base address and /// shuffle the accesses in reverse order into the /// vector. By default we would do only positive /// strides. /// - Value *generateStrideOneLoad(const LoadInst *Load, + Value *generateStrideOneLoad(ScopStmt &Stmt, const LoadInst *Load, VectorValueMapT &ScalarMaps, bool NegativeStride); @@ -291,7 +281,8 @@ private: /// %splat = shufflevector <1 x double> %splat_one, <1 x /// double> %splat_one, <4 x i32> zeroinitializer /// - Value *generateStrideZeroLoad(const LoadInst *Load, ValueMapT &BBMap); + Value *generateStrideZeroLoad(ScopStmt &Stmt, const LoadInst *Load, + ValueMapT &BBMap); /// @brief Load a vector from scalars distributed in memory /// @@ -304,33 +295,33 @@ private: /// %scalar 2 = load double* %p_2 /// %vec_2 = insertelement <2 x double> %vec_1, double %scalar_1, i32 1 /// - Value *generateUnknownStrideLoad(const LoadInst *Load, + Value *generateUnknownStrideLoad(ScopStmt &Stmt, const LoadInst *Load, VectorValueMapT &ScalarMaps); - void generateLoad(const LoadInst *Load, ValueMapT &VectorMap, + void generateLoad(ScopStmt &Stmt, const LoadInst *Load, ValueMapT &VectorMap, VectorValueMapT &ScalarMaps); - void copyUnaryInst(const UnaryInstruction *Inst, ValueMapT &VectorMap, - VectorValueMapT &ScalarMaps); + void copyUnaryInst(ScopStmt &Stmt, const UnaryInstruction *Inst, + ValueMapT &VectorMap, VectorValueMapT &ScalarMaps); - void copyBinaryInst(const BinaryOperator *Inst, ValueMapT &VectorMap, - VectorValueMapT &ScalarMaps); + void copyBinaryInst(ScopStmt &Stmt, const BinaryOperator *Inst, + ValueMapT &VectorMap, VectorValueMapT &ScalarMaps); - void copyStore(const StoreInst *Store, ValueMapT &VectorMap, + void copyStore(ScopStmt &Stmt, const StoreInst *Store, ValueMapT &VectorMap, VectorValueMapT &ScalarMaps); - void copyInstScalarized(const Instruction *Inst, ValueMapT &VectorMap, - VectorValueMapT &ScalarMaps); + void copyInstScalarized(ScopStmt &Stmt, const Instruction *Inst, + ValueMapT &VectorMap, VectorValueMapT &ScalarMaps); bool extractScalarValues(const Instruction *Inst, ValueMapT &VectorMap, VectorValueMapT &ScalarMaps); bool hasVectorOperands(const Instruction *Inst, ValueMapT &VectorMap); - void copyInstruction(const Instruction *Inst, ValueMapT &VectorMap, - VectorValueMapT &ScalarMaps); + void copyInstruction(ScopStmt &Stmt, const Instruction *Inst, + ValueMapT &VectorMap, VectorValueMapT &ScalarMaps); - void copyBB(); + void copyBB(ScopStmt &Stmt); }; } #endif diff --git a/polly/include/polly/ScopInfo.h b/polly/include/polly/ScopInfo.h index df13781421e6..b04a3f511876 100644 --- a/polly/include/polly/ScopInfo.h +++ b/polly/include/polly/ScopInfo.h @@ -46,6 +46,7 @@ struct isl_set; struct isl_union_set; struct isl_union_map; struct isl_space; +struct isl_ast_build; struct isl_constraint; struct isl_pw_multi_aff; @@ -426,6 +427,9 @@ class ScopStmt { /// The BasicBlock represented by this statement. BasicBlock *BB; + /// @brief The isl AST build for the new generated AST. + isl_ast_build *Build; + std::vector NestLoops; std::string BaseName; @@ -559,6 +563,12 @@ public: const char *getBaseName() const; + /// @brief Set the isl AST build. + void setAstBuild(__isl_keep isl_ast_build *B) { Build = B; } + + /// @brief Get the isl AST build. + __isl_keep isl_ast_build *getAstBuild() const { return Build; } + /// @brief Restrict the domain of the statement. /// /// @param NewDomain The new statement domain. diff --git a/polly/lib/Analysis/ScopInfo.cpp b/polly/lib/Analysis/ScopInfo.cpp index f33b3e07f63f..829c11afb91c 100644 --- a/polly/lib/Analysis/ScopInfo.cpp +++ b/polly/lib/Analysis/ScopInfo.cpp @@ -907,7 +907,7 @@ void ScopStmt::deriveAssumptions() { ScopStmt::ScopStmt(Scop &parent, TempScop &tempScop, const Region &CurRegion, BasicBlock &bb, SmallVectorImpl &Nest, SmallVectorImpl &Scatter) - : Parent(parent), BB(&bb), NestLoops(Nest.size()) { + : Parent(parent), BB(&bb), Build(nullptr), NestLoops(Nest.size()) { // Setup the induction variables. for (unsigned i = 0, e = Nest.size(); i < e; ++i) NestLoops[i] = Nest[i]; diff --git a/polly/lib/CodeGen/BlockGenerators.cpp b/polly/lib/CodeGen/BlockGenerators.cpp index d7ccea69c7c0..6790b9dbd971 100644 --- a/polly/lib/CodeGen/BlockGenerators.cpp +++ b/polly/lib/CodeGen/BlockGenerators.cpp @@ -76,16 +76,13 @@ bool polly::isIgnoredIntrinsic(const Value *V) { return false; } -BlockGenerator::BlockGenerator(PollyIRBuilder &B, ScopStmt &Stmt, Pass *P, - LoopInfo &LI, ScalarEvolution &SE, - isl_ast_build *Build, - IslExprBuilder *ExprBuilder) - : Builder(B), Statement(Stmt), P(P), LI(LI), SE(SE), Build(Build), - ExprBuilder(ExprBuilder) {} +BlockGenerator::BlockGenerator(PollyIRBuilder &B, Pass *P, LoopInfo &LI, + ScalarEvolution &SE, IslExprBuilder *ExprBuilder) + : Builder(B), P(P), LI(LI), SE(SE), ExprBuilder(ExprBuilder) {} -Value *BlockGenerator::getNewValue(const Value *Old, ValueMapT &BBMap, - ValueMapT &GlobalMap, LoopToScevMapT <S, - Loop *L) const { +Value *BlockGenerator::getNewValue(ScopStmt &Stmt, const Value *Old, + ValueMapT &BBMap, ValueMapT &GlobalMap, + LoopToScevMapT <S, Loop *L) const { // We assume constants never change. // This avoids map lookups for many calls to this function. if (isa(Old)) @@ -125,7 +122,7 @@ Value *BlockGenerator::getNewValue(const Value *Old, ValueMapT &BBMap, // A scop-constant value defined by an instruction executed outside the scop. if (const Instruction *Inst = dyn_cast(Old)) - if (!Statement.getParent()->getRegion().contains(Inst->getParent())) + if (!Stmt.getParent()->getRegion().contains(Inst->getParent())) return const_cast(Old); // The scalar dependence is neither available nor SCEVCodegenable. @@ -133,8 +130,9 @@ Value *BlockGenerator::getNewValue(const Value *Old, ValueMapT &BBMap, return nullptr; } -void BlockGenerator::copyInstScalar(const Instruction *Inst, ValueMapT &BBMap, - ValueMapT &GlobalMap, LoopToScevMapT <S) { +void BlockGenerator::copyInstScalar(ScopStmt &Stmt, const Instruction *Inst, + ValueMapT &BBMap, ValueMapT &GlobalMap, + LoopToScevMapT <S) { // We do not generate debug intrinsics as we did not investigate how to // copy them correctly. At the current state, they just crash the code // generation as the meta-data operands are not correctly copied. @@ -145,8 +143,8 @@ void BlockGenerator::copyInstScalar(const Instruction *Inst, ValueMapT &BBMap, // Replace old operands with the new ones. for (Value *OldOperand : Inst->operands()) { - Value *NewOperand = - getNewValue(OldOperand, BBMap, GlobalMap, LTS, getLoopForInst(Inst)); + Value *NewOperand = getNewValue(Stmt, OldOperand, BBMap, GlobalMap, LTS, + getLoopForInst(Inst)); if (!NewOperand) { assert(!isa(NewInst) && @@ -165,10 +163,12 @@ void BlockGenerator::copyInstScalar(const Instruction *Inst, ValueMapT &BBMap, NewInst->setName("p_" + Inst->getName()); } -Value *BlockGenerator::getNewAccessOperand(const MemoryAccess &MA) { +Value *BlockGenerator::getNewAccessOperand(ScopStmt &Stmt, + const MemoryAccess &MA) { isl_pw_multi_aff *PWAccRel; isl_union_map *Schedule; isl_ast_expr *Expr; + isl_ast_build *Build = Stmt.getAstBuild(); assert(ExprBuilder && Build && "Cannot generate new value without IslExprBuilder!"); @@ -182,19 +182,17 @@ Value *BlockGenerator::getNewAccessOperand(const MemoryAccess &MA) { return ExprBuilder->create(Expr); } -Value *BlockGenerator::generateLocationAccessed(const Instruction *Inst, - const Value *Pointer, - ValueMapT &BBMap, - ValueMapT &GlobalMap, - LoopToScevMapT <S) { - const MemoryAccess &MA = Statement.getAccessFor(Inst); +Value *BlockGenerator::generateLocationAccessed( + ScopStmt &Stmt, const Instruction *Inst, const Value *Pointer, + ValueMapT &BBMap, ValueMapT &GlobalMap, LoopToScevMapT <S) { + const MemoryAccess &MA = Stmt.getAccessFor(Inst); Value *NewPointer; if (MA.hasNewAccessRelation()) - NewPointer = getNewAccessOperand(MA); + NewPointer = getNewAccessOperand(Stmt, MA); else NewPointer = - getNewValue(Pointer, BBMap, GlobalMap, LTS, getLoopForInst(Inst)); + getNewValue(Stmt, Pointer, BBMap, GlobalMap, LTS, getLoopForInst(Inst)); return NewPointer; } @@ -203,35 +201,36 @@ Loop *BlockGenerator::getLoopForInst(const llvm::Instruction *Inst) { return LI.getLoopFor(Inst->getParent()); } -Value *BlockGenerator::generateScalarLoad(const LoadInst *Load, +Value *BlockGenerator::generateScalarLoad(ScopStmt &Stmt, const LoadInst *Load, ValueMapT &BBMap, ValueMapT &GlobalMap, LoopToScevMapT <S) { const Value *Pointer = Load->getPointerOperand(); Value *NewPointer = - generateLocationAccessed(Load, Pointer, BBMap, GlobalMap, LTS); + generateLocationAccessed(Stmt, Load, Pointer, BBMap, GlobalMap, LTS); Value *ScalarLoad = Builder.CreateAlignedLoad( NewPointer, Load->getAlignment(), Load->getName() + "_p_scalar_"); return ScalarLoad; } -Value *BlockGenerator::generateScalarStore(const StoreInst *Store, +Value *BlockGenerator::generateScalarStore(ScopStmt &Stmt, + const StoreInst *Store, ValueMapT &BBMap, ValueMapT &GlobalMap, LoopToScevMapT <S) { const Value *Pointer = Store->getPointerOperand(); Value *NewPointer = - generateLocationAccessed(Store, Pointer, BBMap, GlobalMap, LTS); - Value *ValueOperand = getNewValue(Store->getValueOperand(), BBMap, GlobalMap, - LTS, getLoopForInst(Store)); + generateLocationAccessed(Stmt, Store, Pointer, BBMap, GlobalMap, LTS); + Value *ValueOperand = getNewValue(Stmt, Store->getValueOperand(), BBMap, + GlobalMap, LTS, getLoopForInst(Store)); Value *NewStore = Builder.CreateAlignedStore(ValueOperand, NewPointer, Store->getAlignment()); return NewStore; } -void BlockGenerator::copyInstruction(const Instruction *Inst, ValueMapT &BBMap, - ValueMapT &GlobalMap, +void BlockGenerator::copyInstruction(ScopStmt &Stmt, const Instruction *Inst, + ValueMapT &BBMap, ValueMapT &GlobalMap, LoopToScevMapT <S) { // Terminator instructions control the control flow. They are explicitly // expressed in the clast and do not need to be copied. @@ -239,11 +238,11 @@ void BlockGenerator::copyInstruction(const Instruction *Inst, ValueMapT &BBMap, return; if (canSynthesize(Inst, &P->getAnalysis().getLoopInfo(), - &SE, &Statement.getParent()->getRegion())) + &SE, &Stmt.getParent()->getRegion())) return; if (const LoadInst *Load = dyn_cast(Inst)) { - Value *NewLoad = generateScalarLoad(Load, BBMap, GlobalMap, LTS); + Value *NewLoad = generateScalarLoad(Stmt, Load, BBMap, GlobalMap, LTS); // Compute NewLoad before its insertion in BBMap to make the insertion // deterministic. BBMap[Load] = NewLoad; @@ -251,7 +250,7 @@ void BlockGenerator::copyInstruction(const Instruction *Inst, ValueMapT &BBMap, } if (const StoreInst *Store = dyn_cast(Inst)) { - Value *NewStore = generateScalarStore(Store, BBMap, GlobalMap, LTS); + Value *NewStore = generateScalarStore(Stmt, Store, BBMap, GlobalMap, LTS); // Compute NewStore before its insertion in BBMap to make the insertion // deterministic. BBMap[Store] = NewStore; @@ -282,16 +281,17 @@ void BlockGenerator::copyInstruction(const Instruction *Inst, ValueMapT &BBMap, } } - copyInstScalar(Inst, BBMap, GlobalMap, LTS); + copyInstScalar(Stmt, Inst, BBMap, GlobalMap, LTS); } -void BlockGenerator::copyBB(ValueMapT &GlobalMap, LoopToScevMapT <S) { +void BlockGenerator::copyBB(ScopStmt &Stmt, ValueMapT &GlobalMap, + LoopToScevMapT <S) { auto *DTWP = P->getAnalysisIfAvailable(); auto *DT = DTWP ? &DTWP->getDomTree() : nullptr; auto *LIWP = P->getAnalysisIfAvailable(); auto *LI = LIWP ? &LIWP->getLoopInfo() : nullptr; - BasicBlock *BB = Statement.getBasicBlock(); + BasicBlock *BB = Stmt.getBasicBlock(); BasicBlock *CopyBB = SplitBlock(Builder.GetInsertBlock(), Builder.GetInsertPoint(), DT, LI); CopyBB->setName("polly.stmt." + BB->getName()); @@ -300,21 +300,20 @@ void BlockGenerator::copyBB(ValueMapT &GlobalMap, LoopToScevMapT <S) { ValueMapT BBMap; for (Instruction &Inst : *BB) - copyInstruction(&Inst, BBMap, GlobalMap, LTS); + copyInstruction(Stmt, &Inst, BBMap, GlobalMap, LTS); } -VectorBlockGenerator::VectorBlockGenerator( - PollyIRBuilder &B, VectorValueMapT &GlobalMaps, - std::vector &VLTS, ScopStmt &Stmt, - __isl_keep isl_map *Schedule, Pass *P, LoopInfo &LI, ScalarEvolution &SE, - __isl_keep isl_ast_build *Build, IslExprBuilder *ExprBuilder) - : BlockGenerator(B, Stmt, P, LI, SE, Build, ExprBuilder), - GlobalMaps(GlobalMaps), VLTS(VLTS), Schedule(Schedule) { +VectorBlockGenerator::VectorBlockGenerator(BlockGenerator &BlockGen, + VectorValueMapT &GlobalMaps, + std::vector &VLTS, + isl_map *Schedule) + : BlockGenerator(BlockGen), GlobalMaps(GlobalMaps), VLTS(VLTS), + Schedule(Schedule) { assert(GlobalMaps.size() > 1 && "Only one vector lane found"); assert(Schedule && "No statement domain provided"); } -Value *VectorBlockGenerator::getVectorValue(const Value *Old, +Value *VectorBlockGenerator::getVectorValue(ScopStmt &Stmt, const Value *Old, ValueMapT &VectorMap, VectorValueMapT &ScalarMaps, Loop *L) { @@ -327,8 +326,8 @@ Value *VectorBlockGenerator::getVectorValue(const Value *Old, for (int Lane = 0; Lane < Width; Lane++) Vector = Builder.CreateInsertElement( - Vector, - getNewValue(Old, ScalarMaps[Lane], GlobalMaps[Lane], VLTS[Lane], L), + Vector, getNewValue(Stmt, Old, ScalarMaps[Lane], GlobalMaps[Lane], + VLTS[Lane], L), Builder.getInt32(Lane)); VectorMap[Old] = Vector; @@ -346,17 +345,16 @@ Type *VectorBlockGenerator::getVectorPtrTy(const Value *Val, int Width) { return PointerType::getUnqual(VectorType); } -Value * -VectorBlockGenerator::generateStrideOneLoad(const LoadInst *Load, - VectorValueMapT &ScalarMaps, - bool NegativeStride = false) { +Value *VectorBlockGenerator::generateStrideOneLoad( + ScopStmt &Stmt, const LoadInst *Load, VectorValueMapT &ScalarMaps, + bool NegativeStride = false) { unsigned VectorWidth = getVectorWidth(); const Value *Pointer = Load->getPointerOperand(); Type *VectorPtrType = getVectorPtrTy(Pointer, VectorWidth); unsigned Offset = NegativeStride ? VectorWidth - 1 : 0; Value *NewPointer = nullptr; - NewPointer = generateLocationAccessed(Load, Pointer, ScalarMaps[Offset], + NewPointer = generateLocationAccessed(Stmt, Load, Pointer, ScalarMaps[Offset], GlobalMaps[Offset], VLTS[Offset]); Value *VectorPtr = Builder.CreateBitCast(NewPointer, VectorPtrType, "vector_ptr"); @@ -378,12 +376,13 @@ VectorBlockGenerator::generateStrideOneLoad(const LoadInst *Load, return VecLoad; } -Value *VectorBlockGenerator::generateStrideZeroLoad(const LoadInst *Load, +Value *VectorBlockGenerator::generateStrideZeroLoad(ScopStmt &Stmt, + const LoadInst *Load, ValueMapT &BBMap) { const Value *Pointer = Load->getPointerOperand(); Type *VectorPtrType = getVectorPtrTy(Pointer, 1); - Value *NewPointer = - generateLocationAccessed(Load, Pointer, BBMap, GlobalMaps[0], VLTS[0]); + Value *NewPointer = generateLocationAccessed(Stmt, Load, Pointer, BBMap, + GlobalMaps[0], VLTS[0]); Value *VectorPtr = Builder.CreateBitCast(NewPointer, VectorPtrType, Load->getName() + "_p_vec_p"); LoadInst *ScalarLoad = @@ -400,9 +399,8 @@ Value *VectorBlockGenerator::generateStrideZeroLoad(const LoadInst *Load, return VectorLoad; } -Value * -VectorBlockGenerator::generateUnknownStrideLoad(const LoadInst *Load, - VectorValueMapT &ScalarMaps) { +Value *VectorBlockGenerator::generateUnknownStrideLoad( + ScopStmt &Stmt, const LoadInst *Load, VectorValueMapT &ScalarMaps) { int VectorWidth = getVectorWidth(); const Value *Pointer = Load->getPointerOperand(); VectorType *VectorType = VectorType::get( @@ -411,8 +409,8 @@ VectorBlockGenerator::generateUnknownStrideLoad(const LoadInst *Load, Value *Vector = UndefValue::get(VectorType); for (int i = 0; i < VectorWidth; i++) { - Value *NewPointer = generateLocationAccessed(Load, Pointer, ScalarMaps[i], - GlobalMaps[i], VLTS[i]); + Value *NewPointer = generateLocationAccessed( + Stmt, Load, Pointer, ScalarMaps[i], GlobalMaps[i], VLTS[i]); Value *ScalarLoad = Builder.CreateLoad(NewPointer, Load->getName() + "_p_scalar_"); Vector = Builder.CreateInsertElement( @@ -422,18 +420,18 @@ VectorBlockGenerator::generateUnknownStrideLoad(const LoadInst *Load, return Vector; } -void VectorBlockGenerator::generateLoad(const LoadInst *Load, +void VectorBlockGenerator::generateLoad(ScopStmt &Stmt, const LoadInst *Load, ValueMapT &VectorMap, VectorValueMapT &ScalarMaps) { if (PollyVectorizerChoice >= VECTORIZER_FIRST_NEED_GROUPED_UNROLL || !VectorType::isValidElementType(Load->getType())) { for (int i = 0; i < getVectorWidth(); i++) ScalarMaps[i][Load] = - generateScalarLoad(Load, ScalarMaps[i], GlobalMaps[i], VLTS[i]); + generateScalarLoad(Stmt, Load, ScalarMaps[i], GlobalMaps[i], VLTS[i]); return; } - const MemoryAccess &Access = Statement.getAccessFor(Load); + const MemoryAccess &Access = Stmt.getAccessFor(Load); // Make sure we have scalar values available to access the pointer to // the data location. @@ -441,23 +439,24 @@ void VectorBlockGenerator::generateLoad(const LoadInst *Load, Value *NewLoad; if (Access.isStrideZero(isl_map_copy(Schedule))) - NewLoad = generateStrideZeroLoad(Load, ScalarMaps[0]); + NewLoad = generateStrideZeroLoad(Stmt, Load, ScalarMaps[0]); else if (Access.isStrideOne(isl_map_copy(Schedule))) - NewLoad = generateStrideOneLoad(Load, ScalarMaps); + NewLoad = generateStrideOneLoad(Stmt, Load, ScalarMaps); else if (Access.isStrideX(isl_map_copy(Schedule), -1)) - NewLoad = generateStrideOneLoad(Load, ScalarMaps, true); + NewLoad = generateStrideOneLoad(Stmt, Load, ScalarMaps, true); else - NewLoad = generateUnknownStrideLoad(Load, ScalarMaps); + NewLoad = generateUnknownStrideLoad(Stmt, Load, ScalarMaps); VectorMap[Load] = NewLoad; } -void VectorBlockGenerator::copyUnaryInst(const UnaryInstruction *Inst, +void VectorBlockGenerator::copyUnaryInst(ScopStmt &Stmt, + const UnaryInstruction *Inst, ValueMapT &VectorMap, VectorValueMapT &ScalarMaps) { int VectorWidth = getVectorWidth(); - Value *NewOperand = getVectorValue(Inst->getOperand(0), VectorMap, ScalarMaps, - getLoopForInst(Inst)); + Value *NewOperand = getVectorValue(Stmt, Inst->getOperand(0), VectorMap, + ScalarMaps, getLoopForInst(Inst)); assert(isa(Inst) && "Can not generate vector code for instruction"); @@ -466,7 +465,8 @@ void VectorBlockGenerator::copyUnaryInst(const UnaryInstruction *Inst, VectorMap[Inst] = Builder.CreateCast(Cast->getOpcode(), NewOperand, DestType); } -void VectorBlockGenerator::copyBinaryInst(const BinaryOperator *Inst, +void VectorBlockGenerator::copyBinaryInst(ScopStmt &Stmt, + const BinaryOperator *Inst, ValueMapT &VectorMap, VectorValueMapT &ScalarMaps) { Loop *L = getLoopForInst(Inst); @@ -474,21 +474,21 @@ void VectorBlockGenerator::copyBinaryInst(const BinaryOperator *Inst, Value *OpOne = Inst->getOperand(1); Value *NewOpZero, *NewOpOne; - NewOpZero = getVectorValue(OpZero, VectorMap, ScalarMaps, L); - NewOpOne = getVectorValue(OpOne, VectorMap, ScalarMaps, L); + NewOpZero = getVectorValue(Stmt, OpZero, VectorMap, ScalarMaps, L); + NewOpOne = getVectorValue(Stmt, OpOne, VectorMap, ScalarMaps, L); Value *NewInst = Builder.CreateBinOp(Inst->getOpcode(), NewOpZero, NewOpOne, Inst->getName() + "p_vec"); VectorMap[Inst] = NewInst; } -void VectorBlockGenerator::copyStore(const StoreInst *Store, +void VectorBlockGenerator::copyStore(ScopStmt &Stmt, const StoreInst *Store, ValueMapT &VectorMap, VectorValueMapT &ScalarMaps) { - const MemoryAccess &Access = Statement.getAccessFor(Store); + const MemoryAccess &Access = Stmt.getAccessFor(Store); const Value *Pointer = Store->getPointerOperand(); - Value *Vector = getVectorValue(Store->getValueOperand(), VectorMap, + Value *Vector = getVectorValue(Stmt, Store->getValueOperand(), VectorMap, ScalarMaps, getLoopForInst(Store)); // Make sure we have scalar values available to access the pointer to @@ -497,8 +497,8 @@ void VectorBlockGenerator::copyStore(const StoreInst *Store, if (Access.isStrideOne(isl_map_copy(Schedule))) { Type *VectorPtrType = getVectorPtrTy(Pointer, getVectorWidth()); - Value *NewPointer = generateLocationAccessed(Store, Pointer, ScalarMaps[0], - GlobalMaps[0], VLTS[0]); + Value *NewPointer = generateLocationAccessed( + Stmt, Store, Pointer, ScalarMaps[0], GlobalMaps[0], VLTS[0]); Value *VectorPtr = Builder.CreateBitCast(NewPointer, VectorPtrType, "vector_ptr"); @@ -510,7 +510,7 @@ void VectorBlockGenerator::copyStore(const StoreInst *Store, for (unsigned i = 0; i < ScalarMaps.size(); i++) { Value *Scalar = Builder.CreateExtractElement(Vector, Builder.getInt32(i)); Value *NewPointer = generateLocationAccessed( - Store, Pointer, ScalarMaps[i], GlobalMaps[i], VLTS[i]); + Stmt, Store, Pointer, ScalarMaps[i], GlobalMaps[i], VLTS[i]); Builder.CreateStore(Scalar, NewPointer); } } @@ -556,7 +556,8 @@ bool VectorBlockGenerator::extractScalarValues(const Instruction *Inst, return HasVectorOperand; } -void VectorBlockGenerator::copyInstScalarized(const Instruction *Inst, +void VectorBlockGenerator::copyInstScalarized(ScopStmt &Stmt, + const Instruction *Inst, ValueMapT &VectorMap, VectorValueMapT &ScalarMaps) { bool HasVectorOperand; @@ -565,7 +566,7 @@ void VectorBlockGenerator::copyInstScalarized(const Instruction *Inst, HasVectorOperand = extractScalarValues(Inst, VectorMap, ScalarMaps); for (int VectorLane = 0; VectorLane < getVectorWidth(); VectorLane++) - BlockGenerator::copyInstruction(Inst, ScalarMaps[VectorLane], + BlockGenerator::copyInstruction(Stmt, Inst, ScalarMaps[VectorLane], GlobalMaps[VectorLane], VLTS[VectorLane]); if (!VectorType::isValidElementType(Inst->getType()) || !HasVectorOperand) @@ -584,7 +585,8 @@ void VectorBlockGenerator::copyInstScalarized(const Instruction *Inst, int VectorBlockGenerator::getVectorWidth() { return GlobalMaps.size(); } -void VectorBlockGenerator::copyInstruction(const Instruction *Inst, +void VectorBlockGenerator::copyInstruction(ScopStmt &Stmt, + const Instruction *Inst, ValueMapT &VectorMap, VectorValueMapT &ScalarMaps) { // Terminator instructions control the control flow. They are explicitly @@ -593,27 +595,27 @@ void VectorBlockGenerator::copyInstruction(const Instruction *Inst, return; if (canSynthesize(Inst, &P->getAnalysis().getLoopInfo(), - &SE, &Statement.getParent()->getRegion())) + &SE, &Stmt.getParent()->getRegion())) return; if (const LoadInst *Load = dyn_cast(Inst)) { - generateLoad(Load, VectorMap, ScalarMaps); + generateLoad(Stmt, Load, VectorMap, ScalarMaps); return; } if (hasVectorOperands(Inst, VectorMap)) { if (const StoreInst *Store = dyn_cast(Inst)) { - copyStore(Store, VectorMap, ScalarMaps); + copyStore(Stmt, Store, VectorMap, ScalarMaps); return; } if (const UnaryInstruction *Unary = dyn_cast(Inst)) { - copyUnaryInst(Unary, VectorMap, ScalarMaps); + copyUnaryInst(Stmt, Unary, VectorMap, ScalarMaps); return; } if (const BinaryOperator *Binary = dyn_cast(Inst)) { - copyBinaryInst(Binary, VectorMap, ScalarMaps); + copyBinaryInst(Stmt, Binary, VectorMap, ScalarMaps); return; } @@ -621,16 +623,16 @@ void VectorBlockGenerator::copyInstruction(const Instruction *Inst, // generate vector code. } - copyInstScalarized(Inst, VectorMap, ScalarMaps); + copyInstScalarized(Stmt, Inst, VectorMap, ScalarMaps); } -void VectorBlockGenerator::copyBB() { +void VectorBlockGenerator::copyBB(ScopStmt &Stmt) { auto *DTWP = P->getAnalysisIfAvailable(); auto *DT = DTWP ? &DTWP->getDomTree() : nullptr; auto *LIWP = P->getAnalysisIfAvailable(); auto *LI = LIWP ? &LIWP->getLoopInfo() : nullptr; - BasicBlock *BB = Statement.getBasicBlock(); + BasicBlock *BB = Stmt.getBasicBlock(); BasicBlock *CopyBB = SplitBlock(Builder.GetInsertBlock(), Builder.GetInsertPoint(), DT, LI); CopyBB->setName("polly.stmt." + BB->getName()); @@ -654,5 +656,5 @@ void VectorBlockGenerator::copyBB() { ValueMapT VectorBlockMap; for (Instruction &Inst : *BB) - copyInstruction(&Inst, VectorBlockMap, ScalarBlockMap); + copyInstruction(Stmt, &Inst, VectorBlockMap, ScalarBlockMap); } diff --git a/polly/lib/CodeGen/IslCodeGeneration.cpp b/polly/lib/CodeGen/IslCodeGeneration.cpp index db8712b39209..45e8e9d06f63 100644 --- a/polly/lib/CodeGen/IslCodeGeneration.cpp +++ b/polly/lib/CodeGen/IslCodeGeneration.cpp @@ -64,7 +64,8 @@ public: DominatorTree &DT, Scop &S) : S(S), Builder(Builder), Annotator(Annotator), Rewriter(new SCEVExpander(SE, "polly")), - ExprBuilder(Builder, IDToValue, *Rewriter), P(P), DL(DL), LI(LI), + ExprBuilder(Builder, IDToValue, *Rewriter), + BlockGen(Builder, P, LI, SE, &ExprBuilder), P(P), DL(DL), LI(LI), SE(SE), DT(DT) {} ~IslNodeBuilder() { delete Rewriter; } @@ -82,6 +83,7 @@ private: SCEVExpander *Rewriter; IslExprBuilder ExprBuilder; + BlockGenerator BlockGen; Pass *P; const DataLayout &DL; LoopInfo &LI; @@ -398,6 +400,7 @@ void IslNodeBuilder::createUserVector(__isl_take isl_ast_node *User, isl_id *Id = isl_ast_expr_get_id(StmtExpr); isl_ast_expr_free(StmtExpr); ScopStmt *Stmt = (ScopStmt *)isl_id_get_user(Id); + Stmt->setAstBuild(IslAstInfo::getBuild(User)); VectorValueMapT VectorMap(IVS.size()); std::vector VLTS(IVS.size()); @@ -406,8 +409,7 @@ void IslNodeBuilder::createUserVector(__isl_take isl_ast_node *User, isl_map *S = isl_map_from_union_map(Schedule); createSubstitutionsVector(Expr, Stmt, VectorMap, VLTS, IVS, IteratorID); - VectorBlockGenerator::generate(Builder, *Stmt, VectorMap, VLTS, S, P, LI, SE, - IslAstInfo::getBuild(User), &ExprBuilder); + VectorBlockGenerator::generate(BlockGen, *Stmt, VectorMap, VLTS, S); isl_map_free(S); isl_id_free(Id); @@ -795,10 +797,10 @@ void IslNodeBuilder::createUser(__isl_take isl_ast_node *User) { LTS.insert(OutsideLoopIterations.begin(), OutsideLoopIterations.end()); Stmt = (ScopStmt *)isl_id_get_user(Id); + Stmt->setAstBuild(IslAstInfo::getBuild(User)); createSubstitutions(Expr, Stmt, VMap, LTS); - BlockGenerator::generate(Builder, *Stmt, VMap, LTS, P, LI, SE, - IslAstInfo::getBuild(User), &ExprBuilder); + BlockGen.copyBB(*Stmt, VMap, LTS); isl_ast_node_free(User); isl_id_free(Id);