Add support for loops that don't start with Zero.
This is important for loops in the LAPACK test-suite. These loops start at 1 because they are auto-converted from fortran. llvm-svn: 167084
This commit is contained in:
Nadav Rotem
parent
9a6a190562
commit
ff7889196b
|
|
@ -633,6 +633,10 @@ SingleBlockLoopVectorizer::createEmptyLoop(LoopVectorizationLegality *Legal) {
|
|||
BasicBlock *ExitBlock = OrigLoop->getExitBlock();
|
||||
assert(ExitBlock && "Must have an exit block");
|
||||
|
||||
// The loop index does not have to start at Zero. It starts with this value.
|
||||
OldInduction = Legal->getInduction();
|
||||
Value *StartIdx = OldInduction->getIncomingValueForBlock(BypassBlock);
|
||||
|
||||
assert(OrigLoop->getNumBlocks() == 1 && "Invalid loop");
|
||||
assert(BypassBlock && "Invalid loop structure");
|
||||
|
||||
|
|
@ -648,7 +652,6 @@ SingleBlockLoopVectorizer::createEmptyLoop(LoopVectorizationLegality *Legal) {
|
|||
"scalar.preheader");
|
||||
// Find the induction variable.
|
||||
BasicBlock *OldBasicBlock = OrigLoop->getHeader();
|
||||
OldInduction = Legal->getInduction();
|
||||
assert(OldInduction && "We must have a single phi node.");
|
||||
Type *IdxTy = OldInduction->getType();
|
||||
|
||||
|
|
@ -658,7 +661,6 @@ SingleBlockLoopVectorizer::createEmptyLoop(LoopVectorizationLegality *Legal) {
|
|||
|
||||
// Generate the induction variable.
|
||||
Induction = Builder.CreatePHI(IdxTy, 2, "index");
|
||||
Constant *Zero = ConstantInt::get(IdxTy, 0);
|
||||
Constant *Step = ConstantInt::get(IdxTy, VF);
|
||||
|
||||
// Find the loop boundaries.
|
||||
|
|
@ -682,15 +684,22 @@ SingleBlockLoopVectorizer::createEmptyLoop(LoopVectorizationLegality *Legal) {
|
|||
|
||||
// Count holds the overall loop count (N).
|
||||
Value *Count = Exp.expandCodeFor(ExitCount, Induction->getType(), Loc);
|
||||
|
||||
// Add the start index to the loop count to get the new end index.
|
||||
Value *IdxEnd = BinaryOperator::CreateAdd(Count, StartIdx, "end.idx", Loc);
|
||||
|
||||
// Now we need to generate the expression for N - (N % VF), which is
|
||||
// the part that the vectorized body will execute.
|
||||
Constant *CIVF = ConstantInt::get(IdxTy, VF);
|
||||
Value *R = BinaryOperator::CreateURem(Count, CIVF, "n.mod.vf", Loc);
|
||||
Value *CountRoundDown = BinaryOperator::CreateSub(Count, R, "n.vec", Loc);
|
||||
Value *IdxEndRoundDown = BinaryOperator::CreateAdd(CountRoundDown, StartIdx,
|
||||
"end.idx.rnd.down", Loc);
|
||||
|
||||
// Now, compare the new count to zero. If it is zero, jump to the scalar part.
|
||||
Value *Cmp = CmpInst::Create(Instruction::ICmp, CmpInst::ICMP_EQ,
|
||||
CountRoundDown, ConstantInt::getNullValue(IdxTy),
|
||||
IdxEndRoundDown,
|
||||
StartIdx,
|
||||
"cmp.zero", Loc);
|
||||
BranchInst::Create(MiddleBlock, VectorPH, Cmp, Loc);
|
||||
// Remove the old terminator.
|
||||
|
|
@ -699,8 +708,8 @@ SingleBlockLoopVectorizer::createEmptyLoop(LoopVectorizationLegality *Legal) {
|
|||
// Add a check in the middle block to see if we have completed
|
||||
// all of the iterations in the first vector loop.
|
||||
// If (N - N%VF) == N, then we *don't* need to run the remainder.
|
||||
Value *CmpN = CmpInst::Create(Instruction::ICmp, CmpInst::ICMP_EQ, Count,
|
||||
CountRoundDown, "cmp.n",
|
||||
Value *CmpN = CmpInst::Create(Instruction::ICmp, CmpInst::ICMP_EQ, IdxEnd,
|
||||
IdxEndRoundDown, "cmp.n",
|
||||
MiddleBlock->getTerminator());
|
||||
|
||||
BranchInst::Create(ExitBlock, ScalarPH, CmpN, MiddleBlock->getTerminator());
|
||||
|
|
@ -709,10 +718,10 @@ SingleBlockLoopVectorizer::createEmptyLoop(LoopVectorizationLegality *Legal) {
|
|||
|
||||
// Create i+1 and fill the PHINode.
|
||||
Value *NextIdx = Builder.CreateAdd(Induction, Step, "index.next");
|
||||
Induction->addIncoming(Zero, VectorPH);
|
||||
Induction->addIncoming(StartIdx, VectorPH);
|
||||
Induction->addIncoming(NextIdx, VecBody);
|
||||
// Create the compare.
|
||||
Value *ICmp = Builder.CreateICmpEQ(NextIdx, CountRoundDown);
|
||||
Value *ICmp = Builder.CreateICmpEQ(NextIdx, IdxEndRoundDown);
|
||||
Builder.CreateCondBr(ICmp, MiddleBlock, VecBody);
|
||||
|
||||
// Now we have two terminators. Remove the old one from the block.
|
||||
|
|
@ -720,7 +729,7 @@ SingleBlockLoopVectorizer::createEmptyLoop(LoopVectorizationLegality *Legal) {
|
|||
|
||||
// Fix the scalar body iteration count.
|
||||
unsigned BlockIdx = OldInduction->getBasicBlockIndex(ScalarPH);
|
||||
OldInduction->setIncomingValue(BlockIdx, CountRoundDown);
|
||||
OldInduction->setIncomingValue(BlockIdx, IdxEndRoundDown);
|
||||
|
||||
// Get ready to start creating new instructions into the vectorized body.
|
||||
Builder.SetInsertPoint(VecBody->getFirstInsertionPt());
|
||||
|
|
@ -748,7 +757,6 @@ SingleBlockLoopVectorizer::createEmptyLoop(LoopVectorizationLegality *Legal) {
|
|||
LoopBypassBlock = BypassBlock;
|
||||
}
|
||||
|
||||
|
||||
/// This function returns the identity element (or neutral element) for
|
||||
/// the operation K.
|
||||
static unsigned
|
||||
|
|
@ -1518,10 +1526,9 @@ bool LoopVectorizationLegality::isInductionVariable(PHINode *Phi) {
|
|||
return false;
|
||||
}
|
||||
const SCEV *Step = AR->getStepRecurrence(*SE);
|
||||
const SCEV *Start = AR->getStart();
|
||||
|
||||
if (!Step->isOne() || !Start->isZero()) {
|
||||
DEBUG(dbgs() << "LV: PHI does not start at zero or steps by one.\n");
|
||||
if (!Step->isOne()) {
|
||||
DEBUG(dbgs() << "LV: PHI stride does not equal one.\n");
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
|
|
|
|||
|
|
@ -0,0 +1,49 @@
|
|||
; RUN: opt < %s -loop-vectorize -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx -S | FileCheck %s
|
||||
|
||||
target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
|
||||
target triple = "x86_64-apple-macosx10.8.0"
|
||||
|
||||
;CHECK: @read_mod_write_single_ptr
|
||||
;CHECK: load <8 x float>
|
||||
;CHECK: ret i32
|
||||
define i32 @read_mod_write_single_ptr(float* nocapture %a, i32 %n) nounwind uwtable ssp {
|
||||
%1 = icmp sgt i32 %n, 0
|
||||
br i1 %1, label %.lr.ph, label %._crit_edge
|
||||
|
||||
.lr.ph: ; preds = %0, %.lr.ph
|
||||
%indvars.iv = phi i64 [ %indvars.iv.next, %.lr.ph ], [ 0, %0 ]
|
||||
%2 = getelementptr inbounds float* %a, i64 %indvars.iv
|
||||
%3 = load float* %2, align 4
|
||||
%4 = fmul float %3, 3.000000e+00
|
||||
store float %4, float* %2, align 4
|
||||
%indvars.iv.next = add i64 %indvars.iv, 1
|
||||
%lftr.wideiv = trunc i64 %indvars.iv.next to i32
|
||||
%exitcond = icmp eq i32 %lftr.wideiv, %n
|
||||
br i1 %exitcond, label %._crit_edge, label %.lr.ph
|
||||
|
||||
._crit_edge: ; preds = %.lr.ph, %0
|
||||
ret i32 undef
|
||||
}
|
||||
|
||||
|
||||
;CHECK: @read_mod_i64
|
||||
;CHECK: load <8 x i64>
|
||||
;CHECK: ret i32
|
||||
define i32 @read_mod_i64(i64* nocapture %a, i32 %n) nounwind uwtable ssp {
|
||||
%1 = icmp sgt i32 %n, 0
|
||||
br i1 %1, label %.lr.ph, label %._crit_edge
|
||||
|
||||
.lr.ph: ; preds = %0, %.lr.ph
|
||||
%indvars.iv = phi i64 [ %indvars.iv.next, %.lr.ph ], [ 0, %0 ]
|
||||
%2 = getelementptr inbounds i64* %a, i64 %indvars.iv
|
||||
%3 = load i64* %2, align 4
|
||||
%4 = mul i64 %3, 3
|
||||
store i64 %4, i64* %2, align 4
|
||||
%indvars.iv.next = add i64 %indvars.iv, 1
|
||||
%lftr.wideiv = trunc i64 %indvars.iv.next to i32
|
||||
%exitcond = icmp eq i32 %lftr.wideiv, %n
|
||||
br i1 %exitcond, label %._crit_edge, label %.lr.ph
|
||||
|
||||
._crit_edge: ; preds = %.lr.ph, %0
|
||||
ret i32 undef
|
||||
}
|
||||
|
|
@ -0,0 +1,35 @@
|
|||
; RUN: opt < %s -loop-vectorize -force-vector-width=4 -instcombine -S | FileCheck %s
|
||||
|
||||
target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
|
||||
target triple = "x86_64-apple-macosx10.8.0"
|
||||
|
||||
;CHECK: @start_at_nonzero
|
||||
;CHECK: mul <4 x i32>
|
||||
;CHECK: ret i32
|
||||
define i32 @start_at_nonzero(i32* nocapture %a, i32 %start, i32 %end) nounwind uwtable ssp {
|
||||
entry:
|
||||
%cmp3 = icmp slt i32 %start, %end
|
||||
br i1 %cmp3, label %for.body.lr.ph, label %for.end
|
||||
|
||||
for.body.lr.ph: ; preds = %entry
|
||||
%0 = sext i32 %start to i64
|
||||
br label %for.body
|
||||
|
||||
for.body: ; preds = %for.body.lr.ph, %for.body
|
||||
%indvars.iv = phi i64 [ %0, %for.body.lr.ph ], [ %indvars.iv.next, %for.body ]
|
||||
%arrayidx = getelementptr inbounds i32* %a, i64 %indvars.iv
|
||||
%1 = load i32* %arrayidx, align 4, !tbaa !0
|
||||
%mul = mul nsw i32 %1, 333
|
||||
store i32 %mul, i32* %arrayidx, align 4, !tbaa !0
|
||||
%indvars.iv.next = add i64 %indvars.iv, 1
|
||||
%2 = trunc i64 %indvars.iv.next to i32
|
||||
%cmp = icmp slt i32 %2, %end
|
||||
br i1 %cmp, label %for.body, label %for.end
|
||||
|
||||
for.end: ; preds = %for.body, %entry
|
||||
ret i32 4
|
||||
}
|
||||
|
||||
!0 = metadata !{metadata !"int", metadata !1}
|
||||
!1 = metadata !{metadata !"omnipotent char", metadata !2}
|
||||
!2 = metadata !{metadata !"Simple C/C++ TBAA"}
|
||||
Loading…
Reference in New Issue