LoopVectorize: MaxVF should not be larger than the loop trip count
Summary: Improve how MaxVF is computed while taking into account that MaxVF should not be larger than the loop's trip count. Other than saving on compile-time by pruning the possible MaxVF candidates, this patch fixes pr34438 which exposed the following flow: 1. Short trip count identified -> Don't bail out, set OptForSize:=True to avoid tail-loop and runtime checks. 2. Compute MaxVF returned 16 on a target supporting AVX512. 3. OptForSize -> choose VF:=MaxVF. 4. Bail out because TripCount = 8, VF = 16, TripCount % VF !=0 means we need a tail loop. With this patch step 2. will choose MaxVF=8 based on TripCount. Reviewers: Ayal, dorit, mkuper, hfinkel Reviewed By: hfinkel Subscribers: hfinkel, llvm-commits Differential Revision: https://reviews.llvm.org/D37425 llvm-svn: 312472
This commit is contained in:
parent
7cd826a321
commit
9a087a357a
|
@ -1960,7 +1960,7 @@ public:
|
||||||
private:
|
private:
|
||||||
/// \return An upper bound for the vectorization factor, larger than zero.
|
/// \return An upper bound for the vectorization factor, larger than zero.
|
||||||
/// One is returned if vectorization should best be avoided due to cost.
|
/// One is returned if vectorization should best be avoided due to cost.
|
||||||
unsigned computeFeasibleMaxVF(bool OptForSize);
|
unsigned computeFeasibleMaxVF(bool OptForSize, unsigned ConstTripCount = 0);
|
||||||
|
|
||||||
/// The vectorization cost is a combination of the cost itself and a boolean
|
/// The vectorization cost is a combination of the cost itself and a boolean
|
||||||
/// indicating whether any of the contributing operations will actually
|
/// indicating whether any of the contributing operations will actually
|
||||||
|
@ -6187,7 +6187,7 @@ Optional<unsigned> LoopVectorizationCostModel::computeMaxVF(bool OptForSize) {
|
||||||
return None;
|
return None;
|
||||||
}
|
}
|
||||||
|
|
||||||
unsigned MaxVF = computeFeasibleMaxVF(OptForSize);
|
unsigned MaxVF = computeFeasibleMaxVF(OptForSize, TC);
|
||||||
|
|
||||||
if (TC % MaxVF != 0) {
|
if (TC % MaxVF != 0) {
|
||||||
// If the trip count that we found modulo the vectorization factor is not
|
// If the trip count that we found modulo the vectorization factor is not
|
||||||
|
@ -6208,7 +6208,9 @@ Optional<unsigned> LoopVectorizationCostModel::computeMaxVF(bool OptForSize) {
|
||||||
return MaxVF;
|
return MaxVF;
|
||||||
}
|
}
|
||||||
|
|
||||||
unsigned LoopVectorizationCostModel::computeFeasibleMaxVF(bool OptForSize) {
|
unsigned
|
||||||
|
LoopVectorizationCostModel::computeFeasibleMaxVF(bool OptForSize,
|
||||||
|
unsigned ConstTripCount) {
|
||||||
MinBWs = computeMinimumValueSizes(TheLoop->getBlocks(), *DB, &TTI);
|
MinBWs = computeMinimumValueSizes(TheLoop->getBlocks(), *DB, &TTI);
|
||||||
unsigned SmallestType, WidestType;
|
unsigned SmallestType, WidestType;
|
||||||
std::tie(SmallestType, WidestType) = getSmallestAndWidestTypes();
|
std::tie(SmallestType, WidestType) = getSmallestAndWidestTypes();
|
||||||
|
@ -6237,7 +6239,9 @@ unsigned LoopVectorizationCostModel::computeFeasibleMaxVF(bool OptForSize) {
|
||||||
if (MaxVectorSize == 0) {
|
if (MaxVectorSize == 0) {
|
||||||
DEBUG(dbgs() << "LV: The target has no vector registers.\n");
|
DEBUG(dbgs() << "LV: The target has no vector registers.\n");
|
||||||
MaxVectorSize = 1;
|
MaxVectorSize = 1;
|
||||||
}
|
} else if (ConstTripCount && ConstTripCount < MaxVectorSize &&
|
||||||
|
isPowerOf2_32(ConstTripCount))
|
||||||
|
MaxVectorSize = ConstTripCount;
|
||||||
|
|
||||||
assert(MaxVectorSize <= 64 && "Did not expect to pack so many elements"
|
assert(MaxVectorSize <= 64 && "Did not expect to pack so many elements"
|
||||||
" into one vector!");
|
" into one vector!");
|
||||||
|
|
|
@ -0,0 +1,35 @@
|
||||||
|
; PR34438
|
||||||
|
; Loop has a short trip count of 8 iterations. It should be vectorized because no runtime checks or tail loop are necessary.
|
||||||
|
; Two cases tested AVX (MaxVF=8 = TripCount) and AVX512 (MaxVF=16 > TripCount)
|
||||||
|
|
||||||
|
; RUN: opt < %s -loop-vectorize -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx -S | FileCheck %s
|
||||||
|
; RUN: opt < %s -loop-vectorize -mtriple=x86_64-apple-macosx10.8.0 -mcpu=skylake-avx512 -S | FileCheck %s
|
||||||
|
|
||||||
|
target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
|
||||||
|
target triple = "x86_64-apple-macosx10.8.0"
|
||||||
|
|
||||||
|
define void @small_tc(float* noalias nocapture %A, float* noalias nocapture readonly %B) {
|
||||||
|
; CHECK-LABEL: @small_tc
|
||||||
|
; CHECK: load <8 x float>, <8 x float>*
|
||||||
|
; CHECK: fadd fast <8 x float>
|
||||||
|
entry:
|
||||||
|
br label %for.body
|
||||||
|
|
||||||
|
for.body:
|
||||||
|
%indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
|
||||||
|
%arrayidx = getelementptr inbounds float, float* %B, i64 %indvars.iv
|
||||||
|
%0 = load float, float* %arrayidx, align 4, !llvm.mem.parallel_loop_access !3
|
||||||
|
%arrayidx2 = getelementptr inbounds float, float* %A, i64 %indvars.iv
|
||||||
|
%1 = load float, float* %arrayidx2, align 4, !llvm.mem.parallel_loop_access !3
|
||||||
|
%add = fadd fast float %0, %1
|
||||||
|
store float %add, float* %arrayidx2, align 4, !llvm.mem.parallel_loop_access !3
|
||||||
|
%indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
|
||||||
|
%exitcond = icmp eq i64 %indvars.iv.next, 8
|
||||||
|
br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !4
|
||||||
|
|
||||||
|
for.end:
|
||||||
|
ret void
|
||||||
|
}
|
||||||
|
|
||||||
|
!3 = !{!3}
|
||||||
|
!4 = !{!4}
|
Loading…
Reference in New Issue