diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst index 3f7845f89f7a..77c9c5fc0638 100644 --- a/llvm/docs/LangRef.rst +++ b/llvm/docs/LangRef.rst @@ -2910,37 +2910,49 @@ suggests an unroll factor to the loop unroller: br i1 %exitcond, label %._crit_edge, label %.lr.ph, !llvm.loop !0 ... !0 = metadata !{ metadata !0, metadata !1 } - !1 = metadata !{ metadata !"llvm.loop.vectorize.width", i32 4 } + !1 = metadata !{ metadata !"llvm.loop.unroll.count", i32 4 } -'``llvm.loop.vectorize``' -^^^^^^^^^^^^^^^^^^^^^^^^^ +'``llvm.loop.vectorize``' and '``llvm.loop.interleave``' +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Metadata prefixed with ``llvm.loop.vectorize`` is used to control -per-loop vectorization parameters such as vectorization width and -interleave count. ``llvm.loop.vectorize`` metadata should be used in +Metadata prefixed with ``llvm.loop.vectorize`` or ``llvm.loop.interleave`` are +used to control per-loop vectorization and interleaving parameters such as +vectorization width and interleave count. These metadata should be used in conjunction with ``llvm.loop`` loop identification metadata. The -``llvm.loop.vectorize`` metadata are only optimization hints and the -vectorizer will only vectorize loops if it believes it is safe to do -so. The ``llvm.mem.parallel_loop_access`` metadata which contains -information about loop-carried memory dependencies can be helpful in -determining the safety of loop vectorization. +``llvm.loop.vectorize`` and ``llvm.loop.interleave`` metadata are only +optimization hints and the optimizer will only interleave and vectorize loops if +it believes it is safe to do so. The ``llvm.mem.parallel_loop_access`` metadata +which contains information about loop-carried memory dependencies can be helpful +in determining the safety of these transformations. -'``llvm.loop.vectorize.unroll``' Metadata +'``llvm.loop.interleave.count``' Metadata ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -This metadata suggests an interleave count to the loop vectorizer. -The first operand is the string ``llvm.loop.vectorize.unroll`` and the +This metadata suggests an interleave count to the loop interleaver. +The first operand is the string ``llvm.loop.interleave.count`` and the second operand is an integer specifying the interleave count. For example: .. code-block:: llvm - !0 = metadata !{ metadata !"llvm.loop.vectorize.unroll", i32 4 } + !0 = metadata !{ metadata !"llvm.loop.interleave.count", i32 4 } -Note that setting ``llvm.loop.vectorize.unroll`` to 1 disables -interleaving multiple iterations of the loop. If -``llvm.loop.vectorize.unroll`` is set to 0 then the interleave count -will be determined automatically. +Note that setting ``llvm.loop.interleave.count`` to 1 disables interleaving +multiple iterations of the loop. If ``llvm.loop.interleave.count`` is set to 0 +then the interleave count will be determined automatically. + +'``llvm.loop.vectorize.enable``' Metadata +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +This metadata selectively enables or disables vectorization for the loop. The +first operand is the string ``llvm.loop.vectorize.enable`` and the second operand +is a bit. If the bit operand value is 1 vectorization is enabled. A value of +0 disables vectorization: + +.. code-block:: llvm + + !0 = metadata !{ metadata !"llvm.loop.vectorize.enable", i1 0 } + !1 = metadata !{ metadata !"llvm.loop.vectorize.enable", i1 1 } '``llvm.loop.vectorize.width``' Metadata ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/llvm/docs/ReleaseNotes.rst b/llvm/docs/ReleaseNotes.rst index fb2e248ce36b..c76b9d946073 100644 --- a/llvm/docs/ReleaseNotes.rst +++ b/llvm/docs/ReleaseNotes.rst @@ -57,7 +57,9 @@ Non-comprehensive list of changes in this release unwinding information. * The prefix for loop vectorizer hint metadata has been changed from - ``llvm.vectorizer`` to ``llvm.loop.vectorize``. + ``llvm.vectorizer`` to ``llvm.loop.vectorize``. In addition, + ``llvm.vectorizer.unroll`` metadata has been renamed + ``llvm.loop.interleave.count``. * Some backends previously implemented Atomic NAND(x,y) as ``x & ~y``. Now all backends implement it as ``~(x & y)``, matching the semantics of GCC 4.4 diff --git a/llvm/lib/IR/AutoUpgrade.cpp b/llvm/lib/IR/AutoUpgrade.cpp index 6554b3c5da93..459bd880ccb0 100644 --- a/llvm/lib/IR/AutoUpgrade.cpp +++ b/llvm/lib/IR/AutoUpgrade.cpp @@ -580,7 +580,9 @@ bool llvm::UpgradeDebugInfo(Module &M) { void llvm::UpgradeMDStringConstant(std::string &String) { const std::string OldPrefix = "llvm.vectorizer."; - if (String.find(OldPrefix) == 0) { - String.replace(0, OldPrefix.size(), "llvm.loop.vectorize."); + if (String == "llvm.vectorizer.unroll") { + String = "llvm.loop.interleave.count"; + } else if (String.find(OldPrefix) == 0) { + String.replace(0, OldPrefix.size(), "llvm.loop.vectorize."); } } diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index b66c9ba48076..531d349d845d 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -978,8 +978,8 @@ public: << "LV: Unrolling disabled by the pass manager\n"); } - /// Return the loop vectorizer metadata prefix. - static StringRef Prefix() { return "llvm.loop.vectorize."; } + /// Return the loop metadata prefix. + static StringRef Prefix() { return "llvm.loop."; } MDNode *createHint(LLVMContext &Context, StringRef Name, unsigned V) const { SmallVector Vals; @@ -1001,8 +1001,10 @@ public: for (unsigned i = 1, ie = LoopID->getNumOperands(); i < ie; ++i) Vals.push_back(LoopID->getOperand(i)); - Vals.push_back(createHint(Context, Twine(Prefix(), "width").str(), Width)); - Vals.push_back(createHint(Context, Twine(Prefix(), "unroll").str(), 1)); + Vals.push_back( + createHint(Context, Twine(Prefix(), "vectorize.width").str(), Width)); + Vals.push_back( + createHint(Context, Twine(Prefix(), "interleave.count").str(), 1)); MDNode *NewLoopID = MDNode::get(Context, Vals); // Set operand 0 to refer to the loop id itself. @@ -1073,7 +1075,7 @@ private: if (!S) continue; - // Check if the hint starts with the vectorizer prefix. + // Check if the hint starts with the loop metadata prefix. StringRef Hint = S->getString(); if (!Hint.startswith(Prefix())) continue; @@ -1091,22 +1093,22 @@ private: if (!C) return; unsigned Val = C->getZExtValue(); - if (Hint == "width") { + if (Hint == "vectorize.width") { if (isPowerOf2_32(Val) && Val <= MaxVectorWidth) Width = Val; else DEBUG(dbgs() << "LV: ignoring invalid width hint metadata\n"); - } else if (Hint == "unroll") { - if (isPowerOf2_32(Val) && Val <= MaxUnrollFactor) - Unroll = Val; - else - DEBUG(dbgs() << "LV: ignoring invalid unroll hint metadata\n"); - } else if (Hint == "enable") { + } else if (Hint == "vectorize.enable") { if (C->getBitWidth() == 1) Force = Val == 1 ? LoopVectorizeHints::FK_Enabled : LoopVectorizeHints::FK_Disabled; else DEBUG(dbgs() << "LV: ignoring invalid enable hint metadata\n"); + } else if (Hint == "interleave.count") { + if (isPowerOf2_32(Val) && Val <= MaxUnrollFactor) + Unroll = Val; + else + DEBUG(dbgs() << "LV: ignoring invalid unroll hint metadata\n"); } else { DEBUG(dbgs() << "LV: ignoring unknown hint " << Hint << '\n'); } diff --git a/llvm/test/Assembler/upgrade-loop-metadata.ll b/llvm/test/Assembler/upgrade-loop-metadata.ll index f664bdf476a7..1c0311dd09e9 100644 --- a/llvm/test/Assembler/upgrade-loop-metadata.ll +++ b/llvm/test/Assembler/upgrade-loop-metadata.ll @@ -30,7 +30,7 @@ for.end: ; preds = %for.cond ret void } -; CHECK: !{metadata !"llvm.loop.vectorize.unroll", i32 4} +; CHECK: !{metadata !"llvm.loop.interleave.count", i32 4} ; CHECK: !{metadata !"llvm.loop.vectorize.width", i32 8} ; CHECK: !{metadata !"llvm.loop.vectorize.enable", i1 true} diff --git a/llvm/test/Bitcode/upgrade-loop-metadata.ll b/llvm/test/Bitcode/upgrade-loop-metadata.ll index 1a45056d397c..67a8d3935926 100644 --- a/llvm/test/Bitcode/upgrade-loop-metadata.ll +++ b/llvm/test/Bitcode/upgrade-loop-metadata.ll @@ -26,7 +26,7 @@ for.end: ; preds = %for.cond ret void } -; CHECK: !{metadata !"llvm.loop.vectorize.unroll", i32 4} +; CHECK: !{metadata !"llvm.loop.interleave.count", i32 4} ; CHECK: !{metadata !"llvm.loop.vectorize.width", i32 8} ; CHECK: !{metadata !"llvm.loop.vectorize.enable", i1 true} diff --git a/llvm/test/Transforms/LoopVectorize/X86/already-vectorized.ll b/llvm/test/Transforms/LoopVectorize/X86/already-vectorized.ll index fce3b70f596a..9c69ba87f392 100644 --- a/llvm/test/Transforms/LoopVectorize/X86/already-vectorized.ll +++ b/llvm/test/Transforms/LoopVectorize/X86/already-vectorized.ll @@ -41,6 +41,6 @@ for.end: ; preds = %for.body ; Now, we check for the Hint metadata ; CHECK: [[vect]] = metadata !{metadata [[vect]], metadata [[width:![0-9]+]], metadata [[unroll:![0-9]+]]} ; CHECK: [[width]] = metadata !{metadata !"llvm.loop.vectorize.width", i32 1} -; CHECK: [[unroll]] = metadata !{metadata !"llvm.loop.vectorize.unroll", i32 1} +; CHECK: [[unroll]] = metadata !{metadata !"llvm.loop.interleave.count", i32 1} ; CHECK: [[scalar]] = metadata !{metadata [[scalar]], metadata [[width]], metadata [[unroll]]} diff --git a/llvm/test/Transforms/LoopVectorize/X86/vectorization-remarks-missed.ll b/llvm/test/Transforms/LoopVectorize/X86/vectorization-remarks-missed.ll index 909dbf57cfb9..93f24cbf4909 100644 --- a/llvm/test/Transforms/LoopVectorize/X86/vectorization-remarks-missed.ll +++ b/llvm/test/Transforms/LoopVectorize/X86/vectorization-remarks-missed.ll @@ -150,7 +150,7 @@ attributes #0 = { nounwind } !25 = metadata !{i32 12, i32 8, metadata !26, null} !26 = metadata !{i32 786443, metadata !1, metadata !7, i32 12, i32 3, i32 0, i32 3} !27 = metadata !{metadata !27, metadata !28, metadata !29} -!28 = metadata !{metadata !"llvm.loop.vectorize.unroll", i32 1} +!28 = metadata !{metadata !"llvm.loop.interleave.count", i32 1} !29 = metadata !{metadata !"llvm.loop.vectorize.width", i32 1} !30 = metadata !{i32 13, i32 5, metadata !26, null} !31 = metadata !{i32 14, i32 1, metadata !7, null} diff --git a/llvm/test/Transforms/LoopVectorize/metadata-unroll.ll b/llvm/test/Transforms/LoopVectorize/metadata-unroll.ll index 2fcc53a3156e..848f1f9601b9 100644 --- a/llvm/test/Transforms/LoopVectorize/metadata-unroll.ll +++ b/llvm/test/Transforms/LoopVectorize/metadata-unroll.ll @@ -38,4 +38,4 @@ define void @inc(i32 %n) nounwind uwtable noinline ssp { } !0 = metadata !{metadata !0, metadata !1} -!1 = metadata !{metadata !"llvm.loop.vectorize.unroll", i32 2} +!1 = metadata !{metadata !"llvm.loop.interleave.count", i32 2} diff --git a/llvm/test/Transforms/LoopVectorize/vectorize-once.ll b/llvm/test/Transforms/LoopVectorize/vectorize-once.ll index 47de13dbfe20..97654f495018 100644 --- a/llvm/test/Transforms/LoopVectorize/vectorize-once.ll +++ b/llvm/test/Transforms/LoopVectorize/vectorize-once.ll @@ -70,7 +70,7 @@ attributes #0 = { nounwind readonly ssp uwtable "fp-contract-model"="standard" " ; CHECK: !0 = metadata !{metadata !0, metadata !1, metadata !2} ; CHECK: !1 = metadata !{metadata !"llvm.loop.vectorize.width", i32 1} -; CHECK: !2 = metadata !{metadata !"llvm.loop.vectorize.unroll", i32 1} +; CHECK: !2 = metadata !{metadata !"llvm.loop.interleave.count", i32 1} ; CHECK: !3 = metadata !{metadata !3, metadata !1, metadata !2} !0 = metadata !{metadata !0, metadata !1}