summaryrefslogtreecommitdiff
path: root/external/apache2/llvm/dist/clang/lib/CodeGen/CGLoopInfo.cpp
diff options
context:
space:
mode:
authorjoerg <joerg@NetBSD.org>2021-05-30 01:25:15 +0000
committerjoerg <joerg@NetBSD.org>2021-05-30 01:25:15 +0000
commit13fbcb42fff2606c3ce926a7c4bb8989aa18758c (patch)
treedcace6c89967106796d44b1a7c5e2269a20967e2 /external/apache2/llvm/dist/clang/lib/CodeGen/CGLoopInfo.cpp
parent13338e33506ade63440b4ba74479551bdc21c0f6 (diff)
Import clang 249b40b558955afe5ac2b549edcf2d7f859c8cc9.
Diffstat (limited to 'external/apache2/llvm/dist/clang/lib/CodeGen/CGLoopInfo.cpp')
-rw-r--r--external/apache2/llvm/dist/clang/lib/CodeGen/CGLoopInfo.cpp128
1 files changed, 87 insertions, 41 deletions
diff --git a/external/apache2/llvm/dist/clang/lib/CodeGen/CGLoopInfo.cpp b/external/apache2/llvm/dist/clang/lib/CodeGen/CGLoopInfo.cpp
index 6822c6286fe..12a6cd8da60 100644
--- a/external/apache2/llvm/dist/clang/lib/CodeGen/CGLoopInfo.cpp
+++ b/external/apache2/llvm/dist/clang/lib/CodeGen/CGLoopInfo.cpp
@@ -9,6 +9,8 @@
#include "CGLoopInfo.h"
#include "clang/AST/ASTContext.h"
#include "clang/AST/Attr.h"
+#include "clang/AST/Expr.h"
+#include "clang/Basic/CodeGenOptions.h"
#include "llvm/IR/BasicBlock.h"
#include "llvm/IR/CFG.h"
#include "llvm/IR/Constants.h"
@@ -22,8 +24,7 @@ MDNode *
LoopInfo::createLoopPropertiesMetadata(ArrayRef<Metadata *> LoopProperties) {
LLVMContext &Ctx = Header->getContext();
SmallVector<Metadata *, 4> NewLoopProperties;
- TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
- NewLoopProperties.push_back(TempNode.get());
+ NewLoopProperties.push_back(nullptr);
NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
MDNode *LoopID = MDNode::getDistinct(Ctx, NewLoopProperties);
@@ -56,8 +57,7 @@ MDNode *LoopInfo::createPipeliningMetadata(const LoopAttributes &Attrs,
}
SmallVector<Metadata *, 4> Args;
- TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
- Args.push_back(TempNode.get());
+ Args.push_back(nullptr);
Args.append(LoopProperties.begin(), LoopProperties.end());
if (Attrs.PipelineInitiationInterval > 0) {
@@ -111,8 +111,7 @@ LoopInfo::createPartialUnrollMetadata(const LoopAttributes &Attrs,
FollowupHasTransforms);
SmallVector<Metadata *, 4> Args;
- TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
- Args.push_back(TempNode.get());
+ Args.push_back(nullptr);
Args.append(LoopProperties.begin(), LoopProperties.end());
// Setting unroll.count
@@ -174,8 +173,7 @@ LoopInfo::createUnrollAndJamMetadata(const LoopAttributes &Attrs,
FollowupHasTransforms);
SmallVector<Metadata *, 4> Args;
- TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
- Args.push_back(TempNode.get());
+ Args.push_back(nullptr);
Args.append(LoopProperties.begin(), LoopProperties.end());
// Setting unroll_and_jam.count
@@ -219,7 +217,8 @@ LoopInfo::createLoopVectorizeMetadata(const LoopAttributes &Attrs,
Enabled = false;
else if (Attrs.VectorizeEnable != LoopAttributes::Unspecified ||
Attrs.VectorizePredicateEnable != LoopAttributes::Unspecified ||
- Attrs.InterleaveCount != 0 || Attrs.VectorizeWidth != 0)
+ Attrs.InterleaveCount != 0 || Attrs.VectorizeWidth != 0 ||
+ Attrs.VectorizeScalable != LoopAttributes::Unspecified)
Enabled = true;
if (Enabled != true) {
@@ -248,16 +247,13 @@ LoopInfo::createLoopVectorizeMetadata(const LoopAttributes &Attrs,
FollowupHasTransforms);
SmallVector<Metadata *, 4> Args;
- TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
- Args.push_back(TempNode.get());
+ Args.push_back(nullptr);
Args.append(LoopProperties.begin(), LoopProperties.end());
- // Setting vectorize.predicate
+ // Setting vectorize.predicate when it has been specified and vectorization
+ // has not been disabled.
bool IsVectorPredicateEnabled = false;
- if (Attrs.VectorizePredicateEnable != LoopAttributes::Unspecified &&
- Attrs.VectorizeEnable != LoopAttributes::Disable &&
- Attrs.VectorizeWidth < 1) {
-
+ if (Attrs.VectorizePredicateEnable != LoopAttributes::Unspecified) {
IsVectorPredicateEnabled =
(Attrs.VectorizePredicateEnable == LoopAttributes::Enable);
@@ -274,6 +270,16 @@ LoopInfo::createLoopVectorizeMetadata(const LoopAttributes &Attrs,
MDString::get(Ctx, "llvm.loop.vectorize.width"),
ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
Attrs.VectorizeWidth))};
+
+ Args.push_back(MDNode::get(Ctx, Vals));
+ }
+
+ if (Attrs.VectorizeScalable != LoopAttributes::Unspecified) {
+ bool IsScalable = Attrs.VectorizeScalable == LoopAttributes::Enable;
+ Metadata *Vals[] = {
+ MDString::get(Ctx, "llvm.loop.vectorize.scalable.enable"),
+ ConstantAsMetadata::get(
+ ConstantInt::get(llvm::Type::getInt1Ty(Ctx), IsScalable))};
Args.push_back(MDNode::get(Ctx, Vals));
}
@@ -286,17 +292,25 @@ LoopInfo::createLoopVectorizeMetadata(const LoopAttributes &Attrs,
Args.push_back(MDNode::get(Ctx, Vals));
}
- // Setting vectorize.enable
+ // vectorize.enable is set if:
+ // 1) loop hint vectorize.enable is set, or
+ // 2) it is implied when vectorize.predicate is set, or
+ // 3) it is implied when vectorize.width is set to a value > 1
+ // 4) it is implied when vectorize.scalable.enable is true
+ // 5) it is implied when vectorize.width is unset (0) and the user
+ // explicitly requested fixed-width vectorization, i.e.
+ // vectorize.scalable.enable is false.
if (Attrs.VectorizeEnable != LoopAttributes::Unspecified ||
- IsVectorPredicateEnabled) {
- Metadata *Vals[] = {
- MDString::get(Ctx, "llvm.loop.vectorize.enable"),
- ConstantAsMetadata::get(ConstantInt::get(
- llvm::Type::getInt1Ty(Ctx),
- IsVectorPredicateEnabled
- ? true
- : (Attrs.VectorizeEnable == LoopAttributes::Enable)))};
- Args.push_back(MDNode::get(Ctx, Vals));
+ (IsVectorPredicateEnabled && Attrs.VectorizeWidth != 1) ||
+ Attrs.VectorizeWidth > 1 ||
+ Attrs.VectorizeScalable == LoopAttributes::Enable ||
+ (Attrs.VectorizeScalable == LoopAttributes::Disable &&
+ Attrs.VectorizeWidth != 1)) {
+ bool AttrVal = Attrs.VectorizeEnable != LoopAttributes::Disable;
+ Args.push_back(
+ MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.vectorize.enable"),
+ ConstantAsMetadata::get(ConstantInt::get(
+ llvm::Type::getInt1Ty(Ctx), AttrVal))}));
}
if (FollowupHasTransforms)
@@ -304,7 +318,7 @@ LoopInfo::createLoopVectorizeMetadata(const LoopAttributes &Attrs,
Ctx,
{MDString::get(Ctx, "llvm.loop.vectorize.followup_all"), Followup}));
- MDNode *LoopID = MDNode::get(Ctx, Args);
+ MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
LoopID->replaceOperandWith(0, LoopID);
HasUserTransforms = true;
return LoopID;
@@ -341,8 +355,7 @@ LoopInfo::createLoopDistributeMetadata(const LoopAttributes &Attrs,
createLoopVectorizeMetadata(Attrs, LoopProperties, FollowupHasTransforms);
SmallVector<Metadata *, 4> Args;
- TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
- Args.push_back(TempNode.get());
+ Args.push_back(nullptr);
Args.append(LoopProperties.begin(), LoopProperties.end());
Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.distribute.enable"),
@@ -356,7 +369,7 @@ LoopInfo::createLoopDistributeMetadata(const LoopAttributes &Attrs,
Ctx,
{MDString::get(Ctx, "llvm.loop.distribute.followup_all"), Followup}));
- MDNode *LoopID = MDNode::get(Ctx, Args);
+ MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
LoopID->replaceOperandWith(0, LoopID);
HasUserTransforms = true;
return LoopID;
@@ -386,8 +399,7 @@ MDNode *LoopInfo::createFullUnrollMetadata(const LoopAttributes &Attrs,
}
SmallVector<Metadata *, 4> Args;
- TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
- Args.push_back(TempNode.get());
+ Args.push_back(nullptr);
Args.append(LoopProperties.begin(), LoopProperties.end());
Args.push_back(MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.full")));
@@ -415,10 +427,14 @@ MDNode *LoopInfo::createMetadata(
LoopProperties.push_back(EndLoc.getAsMDNode());
}
+ LLVMContext &Ctx = Header->getContext();
+ if (Attrs.MustProgress)
+ LoopProperties.push_back(
+ MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.mustprogress")));
+
assert(!!AccGroup == Attrs.IsParallel &&
"There must be an access group iff the loop is parallel");
if (Attrs.IsParallel) {
- LLVMContext &Ctx = Header->getContext();
LoopProperties.push_back(MDNode::get(
Ctx, {MDString::get(Ctx, "llvm.loop.parallel_accesses"), AccGroup}));
}
@@ -433,13 +449,15 @@ LoopAttributes::LoopAttributes(bool IsParallel)
UnrollEnable(LoopAttributes::Unspecified),
UnrollAndJamEnable(LoopAttributes::Unspecified),
VectorizePredicateEnable(LoopAttributes::Unspecified), VectorizeWidth(0),
- InterleaveCount(0), UnrollCount(0), UnrollAndJamCount(0),
+ VectorizeScalable(LoopAttributes::Unspecified), InterleaveCount(0),
+ UnrollCount(0), UnrollAndJamCount(0),
DistributeEnable(LoopAttributes::Unspecified), PipelineDisabled(false),
- PipelineInitiationInterval(0) {}
+ PipelineInitiationInterval(0), MustProgress(false) {}
void LoopAttributes::clear() {
IsParallel = false;
VectorizeWidth = 0;
+ VectorizeScalable = LoopAttributes::Unspecified;
InterleaveCount = 0;
UnrollCount = 0;
UnrollAndJamCount = 0;
@@ -450,6 +468,7 @@ void LoopAttributes::clear() {
DistributeEnable = LoopAttributes::Unspecified;
PipelineDisabled = false;
PipelineInitiationInterval = 0;
+ MustProgress = false;
}
LoopInfo::LoopInfo(BasicBlock *Header, const LoopAttributes &Attrs,
@@ -465,6 +484,7 @@ LoopInfo::LoopInfo(BasicBlock *Header, const LoopAttributes &Attrs,
}
if (!Attrs.IsParallel && Attrs.VectorizeWidth == 0 &&
+ Attrs.VectorizeScalable == LoopAttributes::Unspecified &&
Attrs.InterleaveCount == 0 && Attrs.UnrollCount == 0 &&
Attrs.UnrollAndJamCount == 0 && !Attrs.PipelineDisabled &&
Attrs.PipelineInitiationInterval == 0 &&
@@ -473,7 +493,7 @@ LoopInfo::LoopInfo(BasicBlock *Header, const LoopAttributes &Attrs,
Attrs.UnrollEnable == LoopAttributes::Unspecified &&
Attrs.UnrollAndJamEnable == LoopAttributes::Unspecified &&
Attrs.DistributeEnable == LoopAttributes::Unspecified && !StartLoc &&
- !EndLoc)
+ !EndLoc && !Attrs.MustProgress)
return;
TempLoopID = MDNode::getTemporary(Header->getContext(), None);
@@ -500,6 +520,7 @@ void LoopInfo::finish() {
BeforeJam.IsParallel = AfterJam.IsParallel = Attrs.IsParallel;
BeforeJam.VectorizeWidth = Attrs.VectorizeWidth;
+ BeforeJam.VectorizeScalable = Attrs.VectorizeScalable;
BeforeJam.InterleaveCount = Attrs.InterleaveCount;
BeforeJam.VectorizeEnable = Attrs.VectorizeEnable;
BeforeJam.DistributeEnable = Attrs.DistributeEnable;
@@ -542,7 +563,8 @@ void LoopInfo::finish() {
SmallVector<Metadata *, 1> BeforeLoopProperties;
if (BeforeJam.VectorizeEnable != LoopAttributes::Unspecified ||
BeforeJam.VectorizePredicateEnable != LoopAttributes::Unspecified ||
- BeforeJam.InterleaveCount != 0 || BeforeJam.VectorizeWidth != 0)
+ BeforeJam.InterleaveCount != 0 || BeforeJam.VectorizeWidth != 0 ||
+ BeforeJam.VectorizeScalable == LoopAttributes::Enable)
BeforeLoopProperties.push_back(
MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.isvectorized")));
@@ -571,10 +593,10 @@ void LoopInfoStack::push(BasicBlock *Header, const llvm::DebugLoc &StartLoc,
}
void LoopInfoStack::push(BasicBlock *Header, clang::ASTContext &Ctx,
+ const clang::CodeGenOptions &CGOpts,
ArrayRef<const clang::Attr *> Attrs,
const llvm::DebugLoc &StartLoc,
- const llvm::DebugLoc &EndLoc) {
-
+ const llvm::DebugLoc &EndLoc, bool MustProgress) {
// Identify loop hint attributes from Attrs.
for (const auto *Attr : Attrs) {
const LoopHintAttr *LH = dyn_cast<LoopHintAttr>(Attr);
@@ -619,6 +641,7 @@ void LoopInfoStack::push(BasicBlock *Header, clang::ASTContext &Ctx,
case LoopHintAttr::Vectorize:
// Disable vectorization by specifying a width of 1.
setVectorizeWidth(1);
+ setVectorizeScalable(LoopAttributes::Unspecified);
break;
case LoopHintAttr::Interleave:
// Disable interleaving by speciyfing a count of 1.
@@ -720,11 +743,23 @@ void LoopInfoStack::push(BasicBlock *Header, clang::ASTContext &Ctx,
break;
}
break;
- case LoopHintAttr::Numeric:
+ case LoopHintAttr::FixedWidth:
+ case LoopHintAttr::ScalableWidth:
switch (Option) {
case LoopHintAttr::VectorizeWidth:
- setVectorizeWidth(ValueInt);
+ setVectorizeScalable(State == LoopHintAttr::ScalableWidth
+ ? LoopAttributes::Enable
+ : LoopAttributes::Disable);
+ if (LH->getValue())
+ setVectorizeWidth(ValueInt);
+ break;
+ default:
+ llvm_unreachable("Options cannot be used with 'scalable' hint.");
break;
+ }
+ break;
+ case LoopHintAttr::Numeric:
+ switch (Option) {
case LoopHintAttr::InterleaveCount:
setInterleaveCount(ValueInt);
break;
@@ -741,6 +776,7 @@ void LoopInfoStack::push(BasicBlock *Header, clang::ASTContext &Ctx,
case LoopHintAttr::UnrollAndJam:
case LoopHintAttr::VectorizePredicate:
case LoopHintAttr::Vectorize:
+ case LoopHintAttr::VectorizeWidth:
case LoopHintAttr::Interleave:
case LoopHintAttr::Distribute:
case LoopHintAttr::PipelineDisabled:
@@ -751,6 +787,16 @@ void LoopInfoStack::push(BasicBlock *Header, clang::ASTContext &Ctx,
}
}
+ setMustProgress(MustProgress);
+
+ if (CGOpts.OptimizationLevel > 0)
+ // Disable unrolling for the loop, if unrolling is disabled (via
+ // -fno-unroll-loops) and no pragmas override the decision.
+ if (!CGOpts.UnrollLoops &&
+ (StagedAttrs.UnrollEnable == LoopAttributes::Unspecified &&
+ StagedAttrs.UnrollCount == 0))
+ setUnrollState(LoopAttributes::Disable);
+
/// Stage the attributes.
push(Header, StartLoc, EndLoc);
}