diff options
| author | joerg <joerg@NetBSD.org> | 2021-05-30 01:25:15 +0000 |
|---|---|---|
| committer | joerg <joerg@NetBSD.org> | 2021-05-30 01:25:15 +0000 |
| commit | 13fbcb42fff2606c3ce926a7c4bb8989aa18758c (patch) | |
| tree | dcace6c89967106796d44b1a7c5e2269a20967e2 /external/apache2/llvm/dist/clang/lib/CodeGen/CGLoopInfo.cpp | |
| parent | 13338e33506ade63440b4ba74479551bdc21c0f6 (diff) | |
Import clang 249b40b558955afe5ac2b549edcf2d7f859c8cc9.
Diffstat (limited to 'external/apache2/llvm/dist/clang/lib/CodeGen/CGLoopInfo.cpp')
| -rw-r--r-- | external/apache2/llvm/dist/clang/lib/CodeGen/CGLoopInfo.cpp | 128 |
1 files changed, 87 insertions, 41 deletions
diff --git a/external/apache2/llvm/dist/clang/lib/CodeGen/CGLoopInfo.cpp b/external/apache2/llvm/dist/clang/lib/CodeGen/CGLoopInfo.cpp index 6822c6286fe..12a6cd8da60 100644 --- a/external/apache2/llvm/dist/clang/lib/CodeGen/CGLoopInfo.cpp +++ b/external/apache2/llvm/dist/clang/lib/CodeGen/CGLoopInfo.cpp @@ -9,6 +9,8 @@ #include "CGLoopInfo.h" #include "clang/AST/ASTContext.h" #include "clang/AST/Attr.h" +#include "clang/AST/Expr.h" +#include "clang/Basic/CodeGenOptions.h" #include "llvm/IR/BasicBlock.h" #include "llvm/IR/CFG.h" #include "llvm/IR/Constants.h" @@ -22,8 +24,7 @@ MDNode * LoopInfo::createLoopPropertiesMetadata(ArrayRef<Metadata *> LoopProperties) { LLVMContext &Ctx = Header->getContext(); SmallVector<Metadata *, 4> NewLoopProperties; - TempMDTuple TempNode = MDNode::getTemporary(Ctx, None); - NewLoopProperties.push_back(TempNode.get()); + NewLoopProperties.push_back(nullptr); NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end()); MDNode *LoopID = MDNode::getDistinct(Ctx, NewLoopProperties); @@ -56,8 +57,7 @@ MDNode *LoopInfo::createPipeliningMetadata(const LoopAttributes &Attrs, } SmallVector<Metadata *, 4> Args; - TempMDTuple TempNode = MDNode::getTemporary(Ctx, None); - Args.push_back(TempNode.get()); + Args.push_back(nullptr); Args.append(LoopProperties.begin(), LoopProperties.end()); if (Attrs.PipelineInitiationInterval > 0) { @@ -111,8 +111,7 @@ LoopInfo::createPartialUnrollMetadata(const LoopAttributes &Attrs, FollowupHasTransforms); SmallVector<Metadata *, 4> Args; - TempMDTuple TempNode = MDNode::getTemporary(Ctx, None); - Args.push_back(TempNode.get()); + Args.push_back(nullptr); Args.append(LoopProperties.begin(), LoopProperties.end()); // Setting unroll.count @@ -174,8 +173,7 @@ LoopInfo::createUnrollAndJamMetadata(const LoopAttributes &Attrs, FollowupHasTransforms); SmallVector<Metadata *, 4> Args; - TempMDTuple TempNode = MDNode::getTemporary(Ctx, None); - Args.push_back(TempNode.get()); + Args.push_back(nullptr); Args.append(LoopProperties.begin(), LoopProperties.end()); // Setting unroll_and_jam.count @@ -219,7 +217,8 @@ LoopInfo::createLoopVectorizeMetadata(const LoopAttributes &Attrs, Enabled = false; else if (Attrs.VectorizeEnable != LoopAttributes::Unspecified || Attrs.VectorizePredicateEnable != LoopAttributes::Unspecified || - Attrs.InterleaveCount != 0 || Attrs.VectorizeWidth != 0) + Attrs.InterleaveCount != 0 || Attrs.VectorizeWidth != 0 || + Attrs.VectorizeScalable != LoopAttributes::Unspecified) Enabled = true; if (Enabled != true) { @@ -248,16 +247,13 @@ LoopInfo::createLoopVectorizeMetadata(const LoopAttributes &Attrs, FollowupHasTransforms); SmallVector<Metadata *, 4> Args; - TempMDTuple TempNode = MDNode::getTemporary(Ctx, None); - Args.push_back(TempNode.get()); + Args.push_back(nullptr); Args.append(LoopProperties.begin(), LoopProperties.end()); - // Setting vectorize.predicate + // Setting vectorize.predicate when it has been specified and vectorization + // has not been disabled. bool IsVectorPredicateEnabled = false; - if (Attrs.VectorizePredicateEnable != LoopAttributes::Unspecified && - Attrs.VectorizeEnable != LoopAttributes::Disable && - Attrs.VectorizeWidth < 1) { - + if (Attrs.VectorizePredicateEnable != LoopAttributes::Unspecified) { IsVectorPredicateEnabled = (Attrs.VectorizePredicateEnable == LoopAttributes::Enable); @@ -274,6 +270,16 @@ LoopInfo::createLoopVectorizeMetadata(const LoopAttributes &Attrs, MDString::get(Ctx, "llvm.loop.vectorize.width"), ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx), Attrs.VectorizeWidth))}; + + Args.push_back(MDNode::get(Ctx, Vals)); + } + + if (Attrs.VectorizeScalable != LoopAttributes::Unspecified) { + bool IsScalable = Attrs.VectorizeScalable == LoopAttributes::Enable; + Metadata *Vals[] = { + MDString::get(Ctx, "llvm.loop.vectorize.scalable.enable"), + ConstantAsMetadata::get( + ConstantInt::get(llvm::Type::getInt1Ty(Ctx), IsScalable))}; Args.push_back(MDNode::get(Ctx, Vals)); } @@ -286,17 +292,25 @@ LoopInfo::createLoopVectorizeMetadata(const LoopAttributes &Attrs, Args.push_back(MDNode::get(Ctx, Vals)); } - // Setting vectorize.enable + // vectorize.enable is set if: + // 1) loop hint vectorize.enable is set, or + // 2) it is implied when vectorize.predicate is set, or + // 3) it is implied when vectorize.width is set to a value > 1 + // 4) it is implied when vectorize.scalable.enable is true + // 5) it is implied when vectorize.width is unset (0) and the user + // explicitly requested fixed-width vectorization, i.e. + // vectorize.scalable.enable is false. if (Attrs.VectorizeEnable != LoopAttributes::Unspecified || - IsVectorPredicateEnabled) { - Metadata *Vals[] = { - MDString::get(Ctx, "llvm.loop.vectorize.enable"), - ConstantAsMetadata::get(ConstantInt::get( - llvm::Type::getInt1Ty(Ctx), - IsVectorPredicateEnabled - ? true - : (Attrs.VectorizeEnable == LoopAttributes::Enable)))}; - Args.push_back(MDNode::get(Ctx, Vals)); + (IsVectorPredicateEnabled && Attrs.VectorizeWidth != 1) || + Attrs.VectorizeWidth > 1 || + Attrs.VectorizeScalable == LoopAttributes::Enable || + (Attrs.VectorizeScalable == LoopAttributes::Disable && + Attrs.VectorizeWidth != 1)) { + bool AttrVal = Attrs.VectorizeEnable != LoopAttributes::Disable; + Args.push_back( + MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.vectorize.enable"), + ConstantAsMetadata::get(ConstantInt::get( + llvm::Type::getInt1Ty(Ctx), AttrVal))})); } if (FollowupHasTransforms) @@ -304,7 +318,7 @@ LoopInfo::createLoopVectorizeMetadata(const LoopAttributes &Attrs, Ctx, {MDString::get(Ctx, "llvm.loop.vectorize.followup_all"), Followup})); - MDNode *LoopID = MDNode::get(Ctx, Args); + MDNode *LoopID = MDNode::getDistinct(Ctx, Args); LoopID->replaceOperandWith(0, LoopID); HasUserTransforms = true; return LoopID; @@ -341,8 +355,7 @@ LoopInfo::createLoopDistributeMetadata(const LoopAttributes &Attrs, createLoopVectorizeMetadata(Attrs, LoopProperties, FollowupHasTransforms); SmallVector<Metadata *, 4> Args; - TempMDTuple TempNode = MDNode::getTemporary(Ctx, None); - Args.push_back(TempNode.get()); + Args.push_back(nullptr); Args.append(LoopProperties.begin(), LoopProperties.end()); Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.distribute.enable"), @@ -356,7 +369,7 @@ LoopInfo::createLoopDistributeMetadata(const LoopAttributes &Attrs, Ctx, {MDString::get(Ctx, "llvm.loop.distribute.followup_all"), Followup})); - MDNode *LoopID = MDNode::get(Ctx, Args); + MDNode *LoopID = MDNode::getDistinct(Ctx, Args); LoopID->replaceOperandWith(0, LoopID); HasUserTransforms = true; return LoopID; @@ -386,8 +399,7 @@ MDNode *LoopInfo::createFullUnrollMetadata(const LoopAttributes &Attrs, } SmallVector<Metadata *, 4> Args; - TempMDTuple TempNode = MDNode::getTemporary(Ctx, None); - Args.push_back(TempNode.get()); + Args.push_back(nullptr); Args.append(LoopProperties.begin(), LoopProperties.end()); Args.push_back(MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.full"))); @@ -415,10 +427,14 @@ MDNode *LoopInfo::createMetadata( LoopProperties.push_back(EndLoc.getAsMDNode()); } + LLVMContext &Ctx = Header->getContext(); + if (Attrs.MustProgress) + LoopProperties.push_back( + MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.mustprogress"))); + assert(!!AccGroup == Attrs.IsParallel && "There must be an access group iff the loop is parallel"); if (Attrs.IsParallel) { - LLVMContext &Ctx = Header->getContext(); LoopProperties.push_back(MDNode::get( Ctx, {MDString::get(Ctx, "llvm.loop.parallel_accesses"), AccGroup})); } @@ -433,13 +449,15 @@ LoopAttributes::LoopAttributes(bool IsParallel) UnrollEnable(LoopAttributes::Unspecified), UnrollAndJamEnable(LoopAttributes::Unspecified), VectorizePredicateEnable(LoopAttributes::Unspecified), VectorizeWidth(0), - InterleaveCount(0), UnrollCount(0), UnrollAndJamCount(0), + VectorizeScalable(LoopAttributes::Unspecified), InterleaveCount(0), + UnrollCount(0), UnrollAndJamCount(0), DistributeEnable(LoopAttributes::Unspecified), PipelineDisabled(false), - PipelineInitiationInterval(0) {} + PipelineInitiationInterval(0), MustProgress(false) {} void LoopAttributes::clear() { IsParallel = false; VectorizeWidth = 0; + VectorizeScalable = LoopAttributes::Unspecified; InterleaveCount = 0; UnrollCount = 0; UnrollAndJamCount = 0; @@ -450,6 +468,7 @@ void LoopAttributes::clear() { DistributeEnable = LoopAttributes::Unspecified; PipelineDisabled = false; PipelineInitiationInterval = 0; + MustProgress = false; } LoopInfo::LoopInfo(BasicBlock *Header, const LoopAttributes &Attrs, @@ -465,6 +484,7 @@ LoopInfo::LoopInfo(BasicBlock *Header, const LoopAttributes &Attrs, } if (!Attrs.IsParallel && Attrs.VectorizeWidth == 0 && + Attrs.VectorizeScalable == LoopAttributes::Unspecified && Attrs.InterleaveCount == 0 && Attrs.UnrollCount == 0 && Attrs.UnrollAndJamCount == 0 && !Attrs.PipelineDisabled && Attrs.PipelineInitiationInterval == 0 && @@ -473,7 +493,7 @@ LoopInfo::LoopInfo(BasicBlock *Header, const LoopAttributes &Attrs, Attrs.UnrollEnable == LoopAttributes::Unspecified && Attrs.UnrollAndJamEnable == LoopAttributes::Unspecified && Attrs.DistributeEnable == LoopAttributes::Unspecified && !StartLoc && - !EndLoc) + !EndLoc && !Attrs.MustProgress) return; TempLoopID = MDNode::getTemporary(Header->getContext(), None); @@ -500,6 +520,7 @@ void LoopInfo::finish() { BeforeJam.IsParallel = AfterJam.IsParallel = Attrs.IsParallel; BeforeJam.VectorizeWidth = Attrs.VectorizeWidth; + BeforeJam.VectorizeScalable = Attrs.VectorizeScalable; BeforeJam.InterleaveCount = Attrs.InterleaveCount; BeforeJam.VectorizeEnable = Attrs.VectorizeEnable; BeforeJam.DistributeEnable = Attrs.DistributeEnable; @@ -542,7 +563,8 @@ void LoopInfo::finish() { SmallVector<Metadata *, 1> BeforeLoopProperties; if (BeforeJam.VectorizeEnable != LoopAttributes::Unspecified || BeforeJam.VectorizePredicateEnable != LoopAttributes::Unspecified || - BeforeJam.InterleaveCount != 0 || BeforeJam.VectorizeWidth != 0) + BeforeJam.InterleaveCount != 0 || BeforeJam.VectorizeWidth != 0 || + BeforeJam.VectorizeScalable == LoopAttributes::Enable) BeforeLoopProperties.push_back( MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.isvectorized"))); @@ -571,10 +593,10 @@ void LoopInfoStack::push(BasicBlock *Header, const llvm::DebugLoc &StartLoc, } void LoopInfoStack::push(BasicBlock *Header, clang::ASTContext &Ctx, + const clang::CodeGenOptions &CGOpts, ArrayRef<const clang::Attr *> Attrs, const llvm::DebugLoc &StartLoc, - const llvm::DebugLoc &EndLoc) { - + const llvm::DebugLoc &EndLoc, bool MustProgress) { // Identify loop hint attributes from Attrs. for (const auto *Attr : Attrs) { const LoopHintAttr *LH = dyn_cast<LoopHintAttr>(Attr); @@ -619,6 +641,7 @@ void LoopInfoStack::push(BasicBlock *Header, clang::ASTContext &Ctx, case LoopHintAttr::Vectorize: // Disable vectorization by specifying a width of 1. setVectorizeWidth(1); + setVectorizeScalable(LoopAttributes::Unspecified); break; case LoopHintAttr::Interleave: // Disable interleaving by speciyfing a count of 1. @@ -720,11 +743,23 @@ void LoopInfoStack::push(BasicBlock *Header, clang::ASTContext &Ctx, break; } break; - case LoopHintAttr::Numeric: + case LoopHintAttr::FixedWidth: + case LoopHintAttr::ScalableWidth: switch (Option) { case LoopHintAttr::VectorizeWidth: - setVectorizeWidth(ValueInt); + setVectorizeScalable(State == LoopHintAttr::ScalableWidth + ? LoopAttributes::Enable + : LoopAttributes::Disable); + if (LH->getValue()) + setVectorizeWidth(ValueInt); + break; + default: + llvm_unreachable("Options cannot be used with 'scalable' hint."); break; + } + break; + case LoopHintAttr::Numeric: + switch (Option) { case LoopHintAttr::InterleaveCount: setInterleaveCount(ValueInt); break; @@ -741,6 +776,7 @@ void LoopInfoStack::push(BasicBlock *Header, clang::ASTContext &Ctx, case LoopHintAttr::UnrollAndJam: case LoopHintAttr::VectorizePredicate: case LoopHintAttr::Vectorize: + case LoopHintAttr::VectorizeWidth: case LoopHintAttr::Interleave: case LoopHintAttr::Distribute: case LoopHintAttr::PipelineDisabled: @@ -751,6 +787,16 @@ void LoopInfoStack::push(BasicBlock *Header, clang::ASTContext &Ctx, } } + setMustProgress(MustProgress); + + if (CGOpts.OptimizationLevel > 0) + // Disable unrolling for the loop, if unrolling is disabled (via + // -fno-unroll-loops) and no pragmas override the decision. + if (!CGOpts.UnrollLoops && + (StagedAttrs.UnrollEnable == LoopAttributes::Unspecified && + StagedAttrs.UnrollCount == 0)) + setUnrollState(LoopAttributes::Disable); + /// Stage the attributes. push(Header, StartLoc, EndLoc); } |
