llvm-project
1516 строк · 55.5 Кб
1//===----------- VectorUtils.cpp - Vectorizer utility functions -----------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file defines vectorizer utilities.10//11//===----------------------------------------------------------------------===//12/// Maximum factor for an interleaved memory access.37static cl::opt<unsigned> MaxInterleaveGroupFactor(38"max-interleave-group-factor", cl::Hidden,39cl::desc("Maximum factor for an interleaved access group (default = 8)"),40cl::init(8));41/// Return true if all of the intrinsic's arguments and return type are scalars43/// for the scalar form of the intrinsic, and vectors for the vector form of the44/// intrinsic (except operands that are marked as always being scalar by45/// isVectorIntrinsicWithScalarOpAtArg).46bool llvm::isTriviallyVectorizable(Intrinsic::ID ID) {47switch (ID) {48case Intrinsic::abs: // Begin integer bit-manipulation.49case Intrinsic::bswap:50case Intrinsic::bitreverse:51case Intrinsic::ctpop:52case Intrinsic::ctlz:53case Intrinsic::cttz:54case Intrinsic::fshl:55case Intrinsic::fshr:56case Intrinsic::smax:57case Intrinsic::smin:58case Intrinsic::umax:59case Intrinsic::umin:60case Intrinsic::sadd_sat:61case Intrinsic::ssub_sat:62case Intrinsic::uadd_sat:63case Intrinsic::usub_sat:64case Intrinsic::smul_fix:65case Intrinsic::smul_fix_sat:66case Intrinsic::umul_fix:67case Intrinsic::umul_fix_sat:68case Intrinsic::sqrt: // Begin floating-point.69case Intrinsic::sin:70case Intrinsic::cos:71case Intrinsic::tan:72case Intrinsic::exp:73case Intrinsic::exp2:74case Intrinsic::log:75case Intrinsic::log10:76case Intrinsic::log2:77case Intrinsic::fabs:78case Intrinsic::minnum:79case Intrinsic::maxnum:80case Intrinsic::minimum:81case Intrinsic::maximum:82case Intrinsic::copysign:83case Intrinsic::floor:84case Intrinsic::ceil:85case Intrinsic::trunc:86case Intrinsic::rint:87case Intrinsic::nearbyint:88case Intrinsic::round:89case Intrinsic::roundeven:90case Intrinsic::pow:91case Intrinsic::fma:92case Intrinsic::fmuladd:93case Intrinsic::is_fpclass:94case Intrinsic::powi:95case Intrinsic::canonicalize:96case Intrinsic::fptosi_sat:97case Intrinsic::fptoui_sat:98case Intrinsic::lrint:99case Intrinsic::llrint:100return true;101default:102return false;103}104}105/// Identifies if the vector form of the intrinsic has a scalar operand.107bool llvm::isVectorIntrinsicWithScalarOpAtArg(Intrinsic::ID ID,108unsigned ScalarOpdIdx) {109switch (ID) {110case Intrinsic::abs:111case Intrinsic::ctlz:112case Intrinsic::cttz:113case Intrinsic::is_fpclass:114case Intrinsic::powi:115return (ScalarOpdIdx == 1);116case Intrinsic::smul_fix:117case Intrinsic::smul_fix_sat:118case Intrinsic::umul_fix:119case Intrinsic::umul_fix_sat:120return (ScalarOpdIdx == 2);121default:122return false;123}124}125}144/// Returns intrinsic ID for call.146/// For the input call instruction it finds mapping intrinsic and returns147/// its ID, in case it does not found it return not_intrinsic.148Intrinsic::ID llvm::getVectorIntrinsicIDForCall(const CallInst *CI,149const TargetLibraryInfo *TLI) {150Intrinsic::ID ID = getIntrinsicForCallSite(*CI, TLI);151if (ID == Intrinsic::not_intrinsic)152return Intrinsic::not_intrinsic;153}161/// Given a vector and an element number, see if the scalar value is163/// already around as a register, for example if it were inserted then extracted164/// from the vector.165Value *llvm::findScalarElement(Value *V, unsigned EltNo) {166assert(V->getType()->isVectorTy() && "Not looking at a vector?");167VectorType *VTy = cast<VectorType>(V->getType());168// For fixed-length vector, return poison for out of range access.169if (auto *FVTy = dyn_cast<FixedVectorType>(VTy)) {170unsigned Width = FVTy->getNumElements();171if (EltNo >= Width)172return PoisonValue::get(FVTy->getElementType());173}174}228}246/// Get splat value if the input is a splat vector or return nullptr.248/// This function is not fully general. It checks only 2 cases:249/// the input value is (1) a splat constant vector or (2) a sequence250/// of instructions that broadcasts a scalar at element 0.251Value *llvm::getSplatValue(const Value *V) {252if (isa<VectorType>(V->getType()))253if (auto *C = dyn_cast<Constant>(V))254return C->getSplatValue();255}265}311}348}369}420}445}459}569}727/// Add all access groups in @p AccGroups to @p List.729template <typename ListT>730static void addToAccessGroupList(ListT &List, MDNode *AccGroups) {731// Interpret an access group as a list containing itself.732if (AccGroups->getNumOperands() == 0) {733assert(isValidAsAccessGroup(AccGroups) && "Node must be an access group");734List.insert(AccGroups);735return;736}737}744}765}811/// \returns \p I after propagating metadata from \p VL.813Instruction *llvm::propagateMetadata(Instruction *Inst, ArrayRef<Value *> VL) {814if (VL.empty())815return Inst;816Instruction *I0 = cast<Instruction>(VL[0]);817SmallVector<std::pair<unsigned, MDNode *>, 4> Metadata;818I0->getAllMetadataOtherThanDebugLoc(Metadata);819}861}881}891}901}910}923}940/// A helper function for concatenating vectors. This function concatenates two942/// vectors having the same element type. If the second vector has fewer943/// elements than the first, it is padded with undefs.944static Value *concatenateTwoVectors(IRBuilderBase &Builder, Value *V1,945Value *V2) {946VectorType *VecTy1 = dyn_cast<VectorType>(V1->getType());947VectorType *VecTy2 = dyn_cast<VectorType>(V2->getType());948assert(VecTy1 && VecTy2 &&949VecTy1->getScalarType() == VecTy2->getScalarType() &&950"Expect two vectors with the same element type");951}965}993}1019}1045}1070/// TODO: This is a lot like known bits, but for1072/// vectors. Is there something we can common this with?1073APInt llvm::possiblyDemandedEltsInMask(Value *Mask) {1074assert(isa<FixedVectorType>(Mask->getType()) &&1075isa<IntegerType>(Mask->getType()->getScalarType()) &&1076cast<IntegerType>(Mask->getType()->getScalarType())->getBitWidth() ==10771 &&1078"Mask must be a fixed width vector of i1");1079}1089}1094}1137// Analyze interleaved accesses and collect them into interleaved load and1139// store groups.1140//1141// When generating code for an interleaved load group, we effectively hoist all1142// loads in the group to the location of the first load in program order. When1143// generating code for an interleaved store group, we sink all stores to the1144// location of the last store. This code motion can change the order of load1145// and store instructions and may break dependences.1146//1147// The code generation strategy mentioned above ensures that we won't violate1148// any write-after-read (WAR) dependences.1149//1150// E.g., for the WAR dependence: a = A[i]; // (1)1151// A[i] = b; // (2)1152//1153// The store group of (2) is always inserted at or below (2), and the load1154// group of (1) is always inserted at or above (1). Thus, the instructions will1155// never be reordered. All other dependences are checked to ensure the1156// correctness of the instruction reordering.1157//1158// The algorithm visits all memory accesses in the loop in bottom-up program1159// order. Program order is established by traversing the blocks in the loop in1160// reverse postorder when collecting the accesses.1161//1162// We visit the memory accesses in bottom-up order because it can simplify the1163// construction of store groups in the presence of write-after-write (WAW)1164// dependences.1165//1166// E.g., for the WAW dependence: A[i] = a; // (1)1167// A[i] = b; // (2)1168// A[i + 1] = c; // (3)1169//1170// We will first create a store group with (3) and (2). (1) can't be added to1171// this group because it and (2) are dependent. However, (1) can be grouped1172// with other accesses that may precede it in program order. Note that a1173// bottom-up order does not imply that WAW dependences should not be checked.1174void InterleavedAccessInfo::analyzeInterleaving(1175bool EnablePredicatedInterleavedMemAccesses) {1176LLVM_DEBUG(dbgs() << "LV: Analyzing interleaved accesses...\n");1177const auto &Strides = LAI->getSymbolicStrides();1178}1477}1502}1507}1516} // namespace llvm1517