derek.holloway
7502bf64a4
CodeQL / Analyze (actions) (push) Canceled after 0s
CodeQL / Analyze (python) (push) Canceled after 0s
pre-commit / pre-commit (push) Canceled after 0s
Multi-Arch CI / setup (push) Canceled after 0s
Publish build_manylinux_x86_64 images / publish_build_manylinux_x86_64 (push) Canceled after 0s
Setup (Multi-Arch) / setup (push) Canceled after 0s
Multi-Arch CI / Linux::${{ fromJSON(needs.setup.outputs.linux_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubi10 image / publish_no_rocm_image_ubi10 (push) Canceled after 0s
Multi-Arch CI / Windows::${{ fromJSON(needs.setup.outputs.windows_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Multi-Arch CI / Manifest Diff (push) Canceled after 0s
TheRock Manifest Diff Report / Generate Manifest Diff Report (push) Canceled after 0s
Multi-Arch CI / CI Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_ocl_rt images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_rocgdb images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Unit Tests / Unit Tests :: ubuntu-24.04 (push) Canceled after 0s
Unit Tests / Unit Tests :: windows-2022 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_media (push) Canceled after 0s
Unit Tests / Unit Tests Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_openmpi (push) Canceled after 0s
1045 lines
48 KiB
C++
1045 lines
48 KiB
C++
//===-- X86ISelLowering.h - X86 DAG Lowering Interface ----------*- C++ -*-===//
|
|
//
|
|
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
// See https://llvm.org/LICENSE.txt for license information.
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
//
|
|
// This file defines the interfaces that X86 uses to lower LLVM code into a
|
|
// selection DAG.
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
#ifndef LLVM_LIB_TARGET_X86_X86ISELLOWERING_H
|
|
#define LLVM_LIB_TARGET_X86_X86ISELLOWERING_H
|
|
|
|
#include "X86SelectionDAGInfo.h"
|
|
#include "llvm/CodeGen/MachineFunction.h"
|
|
#include "llvm/CodeGen/TargetLowering.h"
|
|
|
|
namespace llvm {
|
|
class X86Subtarget;
|
|
class X86TargetMachine;
|
|
|
|
namespace X86 {
|
|
/// Current rounding mode is represented in bits 11:10 of FPSR. These
|
|
/// values are same as corresponding constants for rounding mode used
|
|
/// in glibc.
|
|
enum RoundingMode {
|
|
rmInvalid = -1, // For handle Invalid rounding mode
|
|
rmToNearest = 0, // FE_TONEAREST
|
|
rmDownward = 1 << 10, // FE_DOWNWARD
|
|
rmUpward = 2 << 10, // FE_UPWARD
|
|
rmTowardZero = 3 << 10, // FE_TOWARDZERO
|
|
rmMask = 3 << 10 // Bit mask selecting rounding mode
|
|
};
|
|
}
|
|
|
|
/// Define some predicates that are used for node matching.
|
|
namespace X86 {
|
|
/// Returns true if Elt is a constant zero or floating point constant +0.0.
|
|
bool isZeroNode(SDValue Elt);
|
|
|
|
/// Returns true of the given offset can be
|
|
/// fit into displacement field of the instruction.
|
|
bool isOffsetSuitableForCodeModel(int64_t Offset, CodeModel::Model M,
|
|
bool hasSymbolicDisplacement);
|
|
|
|
/// Determines whether the callee is required to pop its
|
|
/// own arguments. Callee pop is necessary to support tail calls.
|
|
bool isCalleePop(CallingConv::ID CallingConv,
|
|
bool is64Bit, bool IsVarArg, bool GuaranteeTCO);
|
|
|
|
/// If Op is a constant whose elements are all the same constant or
|
|
/// undefined, return true and return the constant value in \p SplatVal.
|
|
/// If we have undef bits that don't cover an entire element, we treat these
|
|
/// as zero if AllowPartialUndefs is set, else we fail and return false.
|
|
bool isConstantSplat(SDValue Op, APInt &SplatVal,
|
|
bool AllowPartialUndefs = true);
|
|
|
|
/// Check if Op is a load operation that could be folded into some other x86
|
|
/// instruction as a memory operand. Example: vpaddd (%rdi), %xmm0, %xmm0.
|
|
bool mayFoldLoad(SDValue Op, const X86Subtarget &Subtarget,
|
|
bool AssumeSingleUse = false,
|
|
bool IgnoreAlignment = false);
|
|
|
|
/// Check if Op is a load operation that could be folded into a vector splat
|
|
/// instruction as a memory operand. Example: vbroadcastss 16(%rdi), %xmm2.
|
|
bool mayFoldLoadIntoBroadcastFromMem(SDValue Op, MVT EltVT,
|
|
const X86Subtarget &Subtarget,
|
|
bool AssumeSingleUse = false);
|
|
|
|
/// Check if Op is a value that could be used to fold a store into some
|
|
/// other x86 instruction as a memory operand. Ex: pextrb $0, %xmm0, (%rdi).
|
|
bool mayFoldIntoStore(SDValue Op);
|
|
|
|
/// Check if Op is an operation that could be folded into a zero extend x86
|
|
/// instruction.
|
|
bool mayFoldIntoZeroExtend(SDValue Op);
|
|
|
|
/// True if the target supports the extended frame for async Swift
|
|
/// functions.
|
|
bool isExtendedSwiftAsyncFrameSupported(const X86Subtarget &Subtarget,
|
|
const MachineFunction &MF);
|
|
|
|
/// Convert LLVM rounding mode to X86 rounding mode.
|
|
int getRoundingModeX86(unsigned RM);
|
|
|
|
} // end namespace X86
|
|
|
|
//===--------------------------------------------------------------------===//
|
|
// X86 Implementation of the TargetLowering interface
|
|
class X86TargetLowering final : public TargetLowering {
|
|
// Copying needed for an outgoing byval argument.
|
|
enum ByValCopyKind {
|
|
// Argument is already in the correct location, no copy needed.
|
|
NoCopy,
|
|
// Argument value is currently in the local stack frame, needs copying to
|
|
// outgoing arguemnt area.
|
|
CopyOnce,
|
|
// Argument value is currently in the outgoing argument area, but not at
|
|
// the correct offset, so needs copying via a temporary in local stack
|
|
// space.
|
|
CopyViaTemp,
|
|
};
|
|
|
|
public:
|
|
explicit X86TargetLowering(const X86TargetMachine &TM,
|
|
const X86Subtarget &STI);
|
|
|
|
unsigned getJumpTableEncoding() const override;
|
|
bool useSoftFloat() const override;
|
|
|
|
void markLibCallAttributes(MachineFunction *MF, unsigned CC,
|
|
ArgListTy &Args) const override;
|
|
|
|
MVT getScalarShiftAmountTy(const DataLayout &, EVT VT) const override {
|
|
return MVT::i8;
|
|
}
|
|
|
|
const MCExpr *
|
|
LowerCustomJumpTableEntry(const MachineJumpTableInfo *MJTI,
|
|
const MachineBasicBlock *MBB, unsigned uid,
|
|
MCContext &Ctx) const override;
|
|
|
|
/// Returns relocation base for the given PIC jumptable.
|
|
SDValue getPICJumpTableRelocBase(SDValue Table,
|
|
SelectionDAG &DAG) const override;
|
|
const MCExpr *
|
|
getPICJumpTableRelocBaseExpr(const MachineFunction *MF,
|
|
unsigned JTI, MCContext &Ctx) const override;
|
|
|
|
/// Return the desired alignment for ByVal aggregate
|
|
/// function arguments in the caller parameter area. For X86, aggregates
|
|
/// that contains are placed at 16-byte boundaries while the rest are at
|
|
/// 4-byte boundaries.
|
|
Align getByValTypeAlignment(Type *Ty, const DataLayout &DL) const override;
|
|
|
|
EVT getOptimalMemOpType(LLVMContext &Context, const MemOp &Op,
|
|
const AttributeList &FuncAttributes) const override;
|
|
|
|
/// Returns true if it's safe to use load / store of the
|
|
/// specified type to expand memcpy / memset inline. This is mostly true
|
|
/// for all types except for some special cases. For example, on X86
|
|
/// targets without SSE2 f64 load / store are done with fldl / fstpl which
|
|
/// also does type conversion. Note the specified type doesn't have to be
|
|
/// legal as the hook is used before type legalization.
|
|
bool isSafeMemOpType(MVT VT) const override;
|
|
|
|
bool isMemoryAccessFast(EVT VT, Align Alignment) const;
|
|
|
|
/// Returns true if the target allows unaligned memory accesses of the
|
|
/// specified type. Returns whether it is "fast" in the last argument.
|
|
bool allowsMisalignedMemoryAccesses(EVT VT, unsigned AS, Align Alignment,
|
|
MachineMemOperand::Flags Flags,
|
|
unsigned *Fast) const override;
|
|
|
|
/// This function returns true if the memory access is aligned or if the
|
|
/// target allows this specific unaligned memory access. If the access is
|
|
/// allowed, the optional final parameter returns a relative speed of the
|
|
/// access (as defined by the target).
|
|
bool allowsMemoryAccess(
|
|
LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace,
|
|
Align Alignment,
|
|
MachineMemOperand::Flags Flags = MachineMemOperand::MONone,
|
|
unsigned *Fast = nullptr) const override;
|
|
|
|
bool allowsMemoryAccess(LLVMContext &Context, const DataLayout &DL, EVT VT,
|
|
const MachineMemOperand &MMO,
|
|
unsigned *Fast) const {
|
|
return allowsMemoryAccess(Context, DL, VT, MMO.getAddrSpace(),
|
|
MMO.getAlign(), MMO.getFlags(), Fast);
|
|
}
|
|
|
|
/// Provide custom lowering hooks for some operations.
|
|
///
|
|
SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override;
|
|
|
|
/// Replace the results of node with an illegal result
|
|
/// type with new values built out of custom code.
|
|
///
|
|
void ReplaceNodeResults(SDNode *N, SmallVectorImpl<SDValue>&Results,
|
|
SelectionDAG &DAG) const override;
|
|
|
|
SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override;
|
|
|
|
bool preferABDSToABSWithNSW(EVT VT) const override;
|
|
|
|
bool preferSextInRegOfTruncate(EVT TruncVT, EVT VT,
|
|
EVT ExtVT) const override;
|
|
|
|
bool isXAndYEqZeroPreferableToXAndYEqY(ISD::CondCode Cond,
|
|
EVT VT) const override;
|
|
|
|
/// Return true if the target has native support for
|
|
/// the specified value type and it is 'desirable' to use the type for the
|
|
/// given node type. e.g. On x86 i16 is legal, but undesirable since i16
|
|
/// instruction encodings are longer and some i16 instructions are slow.
|
|
bool isTypeDesirableForOp(unsigned Opc, EVT VT) const override;
|
|
|
|
/// Return true if the target has native support for the
|
|
/// specified value type and it is 'desirable' to use the type. e.g. On x86
|
|
/// i16 is legal, but undesirable since i16 instruction encodings are longer
|
|
/// and some i16 instructions are slow.
|
|
bool IsDesirableToPromoteOp(SDValue Op, EVT &PVT) const override;
|
|
|
|
/// Return prefered fold type, Abs if this is a vector, AddAnd if its an
|
|
/// integer, None otherwise.
|
|
TargetLowering::AndOrSETCCFoldKind
|
|
isDesirableToCombineLogicOpOfSETCC(const SDNode *LogicOp,
|
|
const SDNode *SETCC0,
|
|
const SDNode *SETCC1) const override;
|
|
|
|
/// Return the newly negated expression if the cost is not expensive and
|
|
/// set the cost in \p Cost to indicate that if it is cheaper or neutral to
|
|
/// do the negation.
|
|
SDValue getNegatedExpression(SDValue Op, SelectionDAG &DAG,
|
|
bool LegalOperations, bool ForCodeSize,
|
|
NegatibleCost &Cost,
|
|
unsigned Depth) const override;
|
|
|
|
MachineBasicBlock *
|
|
EmitInstrWithCustomInserter(MachineInstr &MI,
|
|
MachineBasicBlock *MBB) const override;
|
|
|
|
/// Do not merge vector stores after legalization because that may conflict
|
|
/// with x86-specific store splitting optimizations.
|
|
bool mergeStoresAfterLegalization(EVT MemVT) const override {
|
|
return !MemVT.isVector();
|
|
}
|
|
|
|
bool canMergeStoresTo(unsigned AddressSpace, EVT MemVT,
|
|
const MachineFunction &MF) const override;
|
|
|
|
bool isCheapToSpeculateCttz(Type *Ty) const override;
|
|
|
|
bool isCheapToSpeculateCtlz(Type *Ty) const override;
|
|
|
|
bool isCtlzFast() const override;
|
|
|
|
bool isMultiStoresCheaperThanBitsMerge(EVT LTy, EVT HTy) const override {
|
|
// If the pair to store is a mixture of float and int values, we will
|
|
// save two bitwise instructions and one float-to-int instruction and
|
|
// increase one store instruction. There is potentially a more
|
|
// significant benefit because it avoids the float->int domain switch
|
|
// for input value. So It is more likely a win.
|
|
if ((LTy.isFloatingPoint() && HTy.isInteger()) ||
|
|
(LTy.isInteger() && HTy.isFloatingPoint()))
|
|
return true;
|
|
// If the pair only contains int values, we will save two bitwise
|
|
// instructions and increase one store instruction (costing one more
|
|
// store buffer). Since the benefit is more blurred so we leave
|
|
// such pair out until we get testcase to prove it is a win.
|
|
return false;
|
|
}
|
|
|
|
bool isMaskAndCmp0FoldingBeneficial(const Instruction &AndI) const override;
|
|
|
|
bool hasAndNotCompare(SDValue Y) const override;
|
|
|
|
bool hasAndNot(SDValue Y) const override;
|
|
|
|
bool hasBitTest(SDValue X, SDValue Y) const override;
|
|
|
|
bool shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(
|
|
SDValue X, ConstantSDNode *XC, ConstantSDNode *CC, SDValue Y,
|
|
unsigned OldShiftOpcode, unsigned NewShiftOpcode,
|
|
SelectionDAG &DAG) const override;
|
|
|
|
unsigned preferedOpcodeForCmpEqPiecesOfOperand(
|
|
EVT VT, unsigned ShiftOpc, bool MayTransformRotate,
|
|
const APInt &ShiftOrRotateAmt,
|
|
const std::optional<APInt> &AndMask) const override;
|
|
|
|
bool preferScalarizeSplat(SDNode *N) const override;
|
|
|
|
CondMergingParams
|
|
getJumpConditionMergingParams(Instruction::BinaryOps Opc, const Value *Lhs,
|
|
const Value *Rhs) const override;
|
|
|
|
bool shouldFoldConstantShiftPairToMask(const SDNode *N) const override;
|
|
|
|
bool shouldFoldMaskToVariableShiftPair(SDValue Y) const override;
|
|
|
|
bool
|
|
shouldTransformSignedTruncationCheck(EVT XVT,
|
|
unsigned KeptBits) const override {
|
|
// For vectors, we don't have a preference..
|
|
if (XVT.isVector())
|
|
return false;
|
|
|
|
auto VTIsOk = [](EVT VT) -> bool {
|
|
return VT == MVT::i8 || VT == MVT::i16 || VT == MVT::i32 ||
|
|
VT == MVT::i64;
|
|
};
|
|
|
|
// We are ok with KeptBitsVT being byte/word/dword, what MOVS supports.
|
|
// XVT will be larger than KeptBitsVT.
|
|
MVT KeptBitsVT = MVT::getIntegerVT(KeptBits);
|
|
return VTIsOk(XVT) && VTIsOk(KeptBitsVT);
|
|
}
|
|
|
|
ShiftLegalizationStrategy
|
|
preferredShiftLegalizationStrategy(SelectionDAG &DAG, SDNode *N,
|
|
unsigned ExpansionFactor) const override;
|
|
|
|
bool shouldSplatInsEltVarIndex(EVT VT) const override;
|
|
|
|
bool shouldConvertFpToSat(unsigned Op, EVT FPVT, EVT VT) const override {
|
|
// Converting to sat variants holds little benefit on X86 as we will just
|
|
// need to saturate the value back using fp arithmatic.
|
|
return Op != ISD::FP_TO_UINT_SAT && isOperationLegalOrCustom(Op, VT);
|
|
}
|
|
|
|
bool convertSetCCLogicToBitwiseLogic(EVT VT) const override {
|
|
return VT.isScalarInteger();
|
|
}
|
|
|
|
/// Vector-sized comparisons are fast using PCMPEQ + PMOVMSK or PTEST.
|
|
MVT hasFastEqualityCompare(unsigned NumBits) const override;
|
|
|
|
/// Return the value type to use for ISD::SETCC.
|
|
EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context,
|
|
EVT VT) const override;
|
|
|
|
bool targetShrinkDemandedConstant(SDValue Op, const APInt &DemandedBits,
|
|
const APInt &DemandedElts,
|
|
TargetLoweringOpt &TLO) const override;
|
|
|
|
/// Determine which of the bits specified in Mask are known to be either
|
|
/// zero or one and return them in the KnownZero/KnownOne bitsets.
|
|
void computeKnownBitsForTargetNode(const SDValue Op,
|
|
KnownBits &Known,
|
|
const APInt &DemandedElts,
|
|
const SelectionDAG &DAG,
|
|
unsigned Depth = 0) const override;
|
|
|
|
/// Determine the number of bits in the operation that are sign bits.
|
|
unsigned ComputeNumSignBitsForTargetNode(SDValue Op,
|
|
const APInt &DemandedElts,
|
|
const SelectionDAG &DAG,
|
|
unsigned Depth) const override;
|
|
|
|
bool SimplifyDemandedVectorEltsForTargetNode(SDValue Op,
|
|
const APInt &DemandedElts,
|
|
APInt &KnownUndef,
|
|
APInt &KnownZero,
|
|
TargetLoweringOpt &TLO,
|
|
unsigned Depth) const override;
|
|
|
|
bool SimplifyDemandedVectorEltsForTargetShuffle(SDValue Op,
|
|
const APInt &DemandedElts,
|
|
unsigned MaskIndex,
|
|
TargetLoweringOpt &TLO,
|
|
unsigned Depth) const;
|
|
|
|
bool SimplifyDemandedBitsForTargetNode(SDValue Op,
|
|
const APInt &DemandedBits,
|
|
const APInt &DemandedElts,
|
|
KnownBits &Known,
|
|
TargetLoweringOpt &TLO,
|
|
unsigned Depth) const override;
|
|
|
|
SDValue SimplifyMultipleUseDemandedBitsForTargetNode(
|
|
SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
|
|
SelectionDAG &DAG, unsigned Depth) const override;
|
|
|
|
bool isGuaranteedNotToBeUndefOrPoisonForTargetNode(
|
|
SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
|
|
UndefPoisonKind Kind, unsigned Depth) const override;
|
|
|
|
bool canCreateUndefOrPoisonForTargetNode(SDValue Op,
|
|
const APInt &DemandedElts,
|
|
const SelectionDAG &DAG,
|
|
UndefPoisonKind Kind,
|
|
bool ConsiderFlags,
|
|
unsigned Depth) const override;
|
|
|
|
bool isSplatValueForTargetNode(SDValue Op, const APInt &DemandedElts,
|
|
APInt &UndefElts, const SelectionDAG &DAG,
|
|
unsigned Depth) const override;
|
|
|
|
bool isTargetCanonicalConstantNode(SDValue Op) const override {
|
|
// Peek through bitcasts/extracts/inserts to see if we have a vector
|
|
// load/broadcast from memory.
|
|
while (Op.getOpcode() == ISD::BITCAST ||
|
|
Op.getOpcode() == ISD::EXTRACT_SUBVECTOR ||
|
|
(Op.getOpcode() == ISD::INSERT_SUBVECTOR &&
|
|
Op.getOperand(0).isUndef()))
|
|
Op = Op.getOperand(Op.getOpcode() == ISD::INSERT_SUBVECTOR ? 1 : 0);
|
|
|
|
return Op.getOpcode() == X86ISD::VBROADCAST_LOAD ||
|
|
Op.getOpcode() == X86ISD::SUBV_BROADCAST_LOAD ||
|
|
(Op.getOpcode() == ISD::LOAD &&
|
|
getTargetConstantFromLoad(cast<LoadSDNode>(Op))) ||
|
|
TargetLowering::isTargetCanonicalConstantNode(Op);
|
|
}
|
|
|
|
bool isTargetCanonicalSelect(SDNode *N) const override;
|
|
|
|
const Constant *getTargetConstantFromLoad(LoadSDNode *LD) const override;
|
|
|
|
SDValue unwrapAddress(SDValue N) const override;
|
|
|
|
SDValue getReturnAddressFrameIndex(SelectionDAG &DAG) const;
|
|
|
|
ConstraintType getConstraintType(StringRef Constraint) const override;
|
|
|
|
/// Examine constraint string and operand type and determine a weight value.
|
|
/// The operand object must already have been set up with the operand type.
|
|
ConstraintWeight
|
|
getSingleConstraintMatchWeight(AsmOperandInfo &Info,
|
|
const char *Constraint) const override;
|
|
|
|
const char *LowerXConstraint(EVT ConstraintVT) const override;
|
|
|
|
/// Lower the specified operand into the Ops vector. If it is invalid, don't
|
|
/// add anything to Ops. If hasMemory is true it means one of the asm
|
|
/// constraint of the inline asm instruction being processed is 'm'.
|
|
void LowerAsmOperandForConstraint(SDValue Op, StringRef Constraint,
|
|
std::vector<SDValue> &Ops,
|
|
SelectionDAG &DAG) const override;
|
|
|
|
InlineAsm::ConstraintCode
|
|
getInlineAsmMemConstraint(StringRef ConstraintCode) const override {
|
|
if (ConstraintCode == "v")
|
|
return InlineAsm::ConstraintCode::v;
|
|
return TargetLowering::getInlineAsmMemConstraint(ConstraintCode);
|
|
}
|
|
|
|
/// Handle Lowering flag assembly outputs.
|
|
SDValue LowerAsmOutputForConstraint(SDValue &Chain, SDValue &Flag,
|
|
const SDLoc &DL,
|
|
const AsmOperandInfo &Constraint,
|
|
SelectionDAG &DAG) const override;
|
|
|
|
/// Given a physical register constraint
|
|
/// (e.g. {edx}), return the register number and the register class for the
|
|
/// register. This should only be used for C_Register constraints. On
|
|
/// error, this returns a register number of 0.
|
|
std::pair<unsigned, const TargetRegisterClass *>
|
|
getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
|
|
StringRef Constraint, MVT VT) const override;
|
|
|
|
/// Return true if the addressing mode represented
|
|
/// by AM is legal for this target, for a load/store of the specified type.
|
|
bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM,
|
|
Type *Ty, unsigned AS,
|
|
Instruction *I = nullptr) const override;
|
|
|
|
bool addressingModeSupportsTLS(const GlobalValue &GV) const override;
|
|
|
|
/// Return true if the specified immediate is legal
|
|
/// icmp immediate, that is the target has icmp instructions which can
|
|
/// compare a register against the immediate without having to materialize
|
|
/// the immediate into a register.
|
|
bool isLegalICmpImmediate(int64_t Imm) const override;
|
|
|
|
/// Return true if the specified immediate is legal
|
|
/// add immediate, that is the target has add instructions which can
|
|
/// add a register and the immediate without having to materialize
|
|
/// the immediate into a register.
|
|
bool isLegalAddImmediate(int64_t Imm) const override;
|
|
|
|
bool isLegalStoreImmediate(int64_t Imm) const override;
|
|
|
|
/// Add x86-specific opcodes to the default list.
|
|
bool isBinOp(unsigned Opcode) const override;
|
|
|
|
/// Returns true if the opcode is a commutative binary operation.
|
|
bool isCommutativeBinOp(unsigned Opcode) const override;
|
|
|
|
/// Return true if it's free to truncate a value of
|
|
/// type Ty1 to type Ty2. e.g. On x86 it's free to truncate a i32 value in
|
|
/// register EAX to i16 by referencing its sub-register AX.
|
|
bool isTruncateFree(Type *Ty1, Type *Ty2) const override;
|
|
bool isTruncateFree(EVT VT1, EVT VT2) const override;
|
|
|
|
bool allowTruncateForTailCall(Type *Ty1, Type *Ty2) const override;
|
|
|
|
/// Return true if any actual instruction that defines a
|
|
/// value of type Ty1 implicit zero-extends the value to Ty2 in the result
|
|
/// register. This does not necessarily include registers defined in
|
|
/// unknown ways, such as incoming arguments, or copies from unknown
|
|
/// virtual registers. Also, if isTruncateFree(Ty2, Ty1) is true, this
|
|
/// does not necessarily apply to truncate instructions. e.g. on x86-64,
|
|
/// all instructions that define 32-bit values implicit zero-extend the
|
|
/// result out to 64 bits.
|
|
bool isZExtFree(Type *Ty1, Type *Ty2) const override;
|
|
bool isZExtFree(EVT VT1, EVT VT2) const override;
|
|
bool isZExtFree(SDValue Val, EVT VT2) const override;
|
|
|
|
bool shouldConvertPhiType(Type *From, Type *To) const override;
|
|
|
|
/// Return true if folding a vector load into ExtVal (a sign, zero, or any
|
|
/// extend node) is profitable.
|
|
bool isVectorLoadExtDesirable(SDValue) const override;
|
|
|
|
/// Return true if an FMA operation is faster than a pair of fmul and fadd
|
|
/// instructions. fmuladd intrinsics will be expanded to FMAs when this
|
|
/// method returns true, otherwise fmuladd is expanded to fmul + fadd.
|
|
bool isFMAFasterThanFMulAndFAdd(const MachineFunction &MF,
|
|
EVT VT) const override;
|
|
|
|
/// Return true if it's profitable to narrow operations of type SrcVT to
|
|
/// DestVT. e.g. on x86, it's profitable to narrow from i32 to i8 but not
|
|
/// from i32 to i16.
|
|
bool isNarrowingProfitable(SDNode *N, EVT SrcVT, EVT DestVT) const override;
|
|
|
|
bool shouldFoldSelectWithIdentityConstant(unsigned BinOpcode, EVT VT,
|
|
unsigned SelectOpcode, SDValue X,
|
|
SDValue Y) const override;
|
|
|
|
/// Given an intrinsic, checks if on the target the intrinsic will need to
|
|
/// map to a MemIntrinsicNode (touches memory). If this is the case, it
|
|
/// returns true and stores the intrinsic information into the IntrinsicInfo
|
|
/// that was passed to the function.
|
|
void getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos,
|
|
const CallBase &I, MachineFunction &MF,
|
|
unsigned Intrinsic) const override;
|
|
|
|
/// Returns true if the target can instruction select the
|
|
/// specified FP immediate natively. If false, the legalizer will
|
|
/// materialize the FP immediate as a load from a constant pool.
|
|
bool isFPImmLegal(const APFloat &Imm, EVT VT,
|
|
bool ForCodeSize) const override;
|
|
|
|
/// Targets can use this to indicate that they only support *some*
|
|
/// VECTOR_SHUFFLE operations, those with specific masks. By default, if a
|
|
/// target supports the VECTOR_SHUFFLE node, all mask values are assumed to
|
|
/// be legal.
|
|
bool isShuffleMaskLegal(ArrayRef<int> Mask, EVT VT) const override;
|
|
|
|
/// Similar to isShuffleMaskLegal. Targets can use this to indicate if there
|
|
/// is a suitable VECTOR_SHUFFLE that can be used to replace a VAND with a
|
|
/// constant pool entry.
|
|
bool isVectorClearMaskLegal(ArrayRef<int> Mask, EVT VT) const override;
|
|
|
|
/// Returns true if lowering to a jump table is allowed.
|
|
bool areJTsAllowed(const Function *Fn) const override;
|
|
|
|
MVT getPreferredSwitchConditionType(LLVMContext &Context,
|
|
EVT ConditionVT) const override;
|
|
|
|
/// If true, then instruction selection should
|
|
/// seek to shrink the FP constant of the specified type to a smaller type
|
|
/// in order to save space and / or reduce runtime.
|
|
bool ShouldShrinkFPConstant(EVT VT) const override;
|
|
|
|
/// Return true if we believe it is correct and profitable to reduce the
|
|
/// load node to a smaller type.
|
|
bool
|
|
shouldReduceLoadWidth(SDNode *Load, ISD::LoadExtType ExtTy, EVT NewVT,
|
|
std::optional<unsigned> ByteOffset) const override;
|
|
|
|
/// Return true if the specified scalar FP type is computed in an SSE
|
|
/// register, not on the X87 floating point stack.
|
|
bool isScalarFPTypeInSSEReg(EVT VT) const;
|
|
|
|
/// Returns true if it is beneficial to convert a load of a constant
|
|
/// to just the constant itself.
|
|
bool shouldConvertConstantLoadToIntImm(const APInt &Imm,
|
|
Type *Ty) const override;
|
|
|
|
bool reduceSelectOfFPConstantLoads(EVT CmpOpVT) const override;
|
|
|
|
bool convertSelectOfConstantsToMath(EVT VT) const override;
|
|
|
|
bool decomposeMulByConstant(LLVMContext &Context, EVT VT,
|
|
SDValue C) const override;
|
|
|
|
/// Return true if EXTRACT_SUBVECTOR is cheap for this result type
|
|
/// with this index.
|
|
bool isExtractSubvectorCheap(EVT ResVT, EVT SrcVT,
|
|
unsigned Index) const override;
|
|
|
|
/// Scalar ops always have equal or better analysis/performance/power than
|
|
/// the vector equivalent, so this always makes sense if the scalar op is
|
|
/// supported.
|
|
bool shouldScalarizeBinop(SDValue) const override;
|
|
|
|
/// Extract of a scalar FP value from index 0 of a vector is free.
|
|
bool isExtractVecEltCheap(EVT VT, unsigned Index) const override {
|
|
EVT EltVT = VT.getScalarType();
|
|
return (EltVT == MVT::f32 || EltVT == MVT::f64) && Index == 0;
|
|
}
|
|
|
|
/// Overflow nodes should get combined/lowered to optimal instructions
|
|
/// (they should allow eliminating explicit compares by getting flags from
|
|
/// math ops).
|
|
bool shouldFormOverflowOp(unsigned Opcode, EVT VT,
|
|
bool MathUsed) const override;
|
|
|
|
bool storeOfVectorConstantIsCheap(bool IsZero, EVT MemVT, unsigned NumElem,
|
|
unsigned AddrSpace) const override {
|
|
// If we can replace more than 2 scalar stores, there will be a reduction
|
|
// in instructions even after we add a vector constant load.
|
|
return IsZero || NumElem > 2;
|
|
}
|
|
|
|
bool isLoadBitCastBeneficial(EVT LoadVT, EVT BitcastVT,
|
|
const SelectionDAG &DAG,
|
|
const MachineMemOperand &MMO) const override;
|
|
|
|
bool isProfitableToCombineMinNumMaxNum(EVT VT) const override {
|
|
// X86 has instructions that correspond to cmp + select, so forming
|
|
// minnum/maxnum is not profitable.
|
|
return false;
|
|
}
|
|
|
|
Register getRegisterByName(const char* RegName, LLT VT,
|
|
const MachineFunction &MF) const override;
|
|
|
|
/// If a physical register, this returns the register that receives the
|
|
/// exception address on entry to an EH pad.
|
|
Register
|
|
getExceptionPointerRegister(const Constant *PersonalityFn) const override;
|
|
|
|
/// If a physical register, this returns the register that receives the
|
|
/// exception typeid on entry to a landing pad.
|
|
Register
|
|
getExceptionSelectorRegister(const Constant *PersonalityFn) const override;
|
|
|
|
bool needsFixedCatchObjects() const override;
|
|
|
|
/// This method returns a target specific FastISel object,
|
|
/// or null if the target does not support "fast" ISel.
|
|
FastISel *
|
|
createFastISel(FunctionLoweringInfo &funcInfo,
|
|
const TargetLibraryInfo *libInfo,
|
|
const LibcallLoweringInfo *libcallLowering) const override;
|
|
|
|
/// If the target has a standard location for the stack protector cookie,
|
|
/// returns the address of that location. Otherwise, returns nullptr.
|
|
Value *getIRStackGuard(IRBuilderBase &IRB,
|
|
const LibcallLoweringInfo &Libcalls) const override;
|
|
|
|
bool useLoadStackGuardNode(const Module &M) const override;
|
|
bool useStackGuardXorFP() const override;
|
|
void
|
|
insertSSPDeclarations(Module &M,
|
|
const LibcallLoweringInfo &Libcalls) const override;
|
|
SDValue emitStackGuardXorFP(SelectionDAG &DAG, SDValue Val,
|
|
const SDLoc &DL) const override;
|
|
|
|
|
|
/// Return true if the target stores SafeStack pointer at a fixed offset in
|
|
/// some non-standard address space, and populates the address space and
|
|
/// offset as appropriate.
|
|
Value *getSafeStackPointerLocation(
|
|
IRBuilderBase &IRB, const LibcallLoweringInfo &Libcalls) const override;
|
|
|
|
std::pair<SDValue, SDValue> BuildFILD(EVT DstVT, EVT SrcVT, const SDLoc &DL,
|
|
SDValue Chain, SDValue Pointer,
|
|
MachinePointerInfo PtrInfo,
|
|
Align Alignment,
|
|
SelectionDAG &DAG) const;
|
|
|
|
/// Customize the preferred legalization strategy for certain types.
|
|
LegalizeTypeAction getPreferredVectorAction(MVT VT) const override;
|
|
|
|
MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC,
|
|
EVT VT) const override;
|
|
|
|
unsigned getNumRegistersForCallingConv(LLVMContext &Context,
|
|
CallingConv::ID CC,
|
|
EVT VT) const override;
|
|
|
|
unsigned getVectorTypeBreakdownForCallingConv(
|
|
LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT,
|
|
unsigned &NumIntermediates, MVT &RegisterVT) const override;
|
|
|
|
bool functionArgumentNeedsConsecutiveRegisters(
|
|
Type *Ty, CallingConv::ID CallConv, bool isVarArg,
|
|
const DataLayout &DL) const override;
|
|
|
|
bool isIntDivCheap(EVT VT, AttributeList Attr) const override;
|
|
|
|
bool supportSwiftError() const override;
|
|
|
|
bool supportKCFIBundles() const override { return true; }
|
|
|
|
MachineInstr *EmitKCFICheck(MachineBasicBlock &MBB,
|
|
MachineBasicBlock::instr_iterator &MBBI,
|
|
const TargetInstrInfo *TII) const override;
|
|
|
|
bool hasStackProbeSymbol(const MachineFunction &MF) const override;
|
|
bool hasInlineStackProbe(const MachineFunction &MF) const override;
|
|
StringRef getStackProbeSymbolName(const MachineFunction &MF) const override;
|
|
|
|
unsigned getStackProbeSize(const MachineFunction &MF) const;
|
|
|
|
bool hasVectorBlend() const override { return true; }
|
|
|
|
unsigned getMaxSupportedInterleaveFactor() const override { return 4; }
|
|
|
|
bool isInlineAsmTargetBranch(const SmallVectorImpl<StringRef> &AsmStrs,
|
|
unsigned OpNo) const override;
|
|
|
|
SDValue visitMaskedLoad(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain,
|
|
MachineMemOperand *MMO, SDValue &NewLoad,
|
|
SDValue Ptr, SDValue PassThru,
|
|
SDValue Mask) const override;
|
|
SDValue visitMaskedStore(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain,
|
|
MachineMemOperand *MMO, SDValue Ptr, SDValue Val,
|
|
SDValue Mask) const override;
|
|
|
|
/// Lower interleaved load(s) into target specific
|
|
/// instructions/intrinsics.
|
|
bool lowerInterleavedLoad(Instruction *Load, Value *Mask,
|
|
ArrayRef<ShuffleVectorInst *> Shuffles,
|
|
ArrayRef<unsigned> Indices, unsigned Factor,
|
|
const APInt &GapMask) const override;
|
|
|
|
/// Lower interleaved store(s) into target specific
|
|
/// instructions/intrinsics.
|
|
bool lowerInterleavedStore(Instruction *Store, Value *Mask,
|
|
ShuffleVectorInst *SVI, unsigned Factor,
|
|
const APInt &GapMask) const override;
|
|
|
|
SDValue expandIndirectJTBranch(const SDLoc &dl, SDValue Value, SDValue Addr,
|
|
int JTI, SelectionDAG &DAG) const override;
|
|
|
|
Align getPrefLoopAlignment(MachineLoop *ML) const override;
|
|
|
|
EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const override {
|
|
if (VT == MVT::f80)
|
|
return EVT::getIntegerVT(Context, 96);
|
|
return TargetLoweringBase::getTypeToTransformTo(Context, VT);
|
|
}
|
|
|
|
protected:
|
|
std::pair<const TargetRegisterClass *, uint8_t>
|
|
findRepresentativeClass(const TargetRegisterInfo *TRI,
|
|
MVT VT) const override;
|
|
|
|
private:
|
|
/// Keep a reference to the X86Subtarget around so that we can
|
|
/// make the right decision when generating code for different targets.
|
|
const X86Subtarget &Subtarget;
|
|
|
|
/// A list of legal FP immediates.
|
|
std::vector<APFloat> LegalFPImmediates;
|
|
|
|
/// Indicate that this x86 target can instruction
|
|
/// select the specified FP immediate natively.
|
|
void addLegalFPImmediate(const APFloat& Imm) {
|
|
LegalFPImmediates.push_back(Imm);
|
|
}
|
|
|
|
SDValue LowerCallResult(SDValue Chain, SDValue InGlue,
|
|
CallingConv::ID CallConv, bool isVarArg,
|
|
const SmallVectorImpl<ISD::InputArg> &Ins,
|
|
const SDLoc &dl, SelectionDAG &DAG,
|
|
SmallVectorImpl<SDValue> &InVals,
|
|
uint32_t *RegMask) const;
|
|
SDValue LowerMemArgument(SDValue Chain, CallingConv::ID CallConv,
|
|
const SmallVectorImpl<ISD::InputArg> &ArgInfo,
|
|
const SDLoc &dl, SelectionDAG &DAG,
|
|
const CCValAssign &VA, MachineFrameInfo &MFI,
|
|
unsigned i) const;
|
|
SDValue LowerMemOpCallTo(SDValue Chain, SDValue StackPtr, SDValue Arg,
|
|
const SDLoc &dl, SelectionDAG &DAG,
|
|
const CCValAssign &VA,
|
|
ISD::ArgFlagsTy Flags, bool isByval) const;
|
|
|
|
// Call lowering helpers.
|
|
|
|
/// Check whether the call is eligible for sibling call optimization.
|
|
bool
|
|
isEligibleForSiblingCallOpt(TargetLowering::CallLoweringInfo &CLI,
|
|
CCState &CCInfo,
|
|
SmallVectorImpl<CCValAssign> &ArgLocs) const;
|
|
SDValue EmitTailCallLoadRetAddr(SelectionDAG &DAG, SDValue &OutRetAddr,
|
|
SDValue Chain, bool IsTailCall,
|
|
bool Is64Bit, int FPDiff,
|
|
const SDLoc &dl) const;
|
|
|
|
unsigned GetAlignedArgumentStackSize(unsigned StackSize,
|
|
SelectionDAG &DAG) const;
|
|
|
|
unsigned getAddressSpace() const;
|
|
|
|
SDValue FP_TO_INTHelper(SDValue Op, SelectionDAG &DAG, bool IsSigned,
|
|
SDValue &Chain) const;
|
|
SDValue LRINT_LLRINTHelper(SDNode *N, SelectionDAG &DAG) const;
|
|
|
|
SDValue LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const;
|
|
SDValue LowerVSELECT(SDValue Op, SelectionDAG &DAG) const;
|
|
SDValue LowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;
|
|
SDValue LowerINSERT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;
|
|
|
|
unsigned getGlobalWrapperKind(const GlobalValue *GV,
|
|
const unsigned char OpFlags) const;
|
|
SDValue LowerConstantPool(SDValue Op, SelectionDAG &DAG) const;
|
|
SDValue LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const;
|
|
SDValue LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const;
|
|
SDValue LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const;
|
|
SDValue LowerExternalSymbol(SDValue Op, SelectionDAG &DAG) const;
|
|
|
|
/// Creates target global address or external symbol nodes for calls or
|
|
/// other uses.
|
|
SDValue LowerGlobalOrExternal(SDValue Op, SelectionDAG &DAG, bool ForCall,
|
|
bool *IsImpCall) const;
|
|
|
|
SDValue LowerSINT_TO_FP(SDValue Op, SelectionDAG &DAG) const;
|
|
SDValue LowerUINT_TO_FP(SDValue Op, SelectionDAG &DAG) const;
|
|
SDValue LowerTRUNCATE(SDValue Op, SelectionDAG &DAG) const;
|
|
SDValue LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG) const;
|
|
SDValue LowerFP_TO_INT_SAT(SDValue Op, SelectionDAG &DAG) const;
|
|
SDValue LowerLRINT_LLRINT(SDValue Op, SelectionDAG &DAG) const;
|
|
SDValue LowerSETCC(SDValue Op, SelectionDAG &DAG) const;
|
|
SDValue LowerSETCCCARRY(SDValue Op, SelectionDAG &DAG) const;
|
|
SDValue LowerSELECT(SDValue Op, SelectionDAG &DAG) const;
|
|
SDValue LowerConditionalBranch(SDValue Op, SelectionDAG &DAG) const;
|
|
SDValue LowerJumpTable(SDValue Op, SelectionDAG &DAG) const;
|
|
SDValue LowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const;
|
|
SDValue LowerVASTART(SDValue Op, SelectionDAG &DAG) const;
|
|
SDValue LowerVAARG(SDValue Op, SelectionDAG &DAG) const;
|
|
SDValue LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const;
|
|
SDValue LowerADDROFRETURNADDR(SDValue Op, SelectionDAG &DAG) const;
|
|
SDValue LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const;
|
|
SDValue LowerFRAME_TO_ARGS_OFFSET(SDValue Op, SelectionDAG &DAG) const;
|
|
ByValCopyKind ByValNeedsCopyForTailCall(SelectionDAG &DAG, SDValue Src,
|
|
SDValue Dst,
|
|
ISD::ArgFlagsTy Flags) const;
|
|
SDValue LowerEH_RETURN(SDValue Op, SelectionDAG &DAG) const;
|
|
SDValue lowerEH_SJLJ_SETJMP(SDValue Op, SelectionDAG &DAG) const;
|
|
SDValue lowerEH_SJLJ_LONGJMP(SDValue Op, SelectionDAG &DAG) const;
|
|
SDValue lowerEH_SJLJ_SETUP_DISPATCH(SDValue Op, SelectionDAG &DAG) const;
|
|
SDValue LowerINIT_TRAMPOLINE(SDValue Op, SelectionDAG &DAG) const;
|
|
SDValue LowerGET_ROUNDING(SDValue Op, SelectionDAG &DAG) const;
|
|
SDValue LowerSET_ROUNDING(SDValue Op, SelectionDAG &DAG) const;
|
|
SDValue LowerGET_FPENV_MEM(SDValue Op, SelectionDAG &DAG) const;
|
|
SDValue LowerSET_FPENV_MEM(SDValue Op, SelectionDAG &DAG) const;
|
|
SDValue LowerRESET_FPENV(SDValue Op, SelectionDAG &DAG) const;
|
|
SDValue LowerWin64_i128OP(SDValue Op, SelectionDAG &DAG) const;
|
|
SDValue LowerWin64_FP_TO_INT128(SDValue Op, SelectionDAG &DAG,
|
|
SDValue &Chain) const;
|
|
SDValue LowerWin64_INT128_TO_FP(SDValue Op, SelectionDAG &DAG) const;
|
|
SDValue LowerGC_TRANSITION(SDValue Op, SelectionDAG &DAG) const;
|
|
SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) const;
|
|
SDValue lowerFaddFsub(SDValue Op, SelectionDAG &DAG) const;
|
|
SDValue LowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const;
|
|
SDValue LowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const;
|
|
SDValue LowerFP_TO_BF16(SDValue Op, SelectionDAG &DAG) const;
|
|
|
|
SDValue
|
|
LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
|
|
const SmallVectorImpl<ISD::InputArg> &Ins,
|
|
const SDLoc &dl, SelectionDAG &DAG,
|
|
SmallVectorImpl<SDValue> &InVals) const override;
|
|
SDValue LowerCall(CallLoweringInfo &CLI,
|
|
SmallVectorImpl<SDValue> &InVals) const override;
|
|
|
|
SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
|
|
const SmallVectorImpl<ISD::OutputArg> &Outs,
|
|
const SmallVectorImpl<SDValue> &OutVals,
|
|
const SDLoc &dl, SelectionDAG &DAG) const override;
|
|
|
|
bool supportSplitCSR(MachineFunction *MF) const override {
|
|
return MF->getFunction().getCallingConv() == CallingConv::CXX_FAST_TLS &&
|
|
MF->getFunction().hasFnAttribute(Attribute::NoUnwind);
|
|
}
|
|
void initializeSplitCSR(MachineBasicBlock *Entry) const override;
|
|
void insertCopiesSplitCSR(
|
|
MachineBasicBlock *Entry,
|
|
const SmallVectorImpl<MachineBasicBlock *> &Exits) const override;
|
|
|
|
bool isUsedByReturnOnly(SDNode *N, SDValue &Chain) const override;
|
|
|
|
bool mayBeEmittedAsTailCall(const CallInst *CI) const override;
|
|
|
|
EVT getTypeForExtReturn(LLVMContext &Context, EVT VT,
|
|
ISD::NodeType ExtendKind) const override;
|
|
|
|
bool CanLowerReturn(CallingConv::ID CallConv, MachineFunction &MF,
|
|
bool isVarArg,
|
|
const SmallVectorImpl<ISD::OutputArg> &Outs,
|
|
LLVMContext &Context,
|
|
const Type *RetTy) const override;
|
|
|
|
const MCPhysReg *getScratchRegisters(CallingConv::ID CC) const override;
|
|
ArrayRef<MCPhysReg> getRoundingControlRegisters() const override;
|
|
|
|
TargetLoweringBase::AtomicExpansionKind
|
|
shouldExpandAtomicLoadInIR(LoadInst *LI) const override;
|
|
|
|
TargetLoweringBase::AtomicExpansionKind
|
|
shouldExpandAtomicStoreInIR(StoreInst *SI) const override;
|
|
TargetLoweringBase::AtomicExpansionKind
|
|
shouldExpandAtomicRMWInIR(const AtomicRMWInst *AI) const override;
|
|
TargetLoweringBase::AtomicExpansionKind
|
|
shouldExpandLogicAtomicRMWInIR(const AtomicRMWInst *AI) const;
|
|
TargetLoweringBase::AtomicExpansionKind
|
|
shouldCastAtomicLoadInIR(LoadInst *LI) const override;
|
|
void emitBitTestAtomicRMWIntrinsic(AtomicRMWInst *AI) const override;
|
|
void emitCmpArithAtomicRMWIntrinsic(AtomicRMWInst *AI) const override;
|
|
|
|
LoadInst *
|
|
lowerIdempotentRMWIntoFencedLoad(AtomicRMWInst *AI) const override;
|
|
|
|
bool shouldIssueAtomicLoadForAtomicEmulationLoop() const override {
|
|
return false;
|
|
}
|
|
|
|
bool needsCmpXchgNb(Type *MemType) const;
|
|
|
|
void SetupEntryBlockForSjLj(MachineInstr &MI, MachineBasicBlock *MBB,
|
|
MachineBasicBlock *DispatchBB, int FI) const;
|
|
|
|
// Utility function to emit the low-level va_arg code for X86-64.
|
|
MachineBasicBlock *
|
|
EmitVAARGWithCustomInserter(MachineInstr &MI, MachineBasicBlock *MBB) const;
|
|
|
|
/// Utility function to emit the xmm reg save portion of va_start.
|
|
MachineBasicBlock *EmitLoweredCascadedSelect(MachineInstr &MI1,
|
|
MachineInstr &MI2,
|
|
MachineBasicBlock *BB) const;
|
|
|
|
MachineBasicBlock *EmitLoweredSelect(MachineInstr &I,
|
|
MachineBasicBlock *BB) const;
|
|
|
|
MachineBasicBlock *EmitLoweredCatchRet(MachineInstr &MI,
|
|
MachineBasicBlock *BB) const;
|
|
|
|
MachineBasicBlock *EmitLoweredSegAlloca(MachineInstr &MI,
|
|
MachineBasicBlock *BB) const;
|
|
|
|
MachineBasicBlock *EmitLoweredProbedAlloca(MachineInstr &MI,
|
|
MachineBasicBlock *BB) const;
|
|
|
|
MachineBasicBlock *EmitLoweredTLSCall(MachineInstr &MI,
|
|
MachineBasicBlock *BB) const;
|
|
|
|
MachineBasicBlock *EmitLoweredIndirectThunk(MachineInstr &MI,
|
|
MachineBasicBlock *BB) const;
|
|
|
|
MachineBasicBlock *emitEHSjLjSetJmp(MachineInstr &MI,
|
|
MachineBasicBlock *MBB) const;
|
|
|
|
void emitSetJmpShadowStackFix(MachineInstr &MI,
|
|
MachineBasicBlock *MBB) const;
|
|
|
|
MachineBasicBlock *emitEHSjLjLongJmp(MachineInstr &MI,
|
|
MachineBasicBlock *MBB) const;
|
|
|
|
MachineBasicBlock *emitLongJmpShadowStackFix(MachineInstr &MI,
|
|
MachineBasicBlock *MBB) const;
|
|
|
|
MachineBasicBlock *EmitSjLjDispatchBlock(MachineInstr &MI,
|
|
MachineBasicBlock *MBB) const;
|
|
|
|
MachineBasicBlock *emitPatchableEventCall(MachineInstr &MI,
|
|
MachineBasicBlock *MBB) const;
|
|
|
|
/// Emit flags for the given setcc condition and operands. Also returns the
|
|
/// corresponding X86 condition code constant in X86CC.
|
|
SDValue emitFlagsForSetcc(SDValue Op0, SDValue Op1, ISD::CondCode CC,
|
|
const SDLoc &dl, SelectionDAG &DAG,
|
|
SDValue &X86CC) const;
|
|
|
|
bool optimizeFMulOrFDivAsShiftAddBitcast(SDNode *N, SDValue FPConst,
|
|
SDValue IntPow2) const override;
|
|
|
|
/// Check if replacement of SQRT with RSQRT should be disabled.
|
|
bool isFsqrtCheap(SDValue Op, SelectionDAG &DAG) const override;
|
|
|
|
/// Use rsqrt* to speed up sqrt calculations.
|
|
SDValue getSqrtEstimate(SDValue Op, SelectionDAG &DAG, int Enabled,
|
|
int &RefinementSteps, bool &UseOneConstNR,
|
|
bool Reciprocal) const override;
|
|
|
|
/// Use rcp* to speed up fdiv calculations.
|
|
SDValue getRecipEstimate(SDValue Op, SelectionDAG &DAG, int Enabled,
|
|
int &RefinementSteps) const override;
|
|
|
|
/// Reassociate floating point divisions into multiply by reciprocal.
|
|
unsigned combineRepeatedFPDivisors() const override;
|
|
|
|
SDValue BuildSDIVPow2(SDNode *N, const APInt &Divisor, SelectionDAG &DAG,
|
|
SmallVectorImpl<SDNode *> &Created) const override;
|
|
|
|
SDValue getMOVL(SelectionDAG &DAG, const SDLoc &dl, MVT VT, SDValue V1,
|
|
SDValue V2) const;
|
|
};
|
|
|
|
namespace X86 {
|
|
FastISel *createFastISel(FunctionLoweringInfo &funcInfo,
|
|
const TargetLibraryInfo *libInfo,
|
|
const LibcallLoweringInfo *libcallLowering);
|
|
} // end namespace X86
|
|
|
|
// X86 specific Gather/Scatter nodes.
|
|
// The class has the same order of operands as MaskedGatherScatterSDNode for
|
|
// convenience.
|
|
class X86MaskedGatherScatterSDNode : public MemIntrinsicSDNode {
|
|
public:
|
|
// This is a intended as a utility and should never be directly created.
|
|
X86MaskedGatherScatterSDNode() = delete;
|
|
~X86MaskedGatherScatterSDNode() = delete;
|
|
|
|
const SDValue &getBasePtr() const { return getOperand(3); }
|
|
const SDValue &getIndex() const { return getOperand(4); }
|
|
const SDValue &getMask() const { return getOperand(2); }
|
|
const SDValue &getScale() const { return getOperand(5); }
|
|
|
|
static bool classof(const SDNode *N) {
|
|
return N->getOpcode() == X86ISD::MGATHER ||
|
|
N->getOpcode() == X86ISD::MSCATTER;
|
|
}
|
|
};
|
|
|
|
class X86MaskedGatherSDNode : public X86MaskedGatherScatterSDNode {
|
|
public:
|
|
const SDValue &getPassThru() const { return getOperand(1); }
|
|
|
|
static bool classof(const SDNode *N) {
|
|
return N->getOpcode() == X86ISD::MGATHER;
|
|
}
|
|
};
|
|
|
|
class X86MaskedScatterSDNode : public X86MaskedGatherScatterSDNode {
|
|
public:
|
|
const SDValue &getValue() const { return getOperand(1); }
|
|
|
|
static bool classof(const SDNode *N) {
|
|
return N->getOpcode() == X86ISD::MSCATTER;
|
|
}
|
|
};
|
|
|
|
/// Generate unpacklo/unpackhi shuffle mask.
|
|
void createUnpackShuffleMask(EVT VT, SmallVectorImpl<int> &Mask, bool Lo,
|
|
bool Unary);
|
|
|
|
/// Similar to unpacklo/unpackhi, but without the 128-bit lane limitation
|
|
/// imposed by AVX and specific to the unary pattern. Example:
|
|
/// v8iX Lo --> <0, 0, 1, 1, 2, 2, 3, 3>
|
|
/// v8iX Hi --> <4, 4, 5, 5, 6, 6, 7, 7>
|
|
void createSplat2ShuffleMask(MVT VT, SmallVectorImpl<int> &Mask, bool Lo);
|
|
|
|
} // end namespace llvm
|
|
|
|
#endif // LLVM_LIB_TARGET_X86_X86ISELLOWERING_H
|