derek.holloway
7502bf64a4
CodeQL / Analyze (actions) (push) Canceled after 0s
CodeQL / Analyze (python) (push) Canceled after 0s
pre-commit / pre-commit (push) Canceled after 0s
Multi-Arch CI / setup (push) Canceled after 0s
Publish build_manylinux_x86_64 images / publish_build_manylinux_x86_64 (push) Canceled after 0s
Setup (Multi-Arch) / setup (push) Canceled after 0s
Multi-Arch CI / Linux::${{ fromJSON(needs.setup.outputs.linux_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubi10 image / publish_no_rocm_image_ubi10 (push) Canceled after 0s
Multi-Arch CI / Windows::${{ fromJSON(needs.setup.outputs.windows_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Multi-Arch CI / Manifest Diff (push) Canceled after 0s
TheRock Manifest Diff Report / Generate Manifest Diff Report (push) Canceled after 0s
Multi-Arch CI / CI Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_ocl_rt images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_rocgdb images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Unit Tests / Unit Tests :: ubuntu-24.04 (push) Canceled after 0s
Unit Tests / Unit Tests :: windows-2022 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_media (push) Canceled after 0s
Unit Tests / Unit Tests Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_openmpi (push) Canceled after 0s
2612 lines
102 KiB
TableGen
2612 lines
102 KiB
TableGen
//===- NVPTXInstrInfo.td - NVPTX Instruction defs -------------*- tblgen-*-===//
|
|
//
|
|
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
// See https://llvm.org/LICENSE.txt for license information.
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
//
|
|
// This file describes the PTX instructions in TableGen format.
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
include "NVPTXInstrFormats.td"
|
|
|
|
let OperandType = "OPERAND_IMMEDIATE" in {
|
|
def f16imm : Operand<f16>;
|
|
def bf16imm : Operand<bf16>;
|
|
}
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// NVPTX Operand Definitions.
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
def brtarget : Operand<OtherVT>;
|
|
|
|
// CVT conversion modes
|
|
// These must match the enum in NVPTX.h
|
|
def CvtNONE : PatLeaf<(i32 0x0)>;
|
|
def CvtRNI : PatLeaf<(i32 0x1)>;
|
|
def CvtRZI : PatLeaf<(i32 0x2)>;
|
|
def CvtRMI : PatLeaf<(i32 0x3)>;
|
|
def CvtRPI : PatLeaf<(i32 0x4)>;
|
|
def CvtRN : PatLeaf<(i32 0x5)>;
|
|
def CvtRZ : PatLeaf<(i32 0x6)>;
|
|
def CvtRM : PatLeaf<(i32 0x7)>;
|
|
def CvtRP : PatLeaf<(i32 0x8)>;
|
|
def CvtRNA : PatLeaf<(i32 0x9)>;
|
|
def CvtRS : PatLeaf<(i32 0xA)>;
|
|
|
|
def CvtNONE_FTZ : PatLeaf<(i32 0x10)>;
|
|
def CvtRNI_FTZ : PatLeaf<(i32 0x11)>;
|
|
def CvtRZI_FTZ : PatLeaf<(i32 0x12)>;
|
|
def CvtRMI_FTZ : PatLeaf<(i32 0x13)>;
|
|
def CvtRPI_FTZ : PatLeaf<(i32 0x14)>;
|
|
def CvtRN_FTZ : PatLeaf<(i32 0x15)>;
|
|
def CvtRZ_FTZ : PatLeaf<(i32 0x16)>;
|
|
def CvtRM_FTZ : PatLeaf<(i32 0x17)>;
|
|
def CvtRP_FTZ : PatLeaf<(i32 0x18)>;
|
|
|
|
def CvtSAT : PatLeaf<(i32 0x20)>;
|
|
def CvtSAT_FTZ : PatLeaf<(i32 0x30)>;
|
|
|
|
def CvtNONE_RELU : PatLeaf<(i32 0x40)>;
|
|
def CvtRN_RELU : PatLeaf<(i32 0x45)>;
|
|
def CvtRZ_RELU : PatLeaf<(i32 0x46)>;
|
|
def CvtRS_RELU : PatLeaf<(i32 0x4A)>;
|
|
|
|
def CvtNONE_SATFINITE : PatLeaf<(i32 0x80)>;
|
|
def CvtRN_SATFINITE : PatLeaf<(i32 0x85)>;
|
|
def CvtRN_RELU_SATFINITE : PatLeaf<(i32 0xC5)>;
|
|
|
|
def CvtMode : Operand<i32> {
|
|
let PrintMethod = "printCvtMode";
|
|
}
|
|
|
|
// FTZ flag
|
|
|
|
def FTZ : PatLeaf<(i1 1)>;
|
|
def NoFTZ : PatLeaf<(i1 0)>;
|
|
|
|
def getFTZFlag : SDNodeXForm<imm, [{
|
|
(void)N;
|
|
return CurDAG->getTargetConstant(useF32FTZ() ? 1 : 0, SDLoc(), MVT::i1);
|
|
}]>;
|
|
|
|
def FTZFlag : OperandWithDefaultOps<i1, (ops (getFTZFlag (i1 0)))> {
|
|
let PrintMethod = "printFTZFlag";
|
|
}
|
|
|
|
// Flag to indicate predicate negation (@!p) for conditional branches.
|
|
def BranchFlag : OperandWithDefaultOps<i1, (ops (i1 0))> {
|
|
let PrintMethod = "printNegatedPredicate";
|
|
}
|
|
|
|
// Compare modes
|
|
// These must match the enum in NVPTX.h
|
|
def CmpEQ : PatLeaf<(i32 0)>;
|
|
def CmpNE : PatLeaf<(i32 1)>;
|
|
|
|
def CmpMode : Operand<i32> {
|
|
let PrintMethod = "printCmpMode";
|
|
}
|
|
|
|
// PRMT modes
|
|
// These must match the enum in NVPTX.h
|
|
def PrmtNONE : PatLeaf<(i32 0x0)>;
|
|
def PrmtF4E : PatLeaf<(i32 0x1)>;
|
|
def PrmtB4E : PatLeaf<(i32 0x2)>;
|
|
def PrmtRC8 : PatLeaf<(i32 0x3)>;
|
|
def PrmtECL : PatLeaf<(i32 0x4)>;
|
|
def PrmtECR : PatLeaf<(i32 0x5)>;
|
|
def PrmtRC16 : PatLeaf<(i32 0x6)>;
|
|
|
|
def PrmtMode : Operand<i32> {
|
|
let PrintMethod = "printPrmtMode";
|
|
}
|
|
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// NVPTX Instruction Predicate Definitions
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
// Checks PTX version and family-specific and architecture-specific SM versions.
|
|
// For example, sm_100{f/a} and any future variants in the same family will match
|
|
// for any PTX version greater than or equal to `PTXVersion`.
|
|
class PTXWithFamilySMs<int PTXVersion, list<int> SMVersions> :
|
|
Predicate<"Subtarget->hasPTXWithFamilySMs(" # PTXVersion # ", {" #
|
|
!interleave(SMVersions, ", ") # "})">;
|
|
|
|
// Checks PTX version and architecture-specific SM versions.
|
|
// For example, sm_100{a} will match for any PTX version
|
|
// greater than or equal to `PTXVersion`.
|
|
class PTXWithAccelSMs<int PTXVersion, list<int> SMVersions> :
|
|
Predicate<"Subtarget->hasPTXWithAccelSMs(" # PTXVersion # ", {" #
|
|
!interleave(SMVersions, ", ") # "})">;
|
|
|
|
// Helper predicate to call a subtarget method.
|
|
class callSubtarget<string SubtargetMethod> : Predicate<"Subtarget->" # SubtargetMethod # "()">;
|
|
|
|
def hasAtomAddF64 : Predicate<"Subtarget->hasAtomAddF64()">;
|
|
def hasAtomScope : Predicate<"Subtarget->hasAtomScope()">;
|
|
def hasAtomBitwise64 : Predicate<"Subtarget->hasAtomBitwise64()">;
|
|
def hasAtomMinMax64 : Predicate<"Subtarget->hasAtomMinMax64()">;
|
|
def hasAtomSwap128 : Predicate<"Subtarget->hasAtomSwap128()">;
|
|
def hasClusters : Predicate<"Subtarget->hasClusters()">;
|
|
def hasPTXASUnreachableBug : Predicate<"Subtarget->hasPTXASUnreachableBug()">;
|
|
def noPTXASUnreachableBug : Predicate<"!Subtarget->hasPTXASUnreachableBug()">;
|
|
def hasOptEnabled : Predicate<"TM.getOptLevel() != CodeGenOptLevel::None">;
|
|
def hasArchAccelFeatures : Predicate<"Subtarget->hasArchAccelFeatures()">;
|
|
def hasFamilySpecificFeatures : Predicate<"Subtarget->hasFamilySpecificFeatures()">;
|
|
|
|
def doF32FTZ : Predicate<"useF32FTZ()">;
|
|
def doNoF32FTZ : Predicate<"!useF32FTZ()">;
|
|
def doRsqrtOpt : Predicate<"doRsqrtOpt()">;
|
|
def doMADWideOpt : Predicate<"doMADWideOpt()">;
|
|
|
|
def hasHWROT32 : Predicate<"Subtarget->hasHWROT32()">;
|
|
def noHWROT32 : Predicate<"!Subtarget->hasHWROT32()">;
|
|
def hasDotInstructions : Predicate<"Subtarget->hasDotInstructions()">;
|
|
def hasF32x2Instructions : Predicate<"Subtarget->hasF32x2Instructions()">;
|
|
|
|
class hasPTX<int version>: Predicate<"Subtarget->getPTXVersion() >= " # version>;
|
|
class hasSM<int version>: Predicate<"Subtarget->getSmVersion() >= " # version>;
|
|
|
|
// Explicit records for arch-accelerated SM versions
|
|
def hasSM90a : Predicate<"Subtarget->getSmVersion() == 90 && Subtarget->hasArchAccelFeatures()">;
|
|
def hasSM100a : Predicate<"Subtarget->getSmVersion() == 100 && Subtarget->hasArchAccelFeatures()">;
|
|
def hasSM101a : Predicate<"Subtarget->getSmVersion() == 101 && Subtarget->hasArchAccelFeatures()">;
|
|
def hasSM120a : Predicate<"Subtarget->getSmVersion() == 120 && Subtarget->hasArchAccelFeatures()">;
|
|
|
|
def hasSM100aOrSM103a :
|
|
Predicate<"(Subtarget->getSmVersion() == 100 || " #
|
|
"Subtarget->getSmVersion() == 103) " #
|
|
"&& Subtarget->hasArchAccelFeatures()">;
|
|
|
|
// non-sync shfl instructions are not available on sm_70+ in PTX6.4+
|
|
def hasSHFL : Predicate<"!(Subtarget->getSmVersion() >= 70"
|
|
"&& Subtarget->getPTXVersion() >= 64)">;
|
|
|
|
def useFP16Math: Predicate<"Subtarget->allowFP16Math()">;
|
|
def hasBF16Math: Predicate<"Subtarget->hasBF16Math()">;
|
|
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// Some Common Instruction Class Templates
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
class OneUse1<SDPatternOperator operator>
|
|
: PatFrag<(ops node:$A), (operator node:$A), [{ return N->hasOneUse(); }]>;
|
|
class OneUse2<SDPatternOperator operator>
|
|
: PatFrag<(ops node:$A, node:$B), (operator node:$A, node:$B), [{ return N->hasOneUse(); }]>;
|
|
|
|
|
|
class zeroinitializer<ValueType vt> :
|
|
PatLeaf<(vt (bitconvert (!cast<ValueType>("i" # vt.Size) 0)))>;
|
|
|
|
|
|
def fpimm_pos_inf : FPImmLeaf<fAny, [{ return Imm.isPosInfinity(); }]>;
|
|
def fpimm_0 : FPImmLeaf<fAny, [{ return Imm.isZero(); }]>;
|
|
def fpimm_1 : FPImmLeaf<fAny, [{ return Imm.isExactlyValue(1.0); }]>;
|
|
def fpimm_neg_1 : FPImmLeaf<fAny, [{ return Imm.isExactlyValue(-1.0); }]>;
|
|
|
|
|
|
// Operands which can hold a Register or an Immediate.
|
|
//
|
|
// Unfortunately, since most register classes can hold multiple types, we must
|
|
// use the 'Any' type for these.
|
|
|
|
def RI1 : Operand<i1>;
|
|
def RI16 : Operand<Any>;
|
|
def RI32 : Operand<Any>;
|
|
def RI64 : Operand<Any>;
|
|
|
|
// Utility class to wrap up information about a register and DAG type for more
|
|
// convenient iteration and parameterization
|
|
class RegTyInfo<ValueType ty, NVPTXRegClass rc, string ptx_type, Operand imm, SDNode imm_node,
|
|
bit supports_imm = 1> {
|
|
ValueType Ty = ty;
|
|
NVPTXRegClass RC = rc;
|
|
Operand Imm = imm;
|
|
SDNode ImmNode = imm_node;
|
|
bit SupportsImm = supports_imm;
|
|
int Size = ty.Size;
|
|
string PtxType = ptx_type;
|
|
}
|
|
|
|
def I1RT : RegTyInfo<i1, B1, "pred", i1imm, imm>;
|
|
def I16RT : RegTyInfo<i16, B16, "b16", i16imm, imm>;
|
|
def I32RT : RegTyInfo<i32, B32, "b32", i32imm, imm>;
|
|
def I64RT : RegTyInfo<i64, B64, "b64", i64imm, imm>;
|
|
|
|
def F32RT : RegTyInfo<f32, B32, "f32", f32imm, fpimm>;
|
|
def F64RT : RegTyInfo<f64, B64, "f64", f64imm, fpimm>;
|
|
def F16RT : RegTyInfo<f16, B16, "f16", f16imm, fpimm, supports_imm = 0>;
|
|
def BF16RT : RegTyInfo<bf16, B16, "bf16", bf16imm, fpimm, supports_imm = 0>;
|
|
|
|
def F16X2RT : RegTyInfo<v2f16, B32, "f16x2", ?, ?, supports_imm = 0>;
|
|
def BF16X2RT : RegTyInfo<v2bf16, B32, "bf16x2", ?, ?, supports_imm = 0>;
|
|
def F32X2RT : RegTyInfo<v2f32, B64, "f32x2", ?, ?, supports_imm = 0>;
|
|
|
|
|
|
// This class provides a basic wrapper around an NVPTXInst that abstracts the
|
|
// specific syntax of most PTX instructions. It automatically handles the
|
|
// construction of the asm string based on the provided dag arguments.
|
|
// For example, the following asm-strings would be computed:
|
|
//
|
|
// * BasicFlagsNVPTXInst<(outs B32:$dst),
|
|
// (ins B32:$a, B32:$b), (ins),
|
|
// "add.s32">;
|
|
// ---> "add.s32 \t$dst, $a, $b;"
|
|
//
|
|
// * BasicFlagsNVPTXInst<(outs B32:$d),
|
|
// (ins B32:$a, B32:$b, Hexu32imm:$c),
|
|
// (ins PrmtMode:$mode),
|
|
// "prmt.b32${mode}">;
|
|
// ---> "prmt.b32${mode} \t$d, $a, $b, $c;"
|
|
//
|
|
// * BasicFlagsNVPTXInst<(outs B64:$state),
|
|
// (ins ADDR:$addr),
|
|
// "mbarrier.arrive.b64">;
|
|
// ---> "mbarrier.arrive.b64 \t$state, [$addr];"
|
|
//
|
|
class BasicFlagsNVPTXInst<dag outs_dag, dag ins_dag, dag flags_dag, string asmstr,
|
|
list<dag> pattern = []>
|
|
: NVPTXInst<
|
|
outs_dag,
|
|
!con(ins_dag, flags_dag),
|
|
!strconcat(
|
|
asmstr,
|
|
!if(!and(!empty(ins_dag), !empty(outs_dag)), "",
|
|
!strconcat(
|
|
" \t",
|
|
!interleave(
|
|
!foreach(i, !range(!size(outs_dag)),
|
|
"$" # !getdagname(outs_dag, i)),
|
|
"|"),
|
|
!if(!or(!empty(ins_dag), !empty(outs_dag)), "", ", "),
|
|
!interleave(
|
|
!foreach(i, !range(!size(ins_dag)),
|
|
!if(!eq(!cast<string>(!getdagarg<DAGOperand>(ins_dag, i)), "ADDR"),
|
|
"[$" # !getdagname(ins_dag, i) # "]",
|
|
"$" # !getdagname(ins_dag, i)
|
|
)
|
|
),
|
|
", "))),
|
|
";"),
|
|
pattern>;
|
|
|
|
class BasicNVPTXInst<dag outs, dag insv, string asmstr, list<dag> pattern = []>
|
|
: BasicFlagsNVPTXInst<outs, insv, (ins), asmstr, pattern>;
|
|
|
|
|
|
multiclass I3Inst<string op_str, SDPatternOperator op_node, RegTyInfo t,
|
|
bit commutative, list<Predicate> requires = []> {
|
|
def rr :
|
|
BasicNVPTXInst<(outs t.RC:$dst), (ins t.RC:$a, t.RC:$b),
|
|
op_str,
|
|
[(set t.Ty:$dst, (op_node t.Ty:$a, t.Ty:$b))]>,
|
|
Requires<requires>;
|
|
def ri :
|
|
BasicNVPTXInst<(outs t.RC:$dst), (ins t.RC:$a, t.Imm:$b),
|
|
op_str,
|
|
[(set t.Ty:$dst, (op_node t.Ty:$a, (t.Ty imm:$b)))]>,
|
|
Requires<requires>;
|
|
if !not(commutative) then
|
|
def ir :
|
|
BasicNVPTXInst<(outs t.RC:$dst), (ins t.Imm:$a, t.RC:$b),
|
|
op_str,
|
|
[(set t.Ty:$dst, (op_node (t.Ty imm:$a), t.Ty:$b))]>,
|
|
Requires<requires>;
|
|
}
|
|
|
|
// Template for instructions which take three int64, int32, or int16 args.
|
|
// The instructions are named "<OpcStr><Width>" (e.g. "add.s64").
|
|
multiclass I3<string op_str, SDPatternOperator op_node, bit commutative> {
|
|
foreach t = [I16RT, I32RT, I64RT] in
|
|
defm t.Size# : I3Inst<op_str # t.Size, op_node, t, commutative>;
|
|
}
|
|
|
|
class I16x2<string OpcStr, SDNode OpNode> :
|
|
BasicNVPTXInst<(outs B32:$dst), (ins B32:$a, B32:$b),
|
|
OpcStr # "16x2",
|
|
[(set v2i16:$dst, (OpNode v2i16:$a, v2i16:$b))]>,
|
|
Requires<[hasPTX<80>, hasSM<90>]>;
|
|
|
|
// Template for instructions which take 3 int args. The instructions are
|
|
// named "<OpcStr>.s32" (e.g. "addc.cc.s32").
|
|
multiclass ADD_SUB_INT_CARRY<string op_str, SDNode op_node, bit commutative> {
|
|
let hasSideEffects = 1 in {
|
|
defm i32 : I3Inst<op_str # ".s32", op_node, I32RT, commutative>;
|
|
defm i64 : I3Inst<op_str # ".s64", op_node, I64RT, commutative,
|
|
requires = [hasPTX<43>]>;
|
|
}
|
|
}
|
|
|
|
// Template for minimum/maximum instructions.
|
|
//
|
|
// Also defines ftz (flush subnormal inputs and results to sign-preserving
|
|
// zero) variants for fp32 functions.
|
|
multiclass FMINIMUMMAXIMUM<string OpcStr, bit NaN, SDPatternOperator OpNode> {
|
|
defvar nan_str = !if(NaN, ".NaN", "");
|
|
if !not(NaN) then {
|
|
def _f64_rr :
|
|
BasicNVPTXInst<(outs B64:$dst),
|
|
(ins B64:$a, B64:$b),
|
|
OpcStr # ".f64",
|
|
[(set f64:$dst, (OpNode f64:$a, f64:$b))]>;
|
|
def _f64_ri :
|
|
BasicNVPTXInst<(outs B64:$dst),
|
|
(ins B64:$a, f64imm:$b),
|
|
OpcStr # ".f64",
|
|
[(set f64:$dst, (OpNode f64:$a, fpimm:$b))]>;
|
|
}
|
|
def _f32_rr :
|
|
BasicFlagsNVPTXInst<(outs B32:$dst),
|
|
(ins B32:$a, B32:$b),
|
|
(ins FTZFlag:$ftz),
|
|
OpcStr # "$ftz" # nan_str # ".f32",
|
|
[(set f32:$dst, (OpNode f32:$a, f32:$b))]>;
|
|
def _f32_ri :
|
|
BasicFlagsNVPTXInst<(outs B32:$dst),
|
|
(ins B32:$a, f32imm:$b),
|
|
(ins FTZFlag:$ftz),
|
|
OpcStr # "$ftz" # nan_str # ".f32",
|
|
[(set f32:$dst, (OpNode f32:$a, fpimm:$b))]>;
|
|
|
|
def _f16_rr :
|
|
BasicFlagsNVPTXInst<(outs B16:$dst),
|
|
(ins B16:$a, B16:$b),
|
|
(ins FTZFlag:$ftz),
|
|
OpcStr # "$ftz" # nan_str # ".f16",
|
|
[(set f16:$dst, (OpNode f16:$a, f16:$b))]>,
|
|
Requires<[useFP16Math]>;
|
|
|
|
def _f16x2_rr :
|
|
BasicFlagsNVPTXInst<(outs B32:$dst),
|
|
(ins B32:$a, B32:$b),
|
|
(ins FTZFlag:$ftz),
|
|
OpcStr # "$ftz" # nan_str # ".f16x2",
|
|
[(set v2f16:$dst, (OpNode v2f16:$a, v2f16:$b))]>,
|
|
Requires<[useFP16Math, hasSM<80>, hasPTX<70>]>;
|
|
def _bf16_rr :
|
|
BasicNVPTXInst<(outs B16:$dst),
|
|
(ins B16:$a, B16:$b),
|
|
OpcStr # nan_str # ".bf16",
|
|
[(set bf16:$dst, (OpNode bf16:$a, bf16:$b))]>,
|
|
Requires<[hasBF16Math, hasSM<80>, hasPTX<70>]>;
|
|
def _bf16x2_rr :
|
|
BasicNVPTXInst<(outs B32:$dst),
|
|
(ins B32:$a, B32:$b),
|
|
OpcStr # nan_str # ".bf16x2",
|
|
[(set v2bf16:$dst, (OpNode v2bf16:$a, v2bf16:$b))]>,
|
|
Requires<[hasBF16Math, hasSM<80>, hasPTX<70>]>;
|
|
}
|
|
|
|
// Template for 3-input minimum/maximum instructions
|
|
// (sm_100+/PTX 8.8 and f32 only)
|
|
//
|
|
// Also defines ftz (flush subnormal inputs and results to sign-preserving
|
|
// zero) variants for fp32 functions.
|
|
multiclass FMINIMUMMAXIMUM3<string OpcStr, bit NaN, SDNode OpNode> {
|
|
defvar nan_str = !if(NaN, ".NaN", "");
|
|
def f32rrr :
|
|
BasicFlagsNVPTXInst<(outs B32:$dst),
|
|
(ins B32:$a, B32:$b, B32:$c),
|
|
(ins FTZFlag:$ftz),
|
|
OpcStr # "$ftz" # nan_str # ".f32",
|
|
[(set f32:$dst, (OpNode f32:$a, f32:$b, f32:$c))]>,
|
|
Requires<[hasPTX<88>, hasSM<100>]>;
|
|
def f32rri :
|
|
BasicFlagsNVPTXInst<(outs B32:$dst),
|
|
(ins B32:$a, B32:$b, f32imm:$c),
|
|
(ins FTZFlag:$ftz),
|
|
OpcStr # "$ftz" # nan_str # ".f32",
|
|
[(set f32:$dst, (OpNode f32:$a, f32:$b, fpimm:$c))]>,
|
|
Requires<[hasPTX<88>, hasSM<100>]>;
|
|
def f32rii :
|
|
BasicFlagsNVPTXInst<(outs B32:$dst),
|
|
(ins B32:$a, f32imm:$b, f32imm:$c),
|
|
(ins FTZFlag:$ftz),
|
|
OpcStr # "$ftz" # nan_str # ".f32",
|
|
[(set f32:$dst, (OpNode f32:$a, fpimm:$b, fpimm:$c))]>,
|
|
Requires<[hasPTX<88>, hasSM<100>]>;
|
|
}
|
|
|
|
// Template for instructions which take three FP args. The
|
|
// instructions are named "<OpcStr>.f<Width>" (e.g. "add.f64").
|
|
//
|
|
// Also defines ftz (flush subnormal inputs and results to sign-preserving
|
|
// zero) variants for fp32/fp16 functions.
|
|
//
|
|
// This multiclass should be used for nodes that can be folded to make fma ops.
|
|
// In this case, we use the ".rn" variant when FMA is disabled, as this behaves
|
|
// just like the non ".rn" op, but prevents ptxas from creating FMAs.
|
|
multiclass F3<string op_str, SDPatternOperator op_pat> {
|
|
def f64rr :
|
|
BasicNVPTXInst<(outs B64:$dst),
|
|
(ins B64:$a, B64:$b),
|
|
op_str # ".f64",
|
|
[(set f64:$dst, (op_pat f64:$a, f64:$b))]>;
|
|
def f64ri :
|
|
BasicNVPTXInst<(outs B64:$dst),
|
|
(ins B64:$a, f64imm:$b),
|
|
op_str # ".f64",
|
|
[(set f64:$dst, (op_pat f64:$a, fpimm:$b))]>;
|
|
def f32rr :
|
|
BasicFlagsNVPTXInst<(outs B32:$dst),
|
|
(ins B32:$a, B32:$b),
|
|
(ins FTZFlag:$ftz),
|
|
op_str # "$ftz.f32",
|
|
[(set f32:$dst, (op_pat f32:$a, f32:$b))]>;
|
|
def f32ri :
|
|
BasicFlagsNVPTXInst<(outs B32:$dst),
|
|
(ins B32:$a, f32imm:$b),
|
|
(ins FTZFlag:$ftz),
|
|
op_str # "$ftz.f32",
|
|
[(set f32:$dst, (op_pat f32:$a, fpimm:$b))]>;
|
|
|
|
def f16rr :
|
|
BasicFlagsNVPTXInst<(outs B16:$dst),
|
|
(ins B16:$a, B16:$b),
|
|
(ins FTZFlag:$ftz),
|
|
op_str # "$ftz.f16",
|
|
[(set f16:$dst, (op_pat f16:$a, f16:$b))]>,
|
|
Requires<[useFP16Math]>;
|
|
def f32x2rr :
|
|
BasicFlagsNVPTXInst<(outs B64:$dst),
|
|
(ins B64:$a, B64:$b),
|
|
(ins FTZFlag:$ftz),
|
|
op_str # "$ftz.f32x2",
|
|
[(set v2f32:$dst, (op_pat v2f32:$a, v2f32:$b))]>,
|
|
Requires<[hasF32x2Instructions]>;
|
|
def f16x2rr :
|
|
BasicFlagsNVPTXInst<(outs B32:$dst),
|
|
(ins B32:$a, B32:$b),
|
|
(ins FTZFlag:$ftz),
|
|
op_str # "$ftz.f16x2",
|
|
[(set v2f16:$dst, (op_pat v2f16:$a, v2f16:$b))]>,
|
|
Requires<[useFP16Math]>;
|
|
def bf16rr :
|
|
BasicNVPTXInst<(outs B16:$dst),
|
|
(ins B16:$a, B16:$b),
|
|
op_str # ".bf16",
|
|
[(set bf16:$dst, (op_pat bf16:$a, bf16:$b))]>,
|
|
Requires<[hasBF16Math]>;
|
|
|
|
def bf16x2rr :
|
|
BasicNVPTXInst<(outs B32:$dst),
|
|
(ins B32:$a, B32:$b),
|
|
op_str # ".bf16x2",
|
|
[(set v2bf16:$dst, (op_pat v2bf16:$a, v2bf16:$b))]>,
|
|
Requires<[hasBF16Math]>;
|
|
}
|
|
|
|
class BinOpAllowsFMA<SDPatternOperator operator>
|
|
: PatFrag<(ops node:$A, node:$B),
|
|
(operator node:$A, node:$B), [{
|
|
return allowFMA() || N->getFlags().hasAllowContract();
|
|
}]>;
|
|
|
|
multiclass F3_fma_component<string op_str, SDNode op_node> {
|
|
defm "" : F3<op_str, BinOpAllowsFMA<op_node>>;
|
|
defm _rn : F3<op_str # ".rn", op_node>;
|
|
}
|
|
|
|
// Template for operations which take two f32 or f64 operands. Provides three
|
|
// instructions: <OpcStr>.f64, <OpcStr>.f32, and <OpcStr>.ftz.f32 (flush
|
|
// subnormal inputs and results to zero).
|
|
multiclass F2<string OpcStr, SDNode OpNode> {
|
|
def f64 : BasicNVPTXInst<(outs B64:$dst), (ins B64:$a),
|
|
OpcStr # ".f64",
|
|
[(set f64:$dst, (OpNode f64:$a))]>;
|
|
def f32 : BasicFlagsNVPTXInst<(outs B32:$dst), (ins B32:$a),
|
|
(ins FTZFlag:$ftz),
|
|
OpcStr # "$ftz.f32",
|
|
[(set f32:$dst, (OpNode f32:$a))]>;
|
|
}
|
|
|
|
multiclass F2_Support_Half<string OpcStr, SDNode OpNode> {
|
|
def bf16 : BasicNVPTXInst<(outs B16:$dst), (ins B16:$a),
|
|
OpcStr # ".bf16",
|
|
[(set bf16:$dst, (OpNode bf16:$a))]>,
|
|
Requires<[hasSM<80>, hasPTX<70>]>;
|
|
def bf16x2 : BasicNVPTXInst<(outs B32:$dst), (ins B32:$a),
|
|
OpcStr # ".bf16x2",
|
|
[(set v2bf16:$dst, (OpNode v2bf16:$a))]>,
|
|
Requires<[hasSM<80>, hasPTX<70>]>;
|
|
def f16 : BasicFlagsNVPTXInst<(outs B16:$dst), (ins B16:$a),
|
|
(ins FTZFlag:$ftz),
|
|
OpcStr # "$ftz.f16",
|
|
[(set f16:$dst, (OpNode f16:$a))]>,
|
|
Requires<[hasSM<53>, hasPTX<65>]>;
|
|
def f16x2 : BasicFlagsNVPTXInst<(outs B32:$dst), (ins B32:$a),
|
|
(ins FTZFlag:$ftz),
|
|
OpcStr # "$ftz.f16x2",
|
|
[(set v2f16:$dst, (OpNode v2f16:$a))]>,
|
|
Requires<[hasSM<53>, hasPTX<65>]>;
|
|
|
|
}
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// NVPTX Instructions.
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
//-----------------------------------
|
|
// Type Conversion
|
|
//-----------------------------------
|
|
|
|
let hasSideEffects = false in {
|
|
// Generate a cvt to the given type from all possible types. Each instance
|
|
// takes a CvtMode immediate that defines the conversion mode to use. It can
|
|
// be CvtNONE to omit a conversion mode.
|
|
multiclass CVT_FROM_ALL<string ToType, RegisterClass RC, list<Predicate> Preds = []> {
|
|
foreach sign = ["s", "u"] in {
|
|
def _ # sign # "8" :
|
|
BasicFlagsNVPTXInst<(outs RC:$dst),
|
|
(ins B16:$src), (ins CvtMode:$mode),
|
|
"cvt${mode:base}${mode:ftz}${mode:sat}." # ToType # "." # sign # "8">,
|
|
Requires<Preds>;
|
|
def _ # sign # "16" :
|
|
BasicFlagsNVPTXInst<(outs RC:$dst),
|
|
(ins B16:$src), (ins CvtMode:$mode),
|
|
"cvt${mode:base}${mode:ftz}${mode:sat}." # ToType # "." # sign # "16">,
|
|
Requires<Preds>;
|
|
def _ # sign # "32" :
|
|
BasicFlagsNVPTXInst<(outs RC:$dst),
|
|
(ins B32:$src), (ins CvtMode:$mode),
|
|
"cvt${mode:base}${mode:ftz}${mode:sat}." # ToType # "." # sign # "32">,
|
|
Requires<Preds>;
|
|
def _ # sign # "64" :
|
|
BasicFlagsNVPTXInst<(outs RC:$dst),
|
|
(ins B64:$src), (ins CvtMode:$mode),
|
|
"cvt${mode:base}${mode:ftz}${mode:sat}." # ToType # "." # sign # "64">,
|
|
Requires<Preds>;
|
|
}
|
|
def _f16 :
|
|
BasicFlagsNVPTXInst<(outs RC:$dst),
|
|
(ins B16:$src), (ins CvtMode:$mode),
|
|
"cvt${mode:base}${mode:ftz}${mode:sat}." # ToType # ".f16">,
|
|
Requires<Preds>;
|
|
def _bf16 :
|
|
BasicFlagsNVPTXInst<(outs RC:$dst),
|
|
(ins B16:$src), (ins CvtMode:$mode),
|
|
"cvt${mode:base}${mode:ftz}${mode:relu}${mode:sat}." # ToType # ".bf16">,
|
|
Requires<!if(!eq(ToType, "f32"),
|
|
// bf16->f32 was introduced early.
|
|
[hasPTX<71>, hasSM<80>],
|
|
// bf16->everything else needs sm90/ptx78
|
|
[hasPTX<78>, hasSM<90>])>;
|
|
def _f32 :
|
|
BasicFlagsNVPTXInst<(outs RC:$dst),
|
|
(ins B32:$src), (ins CvtMode:$mode),
|
|
"cvt${mode:base}${mode:ftz}${mode:relu}${mode:sat}." # ToType # ".f32">,
|
|
Requires<!if(!eq(ToType, "bf16"),
|
|
// f32->bf16 was introduced early.
|
|
[hasPTX<70>, hasSM<80>],
|
|
Preds)>;
|
|
def _f64 :
|
|
BasicFlagsNVPTXInst<(outs RC:$dst),
|
|
(ins B64:$src), (ins CvtMode:$mode),
|
|
"cvt${mode:base}${mode:ftz}${mode:sat}." # ToType # ".f64">,
|
|
Requires<Preds>;
|
|
}
|
|
|
|
// Generate cvts from all types to all types.
|
|
foreach sign = ["s", "u"] in {
|
|
defm CVT_ # sign # "8" : CVT_FROM_ALL<sign # "8", B16>;
|
|
defm CVT_ # sign # "16" : CVT_FROM_ALL<sign # "16", B16>;
|
|
defm CVT_ # sign # "32" : CVT_FROM_ALL<sign # "32", B32>;
|
|
defm CVT_ # sign # "64" : CVT_FROM_ALL<sign # "64", B64>;
|
|
}
|
|
defm CVT_f16 : CVT_FROM_ALL<"f16", B16>;
|
|
defm CVT_bf16 : CVT_FROM_ALL<"bf16", B16, [hasPTX<78>, hasSM<90>]>;
|
|
defm CVT_f32 : CVT_FROM_ALL<"f32", B32>;
|
|
defm CVT_f64 : CVT_FROM_ALL<"f64", B64>;
|
|
|
|
multiclass CVT_FROM_FLOAT_SATFINITE<string ToName, RegisterClass RC> {
|
|
def _f32_sf :
|
|
BasicFlagsNVPTXInst<(outs RC:$dst),
|
|
(ins B32:$src), (ins CvtMode:$mode),
|
|
"cvt${mode:base}${mode:relu}.satfinite." # ToName # ".f32">;
|
|
}
|
|
defm CVT_bf16 : CVT_FROM_FLOAT_SATFINITE<"bf16", B16>;
|
|
defm CVT_f16 : CVT_FROM_FLOAT_SATFINITE<"f16", B16>;
|
|
|
|
// These cvts are different from those above: The source and dest registers
|
|
// are of the same type.
|
|
def CVT_INREG_s16_s8 : BasicNVPTXInst<(outs B16:$dst), (ins B16:$src), "cvt.s16.s8">;
|
|
def CVT_INREG_s32_s8 : BasicNVPTXInst<(outs B32:$dst), (ins B32:$src), "cvt.s32.s8">;
|
|
def CVT_INREG_s32_s16 : BasicNVPTXInst<(outs B32:$dst), (ins B32:$src), "cvt.s32.s16">;
|
|
def CVT_INREG_s64_s8 : BasicNVPTXInst<(outs B64:$dst), (ins B64:$src), "cvt.s64.s8">;
|
|
def CVT_INREG_s64_s16 : BasicNVPTXInst<(outs B64:$dst), (ins B64:$src), "cvt.s64.s16">;
|
|
def CVT_INREG_s64_s32 : BasicNVPTXInst<(outs B64:$dst), (ins B64:$src), "cvt.s64.s32">;
|
|
|
|
multiclass CVT_FROM_FLOAT_V2_SM80<string FromName, RegisterClass RC> {
|
|
def _f32 :
|
|
BasicFlagsNVPTXInst<(outs RC:$dst),
|
|
(ins B32:$src1, B32:$src2), (ins CvtMode:$mode),
|
|
"cvt${mode:base}${mode:relu}." # FromName # ".f32">,
|
|
Requires<[hasPTX<70>, hasSM<80>]>;
|
|
|
|
def _f32_sf :
|
|
BasicFlagsNVPTXInst<(outs RC:$dst),
|
|
(ins B32:$src1, B32:$src2), (ins CvtMode:$mode),
|
|
"cvt${mode:base}${mode:relu}.satfinite." # FromName # ".f32">;
|
|
}
|
|
|
|
defm CVT_f16x2 : CVT_FROM_FLOAT_V2_SM80<"f16x2", B32>;
|
|
defm CVT_bf16x2 : CVT_FROM_FLOAT_V2_SM80<"bf16x2", B32>;
|
|
|
|
multiclass CVT_FROM_FLOAT_V2_RS<string FromName, RegisterClass RC> {
|
|
def _f32_rs :
|
|
BasicFlagsNVPTXInst<(outs RC:$dst),
|
|
(ins B32:$src1, B32:$src2, B32:$src3),
|
|
(ins CvtMode:$mode),
|
|
"cvt${mode:base}${mode:relu}." # FromName # ".f32">;
|
|
|
|
def _f32_rs_sf :
|
|
BasicFlagsNVPTXInst<(outs RC:$dst),
|
|
(ins B32:$src1, B32:$src2, B32:$src3),
|
|
(ins CvtMode:$mode),
|
|
"cvt${mode:base}${mode:relu}.satfinite." # FromName # ".f32">;
|
|
}
|
|
|
|
defm CVT_f16x2 : CVT_FROM_FLOAT_V2_RS<"f16x2", B32>;
|
|
defm CVT_bf16x2 : CVT_FROM_FLOAT_V2_RS<"bf16x2", B32>;
|
|
|
|
// FP8 conversions.
|
|
multiclass CVT_TO_F8X2<string F8Name> {
|
|
def _f32 :
|
|
BasicFlagsNVPTXInst<(outs B16:$dst),
|
|
(ins B32:$src1, B32:$src2), (ins CvtMode:$mode),
|
|
"cvt${mode:base}.satfinite${mode:relu}." # F8Name # "x2.f32">,
|
|
Requires<[hasPTX<81>, hasSM<89>]>;
|
|
def _f16x2 :
|
|
BasicFlagsNVPTXInst<(outs B16:$dst),
|
|
(ins B32:$src), (ins CvtMode:$mode),
|
|
"cvt${mode:base}.satfinite${mode:relu}." # F8Name # "x2.f16x2">,
|
|
Requires<[hasPTX<81>, hasSM<89>]>;
|
|
def _bf16x2 :
|
|
BasicFlagsNVPTXInst<(outs B16:$dst), (ins B32:$src), (ins CvtMode:$mode),
|
|
"cvt${mode:base}.satfinite${mode:relu}." # F8Name # "x2.bf16x2">,
|
|
Requires<[callSubtarget<"hasFP16X2ToNarrowFPConversionSupport">]>;
|
|
}
|
|
|
|
defm CVT_e4m3x2 : CVT_TO_F8X2<"e4m3">;
|
|
defm CVT_e5m2x2 : CVT_TO_F8X2<"e5m2">;
|
|
|
|
multiclass CVT_F8x2_TO_FP16x2<string F8Name> {
|
|
def f16x2_ # F8Name # x2 :
|
|
BasicFlagsNVPTXInst<(outs B32:$dst),
|
|
(ins B16:$src), (ins CvtMode:$mode),
|
|
"cvt${mode:base}${mode:relu}.f16x2." # F8Name # "x2">,
|
|
Requires<[hasPTX<81>, hasSM<89>]>;
|
|
def bf16x2_ # F8Name # x2_scale :
|
|
BasicFlagsNVPTXInst<(outs B32:$dst),
|
|
(ins B16:$src, B16:$src2), (ins CvtMode:$mode),
|
|
"cvt${mode:base}${mode:relu}${mode:satfinite}.scaled::n2::ue8m0.bf16x2." # F8Name # "x2">,
|
|
Requires<[callSubtarget<"hasNarrowFPToBF16x2ConversionSupport">]>;
|
|
}
|
|
|
|
defm CVT_ : CVT_F8x2_TO_FP16x2<"e4m3">;
|
|
defm CVT_ : CVT_F8x2_TO_FP16x2<"e5m2">;
|
|
|
|
class CVT_TO_FP8X4<string F8Name> :
|
|
NVPTXInst<(outs B32:$dst),
|
|
(ins B32:$src1, B32:$src2, B32:$src3, B32:$src4, B32:$src5, CvtMode:$mode),
|
|
"cvt${mode:base}${mode:relu}.satfinite." # F8Name #
|
|
"x4.f32 \t$dst, {{$src1, $src2, $src3, $src4}}, $src5;">;
|
|
|
|
def CVT_e4m3x4_f32x4_rs_sf : CVT_TO_FP8X4<"e4m3">;
|
|
def CVT_e5m2x4_f32x4_rs_sf : CVT_TO_FP8X4<"e5m2">;
|
|
|
|
// Float to TF32 conversions
|
|
multiclass CVT_TO_TF32<string Modifier, list<Predicate> Preds = [hasPTX<78>, hasSM<90>]> {
|
|
defvar Intr = !cast<Intrinsic>("int_nvvm_f2tf32_" # !subst(".", "_", Modifier));
|
|
|
|
def NAME : BasicNVPTXInst<(outs B32:$dst), (ins B32:$src),
|
|
"cvt." # Modifier # ".tf32.f32",
|
|
[(set i32:$dst, (Intr f32:$src))]>,
|
|
Requires<Preds>;
|
|
}
|
|
|
|
defm CVT_to_tf32_rn : CVT_TO_TF32<"rn">;
|
|
defm CVT_to_tf32_rz : CVT_TO_TF32<"rz">;
|
|
defm CVT_to_tf32_rn_relu : CVT_TO_TF32<"rn.relu">;
|
|
defm CVT_to_tf32_rz_relu : CVT_TO_TF32<"rz.relu">;
|
|
defm CVT_to_tf32_rna : CVT_TO_TF32<"rna", [hasPTX<70>, hasSM<80>]>;
|
|
defm CVT_to_tf32_rna_satf : CVT_TO_TF32<"rna.satfinite", [hasPTX<81>, hasSM<80>]>;
|
|
|
|
defm CVT_to_tf32_rn_satf : CVT_TO_TF32<"rn.satfinite", [hasPTX<86>, hasSM<100>]>;
|
|
defm CVT_to_tf32_rz_satf : CVT_TO_TF32<"rz.satfinite", [hasPTX<86>, hasSM<100>]>;
|
|
defm CVT_to_tf32_rn_relu_satf : CVT_TO_TF32<"rn.relu.satfinite", [hasPTX<86>, hasSM<100>]>;
|
|
defm CVT_to_tf32_rz_relu_satf : CVT_TO_TF32<"rz.relu.satfinite", [hasPTX<86>, hasSM<100>]>;
|
|
|
|
let Predicates = [callSubtarget<"hasS2F6X2ConversionSupport">] in {
|
|
def CVT_s2f6x2_f32_sf_scale : BasicFlagsNVPTXInst<(outs B16:$dst),
|
|
(ins B32:$src1, B32:$src2, B16:$scale), (ins CvtMode:$mode),
|
|
"cvt${mode:base}.satfinite${mode:relu}.scaled::n2::ue8m0.s2f6x2.f32">;
|
|
def CVT_s2f6x2_bf16x2_sf_scale : BasicFlagsNVPTXInst<(outs B16:$dst),
|
|
(ins B32:$src, B16:$scale), (ins CvtMode:$mode),
|
|
"cvt${mode:base}.satfinite${mode:relu}.scaled::n2::ue8m0.s2f6x2.bf16x2">;
|
|
def CVT_bf16x2_s2f6x2_scale : BasicFlagsNVPTXInst<(outs B32:$dst),
|
|
(ins B16:$src, B16:$scale), (ins CvtMode:$mode),
|
|
"cvt${mode:base}${mode:relu}.scaled::n2::ue8m0.bf16x2.s2f6x2">;
|
|
def CVT_bf16x2_s2f6x2_sf_scale : BasicFlagsNVPTXInst<(outs B32:$dst),
|
|
(ins B16:$src, B16:$scale), (ins CvtMode:$mode),
|
|
"cvt${mode:base}.satfinite${mode:relu}.scaled::n2::ue8m0.bf16x2.s2f6x2">;
|
|
}
|
|
|
|
// FP6 conversions.
|
|
multiclass CVT_F6x2_TO_FP16x2<string F6Name> {
|
|
def f16x2_ # F6Name # x2 :
|
|
BasicFlagsNVPTXInst<(outs B32:$dst),
|
|
(ins B16:$src), (ins CvtMode:$mode),
|
|
"cvt${mode:base}${mode:relu}.f16x2." # F6Name # "x2">,
|
|
Requires<[callSubtarget<"hasNarrowFPConversionSupport">]>;
|
|
def bf16x2_ # F6Name # x2_scale :
|
|
BasicFlagsNVPTXInst<(outs B32:$dst),
|
|
(ins B16:$src, B16:$src2), (ins CvtMode:$mode),
|
|
"cvt${mode:base}${mode:relu}${mode:satfinite}.scaled::n2::ue8m0.bf16x2." # F6Name # "x2">,
|
|
Requires<[callSubtarget<"hasNarrowFPToBF16x2ConversionSupport">]>;
|
|
}
|
|
|
|
defm CVT_ : CVT_F6x2_TO_FP16x2<"e2m3">;
|
|
defm CVT_ : CVT_F6x2_TO_FP16x2<"e3m2">;
|
|
|
|
class CVT_TO_FP6X4<string F6Name> :
|
|
NVPTXInst<(outs B32:$dst),
|
|
(ins B32:$src1, B32:$src2, B32:$src3, B32:$src4, B32:$src5, CvtMode:$mode),
|
|
"cvt${mode:base}${mode:relu}.satfinite." # F6Name #
|
|
"x4.f32 \t$dst, {{$src1, $src2, $src3, $src4}}, $src5;">;
|
|
|
|
def CVT_e2m3x4_f32x4_rs_sf : CVT_TO_FP6X4<"e2m3">;
|
|
def CVT_e3m2x4_f32x4_rs_sf : CVT_TO_FP6X4<"e3m2">;
|
|
|
|
multiclass CVT_TO_FP6X2<string FP6Name> {
|
|
def _f32_sf :
|
|
BasicFlagsNVPTXInst<(outs B16:$dst),
|
|
(ins B32:$src1, B32:$src2), (ins CvtMode:$mode),
|
|
"cvt${mode:base}.satfinite${mode:relu}." # FP6Name # "x2.f32">;
|
|
def _f16x2_sf :
|
|
BasicFlagsNVPTXInst<(outs B16:$dst), (ins B32:$src), (ins CvtMode:$mode),
|
|
"cvt${mode:base}.satfinite${mode:relu}." # FP6Name # "x2.f16x2">,
|
|
Requires<[callSubtarget<"hasFP16X2ToNarrowFPConversionSupport">]>;
|
|
def _bf16x2_sf :
|
|
BasicFlagsNVPTXInst<(outs B16:$dst), (ins B32:$src), (ins CvtMode:$mode),
|
|
"cvt${mode:base}.satfinite${mode:relu}." # FP6Name # "x2.bf16x2">,
|
|
Requires<[callSubtarget<"hasFP16X2ToNarrowFPConversionSupport">]>;
|
|
}
|
|
|
|
defm CVT_e2m3x2 : CVT_TO_FP6X2<"e2m3">;
|
|
defm CVT_e3m2x2 : CVT_TO_FP6X2<"e3m2">;
|
|
|
|
// FP4 conversions.
|
|
def CVT_e2m1x2_f32_sf : NVPTXInst<(outs B16:$dst),
|
|
(ins B32:$src1, B32:$src2, CvtMode:$mode),
|
|
!strconcat("{{ \n\t",
|
|
".reg .b8 \t%e2m1x2_out; \n\t",
|
|
"cvt${mode:base}.satfinite${mode:relu}.e2m1x2.f32 \t%e2m1x2_out, $src1, $src2; \n\t",
|
|
"cvt.u16.u8 \t$dst, %e2m1x2_out; \n\t",
|
|
"}}"), []>;
|
|
|
|
def CVT_f16x2_e2m1x2 : NVPTXInst<(outs B32:$dst),
|
|
(ins B16:$src, CvtMode:$mode),
|
|
!strconcat("{{ \n\t",
|
|
".reg .b8 \t%e2m1x2_in; \n\t",
|
|
"cvt.u8.u16 \t%e2m1x2_in, $src; \n\t",
|
|
"cvt${mode:base}${mode:relu}.f16x2.e2m1x2 \t$dst, %e2m1x2_in; \n\t",
|
|
"}}"), []>;
|
|
|
|
def CVT_bf16x2_e2m1x2_scale :
|
|
NVPTXInst<(outs B32:$dst),
|
|
(ins B16:$src1, B16:$src2, CvtMode:$mode),
|
|
"{{ \n\t" #
|
|
".reg .b8 \t%e2m1x2_in; \n\t" #
|
|
"cvt.u8.u16 \t%e2m1x2_in, $src1; \n\t" #
|
|
"cvt${mode:base}${mode:relu}${mode:satfinite}.scaled::n2::ue8m0.bf16x2.e2m1x2 \t$dst, %e2m1x2_in, $src2; \n\t" #
|
|
"}}", []>,
|
|
Requires<[callSubtarget<"hasNarrowFPToBF16x2ConversionSupport">]>;
|
|
|
|
def CVT_e2m1x4_f32x4_rs_sf :
|
|
NVPTXInst<(outs B16:$dst),
|
|
(ins B32:$src1, B32:$src2, B32:$src3, B32:$src4, B32:$src5, CvtMode:$mode),
|
|
"cvt${mode:base}${mode:relu}.satfinite.e2m1x4.f32 \t" #
|
|
"$dst, {{$src1, $src2, $src3, $src4}}, $src5;">;
|
|
|
|
def CVT_e2m1x2_f16x2_sf : NVPTXInst<(outs B16:$dst),
|
|
(ins B32:$src, CvtMode:$mode),
|
|
"{{ \n\t" #
|
|
".reg .b8 \t%e2m1x2_out; \n\t" #
|
|
"cvt${mode:base}.satfinite${mode:relu}.e2m1x2.f16x2 \t%e2m1x2_out, $src; \n\t" #
|
|
"cvt.u16.u8 \t$dst, %e2m1x2_out; \n\t" #
|
|
"}}", []>,
|
|
Requires<[callSubtarget<"hasFP16X2ToNarrowFPConversionSupport">]>;
|
|
|
|
def CVT_e2m1x2_bf16x2_sf : NVPTXInst<(outs B16:$dst),
|
|
(ins B32:$src, CvtMode:$mode),
|
|
"{{ \n\t" #
|
|
".reg .b8 \t%e2m1x2_out; \n\t" #
|
|
"cvt${mode:base}.satfinite${mode:relu}.e2m1x2.bf16x2 \t%e2m1x2_out, $src; \n\t" #
|
|
"cvt.u16.u8 \t$dst, %e2m1x2_out; \n\t" #
|
|
"}}", []>,
|
|
Requires<[callSubtarget<"hasFP16X2ToNarrowFPConversionSupport">]>;
|
|
|
|
// UE8M0x2 conversions.
|
|
class CVT_f32_to_ue8m0x2<string sat = ""> :
|
|
BasicFlagsNVPTXInst<(outs B16:$dst),
|
|
(ins B32:$src1, B32:$src2), (ins CvtMode:$mode),
|
|
"cvt${mode:base}" # sat # ".ue8m0x2.f32">;
|
|
|
|
class CVT_bf16x2_to_ue8m0x2<string sat = ""> :
|
|
BasicFlagsNVPTXInst<(outs B16:$dst),
|
|
(ins B32:$src), (ins CvtMode:$mode),
|
|
"cvt${mode:base}" # sat # ".ue8m0x2.bf16x2">;
|
|
|
|
def CVT_ue8m0x2_f32 : CVT_f32_to_ue8m0x2;
|
|
def CVT_ue8m0x2_f32_sf : CVT_f32_to_ue8m0x2<".satfinite">;
|
|
def CVT_ue8m0x2_bf16x2 : CVT_bf16x2_to_ue8m0x2;
|
|
def CVT_ue8m0x2_bf16x2_sf : CVT_bf16x2_to_ue8m0x2<".satfinite">;
|
|
|
|
def CVT_bf16x2_ue8m0x2 :
|
|
BasicNVPTXInst<(outs B32:$dst),
|
|
(ins B16:$src),
|
|
"cvt.rn.bf16x2.ue8m0x2">;
|
|
|
|
}
|
|
|
|
def fpround_oneuse : OneUse1<fpround>;
|
|
def : Pat<(v2bf16 (build_vector (bf16 (fpround_oneuse f32:$lo)),
|
|
(bf16 (fpround_oneuse f32:$hi)))),
|
|
(CVT_bf16x2_f32 $hi, $lo, CvtRN)>,
|
|
Requires<[hasPTX<70>, hasSM<80>, hasBF16Math]>;
|
|
|
|
def : Pat<(v2f16 (build_vector (f16 (fpround_oneuse f32:$lo)),
|
|
(f16 (fpround_oneuse f32:$hi)))),
|
|
(CVT_f16x2_f32 $hi, $lo, CvtRN)>,
|
|
Requires<[hasPTX<70>, hasSM<80>, useFP16Math]>;
|
|
|
|
//-----------------------------------
|
|
// Selection instructions (selp)
|
|
//-----------------------------------
|
|
|
|
// TODO: Missing slct
|
|
|
|
// selp instructions that don't have any pattern matches; we explicitly use
|
|
// them within this file.
|
|
let hasSideEffects = false in {
|
|
multiclass SELP_PATTERN<string TypeStr, RegTyInfo t> {
|
|
defvar asm_str = "selp." # TypeStr;
|
|
def rr :
|
|
BasicNVPTXInst<(outs t.RC:$dst),
|
|
(ins t.RC:$a, t.RC:$b, B1:$p),
|
|
asm_str,
|
|
[(set t.Ty:$dst, (select i1:$p, t.Ty:$a, t.Ty:$b))]>;
|
|
def ri :
|
|
BasicNVPTXInst<(outs t.RC:$dst),
|
|
(ins t.RC:$a, t.Imm:$b, B1:$p),
|
|
asm_str,
|
|
[(set t.Ty:$dst, (select i1:$p, t.Ty:$a, t.ImmNode:$b))]>;
|
|
def ir :
|
|
BasicNVPTXInst<(outs t.RC:$dst),
|
|
(ins t.Imm:$a, t.RC:$b, B1:$p),
|
|
asm_str,
|
|
[(set t.Ty:$dst, (select i1:$p, t.ImmNode:$a, t.Ty:$b))]>;
|
|
def ii :
|
|
BasicNVPTXInst<(outs t.RC:$dst),
|
|
(ins t.Imm:$a, t.Imm:$b, B1:$p),
|
|
asm_str,
|
|
[(set t.Ty:$dst, (select i1:$p, t.ImmNode:$a, t.ImmNode:$b))]>;
|
|
}
|
|
}
|
|
|
|
// Don't pattern match on selp.{s,u}{16,32,64} -- selp.b{16,32,64} is just as
|
|
// good.
|
|
defm SELP_b16 : SELP_PATTERN<"b16", I16RT>;
|
|
defm SELP_b32 : SELP_PATTERN<"b32", I32RT>;
|
|
defm SELP_b64 : SELP_PATTERN<"b64", I64RT>;
|
|
defm SELP_f16 : SELP_PATTERN<"b16", F16RT>;
|
|
defm SELP_bf16 : SELP_PATTERN<"b16", BF16RT>;
|
|
defm SELP_f32 : SELP_PATTERN<"f32", F32RT>;
|
|
defm SELP_f64 : SELP_PATTERN<"f64", F64RT>;
|
|
|
|
// This does not work as tablegen fails to infer the type of 'imm'.
|
|
// def v2f16imm : Operand<v2f16>;
|
|
// defm SELP_f16x2 : SELP_PATTERN<"b32", v2f16, B32, v2f16imm, imm>;
|
|
|
|
foreach vt = [v2f16, v2bf16, v2i16, v4i8] in {
|
|
def : Pat<(vt (select i1:$p, vt:$a, vt:$b)),
|
|
(SELP_b32rr $a, $b, $p)>;
|
|
}
|
|
|
|
foreach vt = [v2f32, v2i32] in {
|
|
def : Pat<(vt (select i1:$p, vt:$a, vt:$b)),
|
|
(SELP_b64rr $a, $b, $p)>;
|
|
}
|
|
|
|
//-----------------------------------
|
|
// Test Instructions
|
|
//-----------------------------------
|
|
|
|
def fabs_oneuse : OneUse1<fabs>;
|
|
|
|
def TESTINF_f32r : BasicNVPTXInst<(outs B1:$p), (ins B32:$a),
|
|
"testp.infinite.f32",
|
|
[(set i1:$p, (seteq (fabs_oneuse f32:$a), fpimm_pos_inf))]>;
|
|
def TESTINF_f64r : BasicNVPTXInst<(outs B1:$p), (ins B64:$a),
|
|
"testp.infinite.f64",
|
|
[(set i1:$p, (seteq (fabs_oneuse f64:$a), fpimm_pos_inf))]>;
|
|
|
|
//-----------------------------------
|
|
// Integer Arithmetic
|
|
//-----------------------------------
|
|
|
|
// int16, int32, and int64 signed addition. Since nvptx is 2's complement, we
|
|
// also use these for unsigned arithmetic.
|
|
defm ADD : I3<"add.s", add, commutative = true>;
|
|
defm SUB : I3<"sub.s", sub, commutative = false>;
|
|
|
|
def ADD16x2 : I16x2<"add.s", add>;
|
|
|
|
// int32 and int64 addition and subtraction with carry-out.
|
|
defm ADDCC : ADD_SUB_INT_CARRY<"add.cc", addc, commutative = true>;
|
|
defm SUBCC : ADD_SUB_INT_CARRY<"sub.cc", subc, commutative = false>;
|
|
|
|
// int32 and int64 addition and subtraction with carry-in and carry-out.
|
|
defm ADDCCC : ADD_SUB_INT_CARRY<"addc.cc", adde, commutative = true>;
|
|
defm SUBCCC : ADD_SUB_INT_CARRY<"subc.cc", sube, commutative = false>;
|
|
|
|
defm MULT : I3<"mul.lo.s", mul, commutative = true>;
|
|
|
|
defm MUL_HI_S : I3<"mul.hi.s", mulhs, commutative = true>;
|
|
defm MUL_HI_U : I3<"mul.hi.u", mulhu, commutative = true>;
|
|
|
|
defm SDIV : I3<"div.s", sdiv, commutative = false>;
|
|
defm UDIV : I3<"div.u", udiv, commutative = false>;
|
|
|
|
// The ri versions of rem.s and rem.u won't be selected; DAGCombiner::visitSREM
|
|
// will lower it.
|
|
defm SREM : I3<"rem.s", srem, commutative = false>;
|
|
defm UREM : I3<"rem.u", urem, commutative = false>;
|
|
|
|
foreach t = [I16RT, I32RT, I64RT] in {
|
|
// PTX abs.s is undefined for INT_MIN, so it matches ABS_MIN_POISON
|
|
// semantics (where INT_MIN input is poison). ISD::ABS requires
|
|
// abs(INT_MIN) == INT_MIN and is expanded separately.
|
|
def ABS_S # t.Size :
|
|
BasicNVPTXInst<(outs t.RC:$dst), (ins t.RC:$a),
|
|
"abs.s" # t.Size,
|
|
[(set t.Ty:$dst, (abs_min_poison t.Ty:$a))]>;
|
|
|
|
def NEG_S # t.Size :
|
|
BasicNVPTXInst<(outs t.RC:$dst), (ins t.RC:$src),
|
|
"neg.s" # t.Size,
|
|
[(set t.Ty:$dst, (ineg t.Ty:$src))]>;
|
|
}
|
|
|
|
// Integer min/max.
|
|
defm SMAX : I3<"max.s", smax, commutative = true>;
|
|
defm UMAX : I3<"max.u", umax, commutative = true>;
|
|
defm SMIN : I3<"min.s", smin, commutative = true>;
|
|
defm UMIN : I3<"min.u", umin, commutative = true>;
|
|
|
|
def SMAX16x2 : I16x2<"max.s", smax>;
|
|
def UMAX16x2 : I16x2<"max.u", umax>;
|
|
def SMIN16x2 : I16x2<"min.s", smin>;
|
|
def UMIN16x2 : I16x2<"min.u", umin>;
|
|
|
|
let Predicates = [hasPTX<80>, hasSM<90>] in {
|
|
|
|
def MIN_RELU_S32 : BasicNVPTXInst<(outs B32:$dst), (ins B32:$a, B32:$b),
|
|
"min.relu.s32",
|
|
[(set i32:$dst, (smax (smin i32:$a, i32:$b), 0))]>;
|
|
def MAX_RELU_S32 : BasicNVPTXInst<(outs B32:$dst), (ins B32:$a, B32:$b),
|
|
"max.relu.s32",
|
|
[(set i32:$dst, (smax (smax i32:$a, i32:$b), 0))]>;
|
|
def MIN_RELU_S16x2 : BasicNVPTXInst<(outs B32:$dst), (ins B32:$a, B32:$b),
|
|
"min.relu.s16x2",
|
|
[(set v2i16:$dst, (smax (smin v2i16:$a, v2i16:$b),
|
|
zeroinitializer<v2i16>))]>;
|
|
def MAX_RELU_S16x2 : BasicNVPTXInst<(outs B32:$dst), (ins B32:$a, B32:$b),
|
|
"max.relu.s16x2",
|
|
[(set v2i16:$dst, (smax (smax v2i16:$a, v2i16:$b),
|
|
zeroinitializer<v2i16>))]>;
|
|
}
|
|
|
|
//
|
|
// Wide multiplication
|
|
//
|
|
|
|
def SDTMulWide : SDTypeProfile<1, 2, [SDTCisInt<0>, SDTCisInt<1>, SDTCisSameAs<1, 2>]>;
|
|
def smul_wide : SDNode<"NVPTXISD::MUL_WIDE_SIGNED", SDTMulWide, [SDNPCommutative]>;
|
|
def umul_wide : SDNode<"NVPTXISD::MUL_WIDE_UNSIGNED", SDTMulWide, [SDNPCommutative]>;
|
|
|
|
|
|
multiclass MULWIDEInst<string suffix, SDPatternOperator op, RegTyInfo big_t, RegTyInfo small_t> {
|
|
def suffix # _rr :
|
|
BasicNVPTXInst<(outs big_t.RC:$dst), (ins small_t.RC:$a, small_t.RC:$b),
|
|
"mul.wide." # suffix,
|
|
[(set big_t.Ty:$dst, (op small_t.Ty:$a, small_t.Ty:$b))]>;
|
|
def suffix # _ri :
|
|
BasicNVPTXInst<(outs big_t.RC:$dst), (ins small_t.RC:$a, small_t.Imm:$b),
|
|
"mul.wide." # suffix,
|
|
[(set big_t.Ty:$dst, (op small_t.Ty:$a, imm:$b))]>;
|
|
}
|
|
|
|
defm MUL_WIDE : MULWIDEInst<"s32", smul_wide, I64RT, I32RT>;
|
|
defm MUL_WIDE : MULWIDEInst<"u32", umul_wide, I64RT, I32RT>;
|
|
defm MUL_WIDE : MULWIDEInst<"s16", smul_wide, I32RT, I16RT>;
|
|
defm MUL_WIDE : MULWIDEInst<"u16", umul_wide, I32RT, I16RT>;
|
|
|
|
//
|
|
// Integer multiply-add
|
|
//
|
|
multiclass MADInst<string suffix, SDPatternOperator op, RegTyInfo big_t, RegTyInfo small_t> {
|
|
def rrr:
|
|
BasicNVPTXInst<(outs big_t.RC:$dst),
|
|
(ins small_t.RC:$a, small_t.RC:$b, big_t.RC:$c),
|
|
"mad." # suffix,
|
|
[(set big_t.Ty:$dst, (add (OneUse2<op> small_t.Ty:$a, small_t.Ty:$b), big_t.Ty:$c))]>;
|
|
def rri:
|
|
BasicNVPTXInst<(outs big_t.RC:$dst),
|
|
(ins small_t.RC:$a, small_t.RC:$b, big_t.Imm:$c),
|
|
"mad." # suffix,
|
|
[(set big_t.Ty:$dst, (add (OneUse2<op> small_t.Ty:$a, small_t.Ty:$b), imm:$c))]>;
|
|
def rir:
|
|
BasicNVPTXInst<(outs big_t.RC:$dst),
|
|
(ins small_t.RC:$a, small_t.Imm:$b, big_t.RC:$c),
|
|
"mad." # suffix,
|
|
[(set big_t.Ty:$dst, (add (OneUse2<op> small_t.Ty:$a, imm:$b), big_t.Ty:$c))]>;
|
|
def rii:
|
|
BasicNVPTXInst<(outs big_t.RC:$dst),
|
|
(ins small_t.RC:$a, small_t.Imm:$b, big_t.Imm:$c),
|
|
"mad." # suffix,
|
|
[(set big_t.Ty:$dst, (add (OneUse2<op> small_t.Ty:$a, imm:$b), imm:$c))]>;
|
|
}
|
|
|
|
let Predicates = [hasOptEnabled] in {
|
|
defm MAD_LO_S16 : MADInst<"lo.s16", mul, I16RT, I16RT>;
|
|
defm MAD_LO_S32 : MADInst<"lo.s32", mul, I32RT, I32RT>;
|
|
defm MAD_LO_S64 : MADInst<"lo.s64", mul, I64RT, I64RT>;
|
|
|
|
// Generating mad.wide causes a regression in some cases:
|
|
// https://github.com/llvm/llvm-project/pull/150477#issuecomment-3191367837
|
|
// Only do so when the user requests it.
|
|
let Predicates = [doMADWideOpt] in {
|
|
defm MAD_WIDE_U16 : MADInst<"wide.u16", umul_wide, I32RT, I16RT>;
|
|
defm MAD_WIDE_S16 : MADInst<"wide.s16", smul_wide, I32RT, I16RT>;
|
|
defm MAD_WIDE_U32 : MADInst<"wide.u32", umul_wide, I64RT, I32RT>;
|
|
defm MAD_WIDE_S32 : MADInst<"wide.s32", smul_wide, I64RT, I32RT>;
|
|
}
|
|
}
|
|
|
|
//-----------------------------------
|
|
// Floating Point Arithmetic
|
|
//-----------------------------------
|
|
|
|
defm FADD : F3_fma_component<"add", fadd>;
|
|
defm FSUB : F3_fma_component<"sub", fsub>;
|
|
defm FMUL : F3_fma_component<"mul", fmul>;
|
|
|
|
def fminnum_or_fminimumnum : PatFrags<(ops node:$a, node:$b),
|
|
[(fminnum node:$a, node:$b),
|
|
(fminimumnum node:$a, node:$b)]>;
|
|
def fmaxnum_or_fmaximumnum : PatFrags<(ops node:$a, node:$b),
|
|
[(fmaxnum node:$a, node:$b),
|
|
(fmaximumnum node:$a, node:$b)]>;
|
|
|
|
defm MIN : FMINIMUMMAXIMUM<"min", /* NaN */ false, fminnum_or_fminimumnum>;
|
|
defm MAX : FMINIMUMMAXIMUM<"max", /* NaN */ false, fmaxnum_or_fmaximumnum>;
|
|
defm MIN_NAN : FMINIMUMMAXIMUM<"min", /* NaN */ true, fminimum>;
|
|
defm MAX_NAN : FMINIMUMMAXIMUM<"max", /* NaN */ true, fmaximum>;
|
|
|
|
def nvptx_fminnum3 : SDNode<"NVPTXISD::FMINNUM3", SDTFPTernaryOp,
|
|
[SDNPCommutative]>;
|
|
def nvptx_fmaxnum3 : SDNode<"NVPTXISD::FMAXNUM3", SDTFPTernaryOp,
|
|
[SDNPCommutative]>;
|
|
def nvptx_fminimum3 : SDNode<"NVPTXISD::FMINIMUM3", SDTFPTernaryOp,
|
|
[SDNPCommutative]>;
|
|
def nvptx_fmaximum3 : SDNode<"NVPTXISD::FMAXIMUM3", SDTFPTernaryOp,
|
|
[SDNPCommutative]>;
|
|
|
|
defm FMIN3 : FMINIMUMMAXIMUM3<"min", /* NaN */ false, nvptx_fminnum3>;
|
|
defm FMAX3 : FMINIMUMMAXIMUM3<"max", /* NaN */ false, nvptx_fmaxnum3>;
|
|
defm FMINNAN3 : FMINIMUMMAXIMUM3<"min", /* NaN */ true, nvptx_fminimum3>;
|
|
defm FMAXNAN3 : FMINIMUMMAXIMUM3<"max", /* NaN */ true, nvptx_fmaximum3>;
|
|
|
|
defm FABS : F2<"abs", fabs>;
|
|
defm FNEG : F2<"neg", fneg>;
|
|
defm FABS_H: F2_Support_Half<"abs", fabs>;
|
|
defm FNEG_H: F2_Support_Half<"neg", fneg>;
|
|
|
|
defm FSQRT : F2<"sqrt.rn", fsqrt>;
|
|
|
|
//
|
|
// F16 NEG
|
|
//
|
|
class FNEG16<RegTyInfo t> :
|
|
BasicFlagsNVPTXInst<(outs t.RC:$dst), (ins t.RC:$src), (ins FTZFlag:$ftz),
|
|
"neg$ftz." # t.PtxType,
|
|
[(set t.Ty:$dst, (fneg t.Ty:$src))]>;
|
|
|
|
let Predicates = [useFP16Math, hasPTX<60>, hasSM<53>] in {
|
|
def NEG_F16 : FNEG16<F16RT>;
|
|
def NEG_F16x2 : FNEG16<F16X2RT>;
|
|
}
|
|
let Predicates = [hasBF16Math, hasPTX<70>, hasSM<80>] in {
|
|
def NEG_BF16 : FNEG16<BF16RT>;
|
|
def NEG_BF16x2 : FNEG16<BF16X2RT>;
|
|
}
|
|
|
|
//
|
|
// EX2
|
|
//
|
|
|
|
class FEXP2Inst<RegTyInfo t, dag flags, string flag_str> :
|
|
BasicFlagsNVPTXInst<(outs t.RC:$dst), (ins t.RC:$src),
|
|
flags, "ex2.approx" # flag_str # "." # t.PtxType,
|
|
[(set t.Ty:$dst, (fexp2 t.Ty:$src))]>;
|
|
|
|
def EX2_APPROX_f32 : FEXP2Inst<F32RT, (ins FTZFlag:$ftz), "$ftz">;
|
|
|
|
let Predicates = [useFP16Math, hasPTX<70>, hasSM<75>] in {
|
|
def EX2_APPROX_f16 : FEXP2Inst<F16RT, (ins), "">;
|
|
def EX2_APPROX_f16x2 : FEXP2Inst<F16X2RT, (ins), "">;
|
|
}
|
|
let Predicates = [hasPTX<78>, hasSM<90>] in {
|
|
def EX2_APPROX_bf16 : FEXP2Inst<BF16RT, (ins), ".ftz">;
|
|
def EX2_APPROX_bf16x2 : FEXP2Inst<BF16X2RT, (ins), ".ftz">;
|
|
}
|
|
|
|
// F64 division
|
|
//
|
|
def FRCP64r :
|
|
BasicNVPTXInst<(outs B64:$dst),
|
|
(ins B64:$b),
|
|
"rcp.rn.f64",
|
|
[(set f64:$dst, (fdiv fpimm_1, f64:$b))]>;
|
|
def FDIV64rr :
|
|
BasicNVPTXInst<(outs B64:$dst),
|
|
(ins B64:$a, B64:$b),
|
|
"div.rn.f64",
|
|
[(set f64:$dst, (fdiv f64:$a, f64:$b))]>;
|
|
def FDIV64ri :
|
|
BasicNVPTXInst<(outs B64:$dst),
|
|
(ins B64:$a, f64imm:$b),
|
|
"div.rn.f64",
|
|
[(set f64:$dst, (fdiv f64:$a, fpimm:$b))]>;
|
|
|
|
// fdiv will be converted to rcp
|
|
// fneg (fdiv 1.0, X) => fneg (rcp.rn X)
|
|
def : Pat<(fdiv fpimm_neg_1, f64:$b),
|
|
(FNEGf64 (FRCP64r $b))>;
|
|
|
|
//
|
|
// F32 Approximate reciprocal
|
|
//
|
|
|
|
def fdiv_approx : PatFrag<(ops node:$a, node:$b),
|
|
(fdiv node:$a, node:$b), [{
|
|
return getDivF32Level(N) == NVPTX::DivPrecisionLevel::Approx;
|
|
}]>;
|
|
|
|
|
|
def RCP_APPROX_F32_r :
|
|
BasicFlagsNVPTXInst<(outs B32:$dst),
|
|
(ins B32:$b), (ins FTZFlag:$ftz),
|
|
"rcp.approx$ftz.f32",
|
|
[(set f32:$dst, (fdiv_approx fpimm_1, f32:$b))]>;
|
|
|
|
//
|
|
// F32 Approximate division
|
|
//
|
|
def DIV_APPROX_F32_rr :
|
|
BasicFlagsNVPTXInst<(outs B32:$dst),
|
|
(ins B32:$a, B32:$b), (ins FTZFlag:$ftz),
|
|
"div.approx$ftz.f32",
|
|
[(set f32:$dst, (fdiv_approx f32:$a, f32:$b))]>;
|
|
def DIV_APPROX_F32_ri :
|
|
BasicFlagsNVPTXInst<(outs B32:$dst),
|
|
(ins B32:$a, f32imm:$b), (ins FTZFlag:$ftz),
|
|
"div.approx$ftz.f32",
|
|
[(set f32:$dst, (fdiv_approx f32:$a, fpimm:$b))]>;
|
|
//
|
|
// F32 Semi-accurate reciprocal
|
|
//
|
|
// rcp.approx gives the same result as div.full(1.0f, a) and is faster.
|
|
//
|
|
|
|
def fdiv_full : PatFrag<(ops node:$a, node:$b),
|
|
(fdiv node:$a, node:$b), [{
|
|
return getDivF32Level(N) == NVPTX::DivPrecisionLevel::Full;
|
|
}]>;
|
|
|
|
|
|
def : Pat<(fdiv_full fpimm_1, f32:$b),
|
|
(RCP_APPROX_F32_r $b)>;
|
|
|
|
//
|
|
// F32 Semi-accurate division
|
|
//
|
|
def FDIV32rr :
|
|
BasicFlagsNVPTXInst<(outs B32:$dst),
|
|
(ins B32:$a, B32:$b), (ins FTZFlag:$ftz),
|
|
"div.full$ftz.f32",
|
|
[(set f32:$dst, (fdiv_full f32:$a, f32:$b))]>;
|
|
def FDIV32ri :
|
|
BasicFlagsNVPTXInst<(outs B32:$dst),
|
|
(ins B32:$a, f32imm:$b), (ins FTZFlag:$ftz),
|
|
"div.full$ftz.f32",
|
|
[(set f32:$dst, (fdiv_full f32:$a, fpimm:$b))]>;
|
|
//
|
|
// F32 Accurate reciprocal
|
|
//
|
|
|
|
def fdiv_ftz : PatFrag<(ops node:$a, node:$b),
|
|
(fdiv node:$a, node:$b), [{
|
|
return getDivF32Level(N) == NVPTX::DivPrecisionLevel::IEEE754;
|
|
}]>;
|
|
|
|
def FRCP32r_prec :
|
|
BasicFlagsNVPTXInst<(outs B32:$dst),
|
|
(ins B32:$b), (ins FTZFlag:$ftz),
|
|
"rcp.rn$ftz.f32",
|
|
[(set f32:$dst, (fdiv_ftz fpimm_1, f32:$b))]>;
|
|
//
|
|
// F32 Accurate division
|
|
//
|
|
def FDIV32rr_prec :
|
|
BasicFlagsNVPTXInst<(outs B32:$dst),
|
|
(ins B32:$a, B32:$b), (ins FTZFlag:$ftz),
|
|
"div.rn$ftz.f32",
|
|
[(set f32:$dst, (fdiv_ftz f32:$a, f32:$b))]>;
|
|
def FDIV32ri_prec :
|
|
BasicFlagsNVPTXInst<(outs B32:$dst),
|
|
(ins B32:$a, f32imm:$b), (ins FTZFlag:$ftz),
|
|
"div.rn$ftz.f32",
|
|
[(set f32:$dst, (fdiv_ftz f32:$a, fpimm:$b))]>;
|
|
|
|
def : Pat<(fdiv fpimm_1, f32:$b), (FRCP32r_prec $b, NoFTZ)>;
|
|
def : Pat<(fdiv f32:$a, f32:$b), (FDIV32rr_prec $a, $b, NoFTZ)>;
|
|
def : Pat<(fdiv f32:$a, fpimm:$b), (FDIV32ri_prec $a, fpimm:$b, NoFTZ)>;
|
|
|
|
//
|
|
// FMA
|
|
//
|
|
|
|
multiclass FMA<RegTyInfo t, bit allow_ftz = true, list<Predicate> preds = []> {
|
|
defvar flag_str = !if(allow_ftz, "$ftz", "");
|
|
defvar flag_ops = !if(allow_ftz, (ins FTZFlag:$ftz), (ins));
|
|
defvar op_str = "fma.rn" # flag_str # "." # t.PtxType;
|
|
|
|
let Predicates = preds in {
|
|
def rrr : BasicFlagsNVPTXInst<(outs t.RC:$dst), (ins t.RC:$a, t.RC:$b, t.RC:$c),
|
|
flag_ops, op_str,
|
|
[(set t.Ty:$dst, (fma t.Ty:$a, t.Ty:$b, t.Ty:$c))]>;
|
|
|
|
if t.SupportsImm then {
|
|
def rri : BasicFlagsNVPTXInst<(outs t.RC:$dst),
|
|
(ins t.RC:$a, t.RC:$b, t.Imm:$c),
|
|
flag_ops, op_str,
|
|
[(set t.Ty:$dst, (fma t.Ty:$a, t.Ty:$b, fpimm:$c))]>;
|
|
def rir : BasicFlagsNVPTXInst<(outs t.RC:$dst),
|
|
(ins t.RC:$a, t.Imm:$b, t.RC:$c),
|
|
flag_ops, op_str,
|
|
[(set t.Ty:$dst, (fma t.Ty:$a, fpimm:$b, t.Ty:$c))]>;
|
|
def rii : BasicFlagsNVPTXInst<(outs t.RC:$dst),
|
|
(ins t.RC:$a, t.Imm:$b, t.Imm:$c),
|
|
flag_ops, op_str,
|
|
[(set t.Ty:$dst, (fma t.Ty:$a, fpimm:$b, fpimm:$c))]>;
|
|
def iir : BasicFlagsNVPTXInst<(outs t.RC:$dst),
|
|
(ins t.Imm:$a, t.Imm:$b, t.RC:$c),
|
|
flag_ops, op_str,
|
|
[(set t.Ty:$dst, (fma fpimm:$a, fpimm:$b, t.Ty:$c))]>;
|
|
}
|
|
}
|
|
}
|
|
|
|
defm FMA_F16 : FMA<F16RT, allow_ftz = true, preds = [useFP16Math]>;
|
|
defm FMA_F16x2 : FMA<F16X2RT, allow_ftz = true, preds = [useFP16Math]>;
|
|
defm FMA_BF16 : FMA<BF16RT, allow_ftz = false, preds = [hasBF16Math]>;
|
|
defm FMA_BF16x2 : FMA<BF16X2RT, allow_ftz = false, preds = [hasBF16Math]>;
|
|
defm FMA_F32 : FMA<F32RT, allow_ftz = true>;
|
|
defm FMA_F32x2 : FMA<F32X2RT, allow_ftz = true, preds = [hasF32x2Instructions]>;
|
|
defm FMA_F64 : FMA<F64RT, allow_ftz = false>;
|
|
|
|
// sin/cos/tanh
|
|
|
|
class UnaryOpAllowsApproxFn<SDPatternOperator operator>
|
|
: PatFrag<(ops node:$A), (operator node:$A), [{
|
|
return N->getFlags().hasApproximateFuncs();
|
|
}]>;
|
|
|
|
def SIN_APPROX_f32 :
|
|
BasicFlagsNVPTXInst<(outs B32:$dst), (ins B32:$src), (ins FTZFlag:$ftz),
|
|
"sin.approx$ftz.f32",
|
|
[(set f32:$dst, (UnaryOpAllowsApproxFn<fsin> f32:$src))]>;
|
|
def COS_APPROX_f32 :
|
|
BasicFlagsNVPTXInst<(outs B32:$dst), (ins B32:$src), (ins FTZFlag:$ftz),
|
|
"cos.approx$ftz.f32",
|
|
[(set f32:$dst, (UnaryOpAllowsApproxFn<fcos> f32:$src))]>;
|
|
def TANH_APPROX_f32 :
|
|
BasicNVPTXInst<(outs B32:$dst), (ins B32:$src), "tanh.approx.f32",
|
|
[(set f32:$dst, (UnaryOpAllowsApproxFn<ftanh> f32:$src))]>,
|
|
Requires<[hasPTX<70>, hasSM<75>]>;
|
|
|
|
//-----------------------------------
|
|
// Bitwise operations
|
|
//-----------------------------------
|
|
|
|
// Template for three-arg bitwise operations. Takes three args, Creates .b16,
|
|
// .b32, .b64, and .pred (predicate registers -- i.e., i1) versions of OpcStr.
|
|
multiclass BITWISE<string OpcStr, SDNode OpNode> {
|
|
foreach t = [I1RT, I16RT, I32RT, I64RT] in
|
|
defm _ # t.PtxType : I3Inst<OpcStr # "." # t.PtxType, OpNode, t, commutative = true>;
|
|
}
|
|
|
|
defm OR : BITWISE<"or", or>;
|
|
defm AND : BITWISE<"and", and>;
|
|
defm XOR : BITWISE<"xor", xor>;
|
|
|
|
// PTX does not support mul on predicates, convert to and instructions
|
|
def : Pat<(mul i1:$a, i1:$b), (AND_predrr $a, $b)>;
|
|
def : Pat<(mul i1:$a, imm:$b), (AND_predri $a, imm:$b)>;
|
|
|
|
foreach op = [add, sub] in {
|
|
def : Pat<(op i1:$a, i1:$b), (XOR_predrr $a, $b)>;
|
|
def : Pat<(op i1:$a, imm:$b), (XOR_predri $a, imm:$b)>;
|
|
}
|
|
|
|
// These transformations were once reliably performed by instcombine, but thanks
|
|
// to poison semantics they are no longer safe for LLVM IR, perform them here
|
|
// instead.
|
|
def : Pat<(select i1:$a, i1:$b, 0), (AND_predrr $a, $b)>;
|
|
def : Pat<(select i1:$a, 1, i1:$b), (OR_predrr $a, $b)>;
|
|
|
|
// Lower logical v2i16/v4i8 ops as bitwise ops on b32.
|
|
foreach vt = [v2i16, v4i8] in {
|
|
def : Pat<(or vt:$a, vt:$b), (OR_b32rr $a, $b)>;
|
|
def : Pat<(xor vt:$a, vt:$b), (XOR_b32rr $a, $b)>;
|
|
def : Pat<(and vt:$a, vt:$b), (AND_b32rr $a, $b)>;
|
|
|
|
// The constants get legalized into a bitcast from i32, so that's what we need
|
|
// to match here.
|
|
def: Pat<(or vt:$a, (vt (bitconvert (i32 imm:$b)))),
|
|
(OR_b32ri $a, imm:$b)>;
|
|
def: Pat<(xor vt:$a, (vt (bitconvert (i32 imm:$b)))),
|
|
(XOR_b32ri $a, imm:$b)>;
|
|
def: Pat<(and vt:$a, (vt (bitconvert (i32 imm:$b)))),
|
|
(AND_b32ri $a, imm:$b)>;
|
|
}
|
|
|
|
foreach t = [I1RT, I16RT, I32RT, I64RT] in
|
|
def NOT_ # t.PtxType : BasicNVPTXInst<(outs t.RC:$dst), (ins t.RC:$src),
|
|
"not." # t.PtxType,
|
|
[(set t.Ty:$dst, (not t.Ty:$src))]>;
|
|
|
|
// Template for left/right shifts. Takes three operands,
|
|
// [dest (reg), src (reg), shift (reg or imm)].
|
|
// dest and src may be int64, int32, or int16, but shift is always int32.
|
|
//
|
|
// This template also defines a 32-bit shift (imm, imm) instruction.
|
|
multiclass SHIFT<string OpcStr, SDNode OpNode> {
|
|
let hasSideEffects = false in {
|
|
foreach t = [I64RT, I32RT, I16RT] in {
|
|
def t.Size # _rr :
|
|
BasicNVPTXInst<(outs t.RC:$dst), (ins t.RC:$a, B32:$b),
|
|
OpcStr # t.Size,
|
|
[(set t.Ty:$dst, (OpNode t.Ty:$a, i32:$b))]>;
|
|
def t.Size # _ri :
|
|
BasicNVPTXInst<(outs t.RC:$dst), (ins t.RC:$a, i32imm:$b),
|
|
OpcStr # t.Size,
|
|
[(set t.Ty:$dst, (OpNode t.Ty:$a, (i32 imm:$b)))]>;
|
|
def t.Size # _ii :
|
|
BasicNVPTXInst<(outs t.RC:$dst), (ins t.RC:$a, i32imm:$b),
|
|
OpcStr # t.Size,
|
|
[(set t.Ty:$dst, (OpNode (t.Ty imm:$a), (i32 imm:$b)))]>;
|
|
}
|
|
}
|
|
}
|
|
|
|
defm SHL : SHIFT<"shl.b", shl>;
|
|
defm SRA : SHIFT<"shr.s", sra>;
|
|
defm SRL : SHIFT<"shr.u", srl>;
|
|
|
|
// Shift with clamping semantics - these have defined behavior for shift amounts
|
|
// >= BitWidth (returning 0 for logical shifts). Used to optimize guarded shift
|
|
// patterns like `shift >= 32 ? 0 : x >> shift`.
|
|
def shl_clamp : SDNode<"NVPTXISD::SHL_CLAMP", SDTIntShiftOp, []>;
|
|
def srl_clamp : SDNode<"NVPTXISD::SRL_CLAMP", SDTIntShiftOp, []>;
|
|
|
|
defm SHL_CLAMP : SHIFT<"shl.b", shl_clamp>;
|
|
defm SRL_CLAMP : SHIFT<"shr.u", srl_clamp>;
|
|
|
|
// Bit-reverse
|
|
foreach t = [I64RT, I32RT] in
|
|
def BREV_ # t.PtxType :
|
|
BasicNVPTXInst<(outs t.RC:$dst), (ins t.RC:$a),
|
|
"brev." # t.PtxType,
|
|
[(set t.Ty:$dst, (bitreverse t.Ty:$a))]>;
|
|
|
|
|
|
//
|
|
// BFE - bit-field extract
|
|
//
|
|
|
|
// Template for BFE/BFI instructions.
|
|
// Args: [dest (reg), src (reg), start (reg or imm), end (reg or imm)].
|
|
// Start may be an imm only if end is also an imm. FIXME: Is this a
|
|
// restriction in PTX?
|
|
//
|
|
// dest and src may be int32 or int64, but start and end are always int32.
|
|
def SDTBFI :
|
|
SDTypeProfile<1, 4, [SDTCisInt<0>, SDTCisSameAs<0, 1>, SDTCisSameAs<0, 2>,
|
|
SDTCisVT<3, i32>, SDTCisVT<4, i32>]>;
|
|
def bfi : SDNode<"NVPTXISD::BFI", SDTBFI>;
|
|
|
|
def SDTPRMT :
|
|
SDTypeProfile<1, 4, [SDTCisVT<0, i32>, SDTCisVT<1, i32>,
|
|
SDTCisVT<2, i32>, SDTCisVT<3, i32>, SDTCisVT<4, i32>]>;
|
|
def prmt : SDNode<"NVPTXISD::PRMT", SDTPRMT>;
|
|
|
|
multiclass BFE<string Instr, RegisterClass RC> {
|
|
def rrr
|
|
: BasicNVPTXInst<(outs RC:$d), (ins RC:$a, B32:$b, B32:$c), Instr>;
|
|
def rri
|
|
: BasicNVPTXInst<(outs RC:$d), (ins RC:$a, B32:$b, i32imm:$c), Instr>;
|
|
def rii
|
|
: BasicNVPTXInst<(outs RC:$d), (ins RC:$a, i32imm:$b, i32imm:$c), Instr>;
|
|
}
|
|
|
|
multiclass BFI<string Instr, ValueType T, RegisterClass RC, Operand ImmCls> {
|
|
def rrrr
|
|
: BasicNVPTXInst<(outs RC:$f),
|
|
(ins RC:$a, RC:$b, B32:$c, B32:$d),
|
|
Instr,
|
|
[(set T:$f, (bfi T:$a, T:$b, i32:$c, i32:$d))]>;
|
|
def rrri
|
|
: BasicNVPTXInst<(outs RC:$f),
|
|
(ins RC:$a, RC:$b, B32:$c, i32imm:$d),
|
|
Instr,
|
|
[(set T:$f, (bfi T:$a, T:$b, i32:$c, imm:$d))]>;
|
|
def rrii
|
|
: BasicNVPTXInst<(outs RC:$f),
|
|
(ins RC:$a, RC:$b, i32imm:$c, i32imm:$d),
|
|
Instr,
|
|
[(set T:$f, (bfi T:$a, T:$b, imm:$c, imm:$d))]>;
|
|
def irrr
|
|
: BasicNVPTXInst<(outs RC:$f),
|
|
(ins ImmCls:$a, RC:$b, B32:$c, B32:$d),
|
|
Instr,
|
|
[(set T:$f, (bfi (T imm:$a), T:$b, i32:$c, i32:$d))]>;
|
|
def irri
|
|
: BasicNVPTXInst<(outs RC:$f),
|
|
(ins ImmCls:$a, RC:$b, B32:$c, i32imm:$d),
|
|
Instr,
|
|
[(set T:$f, (bfi (T imm:$a), T:$b, i32:$c, imm:$d))]>;
|
|
def irii
|
|
: BasicNVPTXInst<(outs RC:$f),
|
|
(ins ImmCls:$a, RC:$b, i32imm:$c, i32imm:$d),
|
|
Instr,
|
|
[(set T:$f, (bfi (T imm:$a), T:$b, imm:$c, imm:$d))]>;
|
|
}
|
|
|
|
def Hexu32imm : Operand<i32> {
|
|
let PrintMethod = "printHexu32imm";
|
|
}
|
|
|
|
let hasSideEffects = false in {
|
|
// order is somewhat important here. signed/unsigned variants match
|
|
// the same patterns, so the first one wins. Having unsigned byte extraction
|
|
// has the benefit of always having zero in unused bits, which makes some
|
|
// optimizations easier (e.g. no need to mask them).
|
|
defm BFE_U32 : BFE<"bfe.u32", B32>;
|
|
defm BFE_S32 : BFE<"bfe.s32", B32>;
|
|
defm BFE_U64 : BFE<"bfe.u64", B64>;
|
|
defm BFE_S64 : BFE<"bfe.s64", B64>;
|
|
|
|
defm BFI_B32 : BFI<"bfi.b32", i32, B32, i32imm>;
|
|
defm BFI_B64 : BFI<"bfi.b64", i64, B64, i64imm>;
|
|
|
|
def PRMT_B32rrr
|
|
: BasicFlagsNVPTXInst<(outs B32:$d),
|
|
(ins B32:$a, B32:$b, B32:$c),
|
|
(ins PrmtMode:$mode),
|
|
"prmt.b32$mode",
|
|
[(set i32:$d, (prmt i32:$a, i32:$b, i32:$c, imm:$mode))]>;
|
|
def PRMT_B32rri
|
|
: BasicFlagsNVPTXInst<(outs B32:$d),
|
|
(ins B32:$a, B32:$b, Hexu32imm:$c),
|
|
(ins PrmtMode:$mode),
|
|
"prmt.b32$mode",
|
|
[(set i32:$d, (prmt i32:$a, i32:$b, imm:$c, imm:$mode))]>;
|
|
def PRMT_B32rir
|
|
: BasicFlagsNVPTXInst<(outs B32:$d),
|
|
(ins B32:$a, i32imm:$b, B32:$c),
|
|
(ins PrmtMode:$mode),
|
|
"prmt.b32$mode",
|
|
[(set i32:$d, (prmt i32:$a, imm:$b, i32:$c, imm:$mode))]>;
|
|
def PRMT_B32rii
|
|
: BasicFlagsNVPTXInst<(outs B32:$d),
|
|
(ins B32:$a, i32imm:$b, Hexu32imm:$c),
|
|
(ins PrmtMode:$mode),
|
|
"prmt.b32$mode",
|
|
[(set i32:$d, (prmt i32:$a, imm:$b, imm:$c, imm:$mode))]>;
|
|
def PRMT_B32irr
|
|
: BasicFlagsNVPTXInst<(outs B32:$d),
|
|
(ins i32imm:$a, B32:$b, B32:$c), (ins PrmtMode:$mode),
|
|
"prmt.b32$mode",
|
|
[(set i32:$d, (prmt imm:$a, i32:$b, i32:$c, imm:$mode))]>;
|
|
def PRMT_B32iri
|
|
: BasicFlagsNVPTXInst<(outs B32:$d),
|
|
(ins i32imm:$a, B32:$b, Hexu32imm:$c), (ins PrmtMode:$mode),
|
|
"prmt.b32$mode",
|
|
[(set i32:$d, (prmt imm:$a, i32:$b, imm:$c, imm:$mode))]>;
|
|
def PRMT_B32iir
|
|
: BasicFlagsNVPTXInst<(outs B32:$d),
|
|
(ins i32imm:$a, i32imm:$b, B32:$c), (ins PrmtMode:$mode),
|
|
"prmt.b32$mode",
|
|
[(set i32:$d, (prmt imm:$a, imm:$b, i32:$c, imm:$mode))]>;
|
|
|
|
}
|
|
|
|
// PRMT folding patterns
|
|
def : Pat<(fshr i32:$hi, i32:$lo, (shl i32:$amt, (i32 3))),
|
|
(PRMT_B32rrr $lo, $hi, $amt, PrmtF4E)>;
|
|
|
|
|
|
def byte_extract_prmt : ImmLeaf<i32, [{
|
|
return (Imm == 0x7770) || (Imm == 0x7771) || (Imm == 0x7772) || (Imm == 0x7773);
|
|
}]>;
|
|
|
|
def to_sign_extend_selector : SDNodeXForm<imm, [{
|
|
const APInt &V = N->getAPIntValue();
|
|
const APInt B = V.trunc(4);
|
|
const APInt BSext = B | 8;
|
|
const APInt R = BSext.concat(BSext).concat(BSext).concat(B).zext(32);
|
|
return CurDAG->getTargetConstant(R, SDLoc(N), MVT::i32);
|
|
}]>;
|
|
|
|
|
|
// byte extraction + signed/unsigned extension to i32.
|
|
def : Pat<(i32 (sext_inreg (prmt i32:$s, 0, byte_extract_prmt:$sel, PrmtNONE), i8)),
|
|
(PRMT_B32rii $s, 0, (to_sign_extend_selector $sel), PrmtNONE)>;
|
|
|
|
// byte extraction + signed extension to i16
|
|
def : Pat<(i16 (sext_inreg (trunc (prmt i32:$s, 0, byte_extract_prmt:$sel, PrmtNONE)), i8)),
|
|
(CVT_u16_u32 (PRMT_B32rii $s, 0, (to_sign_extend_selector $sel), PrmtNONE), CvtNONE)>;
|
|
|
|
|
|
// Byte extraction via shift/trunc/sext
|
|
def : Pat<(i16 (sext_inreg (trunc i32:$s), i8)), (CVT_s8_s32 $s, CvtNONE)>;
|
|
def : Pat<(i16 (sext_inreg (trunc i64:$s), i8)), (CVT_s8_s64 $s, CvtNONE)>;
|
|
|
|
def : Pat<(sext_inreg (srl i32:$s, (i32 imm:$o)), i8), (BFE_S32rii $s, imm:$o, 8)>;
|
|
def : Pat<(sext_inreg (srl i64:$s, (i32 imm:$o)), i8), (BFE_S64rii $s, imm:$o, 8)>;
|
|
|
|
def : Pat<(i16 (sext_inreg (trunc (srl i32:$s, (i32 imm:$o))), i8)),
|
|
(CVT_s8_s32 (BFE_S32rii $s, imm:$o, 8), CvtNONE)>;
|
|
def : Pat<(i16 (sext_inreg (trunc (srl i64:$s, (i32 imm:$o))), i8)),
|
|
(CVT_s8_s64 (BFE_S64rii $s, imm:$o, 8), CvtNONE)>;
|
|
|
|
def : Pat<(i16 (sra (i16 (trunc i32:$s)), (i32 8))),
|
|
(CVT_s8_s32 (BFE_S32rii $s, 8, 8), CvtNONE)>;
|
|
|
|
//-----------------------------------
|
|
// Comparison instructions (setp, set)
|
|
//-----------------------------------
|
|
|
|
// FIXME: This doesn't cover versions of set and setp that combine with a
|
|
// boolean predicate, e.g. setp.eq.and.b16.
|
|
def cond2cc : SDNodeXForm<cond, [{
|
|
return getPTXCmpMode(*N);
|
|
}]>;
|
|
|
|
multiclass FSETP<RegTyInfo t, bit allow_ftz = true> {
|
|
defvar ftz_str = !if(allow_ftz, "$ftz", "");
|
|
defvar op_str = "setp.${cmp:FCmp}" # ftz_str # "." # t.PtxType;
|
|
defvar flags = !con((ins CmpMode:$cmp), !if(allow_ftz, (ins FTZFlag:$ftz), (ins)));
|
|
let hasSideEffects = false, isCompare = 1, isCommutable = 1 in {
|
|
def rr :
|
|
BasicFlagsNVPTXInst<(outs B1:$dst), (ins t.RC:$a, t.RC:$b),
|
|
flags, op_str>;
|
|
|
|
if t.SupportsImm then {
|
|
def ri :
|
|
BasicFlagsNVPTXInst<(outs B1:$dst), (ins t.RC:$a, t.Imm:$b),
|
|
flags, op_str>;
|
|
def ir :
|
|
BasicFlagsNVPTXInst<(outs B1:$dst), (ins t.Imm:$a, t.RC:$b),
|
|
flags, op_str>;
|
|
}
|
|
}
|
|
def : Pat<(i1 (setcc t.Ty:$a, t.Ty:$b, cond:$cc)),
|
|
(!cast<NVPTXInst>(NAME # "rr") $a, $b, (cond2cc $cc))>;
|
|
if t.SupportsImm then {
|
|
def : Pat<(i1 (setcc t.Ty:$a, fpimm:$b, cond:$cc)),
|
|
(!cast<NVPTXInst>(NAME # "ri") $a, fpimm:$b, (cond2cc $cc))>;
|
|
def : Pat<(i1 (setcc fpimm:$a, t.Ty:$b, cond:$cc)),
|
|
(!cast<NVPTXInst>(NAME # "ir") fpimm:$a, $b, (cond2cc $cc))>;
|
|
}
|
|
}
|
|
|
|
multiclass ISETP<RegTyInfo t> {
|
|
defvar op_str = "setp.${cmp:ICmp}.${cmp:IType}" # t.Size;
|
|
let hasSideEffects = false, isCompare = 1, isCommutable = 1 in {
|
|
def rr :
|
|
BasicFlagsNVPTXInst<(outs B1:$dst), (ins t.RC:$a, t.RC:$b),
|
|
(ins CmpMode:$cmp), op_str>;
|
|
def ri :
|
|
BasicFlagsNVPTXInst<(outs B1:$dst), (ins t.RC:$a, t.Imm:$b),
|
|
(ins CmpMode:$cmp), op_str>;
|
|
def ir :
|
|
BasicFlagsNVPTXInst<(outs B1:$dst), (ins t.Imm:$a, t.RC:$b),
|
|
(ins CmpMode:$cmp), op_str>;
|
|
}
|
|
def : Pat<(i1 (setcc t.Ty:$a, t.Ty:$b, cond:$cc)),
|
|
(!cast<NVPTXInst>(NAME # "rr") $a, $b, (cond2cc $cc))>;
|
|
def : Pat<(i1 (setcc t.Ty:$a, imm:$b, cond:$cc)),
|
|
(!cast<NVPTXInst>(NAME # "ri") $a, imm:$b, (cond2cc $cc))>;
|
|
def : Pat<(i1 (setcc imm:$a, t.Ty:$b, cond:$cc)),
|
|
(!cast<NVPTXInst>(NAME # "ir") imm:$a, $b, (cond2cc $cc))>;
|
|
}
|
|
|
|
defm SETP_i16 : ISETP<I16RT>;
|
|
defm SETP_i32 : ISETP<I32RT>;
|
|
defm SETP_i64 : ISETP<I64RT>;
|
|
|
|
defm SETP_f32 : FSETP<F32RT>;
|
|
defm SETP_f64 : FSETP<F64RT, allow_ftz = false>;
|
|
let Predicates = [useFP16Math] in
|
|
defm SETP_f16 : FSETP<F16RT>;
|
|
let Predicates = [hasBF16Math, hasPTX<78>, hasSM<90>] in
|
|
defm SETP_bf16 : FSETP<BF16RT>;
|
|
|
|
def SETP_f16x2rr :
|
|
BasicFlagsNVPTXInst<(outs B1:$p, B1:$q),
|
|
(ins B32:$a, B32:$b), (ins CmpMode:$cmp, FTZFlag:$ftz),
|
|
"setp.${cmp:FCmp}$ftz.f16x2">,
|
|
Requires<[useFP16Math]>;
|
|
|
|
def SETP_bf16x2rr :
|
|
BasicFlagsNVPTXInst<(outs B1:$p, B1:$q),
|
|
(ins B32:$a, B32:$b), (ins CmpMode:$cmp, FTZFlag:$ftz),
|
|
"setp.${cmp:FCmp}$ftz.bf16x2">,
|
|
Requires<[hasBF16Math, hasPTX<78>, hasSM<90>]>;
|
|
|
|
//-----------------------------------
|
|
// Data Movement (Load / Store, Move)
|
|
//-----------------------------------
|
|
|
|
def addr : ComplexPattern<pAny, 2, "SelectADDR">;
|
|
|
|
def ADDR_base : Operand<pAny>;
|
|
def ADDR : Operand<pAny> {
|
|
let PrintMethod = "printMemOperand";
|
|
let MIOperandInfo = (ops ADDR_base, i32imm);
|
|
}
|
|
|
|
def UsedBytesMask : Operand<i32> {
|
|
let PrintMethod = "printUsedBytesMaskPragma";
|
|
}
|
|
|
|
def RegOrSink : Operand<Any> {
|
|
let PrintMethod = "printRegisterOrSinkSymbol";
|
|
}
|
|
|
|
def AtomicCode : Operand<i32> {
|
|
let PrintMethod = "printAtomicCode";
|
|
}
|
|
|
|
def MmaCode : Operand<i32> {
|
|
let PrintMethod = "printMmaCode";
|
|
}
|
|
|
|
// Get pointer to local stack.
|
|
let hasSideEffects = false in {
|
|
def MOV_DEPOT_ADDR : NVPTXInst<(outs B32:$d), (ins i32imm:$num),
|
|
"mov.b32 \t$d, __local_depot$num;">;
|
|
def MOV_DEPOT_ADDR_64 : NVPTXInst<(outs B64:$d), (ins i32imm:$num),
|
|
"mov.b64 \t$d, __local_depot$num;">;
|
|
}
|
|
|
|
let hasSideEffects = false in {
|
|
let isMoveReg = true, isAsCheapAsAMove = true in
|
|
class MOVr<RegisterClass RC, string OpStr> :
|
|
BasicNVPTXInst<(outs RC:$dst), (ins RC:$src), "mov." # OpStr>;
|
|
|
|
let isMoveImm = true, isAsCheapAsAMove = true in
|
|
class MOVi<RegTyInfo t, string suffix> :
|
|
BasicNVPTXInst<(outs t.RC:$dst), (ins t.Imm:$src),
|
|
"mov." # suffix,
|
|
[(set t.Ty:$dst, t.ImmNode:$src)]>;
|
|
|
|
// We don't want to set isAsCheapAsAMove to true for these instructions as
|
|
// this would prevent CSE and resulted in regressions (see discussion after
|
|
// PR-145581 in llvm-project).
|
|
class MovSymInst<RegTyInfo t> :
|
|
BasicNVPTXInst<(outs t.RC:$dst), (ins Operand<t.Ty>:$src),
|
|
"mov.b" # t.Size>;
|
|
}
|
|
|
|
def MOV_B1_r : MOVr<B1, "pred">;
|
|
def MOV_B16_r : MOVr<B16, "b16">;
|
|
def MOV_B32_r : MOVr<B32, "b32">;
|
|
def MOV_B64_r : MOVr<B64, "b64">;
|
|
def MOV_B128_r : MOVr<B128, "b128">;
|
|
|
|
def MOV_B1_i : MOVi<I1RT, "pred">;
|
|
def MOV_B16_i : MOVi<I16RT, "b16">;
|
|
def MOV_B32_i : MOVi<I32RT, "b32">;
|
|
def MOV_B64_i : MOVi<I64RT, "b64">;
|
|
def MOV_F16_i : MOVi<F16RT, "b16">;
|
|
def MOV_BF16_i : MOVi<BF16RT, "b16">;
|
|
def MOV_F32_i : MOVi<F32RT, "b32">;
|
|
def MOV_F64_i : MOVi<F64RT, "b64">;
|
|
|
|
def MOV_B32_sym : MovSymInst<I32RT>;
|
|
def MOV_B64_sym : MovSymInst<I64RT>;
|
|
|
|
|
|
def to_tglobaladdr : SDNodeXForm<globaladdr, [{
|
|
return CurDAG->getTargetGlobalAddress(N->getGlobal(), SDLoc(N),
|
|
N->getValueType(0), N->getOffset(),
|
|
N->getTargetFlags());
|
|
}]>;
|
|
|
|
def to_texternsym : SDNodeXForm<externalsym, [{
|
|
return CurDAG->getTargetExternalSymbol(N->getSymbol(), N->getValueType(0),
|
|
N->getTargetFlags());
|
|
}]>;
|
|
|
|
def to_tframeindex : SDNodeXForm<frameindex, [{
|
|
return CurDAG->getTargetFrameIndex(N->getIndex(), N->getValueType(0));
|
|
}]>;
|
|
|
|
def : Pat<(i32 globaladdr:$dst), (MOV_B32_sym (to_tglobaladdr $dst))>;
|
|
def : Pat<(i64 globaladdr:$dst), (MOV_B64_sym (to_tglobaladdr $dst))>;
|
|
|
|
def : Pat<(i32 externalsym:$dst), (MOV_B32_sym (to_texternsym $dst))>;
|
|
def : Pat<(i64 externalsym:$dst), (MOV_B64_sym (to_texternsym $dst))>;
|
|
|
|
//---- Copy Frame Index ----
|
|
def LEA_ADDRi : NVPTXInst<(outs B32:$dst), (ins ADDR:$addr),
|
|
"add.u32 \t$dst, ${addr:add};">;
|
|
def LEA_ADDRi64 : NVPTXInst<(outs B64:$dst), (ins ADDR:$addr),
|
|
"add.u64 \t$dst, ${addr:add};">;
|
|
|
|
def : Pat<(i32 frameindex:$fi), (LEA_ADDRi (to_tframeindex $fi), 0)>;
|
|
def : Pat<(i64 frameindex:$fi), (LEA_ADDRi64 (to_tframeindex $fi), 0)>;
|
|
|
|
//-----------------------------------
|
|
// Comparison and Selection
|
|
//-----------------------------------
|
|
// TODO: These patterns seem very specific and brittle. We should try to find
|
|
// a more general solution.
|
|
|
|
def cond_signed : PatLeaf<(cond), [{
|
|
return isSignedIntSetCC(N->get());
|
|
}]>;
|
|
|
|
// A 16-bit signed comparison of sign-extended byte extracts can be converted
|
|
// to 32-bit comparison if we change the PRMT to sign-extend the extracted
|
|
// bytes.
|
|
def : Pat<(setcc (i16 (sext_inreg (trunc (prmt i32:$a, 0, byte_extract_prmt:$sel_a, PrmtNONE)), i8)),
|
|
(i16 (sext_inreg (trunc (prmt i32:$b, 0, byte_extract_prmt:$sel_b, PrmtNONE)), i8)),
|
|
cond_signed:$cc),
|
|
(SETP_i32rr (PRMT_B32rii i32:$a, 0, (to_sign_extend_selector $sel_a), PrmtNONE),
|
|
(PRMT_B32rii i32:$b, 0, (to_sign_extend_selector $sel_b), PrmtNONE),
|
|
(cond2cc $cc))>;
|
|
|
|
def SDTDeclareArrayParam :
|
|
SDTypeProfile<0, 3, [SDTCisVT<0, i32>, SDTCisVT<1, i32>, SDTCisVT<2, i32>]>;
|
|
def SDTDeclareScalarParam :
|
|
SDTypeProfile<0, 2, [SDTCisVT<0, i32>, SDTCisVT<1, i32>]>;
|
|
def SDTMoveParamProfile : SDTypeProfile<1, 1, [SDTCisInt<0>, SDTCisSameAs<0, 1>]>;
|
|
|
|
def SDTProxyReg : SDTypeProfile<1, 1, [SDTCisSameAs<0, 1>]>;
|
|
|
|
// These nodes represent a parameter declaration. In PTX this will look like:
|
|
// .param .align 16 .b8 param0[1024];
|
|
// .param .b32 retval0;
|
|
//
|
|
// DeclareArrayParam(Chain, Externalsym, Align, Size, Glue)
|
|
// DeclareScalarParam(Chain, Externalsym, Size, Glue)
|
|
def declare_array_param :
|
|
SDNode<"NVPTXISD::DeclareArrayParam", SDTDeclareArrayParam,
|
|
[SDNPHasChain, SDNPOutGlue, SDNPInGlue, SDNPSideEffect]>;
|
|
def declare_scalar_param :
|
|
SDNode<"NVPTXISD::DeclareScalarParam", SDTDeclareScalarParam,
|
|
[SDNPHasChain, SDNPOutGlue, SDNPInGlue, SDNPSideEffect]>;
|
|
|
|
def MoveParam :
|
|
SDNode<"NVPTXISD::MoveParam", SDTMoveParamProfile, []>;
|
|
def proxy_reg :
|
|
SDNode<"NVPTXISD::ProxyReg", SDTProxyReg, [SDNPHasChain]>;
|
|
|
|
/// CALL(Chain, IsConvergent, IsIndirectCall/IsUniform, NumReturns,
|
|
/// NumParams, Callee, Proto)
|
|
def SDTCallProfile : SDTypeProfile<0, 6,
|
|
[SDTCisVT<0, i32>, SDTCisVT<1, i32>, SDTCisVT<2, i32>,
|
|
SDTCisVT<3, i32>, SDTCisVT<5, i32>]>;
|
|
def call : SDNode<"NVPTXISD::CALL", SDTCallProfile, [SDNPHasChain, SDNPSideEffect]>;
|
|
|
|
/// CALL(Chain, IsConvergent, IsIndirectCall/IsUniform, NumReturns,
|
|
/// NumParams, Callee, Proto)
|
|
|
|
def CallOperand : Operand<i32> { let PrintMethod = "printCallOperand"; }
|
|
|
|
foreach is_convergent = [0, 1] in {
|
|
defvar convergent_suffix = !if(is_convergent, "_conv", "");
|
|
|
|
let isCall = 1, isConvergent = is_convergent in {
|
|
def CALL # convergent_suffix :
|
|
NVPTXInst<(outs),
|
|
(ins ADDR_base:$addr, CallOperand:$rets, CallOperand:$params,
|
|
i32imm:$proto),
|
|
"call${rets:RetList} $addr, (${params:ParamList}), prototype_$proto;">;
|
|
|
|
def CALL_UNI # convergent_suffix :
|
|
NVPTXInst<(outs),
|
|
(ins ADDR_base:$addr, CallOperand:$rets, CallOperand:$params),
|
|
"call.uni${rets:RetList} $addr, (${params:ParamList});">;
|
|
}
|
|
|
|
defvar call_inst = !cast<NVPTXInst>("CALL" # convergent_suffix);
|
|
def : Pat<(call is_convergent, 1, imm:$rets, imm:$params, i32:$addr, imm:$proto),
|
|
(call_inst $addr, imm:$rets, imm:$params, imm:$proto)>;
|
|
def : Pat<(call is_convergent, 1, imm:$rets, imm:$params, i64:$addr, imm:$proto),
|
|
(call_inst $addr, imm:$rets, imm:$params, imm:$proto)>;
|
|
|
|
defvar call_uni_inst = !cast<NVPTXInst>("CALL_UNI" # convergent_suffix);
|
|
def : Pat<(call is_convergent, 0, imm:$rets, imm:$params, globaladdr:$addr, 0),
|
|
(call_uni_inst (to_tglobaladdr $addr), imm:$rets, imm:$params)>;
|
|
}
|
|
|
|
def DECLARE_PARAM_array :
|
|
NVPTXInst<(outs), (ins i32imm:$a, i32imm:$align, i32imm:$size),
|
|
".param .align $align .b8 \t$a[$size];">;
|
|
def DECLARE_PARAM_scalar :
|
|
NVPTXInst<(outs), (ins i32imm:$a, i32imm:$size),
|
|
".param .b$size \t$a;">;
|
|
|
|
def : Pat<(declare_array_param externalsym:$a, imm:$align, imm:$size),
|
|
(DECLARE_PARAM_array (to_texternsym $a), imm:$align, imm:$size)>;
|
|
def : Pat<(declare_scalar_param externalsym:$a, imm:$size),
|
|
(DECLARE_PARAM_scalar (to_texternsym $a), imm:$size)>;
|
|
|
|
// Call prototype wrapper, this is a dummy instruction that just prints it's
|
|
// operand which is string defining the prototype.
|
|
def SDTCallPrototype : SDTypeProfile<0, 1, [SDTCisInt<0>]>;
|
|
def CallPrototype :
|
|
SDNode<"NVPTXISD::CallPrototype", SDTCallPrototype,
|
|
[SDNPHasChain, SDNPSideEffect]>;
|
|
def ProtoIdent : Operand<i32> { let PrintMethod = "printProtoIdent"; }
|
|
def CALL_PROTOTYPE :
|
|
NVPTXInst<(outs), (ins ProtoIdent:$ident),
|
|
"$ident", [(CallPrototype (i32 texternalsym:$ident))]>;
|
|
|
|
|
|
foreach t = [I32RT, I64RT] in {
|
|
defvar inst_name = "MOV" # t.Size # "_PARAM";
|
|
def inst_name : BasicNVPTXInst<(outs t.RC:$dst), (ins t.RC:$src), "mov.b" # t.Size>;
|
|
def : Pat<(MoveParam (t.Ty externalsym:$src)),
|
|
(!cast<NVPTXInst>(inst_name) (t.Ty (to_texternsym $src)))>;
|
|
}
|
|
|
|
multiclass ProxyRegInst<string SzStr, NVPTXRegClass rc> {
|
|
def NAME : BasicNVPTXInst<(outs rc:$dst), (ins rc:$src),
|
|
"mov." # SzStr>;
|
|
foreach vt = rc.RegTypes in
|
|
def : Pat<(vt (proxy_reg vt:$src)), (!cast<NVPTXInst>(NAME) $src)>;
|
|
}
|
|
|
|
defm ProxyRegB1 : ProxyRegInst<"pred", B1>;
|
|
defm ProxyRegB16 : ProxyRegInst<"b16", B16>;
|
|
defm ProxyRegB32 : ProxyRegInst<"b32", B32>;
|
|
defm ProxyRegB64 : ProxyRegInst<"b64", B64>;
|
|
|
|
|
|
// Callseq start and end
|
|
|
|
// Note: these nodes are marked as SDNPMayStore and SDNPMayLoad because
|
|
// they define the scope in which the declared params may be used. Therefore
|
|
// we add these flags to ensure ld.param and st.param are not sunk or hoisted
|
|
// out of that scope.
|
|
|
|
def callseq_start : SDNode<"ISD::CALLSEQ_START",
|
|
SDCallSeqStart<[SDTCisVT<0, i32>, SDTCisVT<1, i32>]>,
|
|
[SDNPHasChain, SDNPOutGlue,
|
|
SDNPSideEffect, SDNPMayStore, SDNPMayLoad]>;
|
|
def callseq_end : SDNode<"ISD::CALLSEQ_END",
|
|
SDCallSeqEnd<[SDTCisVT<0, i32>, SDTCisVT<1, i32>]>,
|
|
[SDNPHasChain, SDNPOptInGlue, SDNPOutGlue,
|
|
SDNPSideEffect, SDNPMayStore, SDNPMayLoad]>;
|
|
|
|
def Callseq_Start :
|
|
NVPTXInst<(outs), (ins i32imm:$amt1, i32imm:$amt2),
|
|
"\\{ // callseq $amt1, $amt2",
|
|
[(callseq_start timm:$amt1, timm:$amt2)]>;
|
|
def Callseq_End :
|
|
NVPTXInst<(outs), (ins i32imm:$amt1, i32imm:$amt2),
|
|
"\\} // callseq $amt1",
|
|
[(callseq_end timm:$amt1, timm:$amt2)]>;
|
|
|
|
//
|
|
// Load / Store Handling
|
|
//
|
|
class LD<NVPTXRegClass regclass>
|
|
: NVPTXInst<
|
|
(outs regclass:$dst),
|
|
(ins AtomicCode:$sem, AtomicCode:$scope, AtomicCode:$addsp,
|
|
AtomicCode:$Sign, i32imm:$fromWidth, UsedBytesMask:$usedBytes,
|
|
ADDR:$addr),
|
|
"${usedBytes}"
|
|
"ld${sem:sem}${scope:scope}${addsp:addsp}.${Sign:sign}$fromWidth "
|
|
"\t$dst, [$addr];">;
|
|
|
|
let mayLoad=1, hasSideEffects=0 in {
|
|
def LD_i16 : LD<B16>;
|
|
def LD_i32 : LD<B32>;
|
|
def LD_i64 : LD<B64>;
|
|
}
|
|
|
|
class ST<DAGOperand O>
|
|
: NVPTXInst<
|
|
(outs),
|
|
(ins O:$src,
|
|
AtomicCode:$sem, AtomicCode:$scope, AtomicCode:$addsp, i32imm:$toWidth,
|
|
ADDR:$addr),
|
|
"st${sem:sem}${scope:scope}${addsp:addsp}.b$toWidth"
|
|
" \t[$addr], $src;">;
|
|
|
|
let mayStore=1, hasSideEffects=0 in {
|
|
def ST_i16 : ST<RI16>;
|
|
def ST_i32 : ST<RI32>;
|
|
def ST_i64 : ST<RI64>;
|
|
}
|
|
|
|
// The following is used only in and after vector elementizations. Vector
|
|
// elementization happens at the machine instruction level, so the following
|
|
// instructions never appear in the DAG.
|
|
multiclass LD_VEC<NVPTXRegClass regclass, bit support_v8 = false> {
|
|
def _v2 : NVPTXInst<
|
|
(outs regclass:$dst1, regclass:$dst2),
|
|
(ins AtomicCode:$sem, AtomicCode:$scope, AtomicCode:$addsp,
|
|
AtomicCode:$Sign, i32imm:$fromWidth, UsedBytesMask:$usedBytes,
|
|
ADDR:$addr),
|
|
"${usedBytes}"
|
|
"ld${sem:sem}${scope:scope}${addsp:addsp}.v2.${Sign:sign}$fromWidth "
|
|
"\t{{$dst1, $dst2}}, [$addr];">;
|
|
def _v4 : NVPTXInst<
|
|
(outs regclass:$dst1, regclass:$dst2, regclass:$dst3, regclass:$dst4),
|
|
(ins AtomicCode:$sem, AtomicCode:$scope, AtomicCode:$addsp,
|
|
AtomicCode:$Sign, i32imm:$fromWidth, UsedBytesMask:$usedBytes,
|
|
ADDR:$addr),
|
|
"${usedBytes}"
|
|
"ld${sem:sem}${scope:scope}${addsp:addsp}.v4.${Sign:sign}$fromWidth "
|
|
"\t{{$dst1, $dst2, $dst3, $dst4}}, [$addr];">;
|
|
if support_v8 then
|
|
def _v8 : NVPTXInst<
|
|
(outs regclass:$dst1, regclass:$dst2, regclass:$dst3, regclass:$dst4,
|
|
regclass:$dst5, regclass:$dst6, regclass:$dst7, regclass:$dst8),
|
|
(ins AtomicCode:$sem, AtomicCode:$scope, AtomicCode:$addsp,
|
|
AtomicCode:$Sign, i32imm:$fromWidth, UsedBytesMask:$usedBytes,
|
|
ADDR:$addr),
|
|
"${usedBytes}"
|
|
"ld${sem:sem}${scope:scope}${addsp:addsp}.v8.${Sign:sign}$fromWidth "
|
|
"\t{{$dst1, $dst2, $dst3, $dst4, $dst5, $dst6, $dst7, $dst8}}, "
|
|
"[$addr];">;
|
|
}
|
|
let mayLoad=1, hasSideEffects=0 in {
|
|
defm LDV_i16 : LD_VEC<B16>;
|
|
defm LDV_i32 : LD_VEC<B32, support_v8 = true>;
|
|
defm LDV_i64 : LD_VEC<B64>;
|
|
}
|
|
|
|
multiclass ST_VEC<DAGOperand O, bit support_v8 = false> {
|
|
def _v2 : NVPTXInst<
|
|
(outs),
|
|
(ins O:$src1, O:$src2,
|
|
AtomicCode:$sem, AtomicCode:$scope, AtomicCode:$addsp, i32imm:$fromWidth,
|
|
ADDR:$addr),
|
|
"st${sem:sem}${scope:scope}${addsp:addsp}.v2.b$fromWidth "
|
|
"\t[$addr], {{$src1, $src2}};">;
|
|
def _v4 : NVPTXInst<
|
|
(outs),
|
|
(ins RegOrSink:$src1, RegOrSink:$src2, RegOrSink:$src3, RegOrSink:$src4,
|
|
AtomicCode:$sem, AtomicCode:$scope, AtomicCode:$addsp, i32imm:$fromWidth,
|
|
ADDR:$addr),
|
|
"st${sem:sem}${scope:scope}${addsp:addsp}.v4.b$fromWidth "
|
|
"\t[$addr], {{$src1, $src2, $src3, $src4}};">;
|
|
if support_v8 then
|
|
def _v8 : NVPTXInst<
|
|
(outs),
|
|
(ins RegOrSink:$src1, RegOrSink:$src2, RegOrSink:$src3, RegOrSink:$src4,
|
|
RegOrSink:$src5, RegOrSink:$src6, RegOrSink:$src7, RegOrSink:$src8,
|
|
AtomicCode:$sem, AtomicCode:$scope, AtomicCode:$addsp, i32imm:$fromWidth,
|
|
ADDR:$addr),
|
|
"st${sem:sem}${scope:scope}${addsp:addsp}.v8.b$fromWidth "
|
|
"\t[$addr], "
|
|
"{{$src1, $src2, $src3, $src4, $src5, $src6, $src7, $src8}};">;
|
|
}
|
|
|
|
let mayStore=1, hasSideEffects=0 in {
|
|
defm STV_i16 : ST_VEC<RI16>;
|
|
defm STV_i32 : ST_VEC<RI32, support_v8 = true>;
|
|
defm STV_i64 : ST_VEC<RI64>;
|
|
}
|
|
|
|
//---- Conversion ----
|
|
|
|
foreach rc = [B16, B32, B64] in
|
|
foreach ta = rc.RegTypes in
|
|
foreach tb = rc.RegTypes in
|
|
if !ne(ta, tb) then
|
|
def : Pat<(ta (bitconvert tb:$a)),
|
|
(ta rc:$a)>;
|
|
|
|
// NOTE: pred->fp are currently sub-optimal due to an issue in TableGen where
|
|
// we cannot specify floating-point literals in isel patterns. Therefore, we
|
|
// use an integer selp to select either 1 (or -1 in case of signed) or 0
|
|
// and then cvt to floating-point.
|
|
|
|
// sint -> f16
|
|
def : Pat<(f16 (sint_to_fp i1:$a)), (CVT_f16_s32 (SELP_b32ii -1, 0, $a), CvtRN)>;
|
|
def : Pat<(f16 (sint_to_fp i16:$a)), (CVT_f16_s16 $a, CvtRN)>;
|
|
def : Pat<(f16 (sint_to_fp i32:$a)), (CVT_f16_s32 $a, CvtRN)>;
|
|
def : Pat<(f16 (sint_to_fp i64:$a)), (CVT_f16_s64 $a, CvtRN)>;
|
|
|
|
// uint -> f16
|
|
def : Pat<(f16 (uint_to_fp i1:$a)), (CVT_f16_u32 (SELP_b32ii 1, 0, $a), CvtRN)>;
|
|
def : Pat<(f16 (uint_to_fp i16:$a)), (CVT_f16_u16 $a, CvtRN)>;
|
|
def : Pat<(f16 (uint_to_fp i32:$a)), (CVT_f16_u32 $a, CvtRN)>;
|
|
def : Pat<(f16 (uint_to_fp i64:$a)), (CVT_f16_u64 $a, CvtRN)>;
|
|
|
|
// sint -> bf16
|
|
let Predicates = [hasPTX<78>, hasSM<90>] in {
|
|
def : Pat<(bf16 (sint_to_fp i1:$a)), (CVT_bf16_s32 (SELP_b32ii -1, 0, $a), CvtRN)>;
|
|
def : Pat<(bf16 (sint_to_fp i16:$a)), (CVT_bf16_s16 $a, CvtRN)>;
|
|
def : Pat<(bf16 (sint_to_fp i32:$a)), (CVT_bf16_s32 $a, CvtRN)>;
|
|
def : Pat<(bf16 (sint_to_fp i64:$a)), (CVT_bf16_s64 $a, CvtRN)>;
|
|
}
|
|
|
|
// uint -> bf16
|
|
let Predicates = [hasPTX<78>, hasSM<90>] in {
|
|
def : Pat<(bf16 (uint_to_fp i1:$a)), (CVT_bf16_u32 (SELP_b32ii 1, 0, $a), CvtRN)>;
|
|
def : Pat<(bf16 (uint_to_fp i16:$a)), (CVT_bf16_u16 $a, CvtRN)>;
|
|
def : Pat<(bf16 (uint_to_fp i32:$a)), (CVT_bf16_u32 $a, CvtRN)>;
|
|
def : Pat<(bf16 (uint_to_fp i64:$a)), (CVT_bf16_u64 $a, CvtRN)>;
|
|
}
|
|
|
|
// sint -> f32
|
|
def : Pat<(f32 (sint_to_fp i1:$a)), (CVT_f32_s32 (SELP_b32ii -1, 0, $a), CvtRN)>;
|
|
def : Pat<(f32 (sint_to_fp i16:$a)), (CVT_f32_s16 $a, CvtRN)>;
|
|
def : Pat<(f32 (sint_to_fp i32:$a)), (CVT_f32_s32 $a, CvtRN)>;
|
|
def : Pat<(f32 (sint_to_fp i64:$a)), (CVT_f32_s64 $a, CvtRN)>;
|
|
|
|
// uint -> f32
|
|
def : Pat<(f32 (uint_to_fp i1:$a)), (CVT_f32_u32 (SELP_b32ii 1, 0, $a), CvtRN)>;
|
|
def : Pat<(f32 (uint_to_fp i16:$a)), (CVT_f32_u16 $a, CvtRN)>;
|
|
def : Pat<(f32 (uint_to_fp i32:$a)), (CVT_f32_u32 $a, CvtRN)>;
|
|
def : Pat<(f32 (uint_to_fp i64:$a)), (CVT_f32_u64 $a, CvtRN)>;
|
|
|
|
// sint -> f64
|
|
def : Pat<(f64 (sint_to_fp i1:$a)), (CVT_f64_s32 (SELP_b32ii -1, 0, $a), CvtRN)>;
|
|
def : Pat<(f64 (sint_to_fp i16:$a)), (CVT_f64_s16 $a, CvtRN)>;
|
|
def : Pat<(f64 (sint_to_fp i32:$a)), (CVT_f64_s32 $a, CvtRN)>;
|
|
def : Pat<(f64 (sint_to_fp i64:$a)), (CVT_f64_s64 $a, CvtRN)>;
|
|
|
|
// uint -> f64
|
|
def : Pat<(f64 (uint_to_fp i1:$a)), (CVT_f64_u32 (SELP_b32ii 1, 0, $a), CvtRN)>;
|
|
def : Pat<(f64 (uint_to_fp i16:$a)), (CVT_f64_u16 $a, CvtRN)>;
|
|
def : Pat<(f64 (uint_to_fp i32:$a)), (CVT_f64_u32 $a, CvtRN)>;
|
|
def : Pat<(f64 (uint_to_fp i64:$a)), (CVT_f64_u64 $a, CvtRN)>;
|
|
|
|
|
|
// fp_to_sint/uint to i1 is custom-lowered to an fp compare
|
|
// (NVPTXTargetLowering::LowerOperation), so no i1 patterns appear below.
|
|
|
|
// f16 -> sint
|
|
def : Pat<(i16 (fp_to_sint f16:$a)), (CVT_s16_f16 $a, CvtRZI)>;
|
|
def : Pat<(i32 (fp_to_sint f16:$a)), (CVT_s32_f16 $a, CvtRZI)>;
|
|
def : Pat<(i64 (fp_to_sint f16:$a)), (CVT_s64_f16 $a, CvtRZI)>;
|
|
|
|
// f16 -> uint
|
|
def : Pat<(i16 (fp_to_uint f16:$a)), (CVT_u16_f16 $a, CvtRZI)>;
|
|
def : Pat<(i32 (fp_to_uint f16:$a)), (CVT_u32_f16 $a, CvtRZI)>;
|
|
def : Pat<(i64 (fp_to_uint f16:$a)), (CVT_u64_f16 $a, CvtRZI)>;
|
|
|
|
// bf16 -> sint
|
|
def : Pat<(i16 (fp_to_sint bf16:$a)), (CVT_s16_bf16 $a, CvtRZI)>;
|
|
def : Pat<(i32 (fp_to_sint bf16:$a)), (CVT_s32_bf16 $a, CvtRZI)>;
|
|
def : Pat<(i64 (fp_to_sint bf16:$a)), (CVT_s64_bf16 $a, CvtRZI)>;
|
|
|
|
// bf16 -> uint
|
|
def : Pat<(i16 (fp_to_uint bf16:$a)), (CVT_u16_bf16 $a, CvtRZI)>;
|
|
def : Pat<(i32 (fp_to_uint bf16:$a)), (CVT_u32_bf16 $a, CvtRZI)>;
|
|
def : Pat<(i64 (fp_to_uint bf16:$a)), (CVT_u64_bf16 $a, CvtRZI)>;
|
|
// f32 -> sint
|
|
let Predicates = [doF32FTZ] in {
|
|
def : Pat<(i16 (fp_to_sint f32:$a)), (CVT_s16_f32 $a, CvtRZI_FTZ)>;
|
|
def : Pat<(i32 (fp_to_sint f32:$a)), (CVT_s32_f32 $a, CvtRZI_FTZ)>;
|
|
def : Pat<(i64 (fp_to_sint f32:$a)), (CVT_s64_f32 $a, CvtRZI_FTZ)>;
|
|
}
|
|
def : Pat<(i16 (fp_to_sint f32:$a)), (CVT_s16_f32 $a, CvtRZI)>;
|
|
def : Pat<(i32 (fp_to_sint f32:$a)), (CVT_s32_f32 $a, CvtRZI)>;
|
|
def : Pat<(i64 (fp_to_sint f32:$a)), (CVT_s64_f32 $a, CvtRZI)>;
|
|
|
|
// f32 -> uint
|
|
let Predicates = [doF32FTZ] in {
|
|
def : Pat<(i16 (fp_to_uint f32:$a)), (CVT_u16_f32 $a, CvtRZI_FTZ)>;
|
|
def : Pat<(i32 (fp_to_uint f32:$a)), (CVT_u32_f32 $a, CvtRZI_FTZ)>;
|
|
def : Pat<(i64 (fp_to_uint f32:$a)), (CVT_u64_f32 $a, CvtRZI_FTZ)>;
|
|
}
|
|
def : Pat<(i16 (fp_to_uint f32:$a)), (CVT_u16_f32 $a, CvtRZI)>;
|
|
def : Pat<(i32 (fp_to_uint f32:$a)), (CVT_u32_f32 $a, CvtRZI)>;
|
|
def : Pat<(i64 (fp_to_uint f32:$a)), (CVT_u64_f32 $a, CvtRZI)>;
|
|
|
|
// f64 -> sint
|
|
def : Pat<(i16 (fp_to_sint f64:$a)), (CVT_s16_f64 $a, CvtRZI)>;
|
|
def : Pat<(i32 (fp_to_sint f64:$a)), (CVT_s32_f64 $a, CvtRZI)>;
|
|
def : Pat<(i64 (fp_to_sint f64:$a)), (CVT_s64_f64 $a, CvtRZI)>;
|
|
|
|
// f64 -> uint
|
|
def : Pat<(i16 (fp_to_uint f64:$a)), (CVT_u16_f64 $a, CvtRZI)>;
|
|
def : Pat<(i32 (fp_to_uint f64:$a)), (CVT_u32_f64 $a, CvtRZI)>;
|
|
def : Pat<(i64 (fp_to_uint f64:$a)), (CVT_u64_f64 $a, CvtRZI)>;
|
|
|
|
// sext i1
|
|
def : Pat<(i16 (sext i1:$a)), (SELP_b16ii -1, 0, $a)>;
|
|
def : Pat<(i32 (sext i1:$a)), (SELP_b32ii -1, 0, $a)>;
|
|
def : Pat<(i64 (sext i1:$a)), (SELP_b64ii -1, 0, $a)>;
|
|
|
|
// zext i1
|
|
def : Pat<(i16 (zext i1:$a)), (SELP_b16ii 1, 0, $a)>;
|
|
def : Pat<(i32 (zext i1:$a)), (SELP_b32ii 1, 0, $a)>;
|
|
def : Pat<(i64 (zext i1:$a)), (SELP_b64ii 1, 0, $a)>;
|
|
|
|
// anyext i1
|
|
def : Pat<(i16 (anyext i1:$a)), (SELP_b16ii -1, 0, $a)>;
|
|
def : Pat<(i32 (anyext i1:$a)), (SELP_b32ii -1, 0, $a)>;
|
|
def : Pat<(i64 (anyext i1:$a)), (SELP_b64ii -1, 0, $a)>;
|
|
|
|
// sext i16
|
|
def : Pat<(i32 (sext i16:$a)), (CVT_s32_s16 $a, CvtNONE)>;
|
|
def : Pat<(i64 (sext i16:$a)), (CVT_s64_s16 $a, CvtNONE)>;
|
|
|
|
// zext i16
|
|
def : Pat<(i32 (zext i16:$a)), (CVT_u32_u16 $a, CvtNONE)>;
|
|
def : Pat<(i64 (zext i16:$a)), (CVT_u64_u16 $a, CvtNONE)>;
|
|
|
|
// anyext i16
|
|
def : Pat<(i32 (anyext i16:$a)), (CVT_u32_u16 $a, CvtNONE)>;
|
|
def : Pat<(i64 (anyext i16:$a)), (CVT_u64_u16 $a, CvtNONE)>;
|
|
|
|
// sext i32
|
|
def : Pat<(i64 (sext i32:$a)), (CVT_s64_s32 $a, CvtNONE)>;
|
|
|
|
// zext i32
|
|
def : Pat<(i64 (zext i32:$a)), (CVT_u64_u32 $a, CvtNONE)>;
|
|
|
|
// anyext i32
|
|
def : Pat<(i64 (anyext i32:$a)), (CVT_u64_u32 $a, CvtNONE)>;
|
|
|
|
|
|
// truncate i64
|
|
def : Pat<(i32 (trunc i64:$a)), (CVT_u32_u64 $a, CvtNONE)>;
|
|
def : Pat<(i16 (trunc i64:$a)), (CVT_u16_u64 $a, CvtNONE)>;
|
|
def : Pat<(i1 (trunc i64:$a)), (SETP_i64ri (AND_b64ri $a, 1), 0, CmpNE)>;
|
|
|
|
// truncate i32
|
|
def : Pat<(i16 (trunc i32:$a)), (CVT_u16_u32 $a, CvtNONE)>;
|
|
def : Pat<(i1 (trunc i32:$a)), (SETP_i32ri (AND_b32ri $a, 1), 0, CmpNE)>;
|
|
|
|
// truncate i16
|
|
def : Pat<(i1 (trunc i16:$a)), (SETP_i16ri (AND_b16ri $a, 1), 0, CmpNE)>;
|
|
|
|
// sext_inreg
|
|
def : Pat<(sext_inreg i16:$a, i8), (CVT_INREG_s16_s8 $a)>;
|
|
def : Pat<(sext_inreg i32:$a, i8), (CVT_INREG_s32_s8 $a)>;
|
|
def : Pat<(sext_inreg i32:$a, i16), (CVT_INREG_s32_s16 $a)>;
|
|
def : Pat<(sext_inreg i64:$a, i8), (CVT_INREG_s64_s8 $a)>;
|
|
def : Pat<(sext_inreg i64:$a, i16), (CVT_INREG_s64_s16 $a)>;
|
|
def : Pat<(sext_inreg i64:$a, i32), (CVT_INREG_s64_s32 $a)>;
|
|
|
|
let hasSideEffects = false in {
|
|
// pack a set of smaller int registers to a larger int register
|
|
def V4I16toI64 : NVPTXInst<(outs B64:$d),
|
|
(ins B16:$s1, B16:$s2,
|
|
B16:$s3, B16:$s4),
|
|
"mov.b64 \t$d, {{$s1, $s2, $s3, $s4}};">;
|
|
def V2I16toI32 : NVPTXInst<(outs B32:$d),
|
|
(ins B16:$s1, B16:$s2),
|
|
"mov.b32 \t$d, {{$s1, $s2}};">;
|
|
def V2I32toI64 : NVPTXInst<(outs B64:$d),
|
|
(ins B32:$s1, B32:$s2),
|
|
"mov.b64 \t$d, {{$s1, $s2}};">;
|
|
def V2I64toI128 : NVPTXInst<(outs B128:$d),
|
|
(ins B64:$s1, B64:$s2),
|
|
"mov.b128 \t$d, {{$s1, $s2}};">;
|
|
|
|
// unpack a larger int register to a set of smaller int registers
|
|
def I64toV4I16 : NVPTXInst<(outs B16:$d1, B16:$d2,
|
|
B16:$d3, B16:$d4),
|
|
(ins B64:$s),
|
|
"mov.b64 \t{{$d1, $d2, $d3, $d4}}, $s;">;
|
|
def I32toV2I16 : NVPTXInst<(outs B16:$d1, B16:$d2),
|
|
(ins B32:$s),
|
|
"mov.b32 \t{{$d1, $d2}}, $s;">;
|
|
def I64toV2I32 : NVPTXInst<(outs B32:$d1, B32:$d2),
|
|
(ins B64:$s),
|
|
"mov.b64 \t{{$d1, $d2}}, $s;">;
|
|
def I128toV2I64: NVPTXInst<(outs B64:$d1, B64:$d2),
|
|
(ins B128:$s),
|
|
"mov.b128 \t{{$d1, $d2}}, $s;">;
|
|
|
|
def I32toI16H : NVPTXInst<(outs B16:$high), (ins B32:$s),
|
|
"{{ .reg .b16 tmp; mov.b32 {tmp, $high}, $s; }}">;
|
|
def I32toI16L : NVPTXInst<(outs B16:$low), (ins B32:$s),
|
|
"{{ .reg .b16 tmp; mov.b32 {$low, tmp}, $s; }}">;
|
|
def I64toI32H : NVPTXInst<(outs B32:$high), (ins B64:$s),
|
|
"{{ .reg .b32 tmp; mov.b64 {tmp, $high}, $s; }}">;
|
|
def I64toI32L : NVPTXInst<(outs B32:$low), (ins B64:$s),
|
|
"{{ .reg .b32 tmp; mov.b64 {$low, tmp}, $s; }}">;
|
|
|
|
// PTX 7.1 lets you avoid a temp register and just use _ as a "sink" for the
|
|
// unused high/low part.
|
|
let Predicates = [hasPTX<71>] in {
|
|
def I32toI16H_Sink : NVPTXInst<(outs B16:$high), (ins B32:$s),
|
|
"mov.b32 \t{{_, $high}}, $s;">;
|
|
def I32toI16L_Sink : NVPTXInst<(outs B16:$low), (ins B32:$s),
|
|
"mov.b32 \t{{$low, _}}, $s;">;
|
|
def I64toI32H_Sink : NVPTXInst<(outs B32:$high), (ins B64:$s),
|
|
"mov.b64 \t{{_, $high}}, $s;">;
|
|
def I64toI32L_Sink : NVPTXInst<(outs B32:$low), (ins B64:$s),
|
|
"mov.b64 \t{{$low, _}}, $s;">;
|
|
}
|
|
}
|
|
|
|
let Predicates = [hasPTX<71>] in {
|
|
def : Pat<(i16 (trunc (srl i32:$s, (i32 16)))), (I32toI16H_Sink i32:$s)>;
|
|
def : Pat<(i16 (trunc (sra i32:$s, (i32 16)))), (I32toI16H_Sink i32:$s)>;
|
|
def : Pat<(i32 (trunc (srl i64:$s, (i32 32)))), (I64toI32H_Sink i64:$s)>;
|
|
def : Pat<(i32 (trunc (sra i64:$s, (i32 32)))), (I64toI32H_Sink i64:$s)>;
|
|
}
|
|
|
|
// Fall back to the old way if we don't have PTX 7.1.
|
|
def : Pat<(i16 (trunc (srl i32:$s, (i32 16)))), (I32toI16H $s)>;
|
|
def : Pat<(i16 (trunc (sra i32:$s, (i32 16)))), (I32toI16H $s)>;
|
|
def : Pat<(i32 (trunc (srl i64:$s, (i32 32)))), (I64toI32H $s)>;
|
|
def : Pat<(i32 (trunc (sra i64:$s, (i32 32)))), (I64toI32H $s)>;
|
|
|
|
def: Pat<(i32 (sext (extractelt v2i16:$src, 0))),
|
|
(CVT_INREG_s32_s16 $src)>;
|
|
|
|
// Handle extracting one element from the pair (32-bit types)
|
|
foreach vt = [v2f16, v2bf16, v2i16] in {
|
|
def : Pat<(extractelt vt:$src, 0), (I32toI16L_Sink $src)>, Requires<[hasPTX<71>]>;
|
|
def : Pat<(extractelt vt:$src, 1), (I32toI16H_Sink $src)>, Requires<[hasPTX<71>]>;
|
|
|
|
def : Pat<(extractelt vt:$src, 0), (I32toI16L $src)>;
|
|
def : Pat<(extractelt vt:$src, 1), (I32toI16H $src)>;
|
|
|
|
def : Pat<(vt (build_vector vt.ElementType:$a, vt.ElementType:$b)),
|
|
(V2I16toI32 $a, $b)>;
|
|
}
|
|
|
|
// Handle extracting one element from the pair (64-bit types)
|
|
foreach vt = [v2f32, v2i32] in {
|
|
def : Pat<(extractelt vt:$src, 0), (I64toI32L_Sink $src)>, Requires<[hasPTX<71>]>;
|
|
def : Pat<(extractelt vt:$src, 1), (I64toI32H_Sink $src)>, Requires<[hasPTX<71>]>;
|
|
|
|
def : Pat<(extractelt vt:$src, 0), (I64toI32L $src)>;
|
|
def : Pat<(extractelt vt:$src, 1), (I64toI32H $src)>;
|
|
|
|
def : Pat<(vt (build_vector vt.ElementType:$a, vt.ElementType:$b)),
|
|
(V2I32toI64 $a, $b)>;
|
|
}
|
|
|
|
def: Pat<(v2i16 (scalar_to_vector i16:$a)),
|
|
(CVT_u32_u16 $a, CvtNONE)>;
|
|
def: Pat<(v2i32 (scalar_to_vector i32:$a)),
|
|
(CVT_u64_u32 $a, CvtNONE)>;
|
|
def: Pat<(v2f16 (scalar_to_vector f16:$a)),
|
|
(V2I16toI32 $a, 0)>;
|
|
def: Pat<(v2f32 (scalar_to_vector f32:$a)),
|
|
(V2I32toI64 $a, (i32 0))>;
|
|
|
|
// This node is similar to ISD::BUILD_VECTOR except that the output may be
|
|
// implicitly bitcast to a scalar. This allows for the representation of
|
|
// packing move instructions for vector types which are not legal i.e. v2i32
|
|
def nvptx_build_vector : SDNode<"NVPTXISD::BUILD_VECTOR", SDTypeProfile<1, 2, []>, []>;
|
|
|
|
def : Pat<(i64 (nvptx_build_vector i32:$a, i32:$b)),
|
|
(V2I32toI64 $a, $b)>;
|
|
|
|
//
|
|
// Funnel-Shift
|
|
//
|
|
|
|
// Create SDNodes so they can be used in the DAG code, e.g.
|
|
// NVPTXISelLowering (LowerShiftLeftParts and LowerShiftRightParts)
|
|
def fshl_clamp : SDNode<"NVPTXISD::FSHL_CLAMP", SDTIntShiftDOp, []>;
|
|
def fshr_clamp : SDNode<"NVPTXISD::FSHR_CLAMP", SDTIntShiftDOp, []>;
|
|
|
|
// Funnel shift, requires >= sm_32. Does not trap if amt is out of range, so
|
|
// no side effects.
|
|
let hasSideEffects = false in {
|
|
multiclass ShfInst<string mode, SDNode op> {
|
|
def _i
|
|
: BasicNVPTXInst<(outs B32:$dst),
|
|
(ins B32:$lo, B32:$hi, i32imm:$amt),
|
|
"shf." # mode # ".b32",
|
|
[(set i32:$dst,
|
|
(op i32:$hi, i32:$lo, (i32 imm:$amt)))]>,
|
|
Requires<[hasHWROT32]>;
|
|
|
|
def _r
|
|
: BasicNVPTXInst<(outs B32:$dst),
|
|
(ins B32:$lo, B32:$hi, B32:$amt),
|
|
"shf." # mode # ".b32",
|
|
[(set i32:$dst,
|
|
(op i32:$hi, i32:$lo, i32:$amt))]>,
|
|
Requires<[hasHWROT32]>;
|
|
}
|
|
|
|
defm SHF_L_CLAMP : ShfInst<"l.clamp", fshl_clamp>;
|
|
defm SHF_R_CLAMP : ShfInst<"r.clamp", fshr_clamp>;
|
|
defm SHF_L_WRAP : ShfInst<"l.wrap", fshl>;
|
|
defm SHF_R_WRAP : ShfInst<"r.wrap", fshr>;
|
|
}
|
|
|
|
def : Pat<(i32 (int_nvvm_fshl_clamp i32:$hi, i32:$lo, i32:$amt)),
|
|
(SHF_L_CLAMP_r $lo, $hi, $amt)>;
|
|
def : Pat<(i32 (int_nvvm_fshl_clamp i32:$hi, i32:$lo, (i32 imm:$amt))),
|
|
(SHF_L_CLAMP_i $lo, $hi, imm:$amt)>;
|
|
def : Pat<(i32 (int_nvvm_fshr_clamp i32:$hi, i32:$lo, i32:$amt)),
|
|
(SHF_R_CLAMP_r $lo, $hi, $amt)>;
|
|
def : Pat<(i32 (int_nvvm_fshr_clamp i32:$hi, i32:$lo, (i32 imm:$amt))),
|
|
(SHF_R_CLAMP_i $lo, $hi, imm:$amt)>;
|
|
|
|
let hasSideEffects = false in {
|
|
foreach RT = [I32RT, I64RT] in {
|
|
// Count leading zeros
|
|
def CLZr # RT.Size : BasicNVPTXInst<(outs B32:$d), (ins RT.RC:$a),
|
|
"clz.b" # RT.Size,
|
|
[(set i32:$d, (ctlz RT.Ty:$a))]>;
|
|
|
|
// Population count
|
|
def POPCr # RT.Size : BasicNVPTXInst<(outs B32:$d), (ins RT.RC:$a),
|
|
"popc.b" # RT.Size,
|
|
[(set i32:$d, (ctpop RT.Ty:$a))]>;
|
|
}
|
|
}
|
|
|
|
// fpround f32 -> f16
|
|
def : Pat<(f16 (fpround f32:$a)), (CVT_f16_f32 $a, CvtRN)>;
|
|
|
|
// fpround f32 -> bf16
|
|
def : Pat<(bf16 (fpround f32:$a)), (CVT_bf16_f32 $a, CvtRN)>,
|
|
Requires<[hasPTX<70>, hasSM<80>]>;
|
|
|
|
// fpround f64 -> f16
|
|
def : Pat<(f16 (fpround f64:$a)), (CVT_f16_f64 $a, CvtRN)>;
|
|
|
|
// fpround f64 -> bf16
|
|
def : Pat<(bf16 (fpround f64:$a)), (CVT_bf16_f64 $a, CvtRN)>,
|
|
Requires<[hasPTX<78>, hasSM<90>]>;
|
|
|
|
// fpround f64 -> f32
|
|
def : Pat<(f32 (fpround f64:$a)), (CVT_f32_f64 $a, CvtRN_FTZ)>, Requires<[doF32FTZ]>;
|
|
def : Pat<(f32 (fpround f64:$a)), (CVT_f32_f64 $a, CvtRN)>;
|
|
|
|
// fpextend f16 -> f32
|
|
def : Pat<(f32 (fpextend f16:$a)), (CVT_f32_f16 $a, CvtNONE_FTZ)>, Requires<[doF32FTZ]>;
|
|
def : Pat<(f32 (fpextend f16:$a)), (CVT_f32_f16 $a, CvtNONE)>;
|
|
// fpextend bf16 -> f32
|
|
def : Pat<(f32 (fpextend bf16:$a)), (CVT_f32_bf16 $a, CvtNONE_FTZ)>, Requires<[doF32FTZ, hasPTX<78>, hasSM<90>]>;
|
|
def : Pat<(f32 (fpextend bf16:$a)), (CVT_f32_bf16 $a, CvtNONE)>, Requires<[hasPTX<71>, hasSM<80>]>;
|
|
|
|
// fpextend f16 -> f64
|
|
def : Pat<(f64 (fpextend f16:$a)), (CVT_f64_f16 $a, CvtNONE)>;
|
|
|
|
// fpextend bf16 -> f64
|
|
def : Pat<(f64 (fpextend bf16:$a)), (CVT_f64_bf16 $a, CvtNONE)>, Requires<[hasPTX<78>, hasSM<90>]>;
|
|
|
|
// fpextend f32 -> f64
|
|
def : Pat<(f64 (fpextend f32:$a)), (CVT_f64_f32 $a, CvtNONE_FTZ)>, Requires<[doF32FTZ]>;
|
|
def : Pat<(f64 (fpextend f32:$a)), (CVT_f64_f32 $a, CvtNONE)>;
|
|
|
|
def retglue : SDNode<"NVPTXISD::RET_GLUE", SDTNone,
|
|
[SDNPHasChain, SDNPOptInGlue]>;
|
|
|
|
// fceil, ffloor, froundeven, ftrunc.
|
|
|
|
multiclass CVT_ROUND<SDNode OpNode, PatLeaf Mode, PatLeaf ModeFTZ> {
|
|
def : Pat<(OpNode f16:$a), (CVT_f16_f16 $a, Mode)>;
|
|
def : Pat<(OpNode bf16:$a), (CVT_bf16_bf16 $a, Mode)>;
|
|
def : Pat<(OpNode f32:$a), (CVT_f32_f32 $a, ModeFTZ)>, Requires<[doF32FTZ]>;
|
|
def : Pat<(OpNode f32:$a), (CVT_f32_f32 $a, Mode)>, Requires<[doNoF32FTZ]>;
|
|
def : Pat<(OpNode f64:$a), (CVT_f64_f64 $a, Mode)>;
|
|
}
|
|
|
|
defm : CVT_ROUND<fceil, CvtRPI, CvtRPI_FTZ>;
|
|
defm : CVT_ROUND<ffloor, CvtRMI, CvtRMI_FTZ>;
|
|
defm : CVT_ROUND<froundeven, CvtRNI, CvtRNI_FTZ>;
|
|
defm : CVT_ROUND<ftrunc, CvtRZI, CvtRZI_FTZ>;
|
|
|
|
// nearbyint and rint are implemented as rounding to nearest even. This isn't
|
|
// strictly correct, because it causes us to ignore the rounding mode. But it
|
|
// matches what CUDA's "libm" does.
|
|
|
|
defm : CVT_ROUND<fnearbyint, CvtRNI, CvtRNI_FTZ>;
|
|
defm : CVT_ROUND<frint, CvtRNI, CvtRNI_FTZ>;
|
|
|
|
//-----------------------------------
|
|
// Control-flow
|
|
//-----------------------------------
|
|
|
|
let isTerminator=1 in {
|
|
let isReturn=1, isBarrier=1 in
|
|
def Return : BasicNVPTXInst<(outs), (ins), "ret", [(retglue)]>;
|
|
|
|
let isBranch=1 in {
|
|
def CBranch : NVPTXInst<(outs),
|
|
(ins B1:$a, brtarget:$target, BranchFlag:$not),
|
|
"@${not}$a bra \t$target;",
|
|
[(brcond i1:$a, bb:$target)]>;
|
|
|
|
let isBarrier=1 in
|
|
def GOTO : BasicNVPTXInst<(outs), (ins brtarget:$target),
|
|
"bra.uni", [(br bb:$target)]>;
|
|
}
|
|
}
|
|
|
|
|
|
// trap instruction
|
|
def trapinst : BasicNVPTXInst<(outs), (ins), "trap", [(trap)]>, Requires<[noPTXASUnreachableBug]>;
|
|
// Emit an `exit` as well to convey to ptxas that `trap` exits the CFG.
|
|
// This won't be necessary in a future version of ptxas.
|
|
def trapexitinst : NVPTXInst<(outs), (ins), "trap; exit;", [(trap)]>, Requires<[hasPTXASUnreachableBug]>;
|
|
// brkpt instruction
|
|
def debugtrapinst : BasicNVPTXInst<(outs), (ins), "brkpt", [(debugtrap)]>;
|
|
|
|
def SDTDynAllocaOp :
|
|
SDTypeProfile<1, 2, [SDTCisSameAs<0, 1>, SDTCisInt<1>, SDTCisVT<2, i32>]>;
|
|
|
|
def dyn_alloca :
|
|
SDNode<"NVPTXISD::DYNAMIC_STACKALLOC", SDTDynAllocaOp,
|
|
[SDNPHasChain, SDNPSideEffect]>;
|
|
|
|
foreach t = [I32RT, I64RT] in {
|
|
def DYNAMIC_STACKALLOC # t.Size :
|
|
BasicNVPTXInst<(outs t.RC:$ptr),
|
|
(ins t.RC:$size, i32imm:$align),
|
|
"alloca.u" # t.Size,
|
|
[(set t.Ty:$ptr, (dyn_alloca t.Ty:$size, timm:$align))]>,
|
|
Requires<[hasPTX<73>, hasSM<52>]>;
|
|
}
|
|
|
|
//
|
|
// BRX
|
|
//
|
|
|
|
let isTerminator = 1, isBranch = 1, isIndirectBranch = 1, isNotDuplicable = 1 in {
|
|
|
|
def BRX_START :
|
|
NVPTXInst<(outs), (ins i32imm:$id), "$$L_brx_$id: .branchtargets">;
|
|
|
|
def BRX_ITEM :
|
|
NVPTXInst<(outs), (ins brtarget:$target), "\t$target,">;
|
|
|
|
def BRX_END :
|
|
NVPTXInst<(outs), (ins brtarget:$target, B32:$val, i32imm:$id),
|
|
"\t$target;\n\tbrx.idx \t$val, $$L_brx_$id;"> {
|
|
let isBarrier = 1;
|
|
}
|
|
}
|
|
|
|
|
|
foreach a_type = ["s", "u"] in {
|
|
foreach b_type = ["s", "u"] in {
|
|
|
|
def DOT4_ # a_type # b_type :
|
|
BasicNVPTXInst<(outs B32:$dst),
|
|
(ins B32:$a, B32:$b, B32:$c),
|
|
"dp4a." # a_type # "32." # b_type # "32",
|
|
[(set i32:$dst,
|
|
(!cast<Intrinsic>("int_nvvm_idp4a_" # a_type # "_" # b_type)
|
|
i32:$a, i32:$b, i32:$c))]>,
|
|
Requires<[hasDotInstructions]>;
|
|
|
|
foreach is_hi = [0, -1] in {
|
|
defvar lohi_suffix = !if(is_hi, "hi", "lo");
|
|
|
|
def DOT2_ # lohi_suffix # _ # a_type # b_type :
|
|
BasicNVPTXInst<(outs B32:$dst),
|
|
(ins B32:$a, B32:$b, B32:$c),
|
|
"dp2a." # lohi_suffix # "." # a_type # "32." # b_type # "32",
|
|
[(set i32:$dst,
|
|
(!cast<Intrinsic>("int_nvvm_idp2a_" # a_type # "_" # b_type)
|
|
i32:$a, i32:$b, is_hi, i32:$c))]>,
|
|
Requires<[hasDotInstructions]>;
|
|
}
|
|
}
|
|
}
|
|
|
|
//
|
|
// Stack Manipulation
|
|
//
|
|
|
|
def SDTStackRestore : SDTypeProfile<0, 1, [SDTCisInt<0>]>;
|
|
|
|
def stackrestore :
|
|
SDNode<"NVPTXISD::STACKRESTORE", SDTStackRestore,
|
|
[SDNPHasChain, SDNPSideEffect]>;
|
|
|
|
def stacksave :
|
|
SDNode<"NVPTXISD::STACKSAVE", SDTIntLeaf,
|
|
[SDNPHasChain, SDNPSideEffect]>;
|
|
|
|
let Predicates = [hasPTX<73>, hasSM<52>] in {
|
|
foreach t = [I32RT, I64RT] in {
|
|
def STACKRESTORE_ # t.Size :
|
|
BasicNVPTXInst<(outs), (ins t.RC:$ptr),
|
|
"stackrestore.u" # t.Size,
|
|
[(stackrestore t.Ty:$ptr)]>;
|
|
|
|
def STACKSAVE_ # t.Size :
|
|
BasicNVPTXInst<(outs t.RC:$dst), (ins),
|
|
"stacksave.u" # t.Size,
|
|
[(set t.Ty:$dst, (t.Ty stacksave))]>;
|
|
}
|
|
}
|
|
|
|
include "NVPTXIntrinsics.td"
|
|
|
|
////////////////////////////////////////////////////////////////////////////////
|
|
// PTX Fence instructions
|
|
////////////////////////////////////////////////////////////////////////////////
|
|
|
|
class NVPTXFenceInst<string scope, string sem, Predicate ptx>:
|
|
BasicNVPTXInst<(outs), (ins), "fence."#sem#"."#scope>,
|
|
Requires<[ptx, hasSM<70>]>;
|
|
|
|
foreach scope = ["sys", "gpu", "cluster", "cta"] in {
|
|
def atomic_thread_fence_seq_cst_#scope: NVPTXFenceInst<scope, "sc", hasPTX<60>>;
|
|
def atomic_thread_fence_acq_rel_#scope: NVPTXFenceInst<scope, "acq_rel", hasPTX<60>>;
|
|
def atomic_thread_fence_acquire_#scope: NVPTXFenceInst<scope, "acquire", hasPTX<87>>;
|
|
def atomic_thread_fence_release_#scope: NVPTXFenceInst<scope, "release", hasPTX<87>>;
|
|
}
|
|
|
|
// Perform substitution if fma only has one use, and also if instruction has
|
|
// nnan instruction flag
|
|
def NVPTX_fma_oneuse_and_nnan : PatFrag<(ops node:$a, node:$b, node:$c),
|
|
(fma node:$a, node:$b, node:$c), [{
|
|
return N->hasOneUse() && N->getFlags().hasNoNaNs();
|
|
}]>;
|
|
// fmaxnum/fmaximumnum differentiate between signed and unsigned zeros, so this
|
|
// PatFrag is for a fmaxnum/fmaximumnum node with nsz
|
|
def NVPTX_fmaxnum_or_fmaximumnum_nsz : PatFrag<(ops node:$a, node:$b),
|
|
(fmaxnum_or_fmaximumnum node:$a, node:$b), [{
|
|
return N->getFlags().hasNoSignedZeros();
|
|
}]>;
|
|
|
|
class FMARELUInst<RegTyInfo t, bit allow_ftz, PatFrag zero_pat>
|
|
: BasicFlagsNVPTXInst<(outs t.RC:$dst), (ins t.RC:$a, t.RC:$b, t.RC:$c),
|
|
!if(allow_ftz, (ins FTZFlag:$ftz), (ins)),
|
|
"fma.rn" # !if(allow_ftz, "$ftz", "") # ".relu." # t.PtxType,
|
|
[(set t.Ty:$dst, (NVPTX_fmaxnum_or_fmaximumnum_nsz (NVPTX_fma_oneuse_and_nnan t.Ty:$a, t.Ty:$b, t.Ty:$c), zero_pat))]>;
|
|
|
|
let Predicates = [useFP16Math, hasPTX<70>, hasSM<80>] in {
|
|
def FMARELU_F16 : FMARELUInst<F16RT, true, fpimm_0>;
|
|
def FMARELU_F16X2 : FMARELUInst<F16X2RT, true, zeroinitializer<v2f16>>;
|
|
}
|
|
|
|
let Predicates = [hasBF16Math, hasPTX<70>, hasSM<80>] in {
|
|
def FMARELU_BF16 : FMARELUInst<BF16RT, false, fpimm_0>;
|
|
def FMARELU_BF16X2 : FMARELUInst<BF16X2RT, false, zeroinitializer<v2bf16>>;
|
|
}
|