//===- LLVMSPIRVInternal.h - SPIR-V internal header file -------*- C++ -*-===// // // The LLVM/SPIRV Translator // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // // Copyright (c) 2014 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal with the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // Redistributions of source code must retain the above copyright notice, // this list of conditions and the following disclaimers. // Redistributions in binary form must reproduce the above copyright notice, // this list of conditions and the following disclaimers in the documentation // and/or other materials provided with the distribution. // Neither the names of Advanced Micro Devices, Inc., nor the names of its // contributors may be used to endorse or promote products derived from this // Software without specific prior written permission. // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH // THE SOFTWARE. // //===----------------------------------------------------------------------===// /// \file /// /// This file declares classes and functions shared by SPIR-V reader/writer. /// //===----------------------------------------------------------------------===// #ifndef SPIRV_SPIRVINTERNAL_H #define SPIRV_SPIRVINTERNAL_H #include "NameMangleAPI.h" #include "libSPIRV/SPIRVEnum.h" #include "libSPIRV/SPIRVError.h" #include "libSPIRV/SPIRVNameMapEnum.h" #include "libSPIRV/SPIRVType.h" #include "libSPIRV/SPIRVUtil.h" #include "LLVMSPIRVLib.h" #include "llvm/IR/Attributes.h" #include "llvm/IR/Constants.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/TypedPointerType.h" #include #include using namespace SPIRV; using namespace llvm; namespace llvm { class IntrinsicInst; class IRBuilderBase; } // namespace llvm namespace SPIRV { /// The LLVM/SPIR-V translator version used to fill the lower 16 bits of the /// generator's magic number in the generated SPIR-V module. /// This number should be bumped up whenever the generated SPIR-V changes. const static unsigned short KTranslatorVer = 14; class SPIRVOpaqueType; typedef SPIRVMap SPIRVOpaqueTypeOpCodeMap; // Ad hoc function used by LLVM/SPIRV converter for type casting #define SPCV_CAST "spcv.cast" #define LLVM_MEMCPY "llvm.memcpy" // The name of function generated by Clang to initialize sampler(which is // opaqueue type) by 32-bit integer. The name is taken from // CodeGenModule::createOpenCLIntToSamplerConversion(). #define SAMPLER_INIT "__translate_sampler_initializer" template <> inline void SPIRVMap::init() { #define _SPIRV_OP(x, y) add(Instruction::x, Op##y); /* Casts */ _SPIRV_OP(ZExt, UConvert) _SPIRV_OP(SExt, SConvert) _SPIRV_OP(Trunc, UConvert) _SPIRV_OP(FPToUI, ConvertFToU) _SPIRV_OP(FPToSI, ConvertFToS) _SPIRV_OP(UIToFP, ConvertUToF) _SPIRV_OP(SIToFP, ConvertSToF) _SPIRV_OP(FPTrunc, FConvert) _SPIRV_OP(FPExt, FConvert) _SPIRV_OP(PtrToInt, ConvertPtrToU) _SPIRV_OP(IntToPtr, ConvertUToPtr) _SPIRV_OP(BitCast, Bitcast) _SPIRV_OP(AddrSpaceCast, GenericCastToPtr) _SPIRV_OP(GetElementPtr, AccessChain) _SPIRV_OP(FNeg, FNegate) /*Binary*/ _SPIRV_OP(And, BitwiseAnd) _SPIRV_OP(Or, BitwiseOr) _SPIRV_OP(Xor, BitwiseXor) _SPIRV_OP(Add, IAdd) _SPIRV_OP(FAdd, FAdd) _SPIRV_OP(Sub, ISub) _SPIRV_OP(FSub, FSub) _SPIRV_OP(Mul, IMul) _SPIRV_OP(FMul, FMul) _SPIRV_OP(UDiv, UDiv) _SPIRV_OP(SDiv, SDiv) _SPIRV_OP(FDiv, FDiv) _SPIRV_OP(SRem, SRem) _SPIRV_OP(FRem, FRem) _SPIRV_OP(URem, UMod) _SPIRV_OP(Shl, ShiftLeftLogical) _SPIRV_OP(LShr, ShiftRightLogical) _SPIRV_OP(AShr, ShiftRightArithmetic) #undef _SPIRV_OP } typedef SPIRVMap OpCodeMap; template <> inline void SPIRVMap::init() { #define _SPIRV_OP(x, y) add(CmpInst::x, Op##y); _SPIRV_OP(FCMP_OEQ, FOrdEqual) _SPIRV_OP(FCMP_OGT, FOrdGreaterThan) _SPIRV_OP(FCMP_OGE, FOrdGreaterThanEqual) _SPIRV_OP(FCMP_OLT, FOrdLessThan) _SPIRV_OP(FCMP_OLE, FOrdLessThanEqual) _SPIRV_OP(FCMP_ONE, FOrdNotEqual) _SPIRV_OP(FCMP_ORD, Ordered) _SPIRV_OP(FCMP_UNO, Unordered) _SPIRV_OP(FCMP_UEQ, FUnordEqual) _SPIRV_OP(FCMP_UGT, FUnordGreaterThan) _SPIRV_OP(FCMP_UGE, FUnordGreaterThanEqual) _SPIRV_OP(FCMP_ULT, FUnordLessThan) _SPIRV_OP(FCMP_ULE, FUnordLessThanEqual) _SPIRV_OP(FCMP_UNE, FUnordNotEqual) _SPIRV_OP(ICMP_EQ, IEqual) _SPIRV_OP(ICMP_NE, INotEqual) _SPIRV_OP(ICMP_UGT, UGreaterThan) _SPIRV_OP(ICMP_UGE, UGreaterThanEqual) _SPIRV_OP(ICMP_ULT, ULessThan) _SPIRV_OP(ICMP_ULE, ULessThanEqual) _SPIRV_OP(ICMP_SGT, SGreaterThan) _SPIRV_OP(ICMP_SGE, SGreaterThanEqual) _SPIRV_OP(ICMP_SLT, SLessThan) _SPIRV_OP(ICMP_SLE, SLessThanEqual) #undef _SPIRV_OP } typedef SPIRVMap CmpMap; class IntBoolOpMapId; template <> inline void SPIRVMap::init() { add(OpNot, OpLogicalNot); add(OpBitwiseAnd, OpLogicalAnd); add(OpBitwiseOr, OpLogicalOr); add(OpBitwiseXor, OpLogicalNotEqual); add(OpIEqual, OpLogicalEqual); add(OpINotEqual, OpLogicalNotEqual); } typedef SPIRVMap IntBoolOpMap; #define SPIR_TARGETTRIPLE32 "spir-unknown-unknown" #define SPIR_TARGETTRIPLE64 "spir64-unknown-unknown" #define SPIR_DATALAYOUT32 \ "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32" \ "-i64:64:64-f32:32:32-f64:64:64-v16:16:16-v24:32:32" \ "-v32:32:32-v48:64:64-v64:64:64-v96:128:128" \ "-v128:128:128-v192:256:256-v256:256:256" \ "-v512:512:512-v1024:1024:1024" #define SPIR_DATALAYOUT64 \ "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32" \ "-i64:64:64-f32:32:32-f64:64:64-v16:16:16-v24:32:32" \ "-v32:32:32-v48:64:64-v64:64:64-v96:128:128" \ "-v128:128:128-v192:256:256-v256:256:256" \ "-v512:512:512-v1024:1024:1024" template <> inline void SPIRVMap::init() { add(SPIRAS_Private, "Private"); add(SPIRAS_Global, "Global"); add(SPIRAS_Constant, "Constant"); add(SPIRAS_Local, "Local"); add(SPIRAS_Generic, "Generic"); add(SPIRAS_Input, "Input"); add(SPIRAS_CodeSectionINTEL, "CodeSectionINTEL"); add(SPIRAS_GlobalDevice, "GlobalDevice"); add(SPIRAS_GlobalHost, "GlobalHost"); } typedef SPIRVMap SPIRAddrSpaceCapitalizedNameMap; template <> inline void SPIRVMap::init() { add(SPIRAS_Private, StorageClassFunction); add(SPIRAS_Global, StorageClassCrossWorkgroup); add(SPIRAS_Constant, StorageClassUniformConstant); add(SPIRAS_Local, StorageClassWorkgroup); add(SPIRAS_Generic, StorageClassGeneric); add(SPIRAS_Input, StorageClassInput); add(SPIRAS_GlobalDevice, StorageClassDeviceOnlyINTEL); add(SPIRAS_GlobalHost, StorageClassHostOnlyINTEL); add(SPIRAS_CodeSectionINTEL, StorageClassCodeSectionINTEL); } typedef SPIRVMap SPIRSPIRVAddrSpaceMap; // Maps OCL builtin function to SPIRV builtin variable. template <> inline void SPIRVMap::init() { add("read_only", AccessQualifierReadOnly); add("write_only", AccessQualifierWriteOnly); add("read_write", AccessQualifierReadWrite); } typedef SPIRVMap SPIRSPIRVAccessQualifierMap; template <> inline void SPIRVMap::init() { add(Attribute::ZExt, FunctionParameterAttributeZext); add(Attribute::SExt, FunctionParameterAttributeSext); add(Attribute::ByVal, FunctionParameterAttributeByVal); add(Attribute::StructRet, FunctionParameterAttributeSret); add(Attribute::NoAlias, FunctionParameterAttributeNoAlias); add(Attribute::Captures, FunctionParameterAttributeNoCapture); add(Attribute::ReadOnly, FunctionParameterAttributeNoWrite); add(Attribute::ReadNone, FunctionParameterAttributeNoReadWrite); } typedef SPIRVMap SPIRSPIRVFuncParamAttrMap; template <> inline void SPIRVMap::init() { add(Attribute::AlwaysInline, FunctionControlInlineMask); add(Attribute::NoInline, FunctionControlDontInlineMask); add(Attribute::OptimizeNone, FunctionControlOptNoneEXTMask); } typedef SPIRVMap SPIRSPIRVFuncCtlMaskMap; class SPIRVExtSetShortName; template <> inline void SPIRVMap::init() { add(SPIRVEIS_OpenCL, "ocl"); } typedef SPIRVMap SPIRVExtSetShortNameMap; #define SPIR_MD_COMPILER_OPTIONS "opencl.compiler.options" #define SPIR_MD_KERNEL_ARG_ADDR_SPACE "kernel_arg_addr_space" #define SPIR_MD_KERNEL_ARG_ACCESS_QUAL "kernel_arg_access_qual" #define SPIR_MD_KERNEL_ARG_TYPE "kernel_arg_type" #define SPIR_MD_KERNEL_ARG_BASE_TYPE "kernel_arg_base_type" #define SPIR_MD_KERNEL_ARG_TYPE_QUAL "kernel_arg_type_qual" #define SPIR_MD_KERNEL_ARG_NAME "kernel_arg_name" #define SPIRV_MD_PARAMETER_DECORATIONS "spirv.ParameterDecorations" #define SPIRV_MD_DECORATIONS "spirv.Decorations" #define SPIRV_MD_INTEL_CACHE_DECORATIONS "spirv.DecorationCacheControlINTEL" #define OCL_TYPE_NAME_SAMPLER_T "sampler_t" #define SPIR_TYPE_NAME_EVENT_T "opencl.event_t" #define SPIR_TYPE_NAME_CLK_EVENT_T "opencl.clk_event_t" #define SPIR_TYPE_NAME_BLOCK_T "opencl.block" #define SPIR_INTRINSIC_BLOCK_BIND "spir_block_bind" #define SPIR_INTRINSIC_GET_BLOCK_INVOKE "spir_get_block_invoke" #define SPIR_INTRINSIC_GET_BLOCK_CONTEXT "spir_get_block_context" #define SPIR_TEMP_NAME_PREFIX_BLOCK "block" #define SPIR_TEMP_NAME_PREFIX_CALL "call" namespace kLLVMTypeName { const static char StructPrefix[] = "struct."; } // namespace kLLVMTypeName namespace kSPIRVImageSampledTypeName { const static char Float[] = "float"; const static char Half[] = "half"; const static char Int[] = "int"; const static char UInt[] = "uint"; const static char Long[] = "long"; const static char ULong[] = "ulong"; const static char Void[] = "void"; } // namespace kSPIRVImageSampledTypeName namespace kSPIRVTypeName { const static char Delimiter = '.'; const static char DeviceEvent[] = "DeviceEvent"; const static char Event[] = "Event"; const static char Image[] = "Image"; const static char Pipe[] = "Pipe"; const static char PostfixDelim = '_'; const static char Prefix[] = "spirv"; const static char PrefixAndDelim[] = "spirv."; const static char Queue[] = "Queue"; const static char ReserveId[] = "ReserveId"; const static char SampledImg[] = "SampledImage"; const static char Sampler[] = "Sampler"; const static char ConstantSampler[] = "ConstantSampler"; const static char PipeStorage[] = "PipeStorage"; const static char ConstantPipeStorage[] = "ConstantPipeStorage"; const static char VmeImageINTEL[] = "VmeImageINTEL"; const static char CooperativeMatrixKHR[] = "CooperativeMatrixKHR"; const static char BufferSurfaceINTEL[] = "BufferSurfaceINTEL"; } // namespace kSPIRVTypeName namespace kSPR2TypeName { const static char Delimiter = '.'; const static char OCLPrefix[] = "opencl."; const static char ImagePrefix[] = "opencl.image"; const static char PipeRO[] = "opencl.pipe_ro_t"; const static char PipeWO[] = "opencl.pipe_wo_t"; const static char Sampler[] = "opencl.sampler_t"; const static char Event[] = "opencl.event_t"; } // namespace kSPR2TypeName namespace kAccessQualName { const static char ReadOnly[] = "read_only"; const static char WriteOnly[] = "write_only"; const static char ReadWrite[] = "read_write"; } // namespace kAccessQualName namespace kAccessQualPostfix { const static char ReadOnly[] = "_ro"; const static char WriteOnly[] = "_wo"; const static char ReadWrite[] = "_rw"; const static char Type[] = "_t"; } // namespace kAccessQualPostfix namespace kMangledName { const static char Sampler[] = "11ocl_sampler"; const static char AtomicPrefixIncoming[] = "U7_Atomic"; const static char AtomicPrefixInternal[] = "atomic_"; } // namespace kMangledName namespace kSPIRVName { const static char GroupPrefix[] = "group_"; const static char GroupNonUniformPrefix[] = "group_non_uniform_"; const static char ClusteredPrefix[] = "clustered_"; const static char Prefix[] = "__spirv_"; const static char Postfix[] = "__"; const static char ImageQuerySize[] = "ImageQuerySize"; const static char ImageQuerySizeLod[] = "ImageQuerySizeLod"; const static char ImageSampleExplicitLod[] = "ImageSampleExplicitLod"; const static char ReservedPrefix[] = "reserved_"; const static char SampledImage[] = "SampledImage"; const static char TempSampledImage[] = "TempSampledImage"; const static char TranslateOCLMemOrder[] = "__translate_ocl_memory_order"; const static char TranslateOCLMemScope[] = "__translate_ocl_memory_scope"; const static char TranslateSPIRVMemOrder[] = "__translate_spirv_memory_order"; const static char TranslateSPIRVMemScope[] = "__translate_spirv_memory_scope"; const static char TranslateSPIRVMemFence[] = "__translate_spirv_memory_fence"; const static char EntrypointPrefix[] = "__spirv_entry_"; const static char ConvertHandleToImageINTEL[] = "ConvertHandleToImageINTEL"; const static char ConvertHandleToSamplerINTEL[] = "ConvertHandleToSamplerINTEL"; const static char ConvertHandleToSampledImageINTEL[] = "ConvertHandleToSampledImageINTEL"; const static char InternalBuiltinPrefix[] = "__builtin_spirv_"; } // namespace kSPIRVName namespace kSPIRVPostfix { const static char ToGlobal[] = "ToGlobal"; const static char ToLocal[] = "ToLocal"; const static char ToPrivate[] = "ToPrivate"; const static char Sat[] = "sat"; const static char Rtz[] = "rtz"; const static char Rte[] = "rte"; const static char Rtp[] = "rtp"; const static char Rtn[] = "rtn"; const static char Rt[] = "rt"; const static char Return[] = "R"; const static char Divider[] = "_"; /// Divider between extended instruction name and postfix const static char ExtDivider[] = "__"; } // namespace kSPIRVPostfix namespace kSPIRVMD { const static char Capability[] = "spirv.Capability"; const static char EntryPoint[] = "spirv.EntryPoint"; const static char ExecutionMode[] = "spirv.ExecutionMode"; const static char Extension[] = "spirv.Extension"; const static char Generator[] = "spirv.Generator"; const static char Source[] = "spirv.Source"; const static char SourceExtension[] = "spirv.SourceExtension"; const static char MemoryModel[] = "spirv.MemoryModel"; } // namespace kSPIRVMD namespace kSPIR2MD { const static char Extensions[] = "opencl.used.extensions"; const static char FPContract[] = "opencl.enable.FP_CONTRACT"; const static char OCLVer[] = "opencl.ocl.version"; const static char OCLCXXVer[] = "opencl.cxx.version"; const static char OptFeatures[] = "opencl.used.optional.core.features"; const static char SPIRVer[] = "opencl.spir.version"; const static char VecTyHint[] = "vec_type_hint"; const static char WGSize[] = "reqd_work_group_size"; const static char WGSizeHint[] = "work_group_size_hint"; const static char SubgroupSize[] = "intel_reqd_sub_group_size"; const static char MaxWGSize[] = "max_work_group_size"; const static char NoGlobalOffset[] = "no_global_work_offset"; const static char MaxWGDim[] = "max_global_work_dim"; const static char NumSIMD[] = "num_simd_work_items"; const static char StallEnable[] = "stall_enable"; const static char StallFree[] = "stall_free"; const static char FmaxMhz[] = "scheduler_target_fmax_mhz"; const static char LoopFuse[] = "loop_fuse"; const static char PreferDSP[] = "prefer_dsp"; const static char PropDSPPref[] = "propagate_dsp_preference"; const static char InitiationInterval[] = "initiation_interval"; const static char MaxConcurrency[] = "max_concurrency"; const static char PipelineKernel[] = "pipeline_kernel"; const static char IntelFPGAIPInterface[] = "ip_interface"; } // namespace kSPIR2MD enum Spir2SamplerKind { CLK_ADDRESS_NONE = 0x0000, CLK_ADDRESS_CLAMP = 0x0004, CLK_ADDRESS_CLAMP_TO_EDGE = 0x0002, CLK_ADDRESS_REPEAT = 0x0006, CLK_ADDRESS_MIRRORED_REPEAT = 0x0008, CLK_NORMALIZED_COORDS_FALSE = 0x0000, CLK_NORMALIZED_COORDS_TRUE = 0x0001, CLK_FILTER_NEAREST = 0x0010, CLK_FILTER_LINEAR = 0x0020, }; /// Additional information for mangling a function argument type. struct BuiltinArgTypeMangleInfo { bool IsSigned; bool IsVoidPtr; bool IsEnum; bool IsSampler; bool IsAtomic; bool IsLocalArgBlock; SPIR::TypePrimitiveEnum Enum; unsigned Attr; Type *PointerTy; BuiltinArgTypeMangleInfo() : IsSigned(true), IsVoidPtr(false), IsEnum(false), IsSampler(false), IsAtomic(false), IsLocalArgBlock(false), Enum(SPIR::PRIMITIVE_NONE), Attr(0), PointerTy(nullptr) {} }; /// Information for mangling builtin function. class BuiltinFuncMangleInfo { public: /// Translate builtin function name and set /// argument attributes and unsigned args. BuiltinFuncMangleInfo(const std::string &UniqName = "") : VarArgIdx(-1), DontMangle(false) { if (!UniqName.empty()) init(UniqName); } virtual ~BuiltinFuncMangleInfo() {} const std::string &getUnmangledName() const { return UnmangledName; } void addUnsignedArg(int Ndx) { if (Ndx == -1) return addUnsignedArgs(0, 10); // 10 is enough for everybody, right? getTypeMangleInfo(Ndx).IsSigned = false; } void addUnsignedArgs(int StartNdx, int StopNdx) { assert(StartNdx < StopNdx && "wrong parameters"); for (int I = StartNdx; I <= StopNdx; ++I) addUnsignedArg(I); } void addVoidPtrArg(unsigned Ndx) { getTypeMangleInfo(Ndx).IsVoidPtr = true; } void addSamplerArg(unsigned Ndx) { getTypeMangleInfo(Ndx).IsSampler = true; } void addAtomicArg(unsigned Ndx) { getTypeMangleInfo(Ndx).IsAtomic = true; } void setLocalArgBlock(unsigned Ndx) { getTypeMangleInfo(Ndx).IsLocalArgBlock = true; } void setEnumArg(unsigned Ndx, SPIR::TypePrimitiveEnum Enum) { auto &Info = getTypeMangleInfo(Ndx); Info.IsEnum = true; Info.Enum = Enum; } void setArgAttr(unsigned Ndx, unsigned Attr) { getTypeMangleInfo(Ndx).Attr = Attr; } void setVarArg(int Ndx) { assert(0 <= Ndx && "it is not allowed to set less than zero index"); VarArgIdx = Ndx; } void setAsDontMangle() { DontMangle = true; } bool avoidMangling() { return DontMangle; } // get ellipsis index, single ellipsis at the end of the function is possible // only return value < 0 if none int getVarArg() const { return VarArgIdx; } BuiltinArgTypeMangleInfo &getTypeMangleInfo(unsigned Ndx) { while (Ndx >= ArgInfo.size()) ArgInfo.emplace_back(); BuiltinArgTypeMangleInfo &Info = ArgInfo[Ndx]; return Info; } virtual void init(StringRef UniqUnmangledName) { UnmangledName = UniqUnmangledName.str(); } virtual bool isOpenCL() const { return false; } protected: std::string UnmangledName; std::vector ArgInfo; int VarArgIdx; // index of ellipsis argument, idx < 0 if none private: bool DontMangle; // clang doesn't apply mangling for some builtin functions // (i.e. enqueue_kernel) }; /// \returns a vector of types for a collection of values. template std::vector getTypes(llvm::ArrayRef V) { std::vector Tys; for (auto &I : V) Tys.push_back(I->getType()); return Tys; } /// Move elements of std::vector from [begin, end) to target. template void move(std::vector &V, size_t Begin, size_t End, size_t Target) { assert(Begin < End && End <= V.size() && Target <= V.size() && !(Begin < Target && Target < End)); if (Begin <= Target && Target <= End) return; auto B = V.begin() + Begin, E = V.begin() + End; if (Target > V.size()) Target = V.size(); if (Target > End) Target -= (End - Begin); std::vector Segment(B, E); V.erase(B, E); V.insert(V.begin() + Target, Segment.begin(), Segment.end()); } /// Find position of first pointer type value in a vector. template inline unsigned findFirstPtr(const Container &Args) { auto PtArg = std::find_if(Args.begin(), Args.end(), [](Value *V) { return V->getType()->isPointerTy(); }); return PtArg - Args.begin(); } // Utility function to check if a type is a TypedPointerType inline bool isTypedPointerType(llvm::Type *Ty) { return llvm::isa(Ty); } template inline unsigned findFirstPtrType(const Container &Args) { auto PtArg = std::find_if(Args.begin(), Args.end(), [](Type *T) { return T->isPointerTy() || isTypedPointerType(T); }); return PtArg - Args.begin(); } bool isSupportedTriple(Triple T); void removeFnAttr(CallInst *Call, Attribute::AttrKind Attr); void addFnAttr(CallInst *Call, Attribute::AttrKind Attr); void saveLLVMModule(Module *M, const std::string &OutputFile); std::string mapLLVMTypeToOCLType(const Type *Ty, bool Signed, Type *PointerElementType = nullptr); SPIRVDecorate *mapPostfixToDecorate(StringRef Postfix, SPIRVEntry *Target); /// Return vector V extended with poison elements to match the number of /// components of NewType. Value *extendVector(Value *V, FixedVectorType *NewType, IRBuilderBase &Builder); /// Add decorations to a SPIR-V entry. /// \param Decs Each string is a postfix without _ at the beginning. SPIRVValue *addDecorations(SPIRVValue *Target, const SmallVectorImpl &Decs); StructType *getOrCreateOpaqueStructType(Module *M, StringRef Name); void getFunctionTypeParameterTypes(llvm::FunctionType *FT, std::vector &ArgTys); Function *getOrCreateFunction(Module *M, Type *RetTy, ArrayRef ArgTypes, StringRef Name, BuiltinFuncMangleInfo *Mangle = nullptr, AttributeList *Attrs = nullptr, bool TakeName = true); /// Get function call arguments. /// \param Start Starting index. /// \param End Ending index. std::vector getArguments(CallInst *CI, unsigned Start = 0, unsigned End = 0); /// Get constant function call argument as an integer. /// \param I argument index. uint64_t getArgAsInt(CallInst *CI, unsigned I); /// Get constant function call argument as type \param T. /// \param I argument index. template T getArgAs(CallInst *CI, unsigned I) { return static_cast(getArgAsInt(CI, I)); } /// Get constant function call argument as a Scope enum. /// \param I argument index. Scope getArgAsScope(CallInst *CI, unsigned I); /// Check if a type is OCL image type. /// \return type name without "opencl." prefix. bool isOCLImageType(llvm::Type *Ty, StringRef *Name = nullptr); /// \param BaseTyName is the type name as in spirv.BaseTyName.Postfixes /// \param Postfix contains postfixes extracted from the SPIR-V image /// type name as spirv.BaseTyName.Postfixes. bool isSPIRVStructType(llvm::Type *Ty, StringRef BaseTyName, StringRef *Postfix = 0); bool isSYCLHalfType(llvm::Type *Ty); bool isSYCLBfloat16Type(llvm::Type *Ty); /// Check if a function has decorated name as __spirv_{Name}_ /// and get the original name. bool isDecoratedSPIRVFunc(const Function *F, StringRef &UndecName); std::string prefixSPIRVName(const std::string &S); StringRef dePrefixSPIRVName(StringRef R, SmallVectorImpl &Postfix); /// Get a canonical function name for a SPIR-V op code. std::string getSPIRVFuncName(Op OC, StringRef PostFix = ""); std::string getSPIRVFuncName(Op OC, const Type *PRetTy, bool IsSigned = false, Type *PointerElementType = nullptr); std::string getSPIRVFuncName(SPIRVBuiltinVariableKind BVKind); /// Get a canonical function name for a SPIR-V extended instruction std::string getSPIRVExtFuncName(SPIRVExtInstSetKind Set, unsigned ExtOp, StringRef PostFix = ""); /// Get SPIR-V op code given the canonical function name. /// Assume \param Name is either IA64 mangled or unmangled, and the unmangled /// name takes the __spirv_{OpName}_{Postfixes} format. /// \return op code if the unmangled function name is a valid op code name, /// otherwise return OpNop. /// \param Dec contains decorations decoded from function name if it is /// not nullptr. Op getSPIRVFuncOC(StringRef Name, SmallVectorImpl *Dec = nullptr); /// Get SPIR-V builtin variable enum given the canonical builtin name /// Assume \param Name is in format __spirv_BuiltIn{Name} /// \return false if \param Name is not a valid builtin name. bool getSPIRVBuiltin(const std::string &Name, spv::BuiltIn &Builtin); /// \param Name LLVM function name /// \param DemangledName demangled name of the OpenCL built-in function /// \returns true if Name is the name of the OpenCL built-in function, /// false for other functions bool oclIsBuiltin(StringRef Name, StringRef &DemangledName, bool IsCpp = false); /// Check if a function returns void bool isVoidFuncTy(FunctionType *FT); /// \returns true if function \p F has array type argument. bool hasArrayArg(Function *F); /// Mutates function call instruction by changing the arguments. /// \param ArgMutate mutates the function arguments. /// \return mutated call instruction. CallInst *mutateCallInst( Module *M, CallInst *CI, std::function &)> ArgMutate, BuiltinFuncMangleInfo *Mangle = nullptr, AttributeList *Attrs = nullptr, bool TakeName = false); /// Mutates function call instruction by changing the arguments and return /// value. /// \param ArgMutate mutates the function arguments. /// \param RetMutate mutates the return value. /// \return mutated instruction. Instruction *mutateCallInst( Module *M, CallInst *CI, std::function &, Type *&RetTy)> ArgMutate, std::function RetMutate, BuiltinFuncMangleInfo *Mangle = nullptr, AttributeList *Attrs = nullptr, bool TakeName = false); /// Mutate function by change the arguments. /// \param ArgMutate mutates the function arguments. /// \param TakeName Take the original function's name if a new function with /// different type needs to be created. void mutateFunction( Function *F, std::function &)> ArgMutate, BuiltinFuncMangleInfo *Mangle = nullptr, AttributeList *Attrs = nullptr, bool TakeName = true); /// Mutate function by change the arguments & the return type. /// \param ArgMutate mutates the function arguments. /// \param RetMutate mutates the function return value. /// \param TakeName Take the original function's name if a new function with /// different type needs to be created. void mutateFunction( Function *F, std::function &, Type *&RetTy)> ArgMutate, std::function RetMutate, BuiltinFuncMangleInfo *Mangle = nullptr, AttributeList *Attrs = nullptr, bool TakeName = true); /// Add a call instruction at \p Pos. CallInst *addCallInst(Module *M, StringRef FuncName, Type *RetTy, ArrayRef Args, AttributeList *Attrs, Instruction *Pos, BuiltinFuncMangleInfo *Mangle = nullptr, StringRef InstName = SPIR_TEMP_NAME_PREFIX_CALL, bool TakeFuncName = true); /// Check if an LLVM type is spirv.CooperativeMatrixKHR. bool isLLVMCooperativeMatrixType(llvm::Type *Ty); /// Add a call instruction for SPIR-V builtin function. CallInst *addCallInstSPIRV(Module *M, StringRef FuncName, Type *RetTy, ArrayRef Args, AttributeList *Attrs, ArrayRef PointerElementTypes, Instruction *Pos, StringRef InstName); typedef std::pair::iterator, std::vector::iterator> ValueVecRange; /// Add a vector at \param InsPos. Value *addVector(Instruction *InsPos, ValueVecRange Range); /// Replace scalar values with a vector created at \param InsPos. void makeVector(Instruction *InsPos, std::vector &Ops, ValueVecRange Range); /// Get size_t type. IntegerType *getSizetType(Module *M); /// Get a 64 bit integer constant. ConstantInt *getInt64(Module *M, int64_t Value); /// Get a 32 bit integer constant. ConstantInt *getInt32(Module *M, int Value); /// Get a 32 bit unsigned integer constant. ConstantInt *getUInt32(Module *M, unsigned Value); /// Get 32 bit integer constant if the value fits in 32 bits, /// return 64 bit integer constant otherwise ConstantInt *getInt(Module *M, int64_t Value); /// Get 32 bit unsigned integer constant if the value fits in 32 bits, /// return 64 bit unsigned integer constant otherwise ConstantInt *getUInt(Module *M, uint64_t Value); /// Get a 16 bit unsigned integer constant. ConstantInt *getUInt16(Module *M, unsigned short Value); // Get a 32 bit floating point constant. Constant *getFloat32(Module *M, float Value); /// Get a 32 bit integer constant vector. std::vector getInt32(Module *M, const std::vector &Value); /// Get a size_t type constant. ConstantInt *getSizet(Module *M, uint64_t Value); /// Get metadata operand as int. int64_t getMDOperandAsInt(MDNode *N, unsigned I); /// Get metadata operand as string. StringRef getMDOperandAsString(MDNode *N, unsigned I); /// Get metadata operand as another metadata node MDNode *getMDOperandAsMDNode(MDNode *N, unsigned I); /// Get metadata operand as type. Type *getMDOperandAsType(MDNode *N, unsigned I); /// Get a named metadata as a set of string. /// Assume the named metadata has one or more operands each of which might /// contain set of strings. For instance: /// !opencl.used.optional.core.features = !{!0} /// !0 = !{!"cl_doubles", !"cl_images"} /// or if we linked two modules we may have /// !opencl.used.optional.core.features = !{!0, !1} /// !0 = !{!"cl_doubles"} /// !1 = !{!"cl_images"} std::set getNamedMDAsStringSet(Module *M, const std::string &MDName); /// Get SPIR-V language by SPIR-V metadata spirv.Source std::tuple getSPIRVSource(Module *M); /// Get postfix _R{ReturnType} for return type /// The returned postfix does not includ "_" at the beginning std::string getPostfixForReturnType(CallInst *CI, bool IsSigned = false); std::string getPostfixForReturnType(const Type *PRetTy, bool IsSigned = false, Type *PointerElementType = nullptr); Constant *getScalarOrVectorConstantInt(Type *T, uint64_t V, bool IsSigned = false); /// Get a constant int or a constant int array. /// \param T is the type of the constant. It should be an integer type or // an integer pointer type. /// \param Len is the length of the array. /// \param V is the value to fill the array. Value *getScalarOrArrayConstantInt(BasicBlock::iterator P, Type *T, unsigned Len, uint64_t V, bool IsSigned = false); /// Get the array from GEP. /// \param V is a GEP whose pointer operand is a pointer to an array of size /// \param Size. Value *getScalarOrArray(Value *V, unsigned Size, BasicBlock::iterator Pos); void dumpUsers(Value *V, StringRef Prompt = ""); /// Get SPIR-V type name as spirv.BaseTyName.Postfixes. std::string getSPIRVTypeName(StringRef BaseTyName, StringRef Postfixes = ""); /// Checks if given type name is either ConstantSampler or ConsantPipeStorage. bool isSPIRVConstantName(StringRef TyName); /// Get LLVM type for sampled type of SPIR-V image type by postfix. Type *getLLVMTypeForSPIRVImageSampledTypePostfix(StringRef Postfix, LLVMContext &Ctx); /// Convert an LLVM type to a string postfix name. std::string convertTypeToPostfix(Type *T); /// Return the unqualified and unsuffixed base name of an image type. /// E.g. opencl.image2d_ro_t.3 -> image2d_t std::string getImageBaseTypeName(StringRef Name); /// Extract the image type descriptor from the given image type. SPIRVTypeImageDescriptor getImageDescriptor(Type *Ty); /// Return the index of image operands given an image op. size_t getImageOperandsIndex(Op OpCode); /// Check if access qualifier is encoded in the type name. bool hasAccessQualifiedName(StringRef TyName); /// Get access qualifier from the type name. SPIRVAccessQualifierKind getAccessQualifier(StringRef TyName); /// Get access qualifier from the type name. StringRef getAccessQualifierPostfix(SPIRVAccessQualifierKind Access); bool eraseUselessFunctions(Module *M); /// Erase a function if it is declaration, has internal linkage and has no use. bool eraseIfNoUse(Function *F); // Check if a mangled type name is unsigned bool isMangledTypeUnsigned(char Mangled); // Check if a mangled type name is signed bool isMangledTypeSigned(char Mangled); // Check if a mangled type name is floating point (except half) bool isMangledTypeFP(char Mangled); // Check if a mangled type name is half bool isMangledTypeHalf(std::string Mangled); // Check if \param I is valid vector size: 2, 3, 4, 8, 16. bool isValidVectorSize(unsigned I); enum class ParamType { FLOAT = 0, SIGNED = 1, UNSIGNED = 2, UNKNOWN = 3 }; ParamType lastFuncParamType(StringRef MangledName); // Check if the last function parameter is signed bool isLastFuncParamSigned(StringRef MangledName); // Check if a mangled function name contains unsigned atomic type bool containsUnsignedAtomicType(StringRef Name); /// Mangle builtin function name. /// \return \param UniqName if \param BtnInfo is null pointer, otherwise /// return IA64 mangled name. std::string mangleBuiltin(StringRef UniqName, ArrayRef ArgTypes, BuiltinFuncMangleInfo *BtnInfo, const SPIRV::AddrSpaceMap *Map = nullptr); /// Extract the true pointer types, expressed as a TypedPointerType, of /// arguments from a mangled function name. If the corresponding type is not a /// pointer type, its value will be the argument's actual type instead. Returns /// true if the function name was successfully demangled. bool getParameterTypes( Function *F, SmallVectorImpl &ArgTys, std::function StructNameMapFn = nullptr); inline bool getParameterTypes(CallInst *CI, SmallVectorImpl &ArgTys) { return getParameterTypes(CI->getCalledFunction(), ArgTys); } enum class ParamSignedness { Signed = 0, Unsigned, Unknown }; /// Extract signedness of return type and parameter types from a mangled /// function name. bool getRetParamSignedness(Function *F, ParamSignedness &RetSignedness, SmallVectorImpl &ArgSignedness); /// Mangle a function from OpenCL extended instruction set in SPIR-V friendly IR /// manner std::string getSPIRVFriendlyIRFunctionName(OCLExtOpKind ExtOpId, ArrayRef ArgTys, Type *RetTy = nullptr, const SPIRV::AddrSpaceMap *Map = nullptr); /// Mangle a function in SPIR-V friendly IR manner /// \param UniqName full unmangled name of the SPIR-V built-in function that /// contains possible postfixes that depend not on opcode but on decorations or /// return type, for example __spirv_UConvert_Rint_sat. /// \param OC opcode of corresponding built-in instruction. Used to gather info /// for unsigned/constant arguments. /// \param Types of arguments of SPIR-V built-in function /// \param Ops Operands of SPIRVInstruction /// \return IA64 mangled name. std::string getSPIRVFriendlyIRFunctionName( const std::string &UniqName, spv::Op OC, ArrayRef ArgTys, ArrayRef Ops, const SPIRV::AddrSpaceMap *Map = nullptr); /// Get i8* with the same address space. PointerType *getInt8PtrTy(PointerType *T); /// Cast a value to a i8* by inserting a cast instruction. Value *castToInt8Ptr(Value *V, BasicBlock::iterator Pos); template <> inline void SPIRVMap::init() { #define _SPIRV_OP(x) add(#x, OpType##x); _SPIRV_OP(DeviceEvent) _SPIRV_OP(Event) _SPIRV_OP(Image) _SPIRV_OP(Pipe) _SPIRV_OP(Queue) _SPIRV_OP(ReserveId) _SPIRV_OP(Sampler) _SPIRV_OP(SampledImage) _SPIRV_OP(PipeStorage) // SPV_INTEL_device_side_avc_motion_estimation types _SPIRV_OP(AvcMcePayloadINTEL) _SPIRV_OP(AvcImePayloadINTEL) _SPIRV_OP(AvcRefPayloadINTEL) _SPIRV_OP(AvcSicPayloadINTEL) _SPIRV_OP(AvcMceResultINTEL) _SPIRV_OP(AvcImeResultINTEL) _SPIRV_OP(AvcImeResultSingleReferenceStreamoutINTEL) _SPIRV_OP(AvcImeResultDualReferenceStreamoutINTEL) _SPIRV_OP(AvcImeSingleReferenceStreaminINTEL) _SPIRV_OP(AvcImeDualReferenceStreaminINTEL) _SPIRV_OP(AvcRefResultINTEL) _SPIRV_OP(AvcSicResultINTEL) _SPIRV_OP(VmeImageINTEL) _SPIRV_OP(BufferSurfaceINTEL) _SPIRV_OP(CooperativeMatrixKHR) #undef _SPIRV_OP add("TaskSequenceINTEL", internal::OpTypeTaskSequenceINTEL); } // Check if the module contains llvm.loop.* metadata bool hasLoopMetadata(const Module *M); // Check if CI is a call to instruction from OpenCL Extended Instruction Set. // If so, return it's extended opcode in ExtOp. bool isSPIRVOCLExtInst(const CallInst *CI, OCLExtOpKind *ExtOp); /// Returns true if a function name corresponds to an OpenCL builtin that is not /// expected to have name mangling. bool isNonMangledOCLBuiltin(StringRef Name); // check LLVM Intrinsics type(s) for validity bool checkTypeForSPIRVExtendedInstLowering(IntrinsicInst *II, SPIRVModule *BM); /// Decode SPIR-V type name in the format spirv.{TypeName}._{Postfixes} /// where Postfixes are strings separated by underscores. /// \return TypeName. /// \param Strs contains the integers decoded from postfixes. std::string decodeSPIRVTypeName(StringRef Name, SmallVectorImpl &Strs); // Copy attributes from function to call site. CallInst *setAttrByCalledFunc(CallInst *Call); bool isSPIRVBuiltinVariable(GlobalVariable *GV, SPIRVBuiltinVariableKind *Kind); // Transform builtin variable from GlobalVariable to builtin call. // e.g. // - GlobalInvocationId[x] -> _Z33__spirv_BuiltInGlobalInvocationIdi(x) // - WorkDim -> _Z22__spirv_BuiltInWorkDimv() bool lowerBuiltinVariableToCall(GlobalVariable *GV, SPIRVBuiltinVariableKind Kind); // Transform all builtin variables into calls bool lowerBuiltinVariablesToCalls(Module *M); // Transform all builtin calls into variables bool lowerBuiltinCallsToVariables(Module *M); // Transform all builtins into variables or calls // depending on user specification bool lowerBuiltins(SPIRVModule *BM, Module *M); /// \brief Post-process OpenCL or SPIRV builtin function returning struct type. /// /// Some builtin functions are translated to SPIR-V instructions with /// struct type result, e.g. NDRange creation functions. Such functions /// need to be post-processed to return the struct through sret argument. bool postProcessBuiltinReturningStruct(Function *F); /// \brief Post-process OpenCL or SPIRV builtin function having array argument. /// /// These functions are translated to functions with array type argument /// first, then post-processed to have pointer arguments. bool postProcessBuiltinWithArrayArguments(Function *F, StringRef DemangledName); bool postProcessBuiltinsReturningStruct(Module *M, bool IsCpp = false); bool postProcessBuiltinsWithArrayArguments(Module *M, bool IsCpp = false); /// \param MangledName LLVM function name. /// \param DemangledName demangled name of the input function if it is the /// translator's internal built-in function. /// \returns true if MangledName is the name of the translator's internal /// built-in function, false for other functions. /// Used for 'mini'-floats conversion functions bool isInternalSPIRVBuiltin(StringRef MangledName, StringRef &DemangledName); // Wrapper around SPIR-V 1.6.4 FP Encoding to be used in the conversion // descriptor enum FPEncodingWrap { Integer = FPEncoding::FPEncodingMax - 1, IEEE754 = FPEncoding::FPEncodingMax, BF16 = FPEncoding::FPEncodingBFloat16KHR, E4M3 = FPEncoding::FPEncodingFloat8E4M3EXT, E5M2 = FPEncoding::FPEncodingFloat8E5M2EXT, E2M1 = internal::FPEncodingFloat4E2M1INTEL, }; // Structure describing non-trivial conversions (FP8 and int4) struct FPConversionDesc { FPEncodingWrap SrcEncoding; FPEncodingWrap DstEncoding; SPIRVWord ConvOpCode; // To use as a key in std::map bool operator==(const FPConversionDesc &Other) const { return SrcEncoding == Other.SrcEncoding && DstEncoding == Other.DstEncoding && ConvOpCode == Other.ConvOpCode; } bool operator<(const FPConversionDesc &Other) const { if (ConvOpCode != Other.ConvOpCode) return ConvOpCode < Other.ConvOpCode; if (SrcEncoding != Other.SrcEncoding) return SrcEncoding < Other.SrcEncoding; return DstEncoding < Other.DstEncoding; } }; // Maps internal builtin name to conversion descriptor typedef SPIRVMap FPConvertToEncodingMap; // clang-format off template <> inline void FPConvertToEncodingMap::init() { // 4-bit conversions add("ConvertE2M1ToE4M3INTEL", {FPEncodingWrap::E2M1, FPEncodingWrap::E4M3, OpFConvert}); add("ConvertE2M1ToE5M2INTEL", {FPEncodingWrap::E2M1, FPEncodingWrap::E5M2, OpFConvert}); add("ConvertE2M1ToFP16INTEL", {FPEncodingWrap::E2M1, FPEncodingWrap::IEEE754, OpFConvert}); add("ConvertE2M1ToBF16INTEL", {FPEncodingWrap::E2M1, FPEncodingWrap::BF16, OpFConvert}); add("ConvertInt4ToE4M3INTEL", {FPEncodingWrap::Integer, FPEncodingWrap::E4M3, OpConvertSToF}); add("ConvertInt4ToE5M2INTEL", {FPEncodingWrap::Integer, FPEncodingWrap::E5M2, OpConvertSToF}); add("ConvertInt4ToFP16INTEL", {FPEncodingWrap::Integer, FPEncodingWrap::IEEE754, OpConvertSToF}); add("ConvertInt4ToBF16INTEL", {FPEncodingWrap::Integer, FPEncodingWrap::BF16, OpConvertSToF}); add("ConvertInt4ToInt8INTEL", {FPEncodingWrap::Integer, FPEncodingWrap::Integer, OpSConvert}); add("ConvertFP16ToE2M1INTEL", {FPEncodingWrap::IEEE754, FPEncodingWrap::E2M1, OpFConvert}); add("ConvertBF16ToE2M1INTEL", {FPEncodingWrap::BF16, FPEncodingWrap::E2M1, OpFConvert}); add("ConvertFP16ToInt4INTEL", {FPEncodingWrap::IEEE754, FPEncodingWrap::Integer, OpConvertFToS}); add("ConvertBF16ToInt4INTEL", {FPEncodingWrap::BF16, FPEncodingWrap::Integer, OpConvertFToS}); // 8-bit conversions add("ConvertE4M3ToFP16EXT", {FPEncodingWrap::E4M3, FPEncodingWrap::IEEE754, OpFConvert}); add("ConvertE5M2ToFP16EXT", {FPEncodingWrap::E5M2, FPEncodingWrap::IEEE754, OpFConvert}); add("ConvertE4M3ToBF16EXT", {FPEncodingWrap::E4M3, FPEncodingWrap::BF16, OpFConvert}); add("ConvertE5M2ToBF16EXT", {FPEncodingWrap::E5M2, FPEncodingWrap::BF16, OpFConvert}); add("ConvertFP16ToE4M3EXT", {FPEncodingWrap::IEEE754, FPEncodingWrap::E4M3, OpFConvert}); add("ConvertFP16ToE5M2EXT", {FPEncodingWrap::IEEE754, FPEncodingWrap::E5M2, OpFConvert}); add("ConvertBF16ToE4M3EXT", {FPEncodingWrap::BF16, FPEncodingWrap::E4M3, OpFConvert}); add("ConvertBF16ToE5M2EXT", {FPEncodingWrap::BF16, FPEncodingWrap::E5M2, OpFConvert}); // SPV_INTEL_fp_conversions add("ClampConvertFP16ToE2M1INTEL", {FPEncodingWrap::IEEE754, FPEncodingWrap::E2M1, internal::OpClampConvertFToFINTEL}); add("ClampConvertBF16ToE2M1INTEL", {FPEncodingWrap::BF16, FPEncodingWrap::E2M1, internal::OpClampConvertFToFINTEL}); add("ClampConvertFP16ToE4M3INTEL", {FPEncodingWrap::IEEE754, FPEncodingWrap::E4M3, internal::OpClampConvertFToFINTEL}); add("ClampConvertBF16ToE4M3INTEL", {FPEncodingWrap::BF16, FPEncodingWrap::E4M3, internal::OpClampConvertFToFINTEL}); add("ClampConvertFP16ToE5M2INTEL", {FPEncodingWrap::IEEE754, FPEncodingWrap::E5M2, internal::OpClampConvertFToFINTEL}); add("ClampConvertBF16ToE5M2INTEL", {FPEncodingWrap::BF16, FPEncodingWrap::E5M2, internal::OpClampConvertFToFINTEL}); add("ClampConvertFP16ToInt4INTEL", {FPEncodingWrap::IEEE754, FPEncodingWrap::Integer, internal::OpClampConvertFToSINTEL}); add("ClampConvertBF16ToInt4INTEL", {FPEncodingWrap::BF16, FPEncodingWrap::Integer, internal::OpClampConvertFToSINTEL}); add("StochasticRoundFP16ToE5M2INTEL", {FPEncodingWrap::IEEE754, FPEncodingWrap::E5M2, internal::OpStochasticRoundFToFINTEL}); add("StochasticRoundFP16ToE4M3INTEL", {FPEncodingWrap::IEEE754, FPEncodingWrap::E4M3, internal::OpStochasticRoundFToFINTEL}); add("StochasticRoundBF16ToE5M2INTEL", {FPEncodingWrap::BF16, FPEncodingWrap::E5M2, internal::OpStochasticRoundFToFINTEL}); add("StochasticRoundBF16ToE4M3INTEL", {FPEncodingWrap::BF16, FPEncodingWrap::E4M3, internal::OpStochasticRoundFToFINTEL}); add("StochasticRoundFP16ToE2M1INTEL", {FPEncodingWrap::IEEE754, FPEncodingWrap::E2M1, internal::OpStochasticRoundFToFINTEL}); add("StochasticRoundBF16ToE2M1INTEL", {FPEncodingWrap::BF16, FPEncodingWrap::E2M1, internal::OpStochasticRoundFToFINTEL}); add("ClampStochasticRoundFP16ToInt4INTEL", {FPEncodingWrap::IEEE754, FPEncodingWrap::Integer, internal::OpClampStochasticRoundFToSINTEL}); add("ClampStochasticRoundBF16ToInt4INTEL", {FPEncodingWrap::BF16, FPEncodingWrap::Integer, internal::OpClampStochasticRoundFToSINTEL}); add("ClampStochasticRoundFP16ToE5M2INTEL", {FPEncodingWrap::IEEE754, FPEncodingWrap::E5M2, internal::OpClampStochasticRoundFToFINTEL}); add("ClampStochasticRoundFP16ToE4M3INTEL", {FPEncodingWrap::IEEE754, FPEncodingWrap::E4M3, internal::OpClampStochasticRoundFToFINTEL}); add("ClampStochasticRoundBF16ToE5M2INTEL", {FPEncodingWrap::BF16, FPEncodingWrap::E5M2, internal::OpClampStochasticRoundFToFINTEL}); add("ClampStochasticRoundBF16ToE4M3INTEL", {FPEncodingWrap::BF16, FPEncodingWrap::E4M3, internal::OpClampStochasticRoundFToFINTEL}); } // clang-format on } // namespace SPIRV #endif // SPIRV_SPIRVINTERNAL_H