derek.holloway
7502bf64a4
CodeQL / Analyze (actions) (push) Canceled after 0s
CodeQL / Analyze (python) (push) Canceled after 0s
pre-commit / pre-commit (push) Canceled after 0s
Multi-Arch CI / setup (push) Canceled after 0s
Publish build_manylinux_x86_64 images / publish_build_manylinux_x86_64 (push) Canceled after 0s
Setup (Multi-Arch) / setup (push) Canceled after 0s
Multi-Arch CI / Linux::${{ fromJSON(needs.setup.outputs.linux_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubi10 image / publish_no_rocm_image_ubi10 (push) Canceled after 0s
Multi-Arch CI / Windows::${{ fromJSON(needs.setup.outputs.windows_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Multi-Arch CI / Manifest Diff (push) Canceled after 0s
TheRock Manifest Diff Report / Generate Manifest Diff Report (push) Canceled after 0s
Multi-Arch CI / CI Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_ocl_rt images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_rocgdb images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Unit Tests / Unit Tests :: ubuntu-24.04 (push) Canceled after 0s
Unit Tests / Unit Tests :: windows-2022 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_media (push) Canceled after 0s
Unit Tests / Unit Tests Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_openmpi (push) Canceled after 0s
4825 lines
171 KiB
C++
4825 lines
171 KiB
C++
//===- SPIRVInstruction.h - Class to represent SPIRV instruction -- C++ -*-===//
|
|
//
|
|
// The LLVM/SPIRV Translator
|
|
//
|
|
// This file is distributed under the University of Illinois Open Source
|
|
// License. See LICENSE.TXT for details.
|
|
//
|
|
// Copyright (c) 2014 Advanced Micro Devices, Inc. All rights reserved.
|
|
//
|
|
// Permission is hereby granted, free of charge, to any person obtaining a
|
|
// copy of this software and associated documentation files (the "Software"),
|
|
// to deal with the Software without restriction, including without limitation
|
|
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
|
// and/or sell copies of the Software, and to permit persons to whom the
|
|
// Software is furnished to do so, subject to the following conditions:
|
|
//
|
|
// Redistributions of source code must retain the above copyright notice,
|
|
// this list of conditions and the following disclaimers.
|
|
// Redistributions in binary form must reproduce the above copyright notice,
|
|
// this list of conditions and the following disclaimers in the documentation
|
|
// and/or other materials provided with the distribution.
|
|
// Neither the names of Advanced Micro Devices, Inc., nor the names of its
|
|
// contributors may be used to endorse or promote products derived from this
|
|
// Software without specific prior written permission.
|
|
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
// CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH
|
|
// THE SOFTWARE.
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
/// \file
|
|
///
|
|
/// This file defines Instruction class for SPIR-V.
|
|
///
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
#ifndef SPIRV_LIBSPIRV_SPIRVINSTRUCTION_H
|
|
#define SPIRV_LIBSPIRV_SPIRVINSTRUCTION_H
|
|
|
|
#include "SPIRVBasicBlock.h"
|
|
#include "SPIRVEnum.h"
|
|
#include "SPIRVFunction.h"
|
|
#include "SPIRVIsValidEnum.h"
|
|
#include "SPIRVOpCode.h"
|
|
#include "SPIRVStream.h"
|
|
#include "SPIRVValue.h"
|
|
|
|
#include <cassert>
|
|
#include <functional>
|
|
#include <iostream>
|
|
#include <unordered_set>
|
|
#include <utility>
|
|
#include <vector>
|
|
|
|
namespace SPIRV {
|
|
|
|
typedef std::vector<SPIRVValue *> ValueVec;
|
|
typedef std::pair<ValueVec::iterator, ValueVec::iterator> ValueRange;
|
|
|
|
class SPIRVBasicBlock;
|
|
class SPIRVFunction;
|
|
|
|
bool isSpecConstantOpAllowedOp(Op OC);
|
|
|
|
class SPIRVComponentExecutionScope {
|
|
public:
|
|
SPIRVComponentExecutionScope(Scope TheScope = ScopeInvocation)
|
|
: ExecScope(TheScope) {}
|
|
Scope ExecScope;
|
|
};
|
|
|
|
class SPIRVComponentMemorySemanticsMask {
|
|
public:
|
|
SPIRVComponentMemorySemanticsMask(SPIRVWord TheSema = SPIRVWORD_MAX)
|
|
: MemSema(TheSema) {}
|
|
SPIRVWord MemSema;
|
|
};
|
|
|
|
class SPIRVComponentOperands {
|
|
public:
|
|
SPIRVComponentOperands() {}
|
|
SPIRVComponentOperands(const std::vector<SPIRVValue *> &TheOperands)
|
|
: Operands(TheOperands) {}
|
|
SPIRVComponentOperands(std::vector<SPIRVValue *> &&TheOperands)
|
|
: Operands(std::move(TheOperands)) {}
|
|
std::vector<SPIRVValue *> getCompOperands() { return Operands; }
|
|
std::vector<SPIRVType *> getCompOperandTypes() {
|
|
std::vector<SPIRVType *> Tys;
|
|
for (auto &I : getCompOperands())
|
|
Tys.push_back(I->getType());
|
|
return Tys;
|
|
}
|
|
|
|
protected:
|
|
std::vector<SPIRVValue *> Operands;
|
|
};
|
|
|
|
class SPIRVInstruction : public SPIRVValue {
|
|
public:
|
|
// Complete constructor for instruction with type and id
|
|
SPIRVInstruction(unsigned TheWordCount, Op TheOC, SPIRVType *TheType,
|
|
SPIRVId TheId, SPIRVBasicBlock *TheBB);
|
|
// Complete constructor for instruction with module, type and id
|
|
SPIRVInstruction(unsigned TheWordCount, Op TheOC, SPIRVType *TheType,
|
|
SPIRVId TheId, SPIRVBasicBlock *TheBB, SPIRVModule *TheBM);
|
|
// Complete constructor for instruction with id but no type
|
|
SPIRVInstruction(unsigned TheWordCount, Op TheOC, SPIRVId TheId,
|
|
SPIRVBasicBlock *TheBB);
|
|
// Complete constructor for instruction without type and id
|
|
SPIRVInstruction(unsigned TheWordCount, Op TheOC, SPIRVBasicBlock *TheBB);
|
|
// Complete constructor for instruction with type but no id
|
|
SPIRVInstruction(unsigned TheWordCount, Op TheOC, SPIRVType *TheType,
|
|
SPIRVBasicBlock *TheBB);
|
|
// Special constructor for debug instruction
|
|
SPIRVInstruction(unsigned TheWordCount, Op TheOC, SPIRVType *TheType,
|
|
SPIRVId TheId, SPIRVModule *TheBM,
|
|
SPIRVBasicBlock *TheBB = nullptr);
|
|
|
|
// Incomplete constructor
|
|
SPIRVInstruction(Op TheOC = OpNop)
|
|
: SPIRVValue(TheOC), BB(NULL), DebugScope(nullptr) {}
|
|
|
|
bool isInst() const override { return true; }
|
|
SPIRVBasicBlock *getParent() const { return BB; }
|
|
SPIRVInstruction *getPrevious() const { return BB->getPrevious(this); }
|
|
SPIRVInstruction *getNext() const { return BB->getNext(this); }
|
|
virtual std::vector<SPIRVValue *> getOperands();
|
|
static std::vector<SPIRVType *>
|
|
getOperandTypes(const std::vector<SPIRVValue *> &Ops);
|
|
|
|
void setParent(SPIRVBasicBlock *);
|
|
void setScope(SPIRVEntry *) override;
|
|
void addFPRoundingMode(SPIRVFPRoundingModeKind Kind) {
|
|
addDecorate(DecorationFPRoundingMode, Kind);
|
|
}
|
|
void eraseFPRoundingMode() { eraseDecorate(DecorationFPRoundingMode); }
|
|
void setSaturatedConversion(bool Enable) {
|
|
if (Enable)
|
|
addDecorate(DecorationSaturatedConversion);
|
|
else
|
|
eraseDecorate(DecorationSaturatedConversion);
|
|
}
|
|
bool hasFPRoundingMode(SPIRVFPRoundingModeKind *Kind = nullptr) {
|
|
SPIRVWord V;
|
|
auto Found = hasDecorate(DecorationFPRoundingMode, 0, &V);
|
|
if (Found && Kind)
|
|
*Kind = static_cast<SPIRVFPRoundingModeKind>(V);
|
|
return Found;
|
|
}
|
|
bool isSaturatedConversion() {
|
|
return hasDecorate(DecorationSaturatedConversion) ||
|
|
OpCode == OpSatConvertSToU || OpCode == OpSatConvertUToS;
|
|
}
|
|
|
|
SPIRVBasicBlock *getBasicBlock() const { return BB; }
|
|
|
|
void setBasicBlock(SPIRVBasicBlock *TheBB) {
|
|
BB = TheBB;
|
|
if (TheBB)
|
|
setModule(TheBB->getModule());
|
|
}
|
|
|
|
void setDebugScope(SPIRVEntry *Scope) { DebugScope = Scope; }
|
|
|
|
SPIRVEntry *getDebugScope() const { return DebugScope; }
|
|
|
|
protected:
|
|
void validate() const override { SPIRVValue::validate(); }
|
|
|
|
private:
|
|
SPIRVBasicBlock *BB;
|
|
SPIRVEntry *DebugScope;
|
|
};
|
|
|
|
class SPIRVInstTemplateBase : public SPIRVInstruction {
|
|
public:
|
|
/// Create an empty instruction. Mainly for getting format information,
|
|
/// e.g. whether an operand is literal.
|
|
static SPIRVInstTemplateBase *create(Op TheOC) {
|
|
auto Inst = static_cast<SPIRVInstTemplateBase *>(SPIRVEntry::create(TheOC));
|
|
assert(Inst);
|
|
Inst->init();
|
|
return Inst;
|
|
}
|
|
/// Create a instruction without operands.
|
|
static SPIRVInstTemplateBase *create(Op TheOC, SPIRVType *TheType,
|
|
SPIRVId TheId, SPIRVBasicBlock *TheBB,
|
|
SPIRVModule *TheModule) {
|
|
auto Inst = create(TheOC);
|
|
Inst->init(TheType, TheId, TheBB, TheModule);
|
|
return Inst;
|
|
}
|
|
/// Create a complete and valid instruction.
|
|
static SPIRVInstTemplateBase *create(Op TheOC, SPIRVType *TheType,
|
|
SPIRVId TheId,
|
|
const std::vector<SPIRVWord> &TheOps,
|
|
SPIRVBasicBlock *TheBB,
|
|
SPIRVModule *TheModule) {
|
|
auto Inst = create(TheOC);
|
|
Inst->init(TheType, TheId, TheBB, TheModule);
|
|
Inst->setOpWords(TheOps);
|
|
Inst->validate();
|
|
return Inst;
|
|
}
|
|
SPIRVInstTemplateBase(Op OC = OpNop)
|
|
: SPIRVInstruction(OC), HasVariWC(false) {
|
|
init();
|
|
}
|
|
~SPIRVInstTemplateBase() override {}
|
|
SPIRVInstTemplateBase *init(SPIRVType *TheType, SPIRVId TheId,
|
|
SPIRVBasicBlock *TheBB, SPIRVModule *TheModule) {
|
|
assert((TheBB || TheModule) && "Invalid BB or Module");
|
|
if (TheBB)
|
|
setBasicBlock(TheBB);
|
|
else {
|
|
setModule(TheModule);
|
|
}
|
|
setId(hasId() ? TheId : SPIRVID_INVALID);
|
|
setType(hasType() ? TheType : nullptr);
|
|
return this;
|
|
}
|
|
virtual void init() {}
|
|
virtual void initImpl(Op OC, bool HasId = true, SPIRVWord WC = 0,
|
|
bool VariWC = false, unsigned Lit1 = ~0U,
|
|
unsigned Lit2 = ~0U, unsigned Lit3 = ~0U,
|
|
unsigned Lit4 = ~0U) {
|
|
OpCode = OC;
|
|
if (!HasId) {
|
|
setHasNoId();
|
|
setHasNoType();
|
|
}
|
|
if (WC)
|
|
SPIRVEntry::setWordCount(WC);
|
|
setHasVariableWordCount(VariWC);
|
|
addLit(Lit1);
|
|
addLit(Lit2);
|
|
addLit(Lit3);
|
|
addLit(Lit4);
|
|
}
|
|
bool isOperandLiteral(unsigned I) const override { return Lit.count(I); }
|
|
void addLit(unsigned L) {
|
|
if (L != ~0U)
|
|
Lit.insert(L);
|
|
}
|
|
/// \return Expected number of operands. If the instruction has variable
|
|
/// number of words, return the minimum.
|
|
SPIRVWord getExpectedNumOperands() const {
|
|
assert(WordCount > 0 && "Word count not initialized");
|
|
auto Exp = WordCount - 1;
|
|
if (hasId())
|
|
--Exp;
|
|
if (hasType())
|
|
--Exp;
|
|
return Exp;
|
|
}
|
|
virtual void setOpWordsAndValidate(const std::vector<SPIRVWord> &TheOps) {
|
|
setOpWords(TheOps);
|
|
validate();
|
|
}
|
|
virtual void setOpWords(const std::vector<SPIRVWord> &TheOps) {
|
|
SPIRVWord WC = TheOps.size() + 1;
|
|
if (hasId())
|
|
++WC;
|
|
if (hasType())
|
|
++WC;
|
|
if (WordCount) {
|
|
if (WordCount == WC) {
|
|
// do nothing
|
|
} else {
|
|
assert(HasVariWC && WC >= WordCount && "Invalid word count");
|
|
SPIRVEntry::setWordCount(WC);
|
|
}
|
|
} else
|
|
SPIRVEntry::setWordCount(WC);
|
|
Ops = TheOps;
|
|
// The required SPIR-V version depends on the operands for some
|
|
// instructions.
|
|
updateModuleVersion();
|
|
}
|
|
void setWordCount(SPIRVWord TheWordCount) override {
|
|
SPIRVEntry::setWordCount(TheWordCount);
|
|
auto NumOps = WordCount - 1;
|
|
if (hasId())
|
|
--NumOps;
|
|
if (hasType())
|
|
--NumOps;
|
|
Ops.resize(NumOps);
|
|
}
|
|
|
|
std::vector<SPIRVWord> &getOpWords() { return Ops; }
|
|
|
|
const std::vector<SPIRVWord> &getOpWords() const { return Ops; }
|
|
|
|
SPIRVWord getOpWord(int I) const { return Ops[I]; }
|
|
|
|
/// Get operand as value.
|
|
/// If the operand is a literal, return it as a uint32 constant.
|
|
const SPIRVValue *getOpValue(int I) const {
|
|
return isOperandLiteral(I) ? Module->getLiteralAsConstant(Ops[I])
|
|
: getValue(Ops[I]);
|
|
}
|
|
|
|
SPIRVValue *getOpValue(int I) {
|
|
return isOperandLiteral(I) ? Module->getLiteralAsConstant(Ops[I])
|
|
: getValue(Ops[I]);
|
|
}
|
|
|
|
std::vector<SPIRVValue *> getOperands() override {
|
|
std::vector<SPIRVValue *> VOps;
|
|
for (size_t I = 0, E = Ops.size(); I != E; ++I)
|
|
VOps.push_back(getOperand(I));
|
|
return VOps;
|
|
}
|
|
|
|
std::vector<SPIRVEntry *> getNonLiteralOperands() const override {
|
|
std::vector<SPIRVEntry *> Operands;
|
|
for (size_t I = 0, E = Ops.size(); I < E; ++I)
|
|
if (!isOperandLiteral(I))
|
|
Operands.push_back(getEntry(Ops[I]));
|
|
return Operands;
|
|
}
|
|
|
|
virtual const SPIRVValue *getOperand(unsigned I) const {
|
|
return getOpValue(I);
|
|
}
|
|
|
|
virtual SPIRVValue *getOperand(unsigned I) { return getOpValue(I); }
|
|
|
|
bool hasExecScope() const { return SPIRV::hasExecScope(OpCode); }
|
|
|
|
bool hasGroupOperation() const { return SPIRV::hasGroupOperation(OpCode); }
|
|
|
|
bool getSPIRVGroupOperation(SPIRVGroupOperationKind &GroupOp) const {
|
|
if (!hasGroupOperation())
|
|
return false;
|
|
GroupOp = static_cast<SPIRVGroupOperationKind>(Ops[1]);
|
|
return true;
|
|
}
|
|
|
|
Scope getExecutionScope() const {
|
|
if (!hasExecScope())
|
|
return ScopeInvocation;
|
|
return static_cast<Scope>(
|
|
static_cast<SPIRVConstant *>(getValue(Ops[0]))->getZExtIntValue());
|
|
}
|
|
|
|
bool hasVariableWordCount() const { return HasVariWC; }
|
|
|
|
void setHasVariableWordCount(bool VariWC) { HasVariWC = VariWC; }
|
|
|
|
protected:
|
|
void encode(spv_ostream &O) const override {
|
|
auto E = getEncoder(O);
|
|
if (hasType())
|
|
E << Type;
|
|
if (hasId())
|
|
E << Id;
|
|
E << Ops;
|
|
}
|
|
void decode(std::istream &I) override {
|
|
auto D = getDecoder(I);
|
|
if (hasType())
|
|
D >> Type;
|
|
if (hasId())
|
|
D >> Id;
|
|
D >> Ops;
|
|
}
|
|
std::vector<SPIRVWord> Ops;
|
|
bool HasVariWC;
|
|
std::unordered_set<unsigned> Lit; // Literal operand index
|
|
};
|
|
|
|
template <typename BT = SPIRVInstTemplateBase, Op OC = OpNop, bool HasId = true,
|
|
SPIRVWord WC = 0, bool HasVariableWC = false, unsigned Literal1 = ~0U,
|
|
unsigned Literal2 = ~0U, unsigned Literal3 = ~0U,
|
|
unsigned Literal4 = ~0U>
|
|
class SPIRVInstTemplate : public BT {
|
|
public:
|
|
typedef BT BaseTy;
|
|
SPIRVInstTemplate() { init(); }
|
|
~SPIRVInstTemplate() override {}
|
|
void init() override {
|
|
this->initImpl(OC, HasId, WC, HasVariableWC, Literal1, Literal2, Literal3,
|
|
Literal4);
|
|
}
|
|
};
|
|
|
|
class SPIRVMemoryAccess {
|
|
public:
|
|
SPIRVMemoryAccess(const std::vector<SPIRVWord> &TheMemoryAccess)
|
|
: TheMemoryAccessMask(0), Alignment(0), SrcAlignment(0),
|
|
AliasScopeInstID(0), NoAliasInstID(0) {
|
|
memoryAccessUpdate(TheMemoryAccess);
|
|
}
|
|
|
|
SPIRVMemoryAccess()
|
|
: TheMemoryAccessMask(0), Alignment(0), SrcAlignment(0),
|
|
AliasScopeInstID(0), NoAliasInstID(0) {}
|
|
|
|
void memoryAccessUpdate(const std::vector<SPIRVWord> &MemoryAccess) {
|
|
if (!MemoryAccess.size())
|
|
return;
|
|
assert(MemoryAccess.size() > 0 && "Invalid memory access operand size");
|
|
assert(MemoryAccess.size() < 5 && "Invalid memory access operand size");
|
|
TheMemoryAccessMask = MemoryAccess[0];
|
|
size_t MemAccessNumParam = 1;
|
|
if (MemoryAccess[0] & MemoryAccessAlignedMask) {
|
|
assert(MemoryAccess.size() > 1 && "Alignment operand is missing");
|
|
Alignment = MemoryAccess[MemAccessNumParam++];
|
|
}
|
|
if (MemoryAccess[0] & MemoryAccessAliasScopeINTELMaskMask) {
|
|
assert(MemoryAccess.size() > MemAccessNumParam &&
|
|
"Aliasing operand is missing");
|
|
AliasScopeInstID = MemoryAccess[MemAccessNumParam++];
|
|
}
|
|
if (MemoryAccess[0] & MemoryAccessNoAliasINTELMaskMask) {
|
|
assert(MemoryAccess.size() > MemAccessNumParam &&
|
|
"Aliasing operand is missing");
|
|
NoAliasInstID = MemoryAccess[MemAccessNumParam++];
|
|
}
|
|
|
|
// Exit if there is no second memory operand mask
|
|
if (MemoryAccess.size() == MemAccessNumParam)
|
|
return;
|
|
|
|
size_t SecondMaskId = MemAccessNumParam++;
|
|
if (MemoryAccess[SecondMaskId] & MemoryAccessAlignedMask) {
|
|
assert(MemoryAccess.size() > MemAccessNumParam &&
|
|
"Alignment operand is missing");
|
|
SrcAlignment = MemoryAccess[MemAccessNumParam];
|
|
}
|
|
}
|
|
SPIRVWord isVolatile() const {
|
|
return getMemoryAccessMask() & MemoryAccessVolatileMask;
|
|
}
|
|
SPIRVWord isNonTemporal() const {
|
|
return getMemoryAccessMask() & MemoryAccessNontemporalMask;
|
|
}
|
|
SPIRVWord isAliasScope() const {
|
|
return getMemoryAccessMask() & MemoryAccessAliasScopeINTELMaskMask;
|
|
}
|
|
SPIRVWord isNoAlias() const {
|
|
return getMemoryAccessMask() & MemoryAccessNoAliasINTELMaskMask;
|
|
}
|
|
SPIRVWord getMemoryAccessMask() const { return TheMemoryAccessMask; }
|
|
SPIRVWord getAlignment() const { return Alignment; }
|
|
SPIRVWord getSrcAlignment() const { return SrcAlignment; }
|
|
SPIRVWord getAliasScopeInstID() const { return AliasScopeInstID; }
|
|
SPIRVWord getNoAliasInstID() const { return NoAliasInstID; }
|
|
|
|
protected:
|
|
SPIRVWord TheMemoryAccessMask;
|
|
SPIRVWord Alignment;
|
|
SPIRVWord SrcAlignment;
|
|
SPIRVId AliasScopeInstID;
|
|
SPIRVId NoAliasInstID;
|
|
};
|
|
|
|
class SPIRVVariableBase : public SPIRVInstruction {
|
|
public:
|
|
static const SPIRVWord FixedWC = 4;
|
|
// Complete constructor for integer constant
|
|
SPIRVVariableBase(Op OC, SPIRVType *TheType, SPIRVId TheId,
|
|
SPIRVValue *TheInitializer, const std::string &TheName,
|
|
SPIRVStorageClassKind TheStorageClass,
|
|
SPIRVBasicBlock *TheBB, SPIRVModule *TheM, SPIRVWord WC)
|
|
: SPIRVInstruction(WC, OC, TheType, TheId, TheBB, TheM),
|
|
StorageClass(TheStorageClass) {
|
|
if (TheInitializer && !TheInitializer->isUndef())
|
|
Initializer.push_back(TheInitializer->getId());
|
|
Name = TheName;
|
|
validate();
|
|
}
|
|
// Incomplete constructors
|
|
SPIRVVariableBase(Op OC)
|
|
: SPIRVInstruction(OC), StorageClass(StorageClassFunction) {}
|
|
SPIRVVariableBase()
|
|
: SPIRVInstruction(OpNop), StorageClass(StorageClassFunction) {}
|
|
|
|
SPIRVStorageClassKind getStorageClass() const { return StorageClass; }
|
|
SPIRVValue *getInitializer() const {
|
|
if (Initializer.empty())
|
|
return nullptr;
|
|
assert(Initializer.size() == 1);
|
|
return getValue(Initializer[0]);
|
|
}
|
|
bool isConstant() const { return hasDecorate(DecorationConstant); }
|
|
bool isBuiltin(SPIRVBuiltinVariableKind *BuiltinKind = nullptr) const {
|
|
SPIRVWord Kind;
|
|
bool Found = hasDecorate(DecorationBuiltIn, 0, &Kind);
|
|
if (!Found)
|
|
return false;
|
|
if (BuiltinKind)
|
|
*BuiltinKind = static_cast<SPIRVBuiltinVariableKind>(Kind);
|
|
return true;
|
|
}
|
|
void setBuiltin(SPIRVBuiltinVariableKind Kind) {
|
|
assert(isValid(Kind));
|
|
addDecorate(new SPIRVDecorate(DecorationBuiltIn, this, Kind));
|
|
}
|
|
void setIsConstant(bool Is) {
|
|
if (Is)
|
|
addDecorate(new SPIRVDecorate(DecorationConstant, this));
|
|
else
|
|
eraseDecorate(DecorationConstant);
|
|
}
|
|
std::vector<SPIRVEntry *> getNonLiteralOperands() const override {
|
|
if (SPIRVValue *V = getInitializer())
|
|
return std::vector<SPIRVEntry *>(1, V);
|
|
return std::vector<SPIRVEntry *>();
|
|
}
|
|
|
|
protected:
|
|
void validate() const override {
|
|
SPIRVValue::validate();
|
|
assert(isValid(StorageClass));
|
|
assert(Initializer.size() == 1 || Initializer.empty());
|
|
assert(getType()->isTypePointer());
|
|
}
|
|
void setWordCount(SPIRVWord TheWordCount) override {
|
|
SPIRVEntry::setWordCount(TheWordCount);
|
|
SPIRVCK(WordCount >= FixedWC, InvalidWordCount, "");
|
|
Initializer.resize(WordCount - FixedWC);
|
|
}
|
|
_SPIRV_DEF_ENCDEC4(Type, Id, StorageClass, Initializer)
|
|
|
|
SPIRVStorageClassKind StorageClass;
|
|
std::vector<SPIRVId> Initializer;
|
|
};
|
|
|
|
class SPIRVVariable : public SPIRVVariableBase {
|
|
public:
|
|
// Complete constructor for integer constant
|
|
SPIRVVariable(SPIRVType *TheType, SPIRVId TheId, SPIRVValue *TheInitializer,
|
|
const std::string &TheName,
|
|
SPIRVStorageClassKind TheStorageClass, SPIRVBasicBlock *TheBB,
|
|
SPIRVModule *TheM)
|
|
: SPIRVVariableBase(OpVariable, TheType, TheId, TheInitializer, TheName,
|
|
TheStorageClass, TheBB, TheM,
|
|
TheInitializer && !TheInitializer->isUndef() ? 5
|
|
: 4) {}
|
|
// Incomplete constructor
|
|
SPIRVVariable() : SPIRVVariableBase(OpVariable) {}
|
|
};
|
|
|
|
class SPIRVUntypedVariableKHR : public SPIRVVariableBase {
|
|
public:
|
|
SPIRVUntypedVariableKHR(SPIRVType *TheType, SPIRVId TheId,
|
|
SPIRVType *TheDataType, SPIRVValue *TheInitializer,
|
|
const std::string &TheName,
|
|
SPIRVStorageClassKind TheStorageClass,
|
|
SPIRVBasicBlock *TheBB, SPIRVModule *TheM)
|
|
: SPIRVVariableBase(
|
|
OpUntypedVariableKHR, TheType, TheId, TheInitializer, TheName,
|
|
TheStorageClass, TheBB, TheM,
|
|
TheDataType && !TheDataType->isUndef()
|
|
? (TheInitializer && !TheInitializer->isUndef() ? 6 : 5)
|
|
: 4) {
|
|
if (TheDataType && !TheDataType->isUndef())
|
|
DataType.push_back(TheDataType->getId());
|
|
validate();
|
|
}
|
|
SPIRVUntypedVariableKHR() : SPIRVVariableBase(OpUntypedVariableKHR) {}
|
|
SPIRVType *getDataType() const {
|
|
if (DataType.empty())
|
|
return nullptr;
|
|
assert(DataType.size() == 1);
|
|
return get<SPIRVType>(DataType[0]);
|
|
}
|
|
std::vector<SPIRVEntry *> getNonLiteralOperands() const override {
|
|
std::vector<SPIRVEntry *> Vec;
|
|
if (SPIRVType *T = getDataType())
|
|
Vec.push_back(T);
|
|
if (SPIRVValue *V = getInitializer())
|
|
Vec.push_back(V);
|
|
return Vec;
|
|
}
|
|
std::optional<ExtensionID> getRequiredExtension() const override {
|
|
return ExtensionID::SPV_KHR_untyped_pointers;
|
|
}
|
|
SPIRVCapVec getRequiredCapability() const override {
|
|
return getVec(CapabilityUntypedPointersKHR);
|
|
}
|
|
|
|
protected:
|
|
void validate() const override {
|
|
SPIRVVariableBase::validate();
|
|
assert(DataType.size() == 1 || DataType.empty());
|
|
}
|
|
void setWordCount(SPIRVWord TheWordCount) override {
|
|
SPIRVEntry::setWordCount(TheWordCount);
|
|
if (TheWordCount > FixedWC)
|
|
DataType.resize(1);
|
|
if (TheWordCount > FixedWC + 1)
|
|
Initializer.resize(1);
|
|
}
|
|
_SPIRV_DEF_ENCDEC5(Type, Id, StorageClass, DataType, Initializer)
|
|
|
|
std::vector<SPIRVId> DataType;
|
|
};
|
|
|
|
class SPIRVStore : public SPIRVInstruction, public SPIRVMemoryAccess {
|
|
public:
|
|
const static SPIRVWord FixedWords = 3;
|
|
// Complete constructor
|
|
SPIRVStore(SPIRVId PointerId, SPIRVId ValueId,
|
|
const std::vector<SPIRVWord> &TheMemoryAccess,
|
|
SPIRVBasicBlock *TheBB)
|
|
: SPIRVInstruction(FixedWords + TheMemoryAccess.size(), OpStore, TheBB),
|
|
SPIRVMemoryAccess(TheMemoryAccess), MemoryAccess(TheMemoryAccess),
|
|
PtrId(PointerId), ValId(ValueId) {
|
|
setAttr();
|
|
validate();
|
|
assert(TheBB && "Invalid BB");
|
|
}
|
|
// Incomplete constructor
|
|
SPIRVStore()
|
|
: SPIRVInstruction(OpStore), SPIRVMemoryAccess(), PtrId(SPIRVID_INVALID),
|
|
ValId(SPIRVID_INVALID) {
|
|
setAttr();
|
|
}
|
|
|
|
SPIRVValue *getSrc() const { return getValue(ValId); }
|
|
SPIRVValue *getDst() const { return getValue(PtrId); }
|
|
|
|
protected:
|
|
void setAttr() {
|
|
setHasNoType();
|
|
setHasNoId();
|
|
}
|
|
|
|
void setWordCount(SPIRVWord TheWordCount) override {
|
|
SPIRVEntry::setWordCount(TheWordCount);
|
|
SPIRVCK(TheWordCount >= FixedWords, InvalidWordCount, "");
|
|
MemoryAccess.resize(TheWordCount - FixedWords);
|
|
}
|
|
void encode(spv_ostream &O) const override {
|
|
getEncoder(O) << PtrId << ValId << MemoryAccess;
|
|
}
|
|
|
|
void decode(std::istream &I) override {
|
|
getDecoder(I) >> PtrId >> ValId >> MemoryAccess;
|
|
memoryAccessUpdate(MemoryAccess);
|
|
}
|
|
|
|
void validate() const override {
|
|
SPIRVInstruction::validate();
|
|
if (getSrc()->isForward() || getDst()->isForward())
|
|
return;
|
|
assert(
|
|
(getValueType(PtrId)
|
|
->getPointerElementType()
|
|
->isTypeUntypedPointerKHR() ||
|
|
(getValueType(PtrId)->getPointerElementType()->isTypePointer() &&
|
|
getValueType(ValId)->isTypeUntypedPointerKHR()) ||
|
|
getValueType(PtrId)->getPointerElementType() == getValueType(ValId)) &&
|
|
"Inconsistent operand types");
|
|
}
|
|
|
|
private:
|
|
std::vector<SPIRVWord> MemoryAccess;
|
|
SPIRVId PtrId;
|
|
SPIRVId ValId;
|
|
};
|
|
|
|
class SPIRVLoad : public SPIRVInstruction, public SPIRVMemoryAccess {
|
|
public:
|
|
const static SPIRVWord FixedWords = 4;
|
|
// Complete constructor
|
|
SPIRVLoad(SPIRVId TheId, SPIRVId PointerId,
|
|
const std::vector<SPIRVWord> &TheMemoryAccess,
|
|
SPIRVBasicBlock *TheBB, SPIRVType *TheType = nullptr)
|
|
: SPIRVInstruction(
|
|
FixedWords + TheMemoryAccess.size(), OpLoad,
|
|
TheType ? TheType
|
|
: TheBB->getValueType(PointerId)->getPointerElementType(),
|
|
TheId, TheBB),
|
|
SPIRVMemoryAccess(TheMemoryAccess), PtrId(PointerId),
|
|
MemoryAccess(TheMemoryAccess) {
|
|
validate();
|
|
assert(TheBB && "Invalid BB");
|
|
}
|
|
// Incomplete constructor
|
|
SPIRVLoad()
|
|
: SPIRVInstruction(OpLoad), SPIRVMemoryAccess(), PtrId(SPIRVID_INVALID) {}
|
|
|
|
SPIRVValue *getSrc() const { return Module->get<SPIRVValue>(PtrId); }
|
|
|
|
protected:
|
|
void setWordCount(SPIRVWord TheWordCount) override {
|
|
SPIRVEntry::setWordCount(TheWordCount);
|
|
SPIRVCK(TheWordCount >= FixedWords, InvalidWordCount, "");
|
|
MemoryAccess.resize(TheWordCount - FixedWords);
|
|
}
|
|
|
|
void encode(spv_ostream &O) const override {
|
|
getEncoder(O) << Type << Id << PtrId << MemoryAccess;
|
|
}
|
|
|
|
void decode(std::istream &I) override {
|
|
getDecoder(I) >> Type >> Id >> PtrId >> MemoryAccess;
|
|
memoryAccessUpdate(MemoryAccess);
|
|
}
|
|
|
|
void validate() const override {
|
|
SPIRVInstruction::validate();
|
|
assert((getValue(PtrId)->isForward() ||
|
|
getValueType(PtrId)
|
|
->getPointerElementType()
|
|
->isTypeUntypedPointerKHR() ||
|
|
Type->isTypeUntypedPointerKHR() ||
|
|
Type == getValueType(PtrId)->getPointerElementType()) &&
|
|
"Inconsistent types");
|
|
}
|
|
|
|
private:
|
|
SPIRVId PtrId;
|
|
std::vector<SPIRVWord> MemoryAccess;
|
|
};
|
|
|
|
class SPIRVBinary : public SPIRVInstTemplateBase {
|
|
protected:
|
|
void validate() const override {
|
|
SPIRVId Op1 = Ops[0];
|
|
SPIRVId Op2 = Ops[1];
|
|
SPIRVType *Op1Ty, *Op2Ty;
|
|
SPIRVInstruction::validate();
|
|
if (getValue(Op1)->isForward() || getValue(Op2)->isForward())
|
|
return;
|
|
if (getValueType(Op1)->isTypeVector()) {
|
|
Op1Ty = getValueType(Op1)->getVectorComponentType();
|
|
Op2Ty = getValueType(Op2)->getVectorComponentType();
|
|
assert(getValueType(Op1)->getVectorComponentCount() ==
|
|
getValueType(Op2)->getVectorComponentCount() &&
|
|
"Inconsistent Vector component width");
|
|
} else if (getValueType(Op1)->isTypeCooperativeMatrixKHR()) {
|
|
Op1Ty = getValueType(Op1)->getVectorComponentType();
|
|
Op2Ty = getValueType(Op2)->getVectorComponentType();
|
|
assert(Op1Ty == Op2Ty && "Inconsistent Cooperative matrix types");
|
|
} else {
|
|
Op1Ty = getValueType(Op1);
|
|
Op2Ty = getValueType(Op2);
|
|
}
|
|
|
|
(void)Op1Ty;
|
|
(void)Op2Ty;
|
|
if (isBinaryOpCode(OpCode)) {
|
|
assert(getValueType(Op1) == getValueType(Op2) &&
|
|
"Invalid type for binary instruction");
|
|
assert((Op1Ty->isTypeInt() || Op2Ty->isTypeFloat()) &&
|
|
"Invalid type for Binary instruction");
|
|
assert((Op1Ty->getBitWidth() == Op2Ty->getBitWidth()) &&
|
|
"Inconsistent BitWidth");
|
|
} else if (isShiftOpCode(OpCode)) {
|
|
assert((Op1Ty->isTypeInt() || Op2Ty->isTypeInt()) &&
|
|
"Invalid type for shift instruction");
|
|
} else if (isLogicalOpCode(OpCode)) {
|
|
assert((Op1Ty->isTypeBool() || Op2Ty->isTypeBool()) &&
|
|
"Invalid type for logical instruction");
|
|
} else if (isBitwiseOpCode(OpCode)) {
|
|
assert((Op1Ty->isTypeInt() || Op2Ty->isTypeInt()) &&
|
|
"Invalid type for bitwise instruction");
|
|
assert((Op1Ty->getIntegerBitWidth() == Op2Ty->getIntegerBitWidth()) &&
|
|
"Inconsistent BitWidth");
|
|
} else if (isBinaryPtrOpCode(OpCode)) {
|
|
assert((Op1Ty->isTypePointer() && Op2Ty->isTypePointer()) &&
|
|
"Invalid types for PtrEqual, PtrNotEqual, or PtrDiff instruction");
|
|
if (!Op1Ty->isTypeUntypedPointerKHR() &&
|
|
!Op2Ty->isTypeUntypedPointerKHR())
|
|
assert(
|
|
static_cast<SPIRVTypePointer *>(Op1Ty)->getElementType() ==
|
|
static_cast<SPIRVTypePointer *>(Op2Ty)->getElementType() &&
|
|
"Invalid types for PtrEqual, PtrNotEqual, or PtrDiff instruction");
|
|
else if (OpCode == OpPtrDiff)
|
|
assert(Op1Ty == Op2Ty && "Invalid types for PtrDiff instruction");
|
|
} else {
|
|
assert(0 && "Invalid op code!");
|
|
}
|
|
}
|
|
VersionNumber getRequiredSPIRVVersion() const override {
|
|
if (isBinaryPtrOpCode(OpCode))
|
|
return VersionNumber::SPIRV_1_4;
|
|
return VersionNumber::SPIRV_1_0;
|
|
}
|
|
SPIRVCapVec getRequiredCapability() const override {
|
|
if (OpCode == OpDot) {
|
|
const SPIRVType *OpTy = getValueType(Ops[0]);
|
|
if (OpTy && OpTy->isTypeVector()) {
|
|
OpTy = OpTy->getVectorComponentType();
|
|
if (OpTy && OpTy->isTypeFloat(16, FPEncodingBFloat16KHR)) {
|
|
return getVec(CapabilityBFloat16DotProductKHR);
|
|
}
|
|
}
|
|
}
|
|
return SPIRVInstruction::getRequiredCapability();
|
|
}
|
|
};
|
|
|
|
template <Op OC>
|
|
class SPIRVBinaryInst
|
|
: public SPIRVInstTemplate<SPIRVBinary, OC, true, 5, false> {};
|
|
|
|
#define _SPIRV_OP(x) typedef SPIRVBinaryInst<Op##x> SPIRV##x;
|
|
_SPIRV_OP(IAdd)
|
|
_SPIRV_OP(IAddCarry)
|
|
_SPIRV_OP(FAdd)
|
|
_SPIRV_OP(ISub)
|
|
_SPIRV_OP(ISubBorrow)
|
|
_SPIRV_OP(FSub)
|
|
_SPIRV_OP(IMul)
|
|
_SPIRV_OP(FMul)
|
|
_SPIRV_OP(SMulExtended)
|
|
_SPIRV_OP(UMulExtended)
|
|
_SPIRV_OP(UDiv)
|
|
_SPIRV_OP(SDiv)
|
|
_SPIRV_OP(FDiv)
|
|
_SPIRV_OP(SRem)
|
|
_SPIRV_OP(SMod)
|
|
_SPIRV_OP(FRem)
|
|
_SPIRV_OP(FMod)
|
|
_SPIRV_OP(UMod)
|
|
_SPIRV_OP(ShiftLeftLogical)
|
|
_SPIRV_OP(ShiftRightLogical)
|
|
_SPIRV_OP(ShiftRightArithmetic)
|
|
_SPIRV_OP(LogicalAnd)
|
|
_SPIRV_OP(LogicalOr)
|
|
_SPIRV_OP(LogicalEqual)
|
|
_SPIRV_OP(LogicalNotEqual)
|
|
_SPIRV_OP(BitwiseAnd)
|
|
_SPIRV_OP(BitwiseOr)
|
|
_SPIRV_OP(BitwiseXor)
|
|
_SPIRV_OP(Dot)
|
|
_SPIRV_OP(PtrEqual)
|
|
_SPIRV_OP(PtrNotEqual)
|
|
_SPIRV_OP(PtrDiff)
|
|
#undef _SPIRV_OP
|
|
|
|
template <Op TheOpCode> class SPIRVInstNoOperand : public SPIRVInstruction {
|
|
public:
|
|
// Complete constructor
|
|
SPIRVInstNoOperand(SPIRVBasicBlock *TheBB)
|
|
: SPIRVInstruction(1, TheOpCode, TheBB) {
|
|
setAttr();
|
|
validate();
|
|
}
|
|
// Incomplete constructor
|
|
SPIRVInstNoOperand() : SPIRVInstruction(TheOpCode) { setAttr(); }
|
|
|
|
protected:
|
|
void setAttr() {
|
|
setHasNoId();
|
|
setHasNoType();
|
|
}
|
|
_SPIRV_DEF_ENCDEC0
|
|
};
|
|
|
|
typedef SPIRVInstNoOperand<OpReturn> SPIRVReturn;
|
|
typedef SPIRVInstNoOperand<OpUnreachable> SPIRVUnreachable;
|
|
|
|
class SPIRVAbortKHR : public SPIRVInstruction {
|
|
public:
|
|
static const Op OC = OpAbortKHR;
|
|
static const SPIRVWord FixedWordCount = 3;
|
|
// Complete constructor.
|
|
SPIRVAbortKHR(SPIRVValue *TheMessage, SPIRVBasicBlock *TheBB)
|
|
: SPIRVInstruction(FixedWordCount, OC, TheBB),
|
|
MessageTypeId(TheMessage->getType()->getId()),
|
|
MessageId(TheMessage->getId()) {
|
|
setAttr();
|
|
validate();
|
|
assert(TheBB && "Invalid BB");
|
|
}
|
|
// Incomplete constructor.
|
|
SPIRVAbortKHR()
|
|
: SPIRVInstruction(OC), MessageTypeId(SPIRVID_INVALID),
|
|
MessageId(SPIRVID_INVALID) {
|
|
setAttr();
|
|
}
|
|
|
|
SPIRVCapVec getRequiredCapability() const override {
|
|
return getVec(CapabilityAbortKHR);
|
|
}
|
|
|
|
std::optional<ExtensionID> getRequiredExtension() const override {
|
|
return ExtensionID::SPV_KHR_abort;
|
|
}
|
|
|
|
std::vector<SPIRVValue *> getOperands() override {
|
|
return std::vector<SPIRVValue *>(1, getValue(MessageId));
|
|
}
|
|
|
|
_SPIRV_DEF_ENCDEC2(MessageTypeId, MessageId)
|
|
|
|
protected:
|
|
void setAttr() {
|
|
setHasNoId();
|
|
setHasNoType();
|
|
}
|
|
void validate() const override { SPIRVInstruction::validate(); }
|
|
SPIRVId MessageTypeId;
|
|
SPIRVId MessageId;
|
|
};
|
|
|
|
class SPIRVReturnValue : public SPIRVInstruction {
|
|
public:
|
|
static const Op OC = OpReturnValue;
|
|
// Complete constructor
|
|
SPIRVReturnValue(SPIRVValue *TheReturnValue, SPIRVBasicBlock *TheBB)
|
|
: SPIRVInstruction(2, OC, TheBB), ReturnValueId(TheReturnValue->getId()) {
|
|
setAttr();
|
|
validate();
|
|
assert(TheBB && "Invalid BB");
|
|
}
|
|
// Incomplete constructor
|
|
SPIRVReturnValue() : SPIRVInstruction(OC), ReturnValueId(SPIRVID_INVALID) {
|
|
setAttr();
|
|
}
|
|
|
|
SPIRVValue *getReturnValue() const { return getValue(ReturnValueId); }
|
|
|
|
protected:
|
|
void setAttr() {
|
|
setHasNoId();
|
|
setHasNoType();
|
|
}
|
|
_SPIRV_DEF_ENCDEC1(ReturnValueId)
|
|
void validate() const override { SPIRVInstruction::validate(); }
|
|
SPIRVId ReturnValueId;
|
|
};
|
|
|
|
class SPIRVBranch : public SPIRVInstruction {
|
|
public:
|
|
static const Op OC = OpBranch;
|
|
// Complete constructor
|
|
SPIRVBranch(SPIRVLabel *TheTargetLabel, SPIRVBasicBlock *TheBB)
|
|
: SPIRVInstruction(2, OC, TheBB), TargetLabelId(TheTargetLabel->getId()) {
|
|
validate();
|
|
assert(TheBB && "Invalid BB");
|
|
}
|
|
// Incomplete constructor
|
|
SPIRVBranch() : SPIRVInstruction(OC), TargetLabelId(SPIRVID_INVALID) {
|
|
setHasNoId();
|
|
setHasNoType();
|
|
}
|
|
SPIRVValue *getTargetLabel() const { return getValue(TargetLabelId); }
|
|
|
|
protected:
|
|
_SPIRV_DEF_ENCDEC1(TargetLabelId)
|
|
void validate() const override {
|
|
SPIRVInstruction::validate();
|
|
assert(WordCount == 2);
|
|
assert(OpCode == OC);
|
|
assert(getTargetLabel()->isLabel() || getTargetLabel()->isForward());
|
|
}
|
|
SPIRVId TargetLabelId;
|
|
};
|
|
|
|
class SPIRVBranchConditional : public SPIRVInstruction {
|
|
public:
|
|
static const Op OC = OpBranchConditional;
|
|
constexpr static SPIRVWord FixedWC = 4;
|
|
// Complete constructor
|
|
SPIRVBranchConditional(SPIRVValue *TheCondition, SPIRVLabel *TheTrueLabel,
|
|
SPIRVLabel *TheFalseLabel, SPIRVBasicBlock *TheBB)
|
|
: SPIRVInstruction(FixedWC, OC, TheBB),
|
|
ConditionId(TheCondition->getId()), TrueLabelId(TheTrueLabel->getId()),
|
|
FalseLabelId(TheFalseLabel->getId()) {
|
|
validate();
|
|
}
|
|
SPIRVBranchConditional(SPIRVValue *TheCondition, SPIRVLabel *TheTrueLabel,
|
|
SPIRVLabel *TheFalseLabel, SPIRVBasicBlock *TheBB,
|
|
SPIRVWord TrueWeight, SPIRVWord FalseWeight)
|
|
: SPIRVInstruction(6, OC, TheBB), ConditionId(TheCondition->getId()),
|
|
TrueLabelId(TheTrueLabel->getId()),
|
|
FalseLabelId(TheFalseLabel->getId()) {
|
|
BranchWeights.push_back(TrueWeight);
|
|
BranchWeights.push_back(FalseWeight);
|
|
validate();
|
|
assert(TheBB && "Invalid BB");
|
|
}
|
|
// Incomplete constructor
|
|
SPIRVBranchConditional()
|
|
: SPIRVInstruction(OC), ConditionId(SPIRVID_INVALID),
|
|
TrueLabelId(SPIRVID_INVALID), FalseLabelId(SPIRVID_INVALID) {
|
|
setHasNoId();
|
|
setHasNoType();
|
|
}
|
|
SPIRVValue *getCondition() const { return getValue(ConditionId); }
|
|
SPIRVLabel *getTrueLabel() const { return get<SPIRVLabel>(TrueLabelId); }
|
|
SPIRVLabel *getFalseLabel() const { return get<SPIRVLabel>(FalseLabelId); }
|
|
|
|
protected:
|
|
void setWordCount(SPIRVWord TheWordCount) override {
|
|
SPIRVEntry::setWordCount(TheWordCount);
|
|
SPIRVCK(TheWordCount >= FixedWC, InvalidWordCount, "");
|
|
BranchWeights.resize(TheWordCount - FixedWC);
|
|
}
|
|
_SPIRV_DEF_ENCDEC4(ConditionId, TrueLabelId, FalseLabelId, BranchWeights)
|
|
void validate() const override {
|
|
SPIRVInstruction::validate();
|
|
assert(WordCount == FixedWC || WordCount == FixedWC + 2);
|
|
assert(WordCount == BranchWeights.size() + FixedWC);
|
|
assert(OpCode == OC);
|
|
assert(getCondition()->isForward() ||
|
|
getCondition()->getType()->isTypeBool());
|
|
assert(getTrueLabel()->isForward() || getTrueLabel()->isLabel());
|
|
assert(getFalseLabel()->isForward() || getFalseLabel()->isLabel());
|
|
if (Module->isAllowedToUseVersion(VersionNumber::SPIRV_1_6))
|
|
assert(TrueLabelId != FalseLabelId);
|
|
}
|
|
SPIRVId ConditionId;
|
|
SPIRVId TrueLabelId;
|
|
SPIRVId FalseLabelId;
|
|
std::vector<SPIRVWord> BranchWeights;
|
|
};
|
|
|
|
class SPIRVPhi : public SPIRVInstruction {
|
|
public:
|
|
static const Op OC = OpPhi;
|
|
static const SPIRVWord FixedWordCount = 3;
|
|
SPIRVPhi(SPIRVType *TheType, SPIRVId TheId,
|
|
const std::vector<SPIRVValue *> &ThePairs, SPIRVBasicBlock *BB)
|
|
: SPIRVInstruction(ThePairs.size() + FixedWordCount, OC, TheType, TheId,
|
|
BB) {
|
|
Pairs = getIds(ThePairs);
|
|
validate();
|
|
assert(BB && "Invalid BB");
|
|
}
|
|
SPIRVPhi() : SPIRVInstruction(OC) {}
|
|
std::vector<SPIRVValue *> getPairs() { return getValues(Pairs); }
|
|
void addPair(SPIRVValue *Value, SPIRVBasicBlock *BB) {
|
|
Pairs.push_back(Value->getId());
|
|
Pairs.push_back(BB->getId());
|
|
WordCount = Pairs.size() + FixedWordCount;
|
|
validate();
|
|
}
|
|
void setPairs(const std::vector<SPIRVValue *> &ThePairs) {
|
|
Pairs = getIds(ThePairs);
|
|
WordCount = Pairs.size() + FixedWordCount;
|
|
validate();
|
|
}
|
|
void foreachPair(
|
|
std::function<void(SPIRVValue *, SPIRVBasicBlock *, size_t)> Func) {
|
|
for (size_t I = 0, E = Pairs.size() / 2; I != E; ++I) {
|
|
SPIRVEntry *Value, *BB;
|
|
if (!Module->exist(Pairs[2 * I], &Value) ||
|
|
!Module->exist(Pairs[2 * I + 1], &BB))
|
|
continue;
|
|
Func(static_cast<SPIRVValue *>(Value), static_cast<SPIRVBasicBlock *>(BB),
|
|
I);
|
|
}
|
|
}
|
|
void
|
|
foreachPair(std::function<void(SPIRVValue *, SPIRVBasicBlock *)> Func) const {
|
|
for (size_t I = 0, E = Pairs.size() / 2; I != E; ++I) {
|
|
SPIRVEntry *Value, *BB;
|
|
if (!Module->exist(Pairs[2 * I], &Value) ||
|
|
!Module->exist(Pairs[2 * I + 1], &BB))
|
|
continue;
|
|
Func(static_cast<SPIRVValue *>(Value),
|
|
static_cast<SPIRVBasicBlock *>(BB));
|
|
}
|
|
}
|
|
void setWordCount(SPIRVWord TheWordCount) override {
|
|
SPIRVEntry::setWordCount(TheWordCount);
|
|
SPIRVCK(TheWordCount >= FixedWordCount, InvalidWordCount, "");
|
|
Pairs.resize(TheWordCount - FixedWordCount);
|
|
}
|
|
_SPIRV_DEF_ENCDEC3(Type, Id, Pairs)
|
|
void validate() const override {
|
|
assert(WordCount == Pairs.size() + FixedWordCount);
|
|
assert(OpCode == OC);
|
|
assert(Pairs.size() % 2 == 0);
|
|
foreachPair([this](SPIRVValue *IncomingV, SPIRVBasicBlock *IncomingBB) {
|
|
assert(IncomingV->isForward() || IncomingV->getType() == Type ||
|
|
(IncomingV->getType()->isTypePointer() &&
|
|
Type->isTypeUntypedPointerKHR()) ||
|
|
(IncomingV->getType()->isTypeUntypedPointerKHR() &&
|
|
Type->isTypePointer()));
|
|
assert(IncomingBB->isBasicBlock() || IncomingBB->isForward());
|
|
});
|
|
SPIRVInstruction::validate();
|
|
}
|
|
|
|
protected:
|
|
std::vector<SPIRVId> Pairs;
|
|
};
|
|
|
|
class SPIRVFPGARegINTELInstBase : public SPIRVInstTemplateBase {
|
|
public:
|
|
SPIRVCapVec getRequiredCapability() const override {
|
|
return getVec(CapabilityFPGARegINTEL);
|
|
}
|
|
|
|
std::optional<ExtensionID> getRequiredExtension() const override {
|
|
return ExtensionID::SPV_INTEL_fpga_reg;
|
|
}
|
|
|
|
protected:
|
|
void validate() const override {
|
|
SPIRVInstruction::validate();
|
|
|
|
assert(OpCode == OpFPGARegINTEL &&
|
|
"Invalid op code for FPGARegINTEL instruction");
|
|
assert(getType() == getValueType(Ops[0]) && "Inconsistent type");
|
|
}
|
|
};
|
|
|
|
typedef SPIRVInstTemplate<SPIRVFPGARegINTELInstBase, OpFPGARegINTEL, true, 4>
|
|
SPIRVFPGARegINTEL;
|
|
|
|
class SPIRVCompare : public SPIRVInstTemplateBase {
|
|
protected:
|
|
void validate() const override {
|
|
auto Op1 = Ops[0];
|
|
auto Op2 = Ops[1];
|
|
SPIRVType *Op1Ty, *Op2Ty, *ResTy;
|
|
SPIRVInstruction::validate();
|
|
if (getValue(Op1)->isForward() || getValue(Op2)->isForward())
|
|
return;
|
|
|
|
(void)Op1Ty;
|
|
(void)Op2Ty;
|
|
(void)ResTy;
|
|
if (getValueType(Op1)->isTypeVector()) {
|
|
Op1Ty = getValueType(Op1)->getVectorComponentType();
|
|
Op2Ty = getValueType(Op2)->getVectorComponentType();
|
|
ResTy = Type->getVectorComponentType();
|
|
assert(getValueType(Op1)->getVectorComponentCount() ==
|
|
getValueType(Op2)->getVectorComponentCount() &&
|
|
"Inconsistent Vector component width");
|
|
} else {
|
|
Op1Ty = getValueType(Op1);
|
|
Op2Ty = getValueType(Op2);
|
|
ResTy = Type;
|
|
}
|
|
assert(isCmpOpCode(OpCode) && "Invalid op code for cmp inst");
|
|
if (OpCode == OpLessOrGreater)
|
|
assert(this->getModule()->getSPIRVVersion() <= VersionNumber::SPIRV_1_5 &&
|
|
"OpLessOrGreater is removed starting from SPIR-V 1.6");
|
|
assert((ResTy->isTypeBool() || ResTy->isTypeInt()) &&
|
|
"Invalid type for compare instruction");
|
|
assert(Op1Ty == Op2Ty && "Inconsistent types");
|
|
}
|
|
};
|
|
|
|
template <Op OC>
|
|
class SPIRVCmpInst
|
|
: public SPIRVInstTemplate<SPIRVCompare, OC, true, 5, false> {};
|
|
|
|
#define _SPIRV_OP(x) typedef SPIRVCmpInst<Op##x> SPIRV##x;
|
|
_SPIRV_OP(IEqual)
|
|
_SPIRV_OP(FOrdEqual)
|
|
_SPIRV_OP(FUnordEqual)
|
|
_SPIRV_OP(INotEqual)
|
|
_SPIRV_OP(FOrdNotEqual)
|
|
_SPIRV_OP(FUnordNotEqual)
|
|
_SPIRV_OP(ULessThan)
|
|
_SPIRV_OP(SLessThan)
|
|
_SPIRV_OP(FOrdLessThan)
|
|
_SPIRV_OP(FUnordLessThan)
|
|
_SPIRV_OP(UGreaterThan)
|
|
_SPIRV_OP(SGreaterThan)
|
|
_SPIRV_OP(FOrdGreaterThan)
|
|
_SPIRV_OP(FUnordGreaterThan)
|
|
_SPIRV_OP(ULessThanEqual)
|
|
_SPIRV_OP(SLessThanEqual)
|
|
_SPIRV_OP(FOrdLessThanEqual)
|
|
_SPIRV_OP(FUnordLessThanEqual)
|
|
_SPIRV_OP(UGreaterThanEqual)
|
|
_SPIRV_OP(SGreaterThanEqual)
|
|
_SPIRV_OP(FOrdGreaterThanEqual)
|
|
_SPIRV_OP(FUnordGreaterThanEqual)
|
|
_SPIRV_OP(LessOrGreater)
|
|
_SPIRV_OP(Ordered)
|
|
_SPIRV_OP(Unordered)
|
|
#undef _SPIRV_OP
|
|
|
|
class SPIRVSelectBase : public SPIRVInstTemplateBase {
|
|
public:
|
|
SPIRVValue *getCondition() { return getValue(Ops[0]); }
|
|
SPIRVValue *getTrueValue() { return getValue(Ops[1]); }
|
|
SPIRVValue *getFalseValue() { return getValue(Ops[2]); }
|
|
|
|
protected:
|
|
void validate() const override {
|
|
SPIRVId Condition = Ops[0];
|
|
SPIRVId Op1 = Ops[1];
|
|
SPIRVId Op2 = Ops[2];
|
|
|
|
SPIRVInstruction::validate();
|
|
if (getValue(Condition)->isForward() || getValue(Op1)->isForward() ||
|
|
getValue(Op2)->isForward())
|
|
return;
|
|
|
|
SPIRVType *ConTy = getValueType(Condition)->isTypeVector()
|
|
? getValueType(Condition)->getVectorComponentType()
|
|
: getValueType(Condition);
|
|
(void)ConTy;
|
|
assert(ConTy->isTypeBool() && "Invalid type");
|
|
if (getType()->getOpCode() != OpTypeUntypedPointerKHR &&
|
|
getValueType(Op1)->getOpCode() != OpTypeUntypedPointerKHR &&
|
|
getValueType(Op2)->getOpCode() != OpTypeUntypedPointerKHR)
|
|
assert(getType() == getValueType(Op1) && getType() == getValueType(Op2) &&
|
|
"Inconsistent type");
|
|
}
|
|
};
|
|
|
|
typedef SPIRVInstTemplate<SPIRVSelectBase, OpSelect, true, 6> SPIRVSelect;
|
|
|
|
class SPIRVSelectionMerge : public SPIRVInstruction {
|
|
public:
|
|
static const Op OC = OpSelectionMerge;
|
|
static const SPIRVWord FixedWordCount = 3;
|
|
|
|
SPIRVSelectionMerge(SPIRVId TheMergeBlock, SPIRVWord TheSelectionControl,
|
|
SPIRVBasicBlock *BB)
|
|
: SPIRVInstruction(3, OC, BB), MergeBlock(TheMergeBlock),
|
|
SelectionControl(TheSelectionControl) {
|
|
validate();
|
|
assert(BB && "Invalid BB");
|
|
}
|
|
|
|
SPIRVSelectionMerge()
|
|
: SPIRVInstruction(OC), MergeBlock(SPIRVID_INVALID),
|
|
SelectionControl(SPIRVWORD_MAX) {
|
|
setHasNoId();
|
|
setHasNoType();
|
|
}
|
|
|
|
SPIRVId getMergeBlock() { return MergeBlock; }
|
|
SPIRVWord getSelectionControl() { return SelectionControl; }
|
|
|
|
_SPIRV_DEF_ENCDEC2(MergeBlock, SelectionControl)
|
|
|
|
protected:
|
|
SPIRVId MergeBlock;
|
|
SPIRVWord SelectionControl;
|
|
};
|
|
|
|
class SPIRVLoopMerge : public SPIRVInstruction {
|
|
public:
|
|
static const Op OC = OpLoopMerge;
|
|
static const SPIRVWord FixedWordCount = 4;
|
|
|
|
SPIRVLoopMerge(SPIRVId TheMergeBlock, SPIRVId TheContinueTarget,
|
|
SPIRVWord TheLoopControl,
|
|
std::vector<SPIRVWord> TheLoopControlParameters,
|
|
SPIRVBasicBlock *BB)
|
|
: SPIRVInstruction(FixedWordCount + TheLoopControlParameters.size(), OC,
|
|
BB),
|
|
MergeBlock(TheMergeBlock), ContinueTarget(TheContinueTarget),
|
|
LoopControl(TheLoopControl),
|
|
LoopControlParameters(TheLoopControlParameters) {
|
|
validate();
|
|
assert(BB && "Invalid BB");
|
|
}
|
|
|
|
SPIRVLoopMerge()
|
|
: SPIRVInstruction(OC), MergeBlock(SPIRVID_MAX),
|
|
ContinueTarget(SPIRVID_MAX), LoopControl(SPIRVWORD_MAX) {
|
|
setHasNoId();
|
|
setHasNoType();
|
|
}
|
|
|
|
SPIRVId getMergeBlock() { return MergeBlock; }
|
|
SPIRVId getContinueTarget() { return ContinueTarget; }
|
|
SPIRVWord getLoopControl() const { return LoopControl; }
|
|
std::vector<SPIRVWord> getLoopControlParameters() const {
|
|
return LoopControlParameters;
|
|
}
|
|
|
|
void setWordCount(SPIRVWord TheWordCount) override {
|
|
SPIRVEntry::setWordCount(TheWordCount);
|
|
SPIRVCK(TheWordCount >= FixedWordCount, InvalidWordCount, "");
|
|
LoopControlParameters.resize(TheWordCount - FixedWordCount);
|
|
}
|
|
_SPIRV_DEF_ENCDEC4(MergeBlock, ContinueTarget, LoopControl,
|
|
LoopControlParameters)
|
|
|
|
protected:
|
|
SPIRVId MergeBlock;
|
|
SPIRVId ContinueTarget;
|
|
SPIRVWord LoopControl;
|
|
std::vector<SPIRVWord> LoopControlParameters;
|
|
};
|
|
|
|
class SPIRVSwitch : public SPIRVInstruction {
|
|
public:
|
|
static const Op OC = OpSwitch;
|
|
static const SPIRVWord FixedWordCount = 3;
|
|
typedef std::vector<SPIRVWord> LiteralTy;
|
|
typedef std::pair<LiteralTy, SPIRVBasicBlock *> PairTy;
|
|
|
|
SPIRVSwitch(SPIRVValue *TheSelect, SPIRVBasicBlock *TheDefault,
|
|
const std::vector<PairTy> &ThePairs, SPIRVBasicBlock *BB)
|
|
: SPIRVInstruction(FixedWordCount, OC, BB), Select(TheSelect->getId()),
|
|
Default(TheDefault->getId()) {
|
|
|
|
if (!ThePairs.empty())
|
|
SPIRVEntry::setWordCount(
|
|
ThePairs.size() * (ThePairs.at(0).first.size() + 1) + FixedWordCount);
|
|
|
|
for (auto &I : ThePairs) {
|
|
for (auto &U : I.first)
|
|
Pairs.push_back(U);
|
|
Pairs.push_back(I.second->getId());
|
|
}
|
|
validate();
|
|
assert(BB && "Invalid BB");
|
|
}
|
|
SPIRVSwitch()
|
|
: SPIRVInstruction(OC), Select(SPIRVWORD_MAX), Default(SPIRVWORD_MAX) {
|
|
setHasNoId();
|
|
setHasNoType();
|
|
}
|
|
std::vector<SPIRVValue *> getPairs() const { return getValues(Pairs); }
|
|
SPIRVValue *getSelect() const { return getValue(Select); }
|
|
SPIRVBasicBlock *getDefault() const {
|
|
return static_cast<SPIRVBasicBlock *>(getValue(Default));
|
|
}
|
|
size_t getLiteralSize() const {
|
|
unsigned ByteWidth =
|
|
std::max(getSelect()->getType()->getBitWidth() / 8, 1u);
|
|
unsigned Remainder = (ByteWidth % sizeof(SPIRVWord)) != 0;
|
|
return (ByteWidth / sizeof(SPIRVWord)) + Remainder;
|
|
}
|
|
size_t getPairSize() const { return getLiteralSize() + 1; }
|
|
size_t getNumPairs() const { return Pairs.size() / getPairSize(); }
|
|
void
|
|
foreachPair(std::function<void(LiteralTy, SPIRVBasicBlock *)> Func) const {
|
|
unsigned PairSize = getPairSize();
|
|
for (size_t I = 0, E = getNumPairs(); I != E; ++I) {
|
|
SPIRVEntry *BB;
|
|
LiteralTy Literals;
|
|
if (!Module->exist(Pairs[PairSize * I + getLiteralSize()], &BB))
|
|
continue;
|
|
|
|
for (size_t J = 0; J < getLiteralSize(); ++J) {
|
|
Literals.push_back(Pairs.at(PairSize * I + J));
|
|
}
|
|
Func(Literals, static_cast<SPIRVBasicBlock *>(BB));
|
|
}
|
|
}
|
|
void setWordCount(SPIRVWord TheWordCount) override {
|
|
SPIRVEntry::setWordCount(TheWordCount);
|
|
SPIRVCK(TheWordCount >= FixedWordCount, InvalidWordCount, "");
|
|
Pairs.resize(TheWordCount - FixedWordCount);
|
|
}
|
|
_SPIRV_DEF_ENCDEC3(Select, Default, Pairs)
|
|
void validate() const override {
|
|
assert(WordCount == Pairs.size() + FixedWordCount);
|
|
assert(OpCode == OC);
|
|
assert(Pairs.size() % getPairSize() == 0);
|
|
foreachPair([=](LiteralTy Literals, SPIRVBasicBlock *BB) {
|
|
assert(BB->isBasicBlock() || BB->isForward());
|
|
});
|
|
SPIRVInstruction::validate();
|
|
}
|
|
|
|
protected:
|
|
SPIRVId Select;
|
|
SPIRVId Default;
|
|
std::vector<SPIRVWord> Pairs;
|
|
};
|
|
|
|
class SPIRVVectorTimesScalar : public SPIRVInstruction {
|
|
public:
|
|
static const Op OC = OpVectorTimesScalar;
|
|
static const SPIRVWord FixedWordCount = 4;
|
|
// Complete constructor
|
|
SPIRVVectorTimesScalar(SPIRVType *TheType, SPIRVId TheId, SPIRVId TheVector,
|
|
SPIRVId TheScalar, SPIRVBasicBlock *BB)
|
|
: SPIRVInstruction(5, OC, TheType, TheId, BB), Vector(TheVector),
|
|
Scalar(TheScalar) {
|
|
validate();
|
|
assert(BB && "Invalid BB");
|
|
}
|
|
// Incomplete constructor
|
|
SPIRVVectorTimesScalar()
|
|
: SPIRVInstruction(OC), Vector(SPIRVID_INVALID), Scalar(SPIRVID_INVALID) {
|
|
}
|
|
SPIRVValue *getVector() const { return getValue(Vector); }
|
|
SPIRVValue *getScalar() const { return getValue(Scalar); }
|
|
|
|
std::vector<SPIRVValue *> getOperands() override {
|
|
std::vector<SPIRVId> Operands;
|
|
Operands.push_back(Vector);
|
|
Operands.push_back(Scalar);
|
|
return getValues(Operands);
|
|
}
|
|
|
|
void setWordCount(SPIRVWord FixedWordCount) override {
|
|
SPIRVEntry::setWordCount(FixedWordCount);
|
|
}
|
|
_SPIRV_DEF_ENCDEC4(Type, Id, Vector, Scalar)
|
|
void validate() const override {
|
|
SPIRVInstruction::validate();
|
|
if (getValue(Vector)->isForward() || getValue(Scalar)->isForward())
|
|
return;
|
|
|
|
assert(getValueType(Vector)->isTypeVector() &&
|
|
getValueType(Vector)->getVectorComponentType()->isTypeFloat() &&
|
|
"First operand must be a vector of floating-point type");
|
|
assert(getValueType(getId())->isTypeVector() &&
|
|
getValueType(getId())->getVectorComponentType()->isTypeFloat() &&
|
|
"Result type must be a vector of floating-point type");
|
|
assert(
|
|
getValueType(Vector)->getVectorComponentType() ==
|
|
getValueType(getId())->getVectorComponentType() &&
|
|
"Scalar must have the same type as the Component Type in Result Type");
|
|
SPIRVInstruction::validate();
|
|
}
|
|
|
|
protected:
|
|
SPIRVId Vector;
|
|
SPIRVId Scalar;
|
|
};
|
|
|
|
class SPIRVVectorTimesMatrix : public SPIRVInstruction {
|
|
public:
|
|
static const Op OC = OpVectorTimesMatrix;
|
|
static const SPIRVWord FixedWordCount = 4;
|
|
|
|
// Complete constructor
|
|
SPIRVVectorTimesMatrix(SPIRVType *TheType, SPIRVId TheId, SPIRVId TheVector,
|
|
SPIRVId TheMatrix, SPIRVBasicBlock *BB)
|
|
: SPIRVInstruction(5, OC, TheType, TheId, BB), Vector(TheVector),
|
|
Matrix(TheMatrix) {
|
|
validate();
|
|
assert(BB && "Invalid BB");
|
|
}
|
|
|
|
// Incomplete constructor
|
|
SPIRVVectorTimesMatrix()
|
|
: SPIRVInstruction(OC), Vector(SPIRVID_INVALID), Matrix(SPIRVID_INVALID) {
|
|
}
|
|
|
|
SPIRVValue *getVector() const { return getValue(Vector); }
|
|
SPIRVValue *getMatrix() const { return getValue(Matrix); }
|
|
|
|
std::vector<SPIRVValue *> getOperands() override {
|
|
std::vector<SPIRVId> Operands;
|
|
Operands.push_back(Vector);
|
|
Operands.push_back(Matrix);
|
|
return getValues(Operands);
|
|
}
|
|
|
|
void setWordCount(SPIRVWord FixedWordCount) override {
|
|
SPIRVEntry::setWordCount(FixedWordCount);
|
|
}
|
|
|
|
_SPIRV_DEF_ENCDEC4(Type, Id, Vector, Matrix)
|
|
|
|
void validate() const override {
|
|
SPIRVInstruction::validate();
|
|
if (getValue(Vector)->isForward() || getValue(Matrix)->isForward())
|
|
return;
|
|
|
|
SPIRVType *Ty = getType()->getScalarType();
|
|
SPIRVType *MTy = getValueType(Matrix)->getScalarType();
|
|
SPIRVType *VTy = getValueType(Vector)->getScalarType();
|
|
|
|
(void)Ty;
|
|
(void)MTy;
|
|
(void)VTy;
|
|
assert(Ty->isTypeFloat() && "Invalid result type for OpVectorTimesMatrix");
|
|
assert(VTy->isTypeFloat() && "Invalid Vector type for OpVectorTimesMatrix");
|
|
assert(MTy->isTypeFloat() && "Invalid Matrix type for OpVectorTimesMatrix");
|
|
|
|
assert(Ty == MTy && Ty == VTy && "Mismatch float type");
|
|
}
|
|
|
|
private:
|
|
SPIRVId Vector;
|
|
SPIRVId Matrix;
|
|
};
|
|
|
|
class SPIRVMatrixTimesScalar : public SPIRVInstruction {
|
|
public:
|
|
static const Op OC = OpMatrixTimesScalar;
|
|
static const SPIRVWord FixedWordCount = 4;
|
|
// Complete constructor
|
|
SPIRVMatrixTimesScalar(SPIRVType *TheType, SPIRVId TheId, SPIRVId TheMatrix,
|
|
SPIRVId TheScalar, SPIRVBasicBlock *BB)
|
|
: SPIRVInstruction(5, OC, TheType, TheId, BB), Matrix(TheMatrix),
|
|
Scalar(TheScalar) {
|
|
validate();
|
|
assert(BB && "Invalid BB");
|
|
}
|
|
// Incomplete constructor
|
|
SPIRVMatrixTimesScalar()
|
|
: SPIRVInstruction(OC), Matrix(SPIRVID_INVALID), Scalar(SPIRVID_INVALID) {
|
|
}
|
|
SPIRVValue *getMatrix() const { return getValue(Matrix); }
|
|
SPIRVValue *getScalar() const { return getValue(Scalar); }
|
|
|
|
std::vector<SPIRVValue *> getOperands() override {
|
|
std::vector<SPIRVId> Operands;
|
|
Operands.push_back(Matrix);
|
|
Operands.push_back(Scalar);
|
|
return getValues(Operands);
|
|
}
|
|
|
|
void setWordCount(SPIRVWord FixedWordCount) override {
|
|
SPIRVEntry::setWordCount(FixedWordCount);
|
|
}
|
|
|
|
_SPIRV_DEF_ENCDEC4(Type, Id, Matrix, Scalar)
|
|
|
|
void validate() const override {
|
|
SPIRVInstruction::validate();
|
|
if (getValue(Matrix)->isForward() || getValue(Scalar)->isForward())
|
|
return;
|
|
|
|
SPIRVType *Ty = getType()->getScalarType();
|
|
SPIRVType *MTy = getValueType(Matrix)->getScalarType();
|
|
SPIRVType *STy = getValueType(Scalar);
|
|
|
|
(void)Ty;
|
|
(void)MTy;
|
|
(void)STy;
|
|
assert(Ty && Ty->isTypeFloat() &&
|
|
"Invalid result type for OpMatrixTimesScalar");
|
|
assert(MTy && MTy->isTypeFloat() &&
|
|
"Invalid Matrix type for OpMatrixTimesScalar");
|
|
assert(STy->isTypeFloat() && "Invalid Scalar type for OpMatrixTimesScalar");
|
|
assert(Ty == MTy && Ty == STy && "Mismatch float type");
|
|
}
|
|
|
|
private:
|
|
SPIRVId Matrix;
|
|
SPIRVId Scalar;
|
|
};
|
|
|
|
class SPIRVMatrixTimesVector : public SPIRVInstruction {
|
|
public:
|
|
static const Op OC = OpMatrixTimesVector;
|
|
static const SPIRVWord FixedWordCount = 4;
|
|
|
|
// Complete constructor
|
|
SPIRVMatrixTimesVector(SPIRVType *TheType, SPIRVId TheId, SPIRVId TheMatrix,
|
|
SPIRVId TheVector, SPIRVBasicBlock *BB)
|
|
: SPIRVInstruction(5, OC, TheType, TheId, BB), Matrix(TheMatrix),
|
|
Vector(TheVector) {
|
|
validate();
|
|
assert(BB && "Invalid BB");
|
|
}
|
|
|
|
// Incomplete constructor
|
|
SPIRVMatrixTimesVector()
|
|
: SPIRVInstruction(OC), Matrix(SPIRVID_INVALID), Vector(SPIRVID_INVALID) {
|
|
}
|
|
|
|
SPIRVValue *getMatrix() const { return getValue(Matrix); }
|
|
|
|
SPIRVValue *getVector() const { return getValue(Vector); }
|
|
|
|
std::vector<SPIRVValue *> getOperands() override {
|
|
std::vector<SPIRVId> Operands;
|
|
Operands.push_back(Matrix);
|
|
Operands.push_back(Vector);
|
|
return getValues(Operands);
|
|
}
|
|
|
|
void setWordCount(SPIRVWord FixedWordCount) override {
|
|
SPIRVEntry::setWordCount(FixedWordCount);
|
|
}
|
|
|
|
_SPIRV_DEF_ENCDEC4(Type, Id, Matrix, Vector)
|
|
|
|
void validate() const override {
|
|
SPIRVInstruction::validate();
|
|
if (getValue(Matrix)->isForward() || getValue(Vector)->isForward())
|
|
return;
|
|
SPIRVType *Ty = getType()->getScalarType();
|
|
SPIRVType *MTy = getValueType(Matrix)->getScalarType();
|
|
SPIRVType *VTy = getValueType(Vector)->getScalarType();
|
|
|
|
(void)Ty;
|
|
(void)MTy;
|
|
(void)VTy;
|
|
assert(Ty->isTypeFloat() && "Invalid result type for OpMatrixTimesVector");
|
|
assert(MTy->isTypeFloat() && "Invalid Matrix type for OpMatrixTimesVector");
|
|
assert(VTy->isTypeFloat() && "Invalid Vector type for OpMatrixTimesVector");
|
|
|
|
assert(Ty == MTy && Ty == VTy && "Mismatch float type");
|
|
}
|
|
|
|
private:
|
|
SPIRVId Matrix;
|
|
SPIRVId Vector;
|
|
};
|
|
|
|
class SPIRVMatrixTimesMatrix : public SPIRVInstruction {
|
|
public:
|
|
static const Op OC = OpMatrixTimesMatrix;
|
|
static const SPIRVWord FixedWordCount = 4;
|
|
|
|
// Complete constructor
|
|
SPIRVMatrixTimesMatrix(SPIRVType *TheType, SPIRVId TheId, SPIRVId M1,
|
|
SPIRVId M2, SPIRVBasicBlock *BB)
|
|
: SPIRVInstruction(5, OC, TheType, TheId, BB), LeftMatrix(M1),
|
|
RightMatrix(M2) {
|
|
validate();
|
|
assert(BB && "Invalid BB");
|
|
}
|
|
|
|
// Incomplete constructor
|
|
SPIRVMatrixTimesMatrix()
|
|
: SPIRVInstruction(OC), LeftMatrix(SPIRVID_INVALID),
|
|
RightMatrix(SPIRVID_INVALID) {}
|
|
|
|
SPIRVValue *getLeftMatrix() const { return getValue(LeftMatrix); }
|
|
|
|
SPIRVValue *getRightMatrix() const { return getValue(RightMatrix); }
|
|
|
|
std::vector<SPIRVValue *> getOperands() override {
|
|
std::vector<SPIRVId> Operands;
|
|
Operands.push_back(LeftMatrix);
|
|
Operands.push_back(RightMatrix);
|
|
return getValues(Operands);
|
|
}
|
|
|
|
void setWordCount(SPIRVWord FixedWordCount) override {
|
|
SPIRVEntry::setWordCount(FixedWordCount);
|
|
}
|
|
|
|
_SPIRV_DEF_ENCDEC4(Type, Id, LeftMatrix, RightMatrix)
|
|
|
|
void validate() const override {
|
|
SPIRVInstruction::validate();
|
|
if (getValue(LeftMatrix)->isForward() || getValue(RightMatrix)->isForward())
|
|
return;
|
|
|
|
SPIRVType *Ty = getType()->getScalarType();
|
|
SPIRVType *LMTy = getValueType(LeftMatrix)->getScalarType();
|
|
SPIRVType *RMTy = getValueType(RightMatrix)->getScalarType();
|
|
|
|
(void)Ty;
|
|
(void)LMTy;
|
|
(void)RMTy;
|
|
assert(Ty->isTypeFloat() && "Invalid result type for OpMatrixTimesMatrix");
|
|
assert(LMTy->isTypeFloat() &&
|
|
"Invalid Matrix type for OpMatrixTimesMatrix");
|
|
assert(RMTy->isTypeFloat() &&
|
|
"Invalid Matrix type for OpMatrixTimesMatrix");
|
|
|
|
assert(Ty == LMTy && Ty == RMTy && "Mismatch float type");
|
|
}
|
|
|
|
private:
|
|
SPIRVId LeftMatrix;
|
|
SPIRVId RightMatrix;
|
|
};
|
|
|
|
class SPIRVTranspose : public SPIRVInstruction {
|
|
public:
|
|
static const Op OC = OpTranspose;
|
|
static const SPIRVWord FixedWordCount = 3;
|
|
|
|
// Complete constructor
|
|
SPIRVTranspose(SPIRVType *TheType, SPIRVId TheId, SPIRVId TheMatrix,
|
|
SPIRVBasicBlock *BB)
|
|
: SPIRVInstruction(4, OC, TheType, TheId, BB), Matrix(TheMatrix) {
|
|
validate();
|
|
assert(BB && "Invalid BB");
|
|
}
|
|
|
|
// Incomplete constructor
|
|
SPIRVTranspose() : SPIRVInstruction(OC), Matrix(SPIRVID_INVALID) {}
|
|
|
|
SPIRVValue *getMatrix() const { return getValue(Matrix); }
|
|
|
|
std::vector<SPIRVValue *> getOperands() override {
|
|
std::vector<SPIRVId> Operands;
|
|
Operands.push_back(Matrix);
|
|
return getValues(Operands);
|
|
}
|
|
|
|
void setWordCount(SPIRVWord FixedWordCount) override {
|
|
SPIRVEntry::setWordCount(FixedWordCount);
|
|
}
|
|
|
|
_SPIRV_DEF_ENCDEC3(Type, Id, Matrix)
|
|
|
|
void validate() const override {
|
|
SPIRVInstruction::validate();
|
|
if (getValue(Matrix)->isForward())
|
|
return;
|
|
|
|
SPIRVType *Ty = getType()->getScalarType();
|
|
SPIRVType *MTy = getValueType(Matrix)->getScalarType();
|
|
|
|
(void)Ty;
|
|
(void)MTy;
|
|
|
|
assert(Ty->isTypeFloat() && "Invalid result type for OpTranspose");
|
|
assert(Ty == MTy && "Mismatch float type");
|
|
}
|
|
|
|
private:
|
|
SPIRVId Matrix;
|
|
};
|
|
|
|
class SPIRVSizeOfInstBase : public SPIRVInstTemplateBase {
|
|
protected:
|
|
VersionNumber getRequiredSPIRVVersion() const override {
|
|
return VersionNumber::SPIRV_1_1;
|
|
}
|
|
};
|
|
|
|
typedef SPIRVInstTemplate<SPIRVSizeOfInstBase, OpSizeOf, true, 4, false>
|
|
SPIRVSizeOf;
|
|
|
|
class SPIRVUnary : public SPIRVInstTemplateBase {
|
|
protected:
|
|
void validate() const override {
|
|
auto Op = Ops[0];
|
|
SPIRVInstruction::validate();
|
|
if (getValue(Op)->isForward())
|
|
return;
|
|
if (isGenericNegateOpCode(OpCode)) {
|
|
SPIRVType *ResTy =
|
|
Type->isTypeVector() || Type->isTypeCooperativeMatrixKHR()
|
|
? Type->getVectorComponentType()
|
|
: Type;
|
|
SPIRVType *OpTy =
|
|
Type->isTypeVector() || Type->isTypeCooperativeMatrixKHR()
|
|
? getValueType(Op)->getVectorComponentType()
|
|
: getValueType(Op);
|
|
|
|
(void)ResTy;
|
|
(void)OpTy;
|
|
assert(getType() == getValueType(Op) && "Inconsistent type");
|
|
assert((ResTy->isTypeInt() || ResTy->isTypeFloat()) &&
|
|
"Invalid type for Generic Negate instruction");
|
|
assert((ResTy->getBitWidth() == OpTy->getBitWidth()) &&
|
|
"Invalid bitwidth for Generic Negate instruction");
|
|
assert((Type->isTypeVector()
|
|
? (Type->getVectorComponentCount() ==
|
|
getValueType(Op)->getVectorComponentCount())
|
|
: 1) &&
|
|
"Invalid vector component Width for Generic Negate instruction");
|
|
}
|
|
}
|
|
};
|
|
|
|
template <Op OC>
|
|
class SPIRVUnaryInst
|
|
: public SPIRVInstTemplate<SPIRVUnary, OC, true, 4, false> {};
|
|
|
|
#define _SPIRV_OP(x) typedef SPIRVUnaryInst<Op##x> SPIRV##x;
|
|
_SPIRV_OP(ConvertFToU)
|
|
_SPIRV_OP(ConvertFToS)
|
|
_SPIRV_OP(ConvertSToF)
|
|
_SPIRV_OP(ConvertUToF)
|
|
_SPIRV_OP(UConvert)
|
|
_SPIRV_OP(SConvert)
|
|
_SPIRV_OP(FConvert)
|
|
_SPIRV_OP(SatConvertSToU)
|
|
_SPIRV_OP(SatConvertUToS)
|
|
_SPIRV_OP(ConvertPtrToU)
|
|
_SPIRV_OP(ConvertUToPtr)
|
|
_SPIRV_OP(PtrCastToGeneric)
|
|
_SPIRV_OP(GenericCastToPtr)
|
|
_SPIRV_OP(CrossWorkgroupCastToPtrINTEL)
|
|
_SPIRV_OP(PtrCastToCrossWorkgroupINTEL)
|
|
_SPIRV_OP(Bitcast)
|
|
_SPIRV_OP(SNegate)
|
|
_SPIRV_OP(FNegate)
|
|
_SPIRV_OP(Not)
|
|
_SPIRV_OP(LogicalNot)
|
|
_SPIRV_OP(IsNan)
|
|
_SPIRV_OP(IsInf)
|
|
_SPIRV_OP(IsFinite)
|
|
_SPIRV_OP(IsNormal)
|
|
_SPIRV_OP(SignBitSet)
|
|
_SPIRV_OP(Any)
|
|
_SPIRV_OP(All)
|
|
_SPIRV_OP(BitCount)
|
|
_SPIRV_OP(ArithmeticFenceEXT)
|
|
#undef _SPIRV_OP
|
|
|
|
class SPIRVAccessChainBase : public SPIRVInstTemplateBase {
|
|
public:
|
|
SPIRVValue *getBase() {
|
|
if (this->isUntyped())
|
|
return this->getValue(this->Ops[1]);
|
|
return this->getValue(this->Ops[0]);
|
|
}
|
|
SPIRVType *getBaseType() {
|
|
if (this->isUntyped())
|
|
return get<SPIRVType>(this->Ops[0]);
|
|
return this->getValue(this->Ops[0])->getType();
|
|
}
|
|
std::vector<SPIRVValue *> getIndices() const {
|
|
unsigned IdxShift = this->isUntyped() ? 2 : 1;
|
|
std::vector<SPIRVWord> IndexWords(this->Ops.begin() + IdxShift,
|
|
this->Ops.end());
|
|
return this->getValues(IndexWords);
|
|
}
|
|
bool isInBounds() {
|
|
return OpCode == OpInBoundsAccessChain ||
|
|
OpCode == OpInBoundsPtrAccessChain ||
|
|
OpCode == OpUntypedInBoundsAccessChainKHR ||
|
|
OpCode == OpUntypedInBoundsPtrAccessChainKHR;
|
|
}
|
|
bool hasPtrIndex() {
|
|
return OpCode == OpPtrAccessChain || OpCode == OpInBoundsPtrAccessChain ||
|
|
OpCode == OpUntypedPtrAccessChainKHR ||
|
|
OpCode == OpUntypedInBoundsPtrAccessChainKHR;
|
|
}
|
|
bool isUntyped() const {
|
|
return OpCode == OpUntypedAccessChainKHR ||
|
|
OpCode == OpUntypedInBoundsAccessChainKHR ||
|
|
OpCode == OpUntypedPtrAccessChainKHR ||
|
|
OpCode == OpUntypedInBoundsPtrAccessChainKHR;
|
|
}
|
|
SPIRVCapVec getRequiredCapability() const override {
|
|
if (isUntyped())
|
|
return getVec(CapabilityUntypedPointersKHR);
|
|
return {};
|
|
}
|
|
std::optional<ExtensionID> getRequiredExtension() const override {
|
|
if (isUntyped())
|
|
return ExtensionID::SPV_KHR_untyped_pointers;
|
|
return {};
|
|
}
|
|
};
|
|
|
|
template <Op OC, unsigned FixedWC>
|
|
class SPIRVAccessChainGeneric
|
|
: public SPIRVInstTemplate<SPIRVAccessChainBase, OC, true, FixedWC, true> {
|
|
};
|
|
|
|
typedef SPIRVAccessChainGeneric<OpAccessChain, 4> SPIRVAccessChain;
|
|
typedef SPIRVAccessChainGeneric<OpInBoundsAccessChain, 4>
|
|
SPIRVInBoundsAccessChain;
|
|
typedef SPIRVAccessChainGeneric<OpPtrAccessChain, 5> SPIRVPtrAccessChain;
|
|
typedef SPIRVAccessChainGeneric<OpInBoundsPtrAccessChain, 5>
|
|
SPIRVInBoundsPtrAccessChain;
|
|
|
|
typedef SPIRVAccessChainGeneric<OpUntypedAccessChainKHR, 5>
|
|
SPIRVUntypedAccessChainKHR;
|
|
typedef SPIRVAccessChainGeneric<OpUntypedInBoundsAccessChainKHR, 5>
|
|
SPIRVUntypedInBoundsAccessChainKHR;
|
|
typedef SPIRVAccessChainGeneric<OpUntypedPtrAccessChainKHR, 6>
|
|
SPIRVUntypedPtrAccessChainKHR;
|
|
typedef SPIRVAccessChainGeneric<OpUntypedInBoundsPtrAccessChainKHR, 6>
|
|
SPIRVUntypedInBoundsPtrAccessChainKHR;
|
|
|
|
class SPIRVLoopControlINTEL : public SPIRVInstruction {
|
|
public:
|
|
static const Op OC = OpLoopControlINTEL;
|
|
static const SPIRVWord FixedWordCount = 2;
|
|
|
|
SPIRVLoopControlINTEL(SPIRVWord TheLoopControl,
|
|
std::vector<SPIRVWord> TheLoopControlParameters,
|
|
SPIRVBasicBlock *BB)
|
|
: SPIRVInstruction(FixedWordCount + TheLoopControlParameters.size(), OC,
|
|
BB),
|
|
LoopControl(TheLoopControl),
|
|
LoopControlParameters(TheLoopControlParameters) {
|
|
validate();
|
|
assert(BB && "Invalid BB");
|
|
}
|
|
|
|
SPIRVLoopControlINTEL() : SPIRVInstruction(OC), LoopControl(SPIRVWORD_MAX) {
|
|
setHasNoId();
|
|
setHasNoType();
|
|
}
|
|
|
|
SPIRVWord getLoopControl() const { return LoopControl; }
|
|
|
|
std::vector<SPIRVWord> getLoopControlParameters() const {
|
|
return LoopControlParameters;
|
|
}
|
|
|
|
SPIRVCapVec getRequiredCapability() const override {
|
|
return getVec(CapabilityUnstructuredLoopControlsINTEL);
|
|
}
|
|
|
|
std::optional<ExtensionID> getRequiredExtension() const override {
|
|
return ExtensionID::SPV_INTEL_unstructured_loop_controls;
|
|
}
|
|
|
|
void setWordCount(SPIRVWord TheWordCount) override {
|
|
SPIRVEntry::setWordCount(TheWordCount);
|
|
SPIRVCK(TheWordCount >= FixedWordCount, InvalidWordCount, "");
|
|
LoopControlParameters.resize(TheWordCount - FixedWordCount);
|
|
}
|
|
_SPIRV_DEF_ENCDEC2(LoopControl, LoopControlParameters)
|
|
|
|
protected:
|
|
SPIRVWord LoopControl;
|
|
std::vector<SPIRVWord> LoopControlParameters;
|
|
};
|
|
|
|
template <Op OC, SPIRVWord FixedWordCount>
|
|
class SPIRVFunctionCallGeneric : public SPIRVInstruction {
|
|
public:
|
|
SPIRVFunctionCallGeneric(SPIRVType *TheType, SPIRVId TheId,
|
|
const std::vector<SPIRVWord> &TheArgs,
|
|
SPIRVBasicBlock *BB)
|
|
: SPIRVInstruction(TheArgs.size() + FixedWordCount, OC, TheType, TheId,
|
|
BB),
|
|
Args(TheArgs) {
|
|
SPIRVFunctionCallGeneric::validate();
|
|
assert(BB && "Invalid BB");
|
|
}
|
|
SPIRVFunctionCallGeneric(SPIRVType *TheType, SPIRVId TheId,
|
|
const std::vector<SPIRVValue *> &TheArgs,
|
|
SPIRVBasicBlock *BB)
|
|
: SPIRVInstruction(TheArgs.size() + FixedWordCount, OC, TheType, TheId,
|
|
BB) {
|
|
Args = getIds(TheArgs);
|
|
SPIRVFunctionCallGeneric::validate();
|
|
assert(BB && "Invalid BB");
|
|
}
|
|
|
|
SPIRVFunctionCallGeneric(SPIRVModule *BM, SPIRVWord ResId, SPIRVType *TheType,
|
|
const std::vector<SPIRVWord> &TheArgs)
|
|
: SPIRVInstruction(TheArgs.size() + FixedWordCount, OC, TheType, ResId,
|
|
BM),
|
|
Args(TheArgs) {}
|
|
|
|
SPIRVFunctionCallGeneric() : SPIRVInstruction(OC) {}
|
|
const std::vector<SPIRVWord> &getArguments() const { return Args; }
|
|
void setArguments(const std::vector<SPIRVWord> &A) {
|
|
Args = A;
|
|
setWordCount(Args.size() + FixedWordCount);
|
|
}
|
|
std::vector<SPIRVValue *> getArgumentValues() { return getValues(Args); }
|
|
std::vector<SPIRVType *> getArgumentValueTypes() const {
|
|
std::vector<SPIRVType *> ArgTypes;
|
|
for (auto &I : Args)
|
|
ArgTypes.push_back(getValue(I)->getType());
|
|
return ArgTypes;
|
|
}
|
|
void setWordCount(SPIRVWord TheWordCount) override {
|
|
SPIRVEntry::setWordCount(TheWordCount);
|
|
SPIRVCK(TheWordCount >= FixedWordCount, InvalidWordCount, "");
|
|
Args.resize(TheWordCount - FixedWordCount);
|
|
}
|
|
void validate() const override { SPIRVInstruction::validate(); }
|
|
|
|
protected:
|
|
std::vector<SPIRVWord> Args;
|
|
};
|
|
|
|
class SPIRVFunctionCall : public SPIRVFunctionCallGeneric<OpFunctionCall, 4> {
|
|
public:
|
|
SPIRVFunctionCall(SPIRVId TheId, SPIRVFunction *TheFunction,
|
|
const std::vector<SPIRVWord> &TheArgs, SPIRVBasicBlock *BB);
|
|
SPIRVFunctionCall() : FunctionId(SPIRVID_INVALID) {}
|
|
SPIRVFunction *getFunction() const { return get<SPIRVFunction>(FunctionId); }
|
|
_SPIRV_DEF_ENCDEC4(Type, Id, FunctionId, Args)
|
|
void validate() const override;
|
|
bool isOperandLiteral(unsigned Index) const override { return false; }
|
|
|
|
protected:
|
|
SPIRVId FunctionId;
|
|
};
|
|
|
|
class SPIRVFunctionPointerCallINTEL
|
|
: public SPIRVFunctionCallGeneric<OpFunctionPointerCallINTEL, 4> {
|
|
public:
|
|
SPIRVFunctionPointerCallINTEL(SPIRVId TheId, SPIRVValue *TheCalledValue,
|
|
SPIRVType *TheReturnType,
|
|
const std::vector<SPIRVWord> &TheArgs,
|
|
SPIRVBasicBlock *BB);
|
|
SPIRVFunctionPointerCallINTEL() : CalledValueId(SPIRVID_INVALID) {}
|
|
SPIRVValue *getCalledValue() const { return get<SPIRVValue>(CalledValueId); }
|
|
_SPIRV_DEF_ENCDEC4(Type, Id, CalledValueId, Args)
|
|
void validate() const override;
|
|
bool isOperandLiteral(unsigned Index) const override { return false; }
|
|
std::optional<ExtensionID> getRequiredExtension() const override {
|
|
return ExtensionID::SPV_INTEL_function_pointers;
|
|
}
|
|
SPIRVCapVec getRequiredCapability() const override {
|
|
return getVec(CapabilityFunctionPointersINTEL);
|
|
}
|
|
|
|
protected:
|
|
SPIRVId CalledValueId;
|
|
};
|
|
|
|
class SPIRVExtInst : public SPIRVFunctionCallGeneric<OpExtInst, 5> {
|
|
public:
|
|
SPIRVExtInst(SPIRVType *TheType, SPIRVId TheId, SPIRVId TheBuiltinSet,
|
|
SPIRVWord TheEntryPoint, const std::vector<SPIRVWord> &TheArgs,
|
|
SPIRVBasicBlock *BB)
|
|
: SPIRVFunctionCallGeneric(TheType, TheId, TheArgs, BB),
|
|
ExtSetId(TheBuiltinSet), ExtOp(TheEntryPoint) {
|
|
setExtSetKindById();
|
|
SPIRVExtInst::validate();
|
|
}
|
|
SPIRVExtInst(SPIRVType *TheType, SPIRVId TheId, SPIRVId TheBuiltinSet,
|
|
SPIRVWord TheEntryPoint,
|
|
const std::vector<SPIRVValue *> &TheArgs, SPIRVBasicBlock *BB)
|
|
: SPIRVFunctionCallGeneric(TheType, TheId, TheArgs, BB),
|
|
ExtSetId(TheBuiltinSet), ExtOp(TheEntryPoint) {
|
|
setExtSetKindById();
|
|
SPIRVExtInst::validate();
|
|
}
|
|
|
|
SPIRVExtInst(SPIRVModule *BM, SPIRVId ResId, SPIRVType *TheType,
|
|
SPIRVExtInstSetKind SetKind, SPIRVWord SetId, SPIRVWord InstId,
|
|
const std::vector<SPIRVWord> &Args)
|
|
: SPIRVFunctionCallGeneric(BM, ResId, TheType, Args), ExtSetKind(SetKind),
|
|
ExtSetId(SetId), ExtOp(InstId) {}
|
|
|
|
SPIRVExtInst(SPIRVExtInstSetKind SetKind = SPIRVEIS_Count,
|
|
unsigned ExtOC = SPIRVWORD_MAX)
|
|
: ExtSetKind(SetKind), ExtSetId(SPIRVWORD_MAX), ExtOp(ExtOC) {}
|
|
void setExtSetId(unsigned Set) { ExtSetId = Set; }
|
|
void setExtOp(unsigned ExtOC) { ExtOp = ExtOC; }
|
|
SPIRVId getExtSetId() const { return ExtSetId; }
|
|
SPIRVWord getExtOp() const { return ExtOp; }
|
|
|
|
SPIRVExtInstSetKind getExtSetKind() const { return ExtSetKind; }
|
|
|
|
void addContinuedInstruction(SPIRVExtInst *Inst) {
|
|
ContinuedInstructions.push_back(Inst);
|
|
}
|
|
|
|
std::vector<SPIRVExtInst *> getContinuedInstructions() {
|
|
return ContinuedInstructions;
|
|
}
|
|
|
|
void setExtSetKindById() {
|
|
assert(Module && "Invalid module");
|
|
ExtSetKind = Module->getBuiltinSet(ExtSetId);
|
|
assert((ExtSetKind == SPIRVEIS_OpenCL || ExtSetKind == SPIRVEIS_Debug ||
|
|
ExtSetKind == SPIRVEIS_OpenCL_DebugInfo_100 ||
|
|
ExtSetKind == SPIRVEIS_NonSemantic_Shader_DebugInfo_100 ||
|
|
ExtSetKind == SPIRVEIS_NonSemantic_Shader_DebugInfo_200 ||
|
|
ExtSetKind == SPIRVEIS_NonSemantic_AuxData ||
|
|
ExtSetKind == SPIRVEIS_NonSemantic_Unknown) &&
|
|
"not supported");
|
|
}
|
|
void encode(spv_ostream &O) const override {
|
|
getEncoder(O) << Type << Id << ExtSetId;
|
|
switch (ExtSetKind) {
|
|
case SPIRVEIS_OpenCL:
|
|
getEncoder(O) << ExtOpOCL;
|
|
break;
|
|
case SPIRVEIS_Debug:
|
|
case SPIRVEIS_OpenCL_DebugInfo_100:
|
|
case SPIRVEIS_NonSemantic_Shader_DebugInfo_100:
|
|
case SPIRVEIS_NonSemantic_Shader_DebugInfo_200:
|
|
getEncoder(O) << ExtOpDebug;
|
|
break;
|
|
case SPIRVEIS_NonSemantic_AuxData:
|
|
getEncoder(O) << ExtOpNonSemanticAuxData;
|
|
break;
|
|
case SPIRVEIS_NonSemantic_Unknown:
|
|
getEncoder(O) << ExtOp;
|
|
break;
|
|
default:
|
|
assert(0 && "not supported");
|
|
getEncoder(O) << ExtOp;
|
|
}
|
|
getEncoder(O) << Args;
|
|
}
|
|
void decode(std::istream &I) override {
|
|
getDecoder(I) >> Type >> Id >> ExtSetId;
|
|
setExtSetKindById();
|
|
switch (ExtSetKind) {
|
|
case SPIRVEIS_OpenCL:
|
|
getDecoder(I) >> ExtOpOCL;
|
|
break;
|
|
case SPIRVEIS_Debug:
|
|
case SPIRVEIS_OpenCL_DebugInfo_100:
|
|
case SPIRVEIS_NonSemantic_Shader_DebugInfo_100:
|
|
case SPIRVEIS_NonSemantic_Shader_DebugInfo_200:
|
|
getDecoder(I) >> ExtOpDebug;
|
|
break;
|
|
case SPIRVEIS_NonSemantic_AuxData:
|
|
getDecoder(I) >> ExtOpNonSemanticAuxData;
|
|
break;
|
|
case SPIRVEIS_NonSemantic_Unknown:
|
|
getDecoder(I) >> ExtOp;
|
|
break;
|
|
default:
|
|
assert(0 && "not supported");
|
|
getDecoder(I) >> ExtOp;
|
|
}
|
|
SPIRVDecoder Decoder = getDecoder(I);
|
|
Decoder >> Args;
|
|
|
|
if (ExtSetKind == SPIRVEIS_NonSemantic_Shader_DebugInfo_100 ||
|
|
ExtSetKind == SPIRVEIS_NonSemantic_Shader_DebugInfo_200) {
|
|
if (getExtOp() == SPIRVDebug::Instruction::Source) {
|
|
for (SPIRVEntry *E : Decoder.getSourceContinuedInstructions()) {
|
|
addContinuedInstruction(static_cast<SPIRVExtInst *>(E));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
void validate() const override {
|
|
SPIRVFunctionCallGeneric::validate();
|
|
validateBuiltin(ExtSetId, ExtOp);
|
|
}
|
|
bool isOperandLiteral(unsigned Index) const override {
|
|
assert(ExtSetKind == SPIRVEIS_OpenCL &&
|
|
"Unsupported extended instruction set");
|
|
auto EOC = static_cast<OCLExtOpKind>(ExtOp);
|
|
switch (EOC) {
|
|
default:
|
|
return false;
|
|
case OpenCLLIB::Vloadn:
|
|
case OpenCLLIB::Vload_halfn:
|
|
case OpenCLLIB::Vloada_halfn:
|
|
return Index == 2;
|
|
case OpenCLLIB::Vstore_half_r:
|
|
case OpenCLLIB::Vstore_halfn_r:
|
|
case OpenCLLIB::Vstorea_halfn_r:
|
|
return Index == 3;
|
|
}
|
|
}
|
|
std::vector<SPIRVValue *> getArgValues() {
|
|
std::vector<SPIRVValue *> VArgs;
|
|
for (size_t I = 0; I < Args.size(); ++I)
|
|
VArgs.push_back(getArgValue(I));
|
|
return VArgs;
|
|
}
|
|
SPIRVValue *getArgValue(SPIRVWord I) {
|
|
if (isOperandLiteral(I))
|
|
return Module->getLiteralAsConstant(Args[I]);
|
|
return getValue(Args[I]);
|
|
}
|
|
|
|
std::vector<SPIRVType *> getArgTypes() {
|
|
std::vector<SPIRVType *> ArgTypes;
|
|
auto VArgs = getArgValues();
|
|
for (auto VArg : VArgs)
|
|
ArgTypes.push_back(VArg->getType());
|
|
return ArgTypes;
|
|
}
|
|
|
|
std::optional<ExtensionID> getRequiredExtension() const override {
|
|
if (ExtSetKind == SPIRVEIS_NonSemantic_Unknown ||
|
|
SPIRVBuiltinSetNameMap::map(ExtSetKind).find("NonSemantic.") == 0) {
|
|
if (!Module->isAllowedToUseVersion(VersionNumber::SPIRV_1_6))
|
|
return ExtensionID::SPV_KHR_non_semantic_info;
|
|
}
|
|
return {};
|
|
}
|
|
|
|
protected:
|
|
SPIRVExtInstSetKind ExtSetKind;
|
|
SPIRVId ExtSetId;
|
|
union {
|
|
SPIRVWord ExtOp;
|
|
OCLExtOpKind ExtOpOCL;
|
|
SPIRVDebugExtOpKind ExtOpDebug;
|
|
NonSemanticAuxDataOpKind ExtOpNonSemanticAuxData;
|
|
};
|
|
std::vector<SPIRVExtInst *> ContinuedInstructions;
|
|
};
|
|
|
|
class SPIRVCompositeConstruct : public SPIRVInstruction {
|
|
public:
|
|
const static Op OC = OpCompositeConstruct;
|
|
const static SPIRVWord FixedWordCount = 3;
|
|
// Complete constructor
|
|
SPIRVCompositeConstruct(SPIRVType *TheType, SPIRVId TheId,
|
|
const std::vector<SPIRVId> &TheConstituents,
|
|
SPIRVBasicBlock *TheBB)
|
|
: SPIRVInstruction(TheConstituents.size() + FixedWordCount, OC, TheType,
|
|
TheId, TheBB),
|
|
Constituents(TheConstituents) {
|
|
validate();
|
|
assert(TheBB && "Invalid BB");
|
|
}
|
|
// Incomplete constructor
|
|
SPIRVCompositeConstruct() : SPIRVInstruction(OC) {}
|
|
|
|
std::vector<SPIRVValue *> getOperands() override {
|
|
return getValues(Constituents);
|
|
}
|
|
|
|
protected:
|
|
void setWordCount(SPIRVWord TheWordCount) override {
|
|
SPIRVEntry::setWordCount(TheWordCount);
|
|
SPIRVCK(TheWordCount >= FixedWordCount, InvalidWordCount, "");
|
|
Constituents.resize(TheWordCount - FixedWordCount);
|
|
}
|
|
_SPIRV_DEF_ENCDEC3(Type, Id, Constituents)
|
|
void validate() const override {
|
|
SPIRVInstruction::validate();
|
|
size_t TypeOpCode = this->getType()->getOpCode();
|
|
switch (TypeOpCode) {
|
|
case OpTypeVector:
|
|
assert(Constituents.size() > 1 &&
|
|
"There must be at least two Constituent operands in vector");
|
|
break;
|
|
case OpTypeArray:
|
|
case OpTypeStruct:
|
|
case OpTypeCooperativeMatrixKHR:
|
|
break;
|
|
default:
|
|
assert(false && "Invalid type");
|
|
}
|
|
}
|
|
std::vector<SPIRVId> Constituents;
|
|
};
|
|
|
|
class SPIRVCompositeExtractBase : public SPIRVInstTemplateBase {
|
|
public:
|
|
SPIRVValue *getComposite() { return getValue(Ops[0]); }
|
|
const std::vector<SPIRVWord> getIndices() const {
|
|
return std::vector<SPIRVWord>(Ops.begin() + 1, Ops.end());
|
|
}
|
|
|
|
protected:
|
|
// ToDo: validate the result type is consistent with the base type and indices
|
|
// need to trace through the base type for struct types
|
|
void validate() const override {
|
|
SPIRVInstruction::validate();
|
|
assert(OpCode == OpCompositeExtract);
|
|
SPIRVId Composite = Ops[0];
|
|
(void)Composite;
|
|
assert(getValueType(Composite)->isTypeArray() ||
|
|
getValueType(Composite)->isTypeStruct() ||
|
|
getValueType(Composite)->isTypeVector() ||
|
|
getValueType(Composite)->isTypeUntypedPointerKHR());
|
|
}
|
|
};
|
|
|
|
typedef SPIRVInstTemplate<SPIRVCompositeExtractBase, OpCompositeExtract, true,
|
|
4, true>
|
|
SPIRVCompositeExtract;
|
|
|
|
class SPIRVCompositeInsertBase : public SPIRVInstTemplateBase {
|
|
public:
|
|
SPIRVValue *getObject() { return getValue(Ops[0]); }
|
|
SPIRVValue *getComposite() { return getValue(Ops[1]); }
|
|
const std::vector<SPIRVWord> getIndices() const {
|
|
return std::vector<SPIRVWord>(Ops.begin() + 2, Ops.end());
|
|
}
|
|
|
|
protected:
|
|
// ToDo: validate the object type is consistent with the base type and indices
|
|
// need to trace through the base type for struct types
|
|
void validate() const override {
|
|
SPIRVInstruction::validate();
|
|
assert(OpCode == OpCompositeInsert);
|
|
SPIRVId Composite = Ops[1];
|
|
(void)Composite;
|
|
assert(getValueType(Composite)->isTypeArray() ||
|
|
getValueType(Composite)->isTypeStruct() ||
|
|
getValueType(Composite)->isTypeVector() ||
|
|
getValueType(Composite)->isTypeUntypedPointerKHR());
|
|
assert(Type == getValueType(Composite));
|
|
}
|
|
};
|
|
|
|
typedef SPIRVInstTemplate<SPIRVCompositeInsertBase, OpCompositeInsert, true, 5,
|
|
true>
|
|
SPIRVCompositeInsert;
|
|
|
|
class SPIRVCopyObject : public SPIRVInstruction {
|
|
public:
|
|
const static Op OC = OpCopyObject;
|
|
|
|
// Complete constructor
|
|
SPIRVCopyObject(SPIRVType *TheType, SPIRVId TheId, SPIRVValue *TheOperand,
|
|
SPIRVBasicBlock *TheBB)
|
|
: SPIRVInstruction(4, OC, TheType, TheId, TheBB),
|
|
Operand(TheOperand->getId()) {
|
|
validate();
|
|
assert(TheBB && "Invalid BB");
|
|
}
|
|
// Incomplete constructor
|
|
SPIRVCopyObject() : SPIRVInstruction(OC), Operand(SPIRVID_INVALID) {}
|
|
|
|
SPIRVValue *getOperand() { return getValue(Operand); }
|
|
|
|
protected:
|
|
_SPIRV_DEF_ENCDEC3(Type, Id, Operand)
|
|
|
|
void validate() const override { SPIRVInstruction::validate(); }
|
|
SPIRVId Operand;
|
|
};
|
|
|
|
class SPIRVCopyLogical : public SPIRVInstruction {
|
|
public:
|
|
const static Op OC = OpCopyLogical;
|
|
|
|
// Complete constructor
|
|
SPIRVCopyLogical(SPIRVType *TheType, SPIRVId TheId, SPIRVValue *TheOperand,
|
|
SPIRVBasicBlock *TheBB)
|
|
: SPIRVInstruction(4, OC, TheType, TheId, TheBB),
|
|
Operand(TheOperand->getId()) {
|
|
validate();
|
|
assert(TheBB && "Invalid BB");
|
|
}
|
|
// Incomplete constructor
|
|
SPIRVCopyLogical() : SPIRVInstruction(OC), Operand(SPIRVID_INVALID) {}
|
|
|
|
SPIRVValue *getOperand() { return getValue(Operand); }
|
|
std::vector<SPIRVValue *> getOperands() override { return {getOperand()}; }
|
|
|
|
protected:
|
|
_SPIRV_DEF_ENCDEC3(Type, Id, Operand)
|
|
|
|
void validate() const override { SPIRVInstruction::validate(); }
|
|
SPIRVId Operand;
|
|
};
|
|
|
|
class SPIRVCopyMemory : public SPIRVInstruction, public SPIRVMemoryAccess {
|
|
public:
|
|
const static Op OC = OpCopyMemory;
|
|
const static SPIRVWord FixedWords = 3;
|
|
// Complete constructor
|
|
SPIRVCopyMemory(SPIRVValue *TheTarget, SPIRVValue *TheSource,
|
|
const std::vector<SPIRVWord> &TheMemoryAccess,
|
|
SPIRVBasicBlock *TheBB)
|
|
: SPIRVInstruction(FixedWords + TheMemoryAccess.size(), OC, TheBB),
|
|
SPIRVMemoryAccess(TheMemoryAccess), MemoryAccess(TheMemoryAccess),
|
|
Target(TheTarget->getId()), Source(TheSource->getId()) {
|
|
validate();
|
|
assert(TheBB && "Invalid BB");
|
|
}
|
|
|
|
// Incomplete constructor
|
|
SPIRVCopyMemory()
|
|
: SPIRVInstruction(OC), SPIRVMemoryAccess(), Target(SPIRVID_INVALID),
|
|
Source(SPIRVID_INVALID) {
|
|
setHasNoId();
|
|
setHasNoType();
|
|
}
|
|
|
|
SPIRVValue *getSource() { return getValue(Source); }
|
|
SPIRVValue *getTarget() { return getValue(Target); }
|
|
|
|
protected:
|
|
void setWordCount(SPIRVWord TheWordCount) override {
|
|
SPIRVEntry::setWordCount(TheWordCount);
|
|
SPIRVCK(TheWordCount >= FixedWords, InvalidWordCount, "");
|
|
MemoryAccess.resize(TheWordCount - FixedWords);
|
|
}
|
|
|
|
void encode(spv_ostream &O) const override {
|
|
getEncoder(O) << Target << Source << MemoryAccess;
|
|
}
|
|
|
|
void decode(std::istream &I) override {
|
|
getDecoder(I) >> Target >> Source >> MemoryAccess;
|
|
memoryAccessUpdate(MemoryAccess);
|
|
}
|
|
|
|
void validate() const override {
|
|
assert(getValueType(Target)->isTypePointer() && "Invalid Target type");
|
|
assert(getValueType(Source)->isTypePointer() && "Invalid Source type");
|
|
assert(!(getValueType(Target)->getPointerElementType()->isTypeVoid()) &&
|
|
"Invalid Target element type");
|
|
assert(!(getValueType(Source)->getPointerElementType()->isTypeVoid()) &&
|
|
"Invalid Source element type");
|
|
assert(getValueType(Target)->getPointerElementType() ==
|
|
getValueType(Source)->getPointerElementType() &&
|
|
"Mismatching Target and Source element types");
|
|
SPIRVInstruction::validate();
|
|
}
|
|
|
|
std::vector<SPIRVWord> MemoryAccess;
|
|
SPIRVId Target;
|
|
SPIRVId Source;
|
|
};
|
|
|
|
class SPIRVCopyMemorySized : public SPIRVInstruction, public SPIRVMemoryAccess {
|
|
public:
|
|
const static Op OC = OpCopyMemorySized;
|
|
const static SPIRVWord FixedWords = 4;
|
|
// Complete constructor
|
|
SPIRVCopyMemorySized(SPIRVValue *TheTarget, SPIRVValue *TheSource,
|
|
SPIRVValue *TheSize,
|
|
const std::vector<SPIRVWord> &TheMemoryAccess,
|
|
SPIRVBasicBlock *TheBB)
|
|
: SPIRVInstruction(FixedWords + TheMemoryAccess.size(), OC, TheBB),
|
|
SPIRVMemoryAccess(TheMemoryAccess), MemoryAccess(TheMemoryAccess),
|
|
Target(TheTarget->getId()), Source(TheSource->getId()),
|
|
Size(TheSize->getId()) {
|
|
validate();
|
|
assert(TheBB && "Invalid BB");
|
|
updateModuleVersion();
|
|
}
|
|
// Incomplete constructor
|
|
SPIRVCopyMemorySized()
|
|
: SPIRVInstruction(OC), SPIRVMemoryAccess(), Target(SPIRVID_INVALID),
|
|
Source(SPIRVID_INVALID), Size(0) {
|
|
setHasNoId();
|
|
setHasNoType();
|
|
updateModuleVersion();
|
|
}
|
|
|
|
VersionNumber getRequiredSPIRVVersion() const override {
|
|
if (getSrcAlignment())
|
|
return VersionNumber::SPIRV_1_4;
|
|
return VersionNumber::SPIRV_1_0;
|
|
}
|
|
|
|
SPIRVValue *getSource() { return getValue(Source); }
|
|
SPIRVValue *getTarget() { return getValue(Target); }
|
|
SPIRVValue *getSize() { return getValue(Size); }
|
|
|
|
protected:
|
|
void setWordCount(SPIRVWord TheWordCount) override {
|
|
SPIRVEntry::setWordCount(TheWordCount);
|
|
SPIRVCK(TheWordCount >= FixedWords, InvalidWordCount, "");
|
|
MemoryAccess.resize(TheWordCount - FixedWords);
|
|
}
|
|
|
|
void encode(spv_ostream &O) const override {
|
|
getEncoder(O) << Target << Source << Size << MemoryAccess;
|
|
}
|
|
|
|
void decode(std::istream &I) override {
|
|
getDecoder(I) >> Target >> Source >> Size >> MemoryAccess;
|
|
memoryAccessUpdate(MemoryAccess);
|
|
}
|
|
|
|
void validate() const override { SPIRVInstruction::validate(); }
|
|
|
|
std::vector<SPIRVWord> MemoryAccess;
|
|
SPIRVId Target;
|
|
SPIRVId Source;
|
|
SPIRVId Size;
|
|
};
|
|
|
|
class SPIRVVectorExtractDynamic : public SPIRVInstruction {
|
|
public:
|
|
const static Op OC = OpVectorExtractDynamic;
|
|
// Complete constructor
|
|
SPIRVVectorExtractDynamic(SPIRVId TheId, SPIRVValue *TheVector,
|
|
SPIRVValue *TheIndex, SPIRVBasicBlock *TheBB)
|
|
: SPIRVInstruction(5, OC, TheVector->getType()->getVectorComponentType(),
|
|
TheId, TheBB),
|
|
VectorId(TheVector->getId()), IndexId(TheIndex->getId()) {
|
|
validate();
|
|
assert(TheBB && "Invalid BB");
|
|
}
|
|
// Incomplete constructor
|
|
SPIRVVectorExtractDynamic()
|
|
: SPIRVInstruction(OC), VectorId(SPIRVID_INVALID),
|
|
IndexId(SPIRVID_INVALID) {}
|
|
|
|
SPIRVValue *getVector() { return getValue(VectorId); }
|
|
SPIRVValue *getIndex() const { return getValue(IndexId); }
|
|
std::vector<SPIRVValue *> getOperands() override {
|
|
return {getVector(), getIndex()};
|
|
}
|
|
|
|
protected:
|
|
_SPIRV_DEF_ENCDEC4(Type, Id, VectorId, IndexId)
|
|
void validate() const override {
|
|
SPIRVInstruction::validate();
|
|
if (getValue(VectorId)->isForward())
|
|
return;
|
|
assert(getValueType(VectorId)->isTypeVector());
|
|
}
|
|
SPIRVId VectorId;
|
|
SPIRVId IndexId;
|
|
};
|
|
|
|
class SPIRVVectorInsertDynamic : public SPIRVInstruction {
|
|
public:
|
|
const static Op OC = OpVectorInsertDynamic;
|
|
// Complete constructor
|
|
SPIRVVectorInsertDynamic(SPIRVId TheId, SPIRVValue *TheVector,
|
|
SPIRVValue *TheComponent, SPIRVValue *TheIndex,
|
|
SPIRVBasicBlock *TheBB)
|
|
: SPIRVInstruction(6, OC, TheVector->getType(), TheId, TheBB),
|
|
VectorId(TheVector->getId()), IndexId(TheIndex->getId()),
|
|
ComponentId(TheComponent->getId()) {
|
|
validate();
|
|
assert(TheBB && "Invalid BB");
|
|
}
|
|
// Incomplete constructor
|
|
SPIRVVectorInsertDynamic()
|
|
: SPIRVInstruction(OC), VectorId(SPIRVID_INVALID),
|
|
IndexId(SPIRVID_INVALID), ComponentId(SPIRVID_INVALID) {}
|
|
|
|
SPIRVValue *getVector() { return getValue(VectorId); }
|
|
SPIRVValue *getComponent() { return getValue(ComponentId); }
|
|
SPIRVValue *getIndex() const { return getValue(IndexId); }
|
|
std::vector<SPIRVValue *> getOperands() override {
|
|
return {getVector(), getComponent(), getIndex()};
|
|
}
|
|
|
|
protected:
|
|
_SPIRV_DEF_ENCDEC5(Type, Id, VectorId, ComponentId, IndexId)
|
|
void validate() const override {
|
|
SPIRVInstruction::validate();
|
|
if (getValue(VectorId)->isForward())
|
|
return;
|
|
assert(getValueType(VectorId)->isTypeVector());
|
|
}
|
|
SPIRVId VectorId;
|
|
SPIRVId IndexId;
|
|
SPIRVId ComponentId;
|
|
};
|
|
|
|
class SPIRVVectorShuffleBase : public SPIRVInstTemplateBase {
|
|
public:
|
|
SPIRVValue *getVector1() { return getValue(Ops[0]); }
|
|
SPIRVValue *getVector2() { return getValue(Ops[1]); }
|
|
const std::vector<SPIRVWord> getComponents() const {
|
|
return std::vector<SPIRVWord>(Ops.begin() + 2, Ops.end());
|
|
}
|
|
|
|
protected:
|
|
void validate() const override {
|
|
SPIRVInstruction::validate();
|
|
[[maybe_unused]] SPIRVId Vector1 = Ops[0];
|
|
assert(OpCode == OpVectorShuffle);
|
|
assert(Type->isTypeVector());
|
|
assert(Type->getVectorComponentType() ==
|
|
getValueType(Vector1)->getVectorComponentType());
|
|
assert(Ops.size() - 2 == Type->getVectorComponentCount());
|
|
}
|
|
};
|
|
|
|
typedef SPIRVInstTemplate<SPIRVVectorShuffleBase, OpVectorShuffle, true, 5,
|
|
true>
|
|
SPIRVVectorShuffle;
|
|
|
|
class SPIRVControlBarrier : public SPIRVInstruction {
|
|
public:
|
|
static const Op OC = OpControlBarrier;
|
|
// Complete constructor
|
|
SPIRVControlBarrier(SPIRVValue *TheScope, SPIRVValue *TheMemScope,
|
|
SPIRVValue *TheMemSema, SPIRVBasicBlock *TheBB)
|
|
: SPIRVInstruction(4, OC, TheBB), ExecScope(TheScope->getId()),
|
|
MemScope(TheMemScope->getId()), MemSema(TheMemSema->getId()) {
|
|
validate();
|
|
assert(TheBB && "Invalid BB");
|
|
}
|
|
// Incomplete constructor
|
|
SPIRVControlBarrier() : SPIRVInstruction(OC), ExecScope(ScopeInvocation) {
|
|
setHasNoId();
|
|
setHasNoType();
|
|
}
|
|
void setWordCount(SPIRVWord TheWordCount) override {
|
|
SPIRVEntry::setWordCount(TheWordCount);
|
|
}
|
|
SPIRVValue *getExecScope() const { return getValue(ExecScope); }
|
|
SPIRVValue *getMemScope() const { return getValue(MemScope); }
|
|
SPIRVValue *getMemSemantic() const { return getValue(MemSema); }
|
|
std::vector<SPIRVValue *> getOperands() override {
|
|
std::vector<SPIRVId> Operands;
|
|
Operands.push_back(ExecScope);
|
|
Operands.push_back(MemScope);
|
|
Operands.push_back(MemSema);
|
|
return getValues(Operands);
|
|
}
|
|
|
|
SPIRVCapVec getRequiredCapability() const override {
|
|
if (isDeviceBarrier()) {
|
|
return getVec(internal::CapabilityDeviceBarrierINTEL);
|
|
}
|
|
return SPIRVInstruction::getRequiredCapability();
|
|
}
|
|
std::optional<ExtensionID> getRequiredExtension() const override {
|
|
if (isDeviceBarrier()) {
|
|
return ExtensionID::SPV_INTEL_device_barrier;
|
|
}
|
|
return std::nullopt;
|
|
}
|
|
|
|
protected:
|
|
_SPIRV_DEF_ENCDEC3(ExecScope, MemScope, MemSema)
|
|
void validate() const override {
|
|
assert(OpCode == OC);
|
|
assert(WordCount == 4);
|
|
SPIRVInstruction::validate();
|
|
}
|
|
|
|
bool isDeviceBarrier() const {
|
|
if (!getModule()->isAllowedToUseExtension(
|
|
ExtensionID::SPV_INTEL_device_barrier))
|
|
return false;
|
|
SPIRVValue *ESV = getValue(ExecScope);
|
|
if (ESV && ESV->getOpCode() == OpConstant) {
|
|
if (static_cast<SPIRVConstant *>(ESV)->getZExtIntValue() != ScopeDevice) {
|
|
return false;
|
|
}
|
|
}
|
|
return true;
|
|
}
|
|
|
|
SPIRVId ExecScope;
|
|
SPIRVId MemScope = SPIRVID_INVALID;
|
|
SPIRVId MemSema = SPIRVID_INVALID;
|
|
};
|
|
|
|
template <Op OC> class SPIRVLifetime : public SPIRVInstruction {
|
|
public:
|
|
// Complete constructor
|
|
SPIRVLifetime(SPIRVId TheObject, SPIRVWord TheSize, SPIRVBasicBlock *TheBB)
|
|
: SPIRVInstruction(3, OC, TheBB), Object(TheObject), Size(TheSize) {
|
|
auto ObjType = getValue(Object)->getType();
|
|
// Size must be 0 if Pointer is a pointer to a non-void type or the
|
|
// Addresses capability is not being used. If Size is non-zero, it is the
|
|
// number of bytes of memory whose lifetime is starting. Its type must be an
|
|
// integer type scalar. It is treated as unsigned; if its type has
|
|
// Signedness of 1, its sign bit cannot be set.
|
|
if (!(ObjType->getPointerElementType()->isTypeVoid() ||
|
|
// (void *) is i8* in LLVM IR
|
|
ObjType->getPointerElementType()->isTypeInt(8) ||
|
|
ObjType->getPointerElementType()->isTypeUntypedPointerKHR()) ||
|
|
!Module->hasCapability(CapabilityAddresses))
|
|
Size = 0;
|
|
validate();
|
|
assert(TheBB && "Invalid BB");
|
|
}
|
|
// Incomplete constructor
|
|
SPIRVLifetime()
|
|
: SPIRVInstruction(OC), Object(SPIRVID_INVALID), Size(SPIRVWORD_MAX) {
|
|
setHasNoId();
|
|
setHasNoType();
|
|
}
|
|
SPIRVCapVec getRequiredCapability() const override {
|
|
return getVec(CapabilityKernel);
|
|
}
|
|
SPIRVValue *getObject() { return getValue(Object); }
|
|
SPIRVWord getSize() { return Size; }
|
|
|
|
protected:
|
|
void validate() const override {
|
|
[[maybe_unused]] auto ObjType = getValue(Object)->getType();
|
|
// Type must be an OpTypePointer with Storage Class Function.
|
|
assert(ObjType->isTypePointer() && "Objects type must be a pointer");
|
|
assert(static_cast<SPIRVTypePointer *>(ObjType)->getStorageClass() ==
|
|
StorageClassFunction &&
|
|
"Invalid storage class");
|
|
}
|
|
_SPIRV_DEF_ENCDEC2(Object, Size)
|
|
SPIRVId Object;
|
|
SPIRVWord Size;
|
|
};
|
|
|
|
typedef SPIRVLifetime<OpLifetimeStart> SPIRVLifetimeStart;
|
|
typedef SPIRVLifetime<OpLifetimeStop> SPIRVLifetimeStop;
|
|
|
|
class SPIRVGroupAsyncCopy : public SPIRVInstruction {
|
|
public:
|
|
static const Op OC = OpGroupAsyncCopy;
|
|
static const SPIRVWord WC = 9;
|
|
// Complete constructor
|
|
SPIRVGroupAsyncCopy(SPIRVValue *TheScope, SPIRVId TheId, SPIRVValue *TheDest,
|
|
SPIRVValue *TheSrc, SPIRVValue *TheNumElems,
|
|
SPIRVValue *TheStride, SPIRVValue *TheEvent,
|
|
SPIRVBasicBlock *TheBB)
|
|
: SPIRVInstruction(WC, OC, TheEvent->getType(), TheId, TheBB),
|
|
ExecScope(TheScope->getId()), Destination(TheDest->getId()),
|
|
Source(TheSrc->getId()), NumElements(TheNumElems->getId()),
|
|
Stride(TheStride->getId()), Event(TheEvent->getId()) {
|
|
validate();
|
|
assert(TheBB && "Invalid BB");
|
|
}
|
|
// Incomplete constructor
|
|
SPIRVGroupAsyncCopy()
|
|
: SPIRVInstruction(OC), ExecScope(SPIRVID_INVALID),
|
|
Destination(SPIRVID_INVALID), Source(SPIRVID_INVALID),
|
|
NumElements(SPIRVID_INVALID), Stride(SPIRVID_INVALID),
|
|
Event(SPIRVID_INVALID) {}
|
|
SPIRVValue *getExecScope() const { return getValue(ExecScope); }
|
|
SPIRVValue *getDestination() const { return getValue(Destination); }
|
|
SPIRVValue *getSource() const { return getValue(Source); }
|
|
SPIRVValue *getNumElements() const { return getValue(NumElements); }
|
|
SPIRVValue *getStride() const { return getValue(Stride); }
|
|
SPIRVValue *getEvent() const { return getValue(Event); }
|
|
std::vector<SPIRVValue *> getOperands() override {
|
|
std::vector<SPIRVId> Operands;
|
|
Operands.push_back(ExecScope);
|
|
Operands.push_back(Destination);
|
|
Operands.push_back(Source);
|
|
Operands.push_back(NumElements);
|
|
Operands.push_back(Stride);
|
|
Operands.push_back(Event);
|
|
return getValues(Operands);
|
|
}
|
|
|
|
protected:
|
|
_SPIRV_DEF_ENCDEC8(Type, Id, ExecScope, Destination, Source, NumElements,
|
|
Stride, Event)
|
|
void validate() const override {
|
|
assert(OpCode == OC);
|
|
assert(WordCount == WC);
|
|
SPIRVInstruction::validate();
|
|
}
|
|
SPIRVId ExecScope;
|
|
SPIRVId Destination;
|
|
SPIRVId Source;
|
|
SPIRVId NumElements;
|
|
SPIRVId Stride;
|
|
SPIRVId Event;
|
|
};
|
|
|
|
enum SPIRVOpKind { SPIRVOPK_Id, SPIRVOPK_Literal, SPIRVOPK_Count };
|
|
|
|
class SPIRVDevEnqInstBase : public SPIRVInstTemplateBase {
|
|
public:
|
|
SPIRVCapVec getRequiredCapability() const override {
|
|
return getVec(CapabilityDeviceEnqueue);
|
|
}
|
|
};
|
|
|
|
#define _SPIRV_OP(x, ...) \
|
|
typedef SPIRVInstTemplate<SPIRVDevEnqInstBase, Op##x, __VA_ARGS__> SPIRV##x;
|
|
// CL 2.0 enqueue kernel builtins
|
|
_SPIRV_OP(EnqueueMarker, true, 7)
|
|
_SPIRV_OP(EnqueueKernel, true, 13, true)
|
|
_SPIRV_OP(GetKernelNDrangeSubGroupCount, true, 8)
|
|
_SPIRV_OP(GetKernelNDrangeMaxSubGroupSize, true, 8)
|
|
_SPIRV_OP(GetKernelWorkGroupSize, true, 7)
|
|
_SPIRV_OP(GetKernelPreferredWorkGroupSizeMultiple, true, 7)
|
|
_SPIRV_OP(RetainEvent, false, 2)
|
|
_SPIRV_OP(ReleaseEvent, false, 2)
|
|
_SPIRV_OP(CreateUserEvent, true, 3)
|
|
_SPIRV_OP(IsValidEvent, true, 4)
|
|
_SPIRV_OP(SetUserEventStatus, false, 3)
|
|
_SPIRV_OP(CaptureEventProfilingInfo, false, 4)
|
|
_SPIRV_OP(GetDefaultQueue, true, 3)
|
|
_SPIRV_OP(BuildNDRange, true, 6)
|
|
#undef _SPIRV_OP
|
|
|
|
class SPIRVPipeInstBase : public SPIRVInstTemplateBase {
|
|
public:
|
|
SPIRVCapVec getRequiredCapability() const override {
|
|
return getVec(CapabilityPipes);
|
|
}
|
|
};
|
|
|
|
#define _SPIRV_OP(x, ...) \
|
|
typedef SPIRVInstTemplate<SPIRVPipeInstBase, Op##x, __VA_ARGS__> SPIRV##x;
|
|
// CL 2.0 pipe builtins
|
|
_SPIRV_OP(ReadPipe, true, 7)
|
|
_SPIRV_OP(WritePipe, true, 7)
|
|
_SPIRV_OP(ReservedReadPipe, true, 9)
|
|
_SPIRV_OP(ReservedWritePipe, true, 9)
|
|
_SPIRV_OP(ReserveReadPipePackets, true, 7)
|
|
_SPIRV_OP(ReserveWritePipePackets, true, 7)
|
|
_SPIRV_OP(CommitReadPipe, false, 5)
|
|
_SPIRV_OP(CommitWritePipe, false, 5)
|
|
_SPIRV_OP(IsValidReserveId, true, 4)
|
|
_SPIRV_OP(GetNumPipePackets, true, 6)
|
|
_SPIRV_OP(GetMaxPipePackets, true, 6)
|
|
#undef _SPIRV_OP
|
|
|
|
class SPIRVPipeStorageInstBase : public SPIRVInstTemplateBase {
|
|
public:
|
|
SPIRVCapVec getRequiredCapability() const override {
|
|
return getVec(CapabilityPipeStorage, CapabilityPipes);
|
|
}
|
|
};
|
|
|
|
#define _SPIRV_OP(x, ...) \
|
|
typedef SPIRVInstTemplate<SPIRVPipeStorageInstBase, Op##x, __VA_ARGS__> \
|
|
SPIRV##x;
|
|
|
|
_SPIRV_OP(CreatePipeFromPipeStorage, true, 4)
|
|
#undef _SPIRV_OP
|
|
|
|
class SPIRVGroupInstBase : public SPIRVInstTemplateBase {
|
|
public:
|
|
SPIRVCapVec getRequiredCapability() const override {
|
|
return getVec(CapabilityGroups);
|
|
}
|
|
};
|
|
|
|
#define _SPIRV_OP(x, ...) \
|
|
typedef SPIRVInstTemplate<SPIRVGroupInstBase, Op##x, __VA_ARGS__> SPIRV##x;
|
|
// Group instructions.
|
|
// Even though GroupWaitEvents has Group in its name, it doesn't require the
|
|
// Group capability
|
|
typedef SPIRVInstTemplate<SPIRVInstTemplateBase, OpGroupWaitEvents, false, 4>
|
|
SPIRVGroupWaitEvents;
|
|
_SPIRV_OP(GroupAll, true, 5)
|
|
_SPIRV_OP(GroupAny, true, 5)
|
|
_SPIRV_OP(GroupBroadcast, true, 6)
|
|
_SPIRV_OP(GroupIAdd, true, 6, false, 1)
|
|
_SPIRV_OP(GroupFAdd, true, 6, false, 1)
|
|
_SPIRV_OP(GroupFMin, true, 6, false, 1)
|
|
_SPIRV_OP(GroupUMin, true, 6, false, 1)
|
|
_SPIRV_OP(GroupSMin, true, 6, false, 1)
|
|
_SPIRV_OP(GroupFMax, true, 6, false, 1)
|
|
_SPIRV_OP(GroupUMax, true, 6, false, 1)
|
|
_SPIRV_OP(GroupSMax, true, 6, false, 1)
|
|
_SPIRV_OP(GroupReserveReadPipePackets, true, 8)
|
|
_SPIRV_OP(GroupReserveWritePipePackets, true, 8)
|
|
_SPIRV_OP(GroupCommitReadPipe, false, 6)
|
|
_SPIRV_OP(GroupCommitWritePipe, false, 6)
|
|
#undef _SPIRV_OP
|
|
|
|
class SPIRVGroupNonUniformElectInst : public SPIRVInstTemplateBase {
|
|
public:
|
|
SPIRVCapVec getRequiredCapability() const override {
|
|
return getVec(CapabilityGroupNonUniform);
|
|
}
|
|
};
|
|
|
|
#define _SPIRV_OP(x, ...) \
|
|
typedef SPIRVInstTemplate<SPIRVGroupNonUniformElectInst, Op##x, __VA_ARGS__> \
|
|
SPIRV##x;
|
|
_SPIRV_OP(GroupNonUniformElect, true, 4)
|
|
#undef _SPIRV_OP
|
|
|
|
class SPIRVGroupNonUniformVoteInst : public SPIRVInstTemplateBase {
|
|
public:
|
|
SPIRVCapVec getRequiredCapability() const override {
|
|
return getVec(CapabilityGroupNonUniformVote);
|
|
}
|
|
};
|
|
|
|
#define _SPIRV_OP(x, ...) \
|
|
typedef SPIRVInstTemplate<SPIRVGroupNonUniformVoteInst, Op##x, __VA_ARGS__> \
|
|
SPIRV##x;
|
|
_SPIRV_OP(GroupNonUniformAll, true, 5)
|
|
_SPIRV_OP(GroupNonUniformAny, true, 5)
|
|
_SPIRV_OP(GroupNonUniformAllEqual, true, 5)
|
|
#undef _SPIRV_OP
|
|
|
|
class SPIRVGroupNonUniformBallotInst : public SPIRVInstTemplateBase {
|
|
public:
|
|
SPIRVCapVec getRequiredCapability() const override {
|
|
return getVec(CapabilityGroupNonUniformBallot);
|
|
}
|
|
|
|
VersionNumber getRequiredSPIRVVersion() const override {
|
|
switch (OpCode) {
|
|
case OpGroupNonUniformBroadcast: {
|
|
assert(Ops.size() == 3 && "Expecting (Execution, Value, Id) operands");
|
|
if (!isConstantOpCode(getOperand(2)->getOpCode())) {
|
|
// Before version 1.5, Id must come from a constant instruction.
|
|
return VersionNumber::SPIRV_1_5;
|
|
}
|
|
break;
|
|
}
|
|
default:
|
|
break;
|
|
}
|
|
return VersionNumber::SPIRV_1_3;
|
|
}
|
|
};
|
|
|
|
#define _SPIRV_OP(x, ...) \
|
|
typedef SPIRVInstTemplate<SPIRVGroupNonUniformBallotInst, Op##x, \
|
|
__VA_ARGS__> \
|
|
SPIRV##x;
|
|
_SPIRV_OP(GroupNonUniformBroadcast, true, 6)
|
|
_SPIRV_OP(GroupNonUniformBroadcastFirst, true, 5)
|
|
_SPIRV_OP(GroupNonUniformBallot, true, 5)
|
|
_SPIRV_OP(GroupNonUniformInverseBallot, true, 5)
|
|
_SPIRV_OP(GroupNonUniformBallotBitExtract, true, 6)
|
|
_SPIRV_OP(GroupNonUniformBallotBitCount, true, 6, false, 1)
|
|
_SPIRV_OP(GroupNonUniformBallotFindLSB, true, 5)
|
|
_SPIRV_OP(GroupNonUniformBallotFindMSB, true, 5)
|
|
#undef _SPIRV_OP
|
|
|
|
class SPIRVGroupNonUniformArithmeticInst : public SPIRVInstTemplateBase {
|
|
public:
|
|
void setOpWords(const std::vector<SPIRVWord> &Ops) override {
|
|
SPIRVInstTemplateBase::setOpWords(Ops);
|
|
SPIRVGroupOperationKind GroupOp;
|
|
if (getSPIRVGroupOperation(GroupOp)) {
|
|
if (GroupOp == GroupOperationClusteredReduce)
|
|
Module->addCapability(CapabilityGroupNonUniformClustered);
|
|
else
|
|
Module->addCapability(CapabilityGroupNonUniformArithmetic);
|
|
} else
|
|
llvm_unreachable(
|
|
"GroupNonUniformArithmeticInst has no group operation operand!");
|
|
}
|
|
};
|
|
|
|
#define _SPIRV_OP(x, ...) \
|
|
typedef SPIRVInstTemplate<SPIRVGroupNonUniformArithmeticInst, Op##x, \
|
|
__VA_ARGS__> \
|
|
SPIRV##x;
|
|
_SPIRV_OP(GroupNonUniformIAdd, true, 6, true, 1)
|
|
_SPIRV_OP(GroupNonUniformFAdd, true, 6, true, 1)
|
|
_SPIRV_OP(GroupNonUniformIMul, true, 6, true, 1)
|
|
_SPIRV_OP(GroupNonUniformFMul, true, 6, true, 1)
|
|
_SPIRV_OP(GroupNonUniformSMin, true, 6, true, 1)
|
|
_SPIRV_OP(GroupNonUniformUMin, true, 6, true, 1)
|
|
_SPIRV_OP(GroupNonUniformFMin, true, 6, true, 1)
|
|
_SPIRV_OP(GroupNonUniformSMax, true, 6, true, 1)
|
|
_SPIRV_OP(GroupNonUniformUMax, true, 6, true, 1)
|
|
_SPIRV_OP(GroupNonUniformFMax, true, 6, true, 1)
|
|
_SPIRV_OP(GroupNonUniformBitwiseAnd, true, 6, true, 1)
|
|
_SPIRV_OP(GroupNonUniformBitwiseOr, true, 6, true, 1)
|
|
_SPIRV_OP(GroupNonUniformBitwiseXor, true, 6, true, 1)
|
|
_SPIRV_OP(GroupNonUniformLogicalAnd, true, 6, true, 1)
|
|
_SPIRV_OP(GroupNonUniformLogicalOr, true, 6, true, 1)
|
|
_SPIRV_OP(GroupNonUniformLogicalXor, true, 6, true, 1)
|
|
#undef _SPIRV_OP
|
|
|
|
class SPIRVGroupNonUniformShuffleInst : public SPIRVInstTemplateBase {
|
|
public:
|
|
SPIRVCapVec getRequiredCapability() const override {
|
|
return getVec(CapabilityGroupNonUniformShuffle);
|
|
}
|
|
};
|
|
|
|
#define _SPIRV_OP(x, ...) \
|
|
typedef SPIRVInstTemplate<SPIRVGroupNonUniformShuffleInst, Op##x, \
|
|
__VA_ARGS__> \
|
|
SPIRV##x;
|
|
_SPIRV_OP(GroupNonUniformShuffle, true, 6)
|
|
_SPIRV_OP(GroupNonUniformShuffleXor, true, 6)
|
|
#undef _SPIRV_OP
|
|
|
|
class SPIRVGroupNonUniformShuffleRelativeInst : public SPIRVInstTemplateBase {
|
|
public:
|
|
SPIRVCapVec getRequiredCapability() const override {
|
|
return getVec(CapabilityGroupNonUniformShuffleRelative);
|
|
}
|
|
};
|
|
|
|
#define _SPIRV_OP(x, ...) \
|
|
typedef SPIRVInstTemplate<SPIRVGroupNonUniformShuffleRelativeInst, Op##x, \
|
|
__VA_ARGS__> \
|
|
SPIRV##x;
|
|
_SPIRV_OP(GroupNonUniformShuffleUp, true, 6)
|
|
_SPIRV_OP(GroupNonUniformShuffleDown, true, 6)
|
|
#undef _SPIRV_OP
|
|
|
|
class SPIRVGroupNonUniformRotateKHRInst : public SPIRVInstTemplateBase {
|
|
public:
|
|
SPIRVCapVec getRequiredCapability() const override {
|
|
return getVec(CapabilityGroupNonUniformRotateKHR);
|
|
}
|
|
|
|
std::optional<ExtensionID> getRequiredExtension() const override {
|
|
return ExtensionID::SPV_KHR_subgroup_rotate;
|
|
}
|
|
};
|
|
|
|
#define _SPIRV_OP(x, ...) \
|
|
typedef SPIRVInstTemplate<SPIRVGroupNonUniformRotateKHRInst, Op##x, \
|
|
__VA_ARGS__> \
|
|
SPIRV##x;
|
|
_SPIRV_OP(GroupNonUniformRotateKHR, true, 6, true)
|
|
#undef _SPIRV_OP
|
|
|
|
class SPIRVBlockingPipesIntelInst : public SPIRVInstTemplateBase {
|
|
protected:
|
|
SPIRVCapVec getRequiredCapability() const override {
|
|
return getVec(CapabilityBlockingPipesINTEL);
|
|
}
|
|
|
|
std::optional<ExtensionID> getRequiredExtension() const override {
|
|
return ExtensionID::SPV_INTEL_blocking_pipes;
|
|
}
|
|
};
|
|
|
|
#define _SPIRV_OP(x, ...) \
|
|
typedef SPIRVInstTemplate<SPIRVBlockingPipesIntelInst, Op##x, __VA_ARGS__> \
|
|
SPIRV##x;
|
|
_SPIRV_OP(ReadPipeBlockingINTEL, false, 5)
|
|
_SPIRV_OP(WritePipeBlockingINTEL, false, 5)
|
|
#undef _SPIRV_OP
|
|
|
|
class SPIRVFixedPointIntelInst : public SPIRVInstTemplateBase {
|
|
protected:
|
|
SPIRVCapVec getRequiredCapability() const override {
|
|
return getVec(CapabilityArbitraryPrecisionFixedPointINTEL);
|
|
}
|
|
|
|
std::optional<ExtensionID> getRequiredExtension() const override {
|
|
return ExtensionID::SPV_INTEL_arbitrary_precision_fixed_point;
|
|
}
|
|
};
|
|
|
|
#define _SPIRV_OP(x, ...) \
|
|
typedef SPIRVInstTemplate<SPIRVFixedPointIntelInst, Op##x, __VA_ARGS__> \
|
|
SPIRV##x;
|
|
_SPIRV_OP(FixedSqrtINTEL, true, 9)
|
|
_SPIRV_OP(FixedRecipINTEL, true, 9)
|
|
_SPIRV_OP(FixedRsqrtINTEL, true, 9)
|
|
_SPIRV_OP(FixedSinINTEL, true, 9)
|
|
_SPIRV_OP(FixedCosINTEL, true, 9)
|
|
_SPIRV_OP(FixedSinCosINTEL, true, 9)
|
|
_SPIRV_OP(FixedSinPiINTEL, true, 9)
|
|
_SPIRV_OP(FixedCosPiINTEL, true, 9)
|
|
_SPIRV_OP(FixedSinCosPiINTEL, true, 9)
|
|
_SPIRV_OP(FixedLogINTEL, true, 9)
|
|
_SPIRV_OP(FixedExpINTEL, true, 9)
|
|
#undef _SPIRV_OP
|
|
|
|
class SPIRVArbFloatIntelInst : public SPIRVInstTemplateBase {
|
|
protected:
|
|
SPIRVCapVec getRequiredCapability() const override {
|
|
return getVec(CapabilityArbitraryPrecisionFloatingPointINTEL);
|
|
}
|
|
|
|
std::optional<ExtensionID> getRequiredExtension() const override {
|
|
return ExtensionID::SPV_INTEL_arbitrary_precision_floating_point;
|
|
}
|
|
};
|
|
|
|
#define _SPIRV_OP(x, ...) \
|
|
typedef SPIRVInstTemplate<SPIRVArbFloatIntelInst, \
|
|
OpArbitraryFloat##x##INTEL, __VA_ARGS__> \
|
|
SPIRVArbitraryFloat##x##INTEL;
|
|
_SPIRV_OP(Cast, true, 9)
|
|
_SPIRV_OP(CastFromInt, true, 9)
|
|
_SPIRV_OP(CastToInt, true, 9)
|
|
_SPIRV_OP(Add, true, 11)
|
|
_SPIRV_OP(Sub, true, 11)
|
|
_SPIRV_OP(Mul, true, 11)
|
|
_SPIRV_OP(Div, true, 11)
|
|
_SPIRV_OP(GT, true, 7)
|
|
_SPIRV_OP(GE, true, 7)
|
|
_SPIRV_OP(LT, true, 7)
|
|
_SPIRV_OP(LE, true, 7)
|
|
_SPIRV_OP(EQ, true, 7)
|
|
_SPIRV_OP(Recip, true, 9)
|
|
_SPIRV_OP(RSqrt, true, 9)
|
|
_SPIRV_OP(Cbrt, true, 9)
|
|
_SPIRV_OP(Hypot, true, 11)
|
|
_SPIRV_OP(Sqrt, true, 9)
|
|
_SPIRV_OP(Log, true, 9)
|
|
_SPIRV_OP(Log2, true, 9)
|
|
_SPIRV_OP(Log10, true, 9)
|
|
_SPIRV_OP(Log1p, true, 9)
|
|
_SPIRV_OP(Exp, true, 9)
|
|
_SPIRV_OP(Exp2, true, 9)
|
|
_SPIRV_OP(Exp10, true, 9)
|
|
_SPIRV_OP(Expm1, true, 9)
|
|
_SPIRV_OP(Sin, true, 9)
|
|
_SPIRV_OP(Cos, true, 9)
|
|
_SPIRV_OP(SinCos, true, 9)
|
|
_SPIRV_OP(SinPi, true, 9)
|
|
_SPIRV_OP(CosPi, true, 9)
|
|
_SPIRV_OP(SinCosPi, true, 9)
|
|
_SPIRV_OP(ASin, true, 9)
|
|
_SPIRV_OP(ASinPi, true, 9)
|
|
_SPIRV_OP(ACos, true, 9)
|
|
_SPIRV_OP(ACosPi, true, 9)
|
|
_SPIRV_OP(ATan, true, 9)
|
|
_SPIRV_OP(ATanPi, true, 9)
|
|
_SPIRV_OP(ATan2, true, 11)
|
|
_SPIRV_OP(Pow, true, 11)
|
|
_SPIRV_OP(PowR, true, 11)
|
|
_SPIRV_OP(PowN, true, 11)
|
|
#undef _SPIRV_OP
|
|
|
|
class SPIRVAtomicInstBase : public SPIRVInstTemplateBase {
|
|
public:
|
|
SPIRVCapVec getRequiredCapability() const override {
|
|
// Most of the atomic instructions require a specific capability when
|
|
// operating on 64-bit integers.
|
|
// In SPIRV 1.2 spec, only 2 atomic instructions have no result type:
|
|
// 1. OpAtomicStore - need to check type of the Value operand
|
|
// 2. OpAtomicFlagClear - doesn't require Int64Atomics capability.
|
|
// Besides, OpAtomicCompareExchangeWeak, OpAtomicFlagTestAndSet and
|
|
// OpAtomicFlagClear instructions require the "kernel" capability. But this
|
|
// capability should be added by setting the OpenCL memory model.
|
|
if (hasType()) {
|
|
if (getType()->isTypeInt(64))
|
|
return {CapabilityInt64Atomics};
|
|
if (getType()->isTypeInt(16) &&
|
|
Module->isAllowedToUseExtension(
|
|
ExtensionID::SPV_INTEL_16bit_atomics)) {
|
|
Module->addExtension(ExtensionID::SPV_INTEL_16bit_atomics);
|
|
return {internal::CapabilityInt16AtomicsINTEL};
|
|
}
|
|
}
|
|
return {};
|
|
}
|
|
|
|
// Overriding the following method because of particular OpAtomic*
|
|
// instructions that declare additional capabilities, e.g. based on operand
|
|
// types.
|
|
void setOpWords(const std::vector<SPIRVWord> &TheOps) override {
|
|
SPIRVInstTemplateBase::setOpWords(TheOps);
|
|
for (auto RC : getRequiredCapability())
|
|
Module->addCapability(RC);
|
|
}
|
|
|
|
void validate() const override {
|
|
if (OpCode == OpAtomicCompareExchangeWeak)
|
|
assert(this->getModule()->getSPIRVVersion() < VersionNumber::SPIRV_1_4 &&
|
|
"OpAtomicCompareExchangeWeak is removed starting from SPIR-V 1.4");
|
|
}
|
|
|
|
// This method is needed for correct translation of atomic instructions when
|
|
// SPV_KHR_untyped_pointers is enabled.
|
|
// The interpreted data type for untyped pointers is specified by the Result
|
|
// Type if it exists, or from the type of the object being stored in other
|
|
// case.
|
|
SPIRVType *getSemanticType() {
|
|
switch (OpCode) {
|
|
case OpAtomicStore:
|
|
// Get type of Value operand
|
|
return getOperand(3)->getType();
|
|
default: {
|
|
if (hasType())
|
|
return getType();
|
|
return nullptr;
|
|
}
|
|
}
|
|
}
|
|
};
|
|
|
|
// This specialization will handle smaller set of compare-and-swap instructions
|
|
// that require only one capability. The instructions are: OpAtomicLoad,
|
|
// OpAtomicStore, OpAtomicExchange, OpAtomicCompareExchange and
|
|
// OpAtomicCompareExchangeWeak.
|
|
class SPIRVAtomicCompareExchangeInstructions : public SPIRVAtomicInstBase {
|
|
public:
|
|
SPIRVCapVec getRequiredCapability() const override {
|
|
if (hasType() && getType()->isTypeInt(16) &&
|
|
this->getModule()->isAllowedToUseExtension(
|
|
ExtensionID::SPV_INTEL_16bit_atomics)) {
|
|
Module->addExtension(ExtensionID::SPV_INTEL_16bit_atomics);
|
|
return {internal::CapabilityAtomicInt16CompareExchangeINTEL};
|
|
}
|
|
return SPIRVAtomicInstBase::getRequiredCapability();
|
|
}
|
|
};
|
|
|
|
class SPIRVAtomicStoreInst : public SPIRVAtomicCompareExchangeInstructions {
|
|
public:
|
|
// Overriding the following method because of 'const'-related
|
|
// issues with overriding getRequiredCapability(). TODO: Resolve.
|
|
void setOpWords(const std::vector<SPIRVWord> &TheOps) override {
|
|
SPIRVInstTemplateBase::setOpWords(TheOps);
|
|
static const unsigned ValueOperandIndex = 3;
|
|
if (getOperand(ValueOperandIndex)->getType()->isTypeInt(64))
|
|
Module->addCapability(CapabilityInt64Atomics);
|
|
}
|
|
};
|
|
|
|
class SPIRVAtomicFAddEXTInst : public SPIRVAtomicInstBase {
|
|
public:
|
|
std::optional<ExtensionID> getRequiredExtension() const override {
|
|
assert(hasType());
|
|
if (getType()->isTypeFloat(16, FPEncodingBFloat16KHR))
|
|
Module->addExtension(ExtensionID::SPV_INTEL_16bit_atomics);
|
|
if (getType()->isTypeFloat(16))
|
|
return ExtensionID::SPV_EXT_shader_atomic_float16_add;
|
|
return ExtensionID::SPV_EXT_shader_atomic_float_add;
|
|
}
|
|
|
|
SPIRVCapVec getRequiredCapability() const override {
|
|
assert(hasType());
|
|
if (getType()->isTypeFloat(16, FPEncodingBFloat16KHR))
|
|
return {internal::CapabilityAtomicBFloat16AddINTEL};
|
|
if (getType()->isTypeFloat(16))
|
|
return {CapabilityAtomicFloat16AddEXT};
|
|
if (getType()->isTypeFloat(32))
|
|
return {CapabilityAtomicFloat32AddEXT};
|
|
if (getType()->isTypeFloat(64))
|
|
return {CapabilityAtomicFloat64AddEXT};
|
|
// TODO: Temporary workaround to allow both FP16 and BF16 atomics.
|
|
if (getType()->isTypeVectorFloat() &&
|
|
(getType()->getVectorComponentType()->isTypeFloat(16) ||
|
|
getType()->getVectorComponentType()->isTypeFloat(16,
|
|
FPEncodingBFloat16KHR)))
|
|
return {CapabilityAtomicFloat16VectorNV};
|
|
llvm_unreachable(
|
|
"AtomicFAddEXT can only be generated for bf16, f16, f32, f64 types");
|
|
}
|
|
};
|
|
|
|
class SPIRVAtomicFMinMaxEXTBase : public SPIRVAtomicInstBase {
|
|
public:
|
|
std::optional<ExtensionID> getRequiredExtension() const override {
|
|
if (getType()->isTypeFloat(16, FPEncodingBFloat16KHR))
|
|
Module->addExtension(ExtensionID::SPV_INTEL_16bit_atomics);
|
|
return ExtensionID::SPV_EXT_shader_atomic_float_min_max;
|
|
}
|
|
|
|
SPIRVCapVec getRequiredCapability() const override {
|
|
assert(hasType());
|
|
if (getType()->isTypeFloat(16, FPEncodingBFloat16KHR))
|
|
return {internal::CapabilityAtomicBFloat16MinMaxINTEL};
|
|
if (getType()->isTypeFloat(16))
|
|
return {CapabilityAtomicFloat16MinMaxEXT};
|
|
if (getType()->isTypeFloat(32))
|
|
return {CapabilityAtomicFloat32MinMaxEXT};
|
|
if (getType()->isTypeFloat(64))
|
|
return {CapabilityAtomicFloat64MinMaxEXT};
|
|
llvm_unreachable("AtomicF(Min|Max)EXT can only be generated for bf16, f16, "
|
|
"f32, f64 types");
|
|
}
|
|
};
|
|
|
|
#define _SPIRV_OP(x, ...) \
|
|
typedef SPIRVInstTemplate<SPIRVAtomicInstBase, Op##x, __VA_ARGS__> SPIRV##x;
|
|
// Atomic builtins
|
|
_SPIRV_OP(AtomicFlagTestAndSet, true, 6)
|
|
_SPIRV_OP(AtomicFlagClear, false, 4)
|
|
_SPIRV_OP(AtomicIIncrement, true, 6)
|
|
_SPIRV_OP(AtomicIDecrement, true, 6)
|
|
_SPIRV_OP(AtomicIAdd, true, 7)
|
|
_SPIRV_OP(AtomicISub, true, 7)
|
|
_SPIRV_OP(AtomicUMin, true, 7)
|
|
_SPIRV_OP(AtomicUMax, true, 7)
|
|
_SPIRV_OP(AtomicSMin, true, 7)
|
|
_SPIRV_OP(AtomicSMax, true, 7)
|
|
_SPIRV_OP(AtomicAnd, true, 7)
|
|
_SPIRV_OP(AtomicOr, true, 7)
|
|
_SPIRV_OP(AtomicXor, true, 7)
|
|
_SPIRV_OP(MemoryBarrier, false, 3)
|
|
#undef _SPIRV_OP
|
|
#define _SPIRV_OP(x, BaseClass, ...) \
|
|
typedef SPIRVInstTemplate<SPIRV##BaseClass, Op##x, __VA_ARGS__> SPIRV##x;
|
|
// Specialized atomic builtins
|
|
_SPIRV_OP(AtomicLoad, AtomicCompareExchangeInstructions, true, 6)
|
|
_SPIRV_OP(AtomicStore, AtomicStoreInst, false, 5)
|
|
_SPIRV_OP(AtomicExchange, AtomicCompareExchangeInstructions, true, 7)
|
|
_SPIRV_OP(AtomicCompareExchange, AtomicCompareExchangeInstructions, true, 9)
|
|
_SPIRV_OP(AtomicCompareExchangeWeak, AtomicCompareExchangeInstructions, true, 9)
|
|
_SPIRV_OP(AtomicFAddEXT, AtomicFAddEXTInst, true, 7)
|
|
_SPIRV_OP(AtomicFMinEXT, AtomicFMinMaxEXTBase, true, 7)
|
|
_SPIRV_OP(AtomicFMaxEXT, AtomicFMinMaxEXTBase, true, 7)
|
|
#undef _SPIRV_OP
|
|
|
|
class SPIRVImageInstBase : public SPIRVInstTemplateBase {
|
|
public:
|
|
SPIRVCapVec getRequiredCapability() const override {
|
|
return getVec(CapabilityImageBasic);
|
|
}
|
|
bool hasImageOperand(ImageOperandsMask Mask) const;
|
|
|
|
protected:
|
|
void setOpWords(const std::vector<SPIRVWord> &OpsArg) override;
|
|
};
|
|
|
|
#define _SPIRV_OP(x, ...) \
|
|
typedef SPIRVInstTemplate<SPIRVImageInstBase, Op##x, __VA_ARGS__> SPIRV##x;
|
|
// Image instructions
|
|
_SPIRV_OP(SampledImage, true, 5)
|
|
_SPIRV_OP(ImageSampleImplicitLod, true, 5, true)
|
|
_SPIRV_OP(ImageSampleExplicitLod, true, 7, true, 2)
|
|
_SPIRV_OP(ImageRead, true, 5, true, 2)
|
|
_SPIRV_OP(ImageWrite, false, 4, true, 3)
|
|
_SPIRV_OP(Image, true, 4)
|
|
_SPIRV_OP(ImageQueryFormat, true, 4)
|
|
_SPIRV_OP(ImageQueryOrder, true, 4)
|
|
_SPIRV_OP(ImageQuerySizeLod, true, 5)
|
|
_SPIRV_OP(ImageQuerySize, true, 4)
|
|
_SPIRV_OP(ImageQueryLod, true, 5)
|
|
_SPIRV_OP(ImageQueryLevels, true, 4)
|
|
_SPIRV_OP(ImageQuerySamples, true, 4)
|
|
#undef _SPIRV_OP
|
|
|
|
#define _SPIRV_OP(x, ...) \
|
|
typedef SPIRVInstTemplate<SPIRVInstTemplateBase, Op##x, __VA_ARGS__> SPIRV##x;
|
|
// Other instructions
|
|
_SPIRV_OP(GenericPtrMemSemantics, true, 4, false)
|
|
_SPIRV_OP(GenericCastToPtrExplicit, true, 5, false, 1)
|
|
#undef _SPIRV_OP
|
|
|
|
class SPIRVSpecConstantOpBase : public SPIRVInstTemplateBase {
|
|
public:
|
|
bool isOperandLiteral(unsigned I) const override {
|
|
// If SpecConstant results from CompositeExtract/Insert operation, then all
|
|
// operands are expected to be literals.
|
|
switch (Ops[0]) { // Opcode of underlying SpecConstant operation
|
|
case OpCompositeExtract:
|
|
case OpCompositeInsert:
|
|
return true;
|
|
default:
|
|
return SPIRVInstTemplateBase::isOperandLiteral(I);
|
|
}
|
|
}
|
|
};
|
|
|
|
typedef SPIRVInstTemplate<SPIRVSpecConstantOpBase, OpSpecConstantOp, true, 4,
|
|
true, 0>
|
|
SPIRVSpecConstantOp;
|
|
|
|
class SPIRVAssumeTrueKHR : public SPIRVInstruction {
|
|
public:
|
|
static const Op OC = OpAssumeTrueKHR;
|
|
static const SPIRVWord FixedWordCount = 2;
|
|
|
|
SPIRVAssumeTrueKHR(SPIRVId TheCondition, SPIRVBasicBlock *BB)
|
|
: SPIRVInstruction(FixedWordCount, OC, BB), ConditionId(TheCondition) {
|
|
validate();
|
|
setHasNoId();
|
|
setHasNoType();
|
|
assert(BB && "Invalid BB");
|
|
}
|
|
|
|
SPIRVAssumeTrueKHR() : SPIRVInstruction(OC), ConditionId(SPIRVID_MAX) {
|
|
setHasNoId();
|
|
setHasNoType();
|
|
}
|
|
|
|
SPIRVCapVec getRequiredCapability() const override {
|
|
return getVec(CapabilityExpectAssumeKHR);
|
|
}
|
|
|
|
std::optional<ExtensionID> getRequiredExtension() const override {
|
|
return ExtensionID::SPV_KHR_expect_assume;
|
|
}
|
|
|
|
SPIRVValue *getCondition() const { return getValue(ConditionId); }
|
|
|
|
void setWordCount(SPIRVWord TheWordCount) override {
|
|
SPIRVEntry::setWordCount(TheWordCount);
|
|
}
|
|
_SPIRV_DEF_ENCDEC1(ConditionId)
|
|
|
|
protected:
|
|
SPIRVId ConditionId;
|
|
};
|
|
|
|
class SPIRVExpectKHRInstBase : public SPIRVInstTemplateBase {
|
|
protected:
|
|
SPIRVCapVec getRequiredCapability() const override {
|
|
return getVec(CapabilityExpectAssumeKHR);
|
|
}
|
|
|
|
std::optional<ExtensionID> getRequiredExtension() const override {
|
|
return ExtensionID::SPV_KHR_expect_assume;
|
|
}
|
|
};
|
|
|
|
#define _SPIRV_OP(x, ...) \
|
|
typedef SPIRVInstTemplate<SPIRVExpectKHRInstBase, Op##x, __VA_ARGS__> \
|
|
SPIRV##x;
|
|
_SPIRV_OP(ExpectKHR, true, 5)
|
|
#undef _SPIRV_OP
|
|
|
|
class SPIRVDotKHRBase : public SPIRVInstTemplateBase {
|
|
protected:
|
|
SPIRVCapVec getRequiredCapability() const override {
|
|
// Both vector operands must have the same type, so analyzing the
|
|
// first operand will suffice.
|
|
SPIRVCapabilityKind ArgCap = getRequiredCapabilityForOperand(Ops[0]);
|
|
return getVec(ArgCap, CapabilityDotProductKHR);
|
|
}
|
|
|
|
std::optional<ExtensionID> getRequiredExtension() const override {
|
|
if (!Module->isAllowedToUseVersion(VersionNumber::SPIRV_1_6))
|
|
return ExtensionID::SPV_KHR_integer_dot_product;
|
|
return {};
|
|
}
|
|
|
|
void validate() const override {
|
|
SPIRVInstruction::validate();
|
|
SPIRVId Vec1 = Ops[0];
|
|
SPIRVId Vec2 = Ops[1];
|
|
(void)Vec1;
|
|
(void)Vec2;
|
|
|
|
assert(getValueType(Vec1) == getValueType(Vec2) &&
|
|
"Input vectors must have the same type");
|
|
assert(getType()->isTypeInt() && "Result type must be an integer type");
|
|
assert(!getType()->isTypeVector() && "Result type must be scalar");
|
|
}
|
|
|
|
private:
|
|
bool isAccSat() const {
|
|
return (OpCode == OpSDotAccSatKHR || OpCode == OpUDotAccSatKHR ||
|
|
OpCode == OpSUDotAccSatKHR);
|
|
}
|
|
|
|
std::optional<PackedVectorFormat> getPackedVectorFormat() const {
|
|
size_t PackFmtIdx = 2;
|
|
if (isAccSat()) {
|
|
// AccSat instructions have an additional Accumulator operand.
|
|
PackFmtIdx++;
|
|
}
|
|
|
|
if (PackFmtIdx == Ops.size() - 1)
|
|
return static_cast<PackedVectorFormat>(Ops[PackFmtIdx]);
|
|
|
|
return std::nullopt;
|
|
}
|
|
|
|
SPIRVCapabilityKind getRequiredCapabilityForOperand(SPIRVId ArgId) const {
|
|
const SPIRVType *T = getValueType(ArgId);
|
|
if (auto PackFmt = getPackedVectorFormat()) {
|
|
switch (*PackFmt) {
|
|
case PackedVectorFormatPackedVectorFormat4x8BitKHR:
|
|
assert(!T->isTypeVector() && T->isTypeInt() && T->getBitWidth() == 32 &&
|
|
"Type does not match pack format");
|
|
return CapabilityDotProductInput4x8BitPackedKHR;
|
|
case PackedVectorFormatMax:
|
|
break;
|
|
}
|
|
llvm_unreachable("Unknown Packed Vector Format");
|
|
}
|
|
|
|
if (T->isTypeVector()) {
|
|
const SPIRVType *EltT = T->getVectorComponentType();
|
|
if (T->getVectorComponentCount() == 4 && EltT->isTypeInt() &&
|
|
EltT->getBitWidth() == 8)
|
|
return CapabilityDotProductInput4x8BitKHR;
|
|
if (EltT->isTypeInt())
|
|
return CapabilityDotProductInputAllKHR;
|
|
}
|
|
|
|
llvm_unreachable("No mapping for argument type to capability.");
|
|
}
|
|
|
|
VersionNumber getRequiredSPIRVVersion() const override {
|
|
if (Module->isAllowedToUseVersion(VersionNumber::SPIRV_1_6))
|
|
return VersionNumber::SPIRV_1_6;
|
|
return VersionNumber::SPIRV_1_0;
|
|
}
|
|
};
|
|
|
|
#define _SPIRV_OP(x, ...) \
|
|
typedef SPIRVInstTemplate<SPIRVDotKHRBase, Op##x, __VA_ARGS__> SPIRV##x;
|
|
_SPIRV_OP(SDotKHR, true, 5, true, 2)
|
|
_SPIRV_OP(UDotKHR, true, 5, true, 2)
|
|
_SPIRV_OP(SUDotKHR, true, 5, true, 2)
|
|
_SPIRV_OP(SDotAccSatKHR, true, 6, true, 3)
|
|
_SPIRV_OP(UDotAccSatKHR, true, 6, true, 3)
|
|
_SPIRV_OP(SUDotAccSatKHR, true, 6, true, 3)
|
|
#undef _SPIRV_OP
|
|
|
|
class SPIRVBitOp : public SPIRVInstTemplateBase {
|
|
public:
|
|
SPIRVCapVec getRequiredCapability() const override {
|
|
return getVec(CapabilityBitInstructions);
|
|
}
|
|
|
|
std::optional<ExtensionID> getRequiredExtension() const override {
|
|
for (auto Cap : getRequiredCapability()) {
|
|
if (Cap == CapabilityBitInstructions)
|
|
return ExtensionID::SPV_KHR_bit_instructions;
|
|
}
|
|
return std::nullopt;
|
|
}
|
|
};
|
|
|
|
#define _SPIRV_OP(x, ...) \
|
|
typedef SPIRVInstTemplate<SPIRVBitOp, Op##x, __VA_ARGS__> SPIRV##x;
|
|
_SPIRV_OP(BitFieldInsert, true, 7)
|
|
_SPIRV_OP(BitFieldSExtract, true, 6)
|
|
_SPIRV_OP(BitFieldUExtract, true, 6)
|
|
_SPIRV_OP(BitReverse, true, 4)
|
|
#undef _SPIRV_OP
|
|
|
|
class SPIRVSubgroupShuffleINTELInstBase : public SPIRVInstTemplateBase {
|
|
protected:
|
|
SPIRVCapVec getRequiredCapability() const override {
|
|
return getVec(CapabilitySubgroupShuffleINTEL);
|
|
}
|
|
|
|
std::optional<ExtensionID> getRequiredExtension() const override {
|
|
return ExtensionID::SPV_INTEL_subgroups;
|
|
}
|
|
};
|
|
|
|
#define _SPIRV_OP(x, ...) \
|
|
typedef SPIRVInstTemplate<SPIRVSubgroupShuffleINTELInstBase, Op##x, \
|
|
__VA_ARGS__> \
|
|
SPIRV##x;
|
|
// Intel Subgroup Shuffle Instructions
|
|
_SPIRV_OP(SubgroupShuffleINTEL, true, 5)
|
|
_SPIRV_OP(SubgroupShuffleDownINTEL, true, 6)
|
|
_SPIRV_OP(SubgroupShuffleUpINTEL, true, 6)
|
|
_SPIRV_OP(SubgroupShuffleXorINTEL, true, 5)
|
|
#undef _SPIRV_OP
|
|
|
|
class SPIRVSubgroupBufferBlockIOINTELInstBase : public SPIRVInstTemplateBase {
|
|
protected:
|
|
SPIRVCapVec getRequiredCapability() const override {
|
|
return getVec(CapabilitySubgroupBufferBlockIOINTEL);
|
|
}
|
|
|
|
std::optional<ExtensionID> getRequiredExtension() const override {
|
|
return ExtensionID::SPV_INTEL_subgroups;
|
|
}
|
|
};
|
|
|
|
#define _SPIRV_OP(x, ...) \
|
|
typedef SPIRVInstTemplate<SPIRVSubgroupBufferBlockIOINTELInstBase, Op##x, \
|
|
__VA_ARGS__> \
|
|
SPIRV##x;
|
|
// Intel Subgroup Buffer Block Read and Write Instructions
|
|
_SPIRV_OP(SubgroupBlockReadINTEL, true, 4)
|
|
_SPIRV_OP(SubgroupBlockWriteINTEL, false, 3)
|
|
#undef _SPIRV_OP
|
|
|
|
class SPIRVSubgroupImageBlockIOINTELInstBase : public SPIRVInstTemplateBase {
|
|
protected:
|
|
SPIRVCapVec getRequiredCapability() const override {
|
|
return getVec(CapabilitySubgroupImageBlockIOINTEL);
|
|
}
|
|
|
|
std::optional<ExtensionID> getRequiredExtension() const override {
|
|
return ExtensionID::SPV_INTEL_subgroups;
|
|
}
|
|
};
|
|
|
|
#define _SPIRV_OP(x, ...) \
|
|
typedef SPIRVInstTemplate<SPIRVSubgroupImageBlockIOINTELInstBase, Op##x, \
|
|
__VA_ARGS__> \
|
|
SPIRV##x;
|
|
// Intel Subgroup Image Block Read and Write Instructions
|
|
_SPIRV_OP(SubgroupImageBlockReadINTEL, true, 5)
|
|
_SPIRV_OP(SubgroupImageBlockWriteINTEL, false, 4)
|
|
#undef _SPIRV_OP
|
|
|
|
class SPIRVSubgroupImageMediaBlockIOINTELInstBase
|
|
: public SPIRVInstTemplateBase {
|
|
protected:
|
|
SPIRVCapVec getRequiredCapability() const override {
|
|
return getVec(CapabilitySubgroupImageMediaBlockIOINTEL);
|
|
}
|
|
std::optional<ExtensionID> getRequiredExtension() const override {
|
|
return ExtensionID::SPV_INTEL_media_block_io;
|
|
}
|
|
};
|
|
|
|
#define _SPIRV_OP(x, ...) \
|
|
typedef SPIRVInstTemplate<SPIRVSubgroupImageMediaBlockIOINTELInstBase, \
|
|
Op##x, __VA_ARGS__> \
|
|
SPIRV##x;
|
|
// Intel Subgroup Image Media Block Read and Write Instructions
|
|
_SPIRV_OP(SubgroupImageMediaBlockReadINTEL, true, 7)
|
|
_SPIRV_OP(SubgroupImageMediaBlockWriteINTEL, false, 6)
|
|
#undef _SPIRV_OP
|
|
|
|
class SPIRVSubgroupAVCIntelInstBase : public SPIRVInstTemplateBase {
|
|
protected:
|
|
SPIRVCapVec getRequiredCapability() const override {
|
|
return getVec(CapabilitySubgroupAvcMotionEstimationINTEL);
|
|
}
|
|
|
|
std::optional<ExtensionID> getRequiredExtension() const override {
|
|
return ExtensionID::SPV_INTEL_device_side_avc_motion_estimation;
|
|
}
|
|
};
|
|
|
|
// Intel Subgroup AVC Motion Estimation Instructions
|
|
typedef SPIRVInstTemplate<SPIRVSubgroupAVCIntelInstBase, OpVmeImageINTEL, true,
|
|
5>
|
|
SPIRVVmeImageINTEL;
|
|
|
|
#define _SPIRV_OP(x, ...) \
|
|
typedef SPIRVInstTemplate<SPIRVSubgroupAVCIntelInstBase, \
|
|
OpSubgroupAvc##x##INTEL, __VA_ARGS__> \
|
|
SPIRVSubgroupAvc##x##INTEL;
|
|
_SPIRV_OP(MceGetDefaultInterBaseMultiReferencePenalty, true, 5)
|
|
_SPIRV_OP(MceSetInterBaseMultiReferencePenalty, true, 5)
|
|
_SPIRV_OP(MceGetDefaultInterShapePenalty, true, 5)
|
|
_SPIRV_OP(MceSetInterShapePenalty, true, 5)
|
|
_SPIRV_OP(MceGetDefaultInterDirectionPenalty, true, 5)
|
|
_SPIRV_OP(MceSetInterDirectionPenalty, true, 5)
|
|
_SPIRV_OP(MceGetDefaultInterMotionVectorCostTable, true, 5)
|
|
_SPIRV_OP(MceGetDefaultHighPenaltyCostTable, true, 3)
|
|
_SPIRV_OP(MceGetDefaultMediumPenaltyCostTable, true, 3)
|
|
_SPIRV_OP(MceGetDefaultLowPenaltyCostTable, true, 3)
|
|
_SPIRV_OP(MceSetMotionVectorCostFunction, true, 7)
|
|
_SPIRV_OP(MceSetAcOnlyHaar, true, 4)
|
|
_SPIRV_OP(MceSetSourceInterlacedFieldPolarity, true, 5)
|
|
_SPIRV_OP(MceSetSingleReferenceInterlacedFieldPolarity, true, 5)
|
|
_SPIRV_OP(MceSetDualReferenceInterlacedFieldPolarities, true, 6)
|
|
_SPIRV_OP(MceConvertToImePayload, true, 4)
|
|
_SPIRV_OP(MceConvertToImeResult, true, 4)
|
|
_SPIRV_OP(MceConvertToRefPayload, true, 4)
|
|
_SPIRV_OP(MceConvertToRefResult, true, 4)
|
|
_SPIRV_OP(MceConvertToSicPayload, true, 4)
|
|
_SPIRV_OP(MceConvertToSicResult, true, 4)
|
|
_SPIRV_OP(MceGetMotionVectors, true, 4)
|
|
_SPIRV_OP(MceGetInterDistortions, true, 4)
|
|
_SPIRV_OP(MceGetBestInterDistortions, true, 4)
|
|
_SPIRV_OP(MceGetInterMajorShape, true, 4)
|
|
_SPIRV_OP(MceGetInterMinorShape, true, 4)
|
|
_SPIRV_OP(MceGetInterDirections, true, 4)
|
|
_SPIRV_OP(MceGetInterMotionVectorCount, true, 4)
|
|
_SPIRV_OP(MceGetInterReferenceIds, true, 4)
|
|
_SPIRV_OP(MceGetInterReferenceInterlacedFieldPolarities, true, 6)
|
|
_SPIRV_OP(ImeInitialize, true, 6)
|
|
_SPIRV_OP(ImeSetSingleReference, true, 6)
|
|
_SPIRV_OP(ImeSetDualReference, true, 7)
|
|
_SPIRV_OP(ImeRefWindowSize, true, 5)
|
|
_SPIRV_OP(ImeAdjustRefOffset, true, 7)
|
|
_SPIRV_OP(ImeConvertToMcePayload, true, 4)
|
|
_SPIRV_OP(ImeSetMaxMotionVectorCount, true, 5)
|
|
_SPIRV_OP(ImeSetUnidirectionalMixDisable, true, 4)
|
|
_SPIRV_OP(ImeSetEarlySearchTerminationThreshold, true, 5)
|
|
_SPIRV_OP(ImeSetWeightedSad, true, 5)
|
|
_SPIRV_OP(ImeEvaluateWithSingleReference, true, 6)
|
|
_SPIRV_OP(ImeEvaluateWithDualReference, true, 7)
|
|
_SPIRV_OP(ImeEvaluateWithSingleReferenceStreamin, true, 7)
|
|
_SPIRV_OP(ImeEvaluateWithDualReferenceStreamin, true, 8)
|
|
_SPIRV_OP(ImeEvaluateWithSingleReferenceStreamout, true, 6)
|
|
_SPIRV_OP(ImeEvaluateWithDualReferenceStreamout, true, 7)
|
|
_SPIRV_OP(ImeEvaluateWithSingleReferenceStreaminout, true, 7)
|
|
_SPIRV_OP(ImeEvaluateWithDualReferenceStreaminout, true, 8)
|
|
_SPIRV_OP(ImeConvertToMceResult, true, 4)
|
|
_SPIRV_OP(ImeGetSingleReferenceStreamin, true, 4)
|
|
_SPIRV_OP(ImeGetDualReferenceStreamin, true, 4)
|
|
_SPIRV_OP(ImeStripSingleReferenceStreamout, true, 4)
|
|
_SPIRV_OP(ImeStripDualReferenceStreamout, true, 4)
|
|
_SPIRV_OP(ImeGetStreamoutSingleReferenceMajorShapeMotionVectors, true, 5)
|
|
_SPIRV_OP(ImeGetStreamoutSingleReferenceMajorShapeDistortions, true, 5)
|
|
_SPIRV_OP(ImeGetStreamoutSingleReferenceMajorShapeReferenceIds, true, 5)
|
|
_SPIRV_OP(ImeGetStreamoutDualReferenceMajorShapeMotionVectors, true, 6)
|
|
_SPIRV_OP(ImeGetStreamoutDualReferenceMajorShapeDistortions, true, 6)
|
|
_SPIRV_OP(ImeGetStreamoutDualReferenceMajorShapeReferenceIds, true, 6)
|
|
_SPIRV_OP(ImeGetBorderReached, true, 5)
|
|
_SPIRV_OP(ImeGetTruncatedSearchIndication, true, 4)
|
|
_SPIRV_OP(ImeGetUnidirectionalEarlySearchTermination, true, 4)
|
|
_SPIRV_OP(ImeGetWeightingPatternMinimumMotionVector, true, 4)
|
|
_SPIRV_OP(ImeGetWeightingPatternMinimumDistortion, true, 4)
|
|
_SPIRV_OP(FmeInitialize, true, 10)
|
|
_SPIRV_OP(BmeInitialize, true, 11)
|
|
_SPIRV_OP(RefConvertToMcePayload, true, 4)
|
|
_SPIRV_OP(RefSetBidirectionalMixDisable, true, 4)
|
|
_SPIRV_OP(RefSetBilinearFilterEnable, true, 4)
|
|
_SPIRV_OP(RefEvaluateWithSingleReference, true, 6)
|
|
_SPIRV_OP(RefEvaluateWithDualReference, true, 7)
|
|
_SPIRV_OP(RefEvaluateWithMultiReference, true, 6)
|
|
_SPIRV_OP(RefEvaluateWithMultiReferenceInterlaced, true, 7)
|
|
_SPIRV_OP(RefConvertToMceResult, true, 4)
|
|
_SPIRV_OP(SicInitialize, true, 4)
|
|
_SPIRV_OP(SicConfigureSkc, true, 9)
|
|
_SPIRV_OP(SicGetMotionVectorMask, true, 5)
|
|
_SPIRV_OP(SicConvertToMcePayload, true, 4)
|
|
_SPIRV_OP(SicSetIntraLumaShapePenalty, true, 5)
|
|
_SPIRV_OP(SicSetBilinearFilterEnable, true, 4)
|
|
_SPIRV_OP(SicSetSkcForwardTransformEnable, true, 5)
|
|
_SPIRV_OP(SicSetBlockBasedRawSkipSad, true, 5)
|
|
_SPIRV_OP(SicEvaluateWithSingleReference, true, 6)
|
|
_SPIRV_OP(SicEvaluateWithDualReference, true, 7)
|
|
_SPIRV_OP(SicEvaluateWithMultiReference, true, 6)
|
|
_SPIRV_OP(SicEvaluateWithMultiReferenceInterlaced, true, 7)
|
|
_SPIRV_OP(SicConvertToMceResult, true, 4)
|
|
_SPIRV_OP(SicGetInterRawSads, true, 4)
|
|
#undef _SPIRV_OP
|
|
|
|
class SPIRVSubgroupAVCIntelInstBaseIntra : public SPIRVInstTemplateBase {
|
|
protected:
|
|
SPIRVCapVec getRequiredCapability() const override {
|
|
return getVec(CapabilitySubgroupAvcMotionEstimationIntraINTEL);
|
|
}
|
|
};
|
|
|
|
// Intel Subgroup AVC Motion Estimation Intra Instructions
|
|
#define _SPIRV_OP(x, ...) \
|
|
typedef SPIRVInstTemplate<SPIRVSubgroupAVCIntelInstBaseIntra, \
|
|
OpSubgroupAvc##x##INTEL, __VA_ARGS__> \
|
|
SPIRVSubgroupAvc##x##INTEL;
|
|
_SPIRV_OP(MceGetDefaultIntraLumaShapePenalty, true, 5)
|
|
_SPIRV_OP(MceGetDefaultIntraLumaModePenalty, true, 5)
|
|
_SPIRV_OP(MceGetDefaultNonDcLumaIntraPenalty, true, 3)
|
|
_SPIRV_OP(SicConfigureIpeLuma, true, 11)
|
|
_SPIRV_OP(SicSetIntraLumaModeCostFunction, true, 7)
|
|
_SPIRV_OP(SicEvaluateIpe, true, 5)
|
|
_SPIRV_OP(SicGetIpeLumaShape, true, 4)
|
|
_SPIRV_OP(SicGetBestIpeLumaDistortion, true, 4)
|
|
_SPIRV_OP(SicGetPackedIpeLumaModes, true, 4)
|
|
_SPIRV_OP(SicGetPackedSkcLumaCountThreshold, true, 4)
|
|
_SPIRV_OP(SicGetPackedSkcLumaSumThreshold, true, 4)
|
|
#undef _SPIRV_OP
|
|
|
|
class SPIRVSubgroupAVCIntelInstBaseChroma : public SPIRVInstTemplateBase {
|
|
protected:
|
|
SPIRVCapVec getRequiredCapability() const override {
|
|
return getVec(CapabilitySubgroupAvcMotionEstimationChromaINTEL);
|
|
}
|
|
};
|
|
|
|
// Intel Subgroup AVC Motion Estimation Chroma Instructions
|
|
#define _SPIRV_OP(x, ...) \
|
|
typedef SPIRVInstTemplate<SPIRVSubgroupAVCIntelInstBaseChroma, \
|
|
OpSubgroupAvc##x##INTEL, __VA_ARGS__> \
|
|
SPIRVSubgroupAvc##x##INTEL;
|
|
_SPIRV_OP(MceGetDefaultIntraChromaModeBasePenalty, true, 3)
|
|
_SPIRV_OP(SicConfigureIpeLumaChroma, true, 14)
|
|
_SPIRV_OP(SicSetIntraChromaModeCostFunction, true, 5)
|
|
_SPIRV_OP(SicGetBestIpeChromaDistortion, true, 4)
|
|
_SPIRV_OP(SicGetIpeChromaMode, true, 4)
|
|
#undef _SPIRV_OP
|
|
|
|
SPIRVSpecConstantOp *createSpecConstantOpInst(SPIRVInstruction *Inst);
|
|
SPIRVInstruction *createInstFromSpecConstantOp(SPIRVSpecConstantOp *C);
|
|
|
|
class SPIRVVariableLengthArrayINTELInstBase : public SPIRVInstTemplateBase {
|
|
protected:
|
|
SPIRVCapVec getRequiredCapability() const override {
|
|
return getVec(CapabilityVariableLengthArrayINTEL);
|
|
}
|
|
std::optional<ExtensionID> getRequiredExtension() const override {
|
|
return ExtensionID::SPV_INTEL_variable_length_array;
|
|
}
|
|
};
|
|
#define _SPIRV_OP(x, ...) \
|
|
typedef SPIRVInstTemplate<SPIRVVariableLengthArrayINTELInstBase, \
|
|
Op##x##INTEL, __VA_ARGS__> \
|
|
SPIRV##x##INTEL;
|
|
_SPIRV_OP(VariableLengthArray, true, 4)
|
|
_SPIRV_OP(SaveMemory, true, 3)
|
|
_SPIRV_OP(RestoreMemory, false, 2)
|
|
#undef _SPIRV_OP
|
|
|
|
template <Op OC>
|
|
class SPIRVBfloat16ConversionINTELInstBase : public SPIRVUnaryInst<OC> {
|
|
protected:
|
|
SPIRVCapVec getRequiredCapability() const override {
|
|
SPIRVType *ResCompTy = this->getType();
|
|
if (ResCompTy->isTypeCooperativeMatrixKHR())
|
|
return getVec(
|
|
CapabilityBFloat16ConversionINTEL,
|
|
internal::CapabilityCooperativeMatrixBFloat16ComponentTypeINTEL);
|
|
return getVec(CapabilityBFloat16ConversionINTEL);
|
|
}
|
|
|
|
std::optional<ExtensionID> getRequiredExtension() const override {
|
|
SPIRVType *ResCompTy = this->getType();
|
|
if (ResCompTy->isTypeCooperativeMatrixKHR())
|
|
this->getModule()->addExtension(ExtensionID::SPV_INTEL_joint_matrix);
|
|
return ExtensionID::SPV_INTEL_bfloat16_conversion;
|
|
}
|
|
|
|
void validate() const override {
|
|
SPIRVUnaryInst<OC>::validate();
|
|
|
|
SPIRVType *ResCompTy = this->getType();
|
|
SPIRVWord ResCompCount = 1;
|
|
if (ResCompTy->isTypeVector()) {
|
|
ResCompCount = ResCompTy->getVectorComponentCount();
|
|
ResCompTy = ResCompTy->getVectorComponentType();
|
|
}
|
|
|
|
// validate is a const method, whilst getOperand is non-const method
|
|
// because it may call a method of class Module that may modify LiteralMap
|
|
// of Module field. That modification is not impacting validate method for
|
|
// these instructions, so const_cast is safe here.
|
|
using SPVBf16ConvTy = SPIRVBfloat16ConversionINTELInstBase<OC>;
|
|
SPIRVValue *Input = const_cast<SPVBf16ConvTy *>(this)->getOperand(0);
|
|
|
|
SPIRVType *InCompTy = Input->getType();
|
|
SPIRVWord InCompCount = 1;
|
|
if (InCompTy->isTypeVector()) {
|
|
InCompCount = InCompTy->getVectorComponentCount();
|
|
InCompTy = InCompTy->getVectorComponentType();
|
|
}
|
|
|
|
auto InstName = OpCodeNameMap::map(OC);
|
|
auto *Module = this->getModule();
|
|
SPIRVErrorLog &SPVErrLog = Module->getErrorLog();
|
|
|
|
// Cooperative matrix type is allowed as input/output of the instruction
|
|
// if SPV_INTEL_joint_matrix is enabled
|
|
if (ResCompTy->isTypeCooperativeMatrixKHR()) {
|
|
SPVErrLog.checkError(
|
|
Module->isAllowedToUseExtension(ExtensionID::SPV_INTEL_joint_matrix),
|
|
SPIRVEC_InvalidInstruction,
|
|
InstName + "\nCan be used with "
|
|
"cooperative matrices only when SPV_INTEL_joint_matrix is "
|
|
"enabled\n");
|
|
assert(InCompTy->isTypeCooperativeMatrixKHR() &&
|
|
"Input must also be a cooperative matrix");
|
|
ResCompTy = static_cast<SPIRVTypeCooperativeMatrixKHR *>(ResCompTy)
|
|
->getCompType();
|
|
InCompTy =
|
|
static_cast<SPIRVTypeCooperativeMatrixKHR *>(InCompTy)->getCompType();
|
|
}
|
|
if (OC == OpConvertFToBF16INTEL) {
|
|
SPVErrLog.checkError(
|
|
ResCompTy->isTypeInt(16), SPIRVEC_InvalidInstruction,
|
|
InstName + "\nResult value must be a scalar or vector of integer "
|
|
"16-bit type\n");
|
|
SPVErrLog.checkError(
|
|
InCompTy->isTypeFloat(32), SPIRVEC_InvalidInstruction,
|
|
InstName + "\nInput value must be a scalar or vector of "
|
|
"floating-point 32-bit type\n");
|
|
} else {
|
|
SPVErrLog.checkError(
|
|
ResCompTy->isTypeFloat(32), SPIRVEC_InvalidInstruction,
|
|
InstName + "\nResult value must be a scalar or vector of "
|
|
"floating-point 32-bit type\n");
|
|
SPVErrLog.checkError(
|
|
InCompTy->isTypeInt(16), SPIRVEC_InvalidInstruction,
|
|
InstName + "\nInput value must be a scalar or vector of integer "
|
|
"16-bit type\n");
|
|
}
|
|
|
|
SPVErrLog.checkError(
|
|
ResCompCount == InCompCount, SPIRVEC_InvalidInstruction,
|
|
InstName + "\nInput type must have the same number of components as "
|
|
"result type\n");
|
|
}
|
|
};
|
|
|
|
#define _SPIRV_OP(x) \
|
|
typedef SPIRVBfloat16ConversionINTELInstBase<Op##x> SPIRV##x;
|
|
_SPIRV_OP(ConvertFToBF16INTEL)
|
|
_SPIRV_OP(ConvertBF16ToFINTEL)
|
|
#undef _SPIRV_OP
|
|
|
|
class SPIRVJointMatrixINTELInstBase : public SPIRVInstTemplateBase {
|
|
protected:
|
|
std::optional<ExtensionID> getRequiredExtension() const override {
|
|
return ExtensionID::SPV_INTEL_joint_matrix;
|
|
}
|
|
};
|
|
|
|
class SPIRVJointMatrixINTELWorkItemInst : public SPIRVJointMatrixINTELInstBase {
|
|
protected:
|
|
SPIRVCapVec getRequiredCapability() const override {
|
|
return getVec(internal::CapabilityJointMatrixWIInstructionsINTEL);
|
|
}
|
|
};
|
|
|
|
#define _SPIRV_OP(x, ...) \
|
|
typedef SPIRVInstTemplate<SPIRVJointMatrixINTELWorkItemInst, \
|
|
internal::Op##x##INTEL, __VA_ARGS__> \
|
|
SPIRV##x##INTEL;
|
|
_SPIRV_OP(JointMatrixGetElementCoord, true, 5)
|
|
#undef _SPIRV_OP
|
|
|
|
class SPIRVCooperativeMatrixPrefetchINTELInstBase
|
|
: public SPIRVInstTemplateBase {
|
|
protected:
|
|
std::optional<ExtensionID> getRequiredExtension() const override {
|
|
return ExtensionID::SPV_INTEL_joint_matrix;
|
|
}
|
|
SPIRVCapVec getRequiredCapability() const override {
|
|
return getVec(internal::CapabilityCooperativeMatrixPrefetchINTEL);
|
|
}
|
|
};
|
|
|
|
#define _SPIRV_OP(x, ...) \
|
|
typedef SPIRVInstTemplate<SPIRVCooperativeMatrixPrefetchINTELInstBase, \
|
|
internal::Op##x##INTEL, __VA_ARGS__> \
|
|
SPIRV##x##INTEL;
|
|
_SPIRV_OP(CooperativeMatrixPrefetch, false, 6, true, 3)
|
|
#undef _SPIRV_OP
|
|
|
|
class SPIRVCooperativeMatrixCheckedInstructionsINTELInstBase
|
|
: public SPIRVInstTemplateBase {
|
|
protected:
|
|
std::optional<ExtensionID> getRequiredExtension() const override {
|
|
return ExtensionID::SPV_INTEL_joint_matrix;
|
|
}
|
|
SPIRVCapVec getRequiredCapability() const override {
|
|
return getVec(
|
|
internal::CapabilityCooperativeMatrixCheckedInstructionsINTEL);
|
|
}
|
|
};
|
|
|
|
#define _SPIRV_OP(x, ...) \
|
|
typedef SPIRVInstTemplate< \
|
|
SPIRVCooperativeMatrixCheckedInstructionsINTELInstBase, \
|
|
internal::Op##x##INTEL, __VA_ARGS__> \
|
|
SPIRV##x##INTEL;
|
|
_SPIRV_OP(CooperativeMatrixLoadChecked, true, 9, true, 7)
|
|
_SPIRV_OP(CooperativeMatrixStoreChecked, false, 8, true, 8)
|
|
_SPIRV_OP(CooperativeMatrixConstructChecked, true, 8)
|
|
#undef _SPIRV_OP
|
|
|
|
class SPIRVCooperativeMatrixOffsetInstructionsINTELInstBase
|
|
: public SPIRVInstTemplateBase {
|
|
protected:
|
|
std::optional<ExtensionID> getRequiredExtension() const override {
|
|
return ExtensionID::SPV_INTEL_joint_matrix;
|
|
}
|
|
SPIRVCapVec getRequiredCapability() const override {
|
|
return getVec(internal::CapabilityCooperativeMatrixOffsetInstructionsINTEL);
|
|
}
|
|
};
|
|
|
|
#define _SPIRV_OP(x, ...) \
|
|
typedef SPIRVInstTemplate< \
|
|
SPIRVCooperativeMatrixOffsetInstructionsINTELInstBase, \
|
|
internal::Op##x##INTEL, __VA_ARGS__> \
|
|
SPIRV##x##INTEL;
|
|
_SPIRV_OP(CooperativeMatrixLoadOffset, true, 8, true, 5)
|
|
_SPIRV_OP(CooperativeMatrixStoreOffset, false, 7, true, 6)
|
|
#undef _SPIRV_OP
|
|
|
|
class SPIRVCooperativeMatrixInvocationInstructionsINTELInstBase
|
|
: public SPIRVInstTemplateBase {
|
|
protected:
|
|
std::optional<ExtensionID> getRequiredExtension() const override {
|
|
return ExtensionID::SPV_INTEL_joint_matrix;
|
|
}
|
|
SPIRVCapVec getRequiredCapability() const override {
|
|
return getVec(
|
|
internal::CapabilityCooperativeMatrixInvocationInstructionsINTEL);
|
|
}
|
|
};
|
|
|
|
#define _SPIRV_OP(x, ...) \
|
|
typedef SPIRVInstTemplate< \
|
|
SPIRVCooperativeMatrixInvocationInstructionsINTELInstBase, \
|
|
internal::Op##x##INTEL, __VA_ARGS__> \
|
|
SPIRV##x##INTEL;
|
|
_SPIRV_OP(CooperativeMatrixApplyFunction, true, 5)
|
|
#undef _SPIRV_OP
|
|
|
|
class SPIRVCooperativeMatrixKHRInstBase : public SPIRVInstTemplateBase {
|
|
protected:
|
|
std::optional<ExtensionID> getRequiredExtension() const override {
|
|
return ExtensionID::SPV_KHR_cooperative_matrix;
|
|
}
|
|
SPIRVCapVec getRequiredCapability() const override {
|
|
return getVec(CapabilityCooperativeMatrixKHR);
|
|
}
|
|
};
|
|
|
|
#define _SPIRV_OP(x, ...) \
|
|
typedef SPIRVInstTemplate<SPIRVCooperativeMatrixKHRInstBase, Op##x, \
|
|
__VA_ARGS__> \
|
|
SPIRV##x;
|
|
_SPIRV_OP(CooperativeMatrixLoadKHR, true, 5, true, 3)
|
|
_SPIRV_OP(CooperativeMatrixStoreKHR, false, 4, true, 4)
|
|
_SPIRV_OP(CooperativeMatrixLengthKHR, true, 4, false)
|
|
_SPIRV_OP(CooperativeMatrixMulAddKHR, true, 6, true, 3)
|
|
#undef _SPIRV_OP
|
|
|
|
class SPIRVSplitBarrierINTELBase : public SPIRVInstTemplateBase {
|
|
protected:
|
|
SPIRVCapVec getRequiredCapability() const override {
|
|
return getVec(CapabilitySplitBarrierINTEL);
|
|
}
|
|
|
|
std::optional<ExtensionID> getRequiredExtension() const override {
|
|
return ExtensionID::SPV_INTEL_split_barrier;
|
|
}
|
|
};
|
|
|
|
#define _SPIRV_OP(x, ...) \
|
|
typedef SPIRVInstTemplate<SPIRVSplitBarrierINTELBase, Op##x, __VA_ARGS__> \
|
|
SPIRV##x;
|
|
_SPIRV_OP(ControlBarrierArriveINTEL, false, 4)
|
|
_SPIRV_OP(ControlBarrierWaitINTEL, false, 4)
|
|
#undef _SPIRV_OP
|
|
|
|
class SPIRVGroupUniformArithmeticKHRInstBase : public SPIRVInstTemplateBase {
|
|
public:
|
|
SPIRVCapVec getRequiredCapability() const override {
|
|
return getVec(CapabilityGroupUniformArithmeticKHR);
|
|
}
|
|
|
|
std::optional<ExtensionID> getRequiredExtension() const override {
|
|
return ExtensionID::SPV_KHR_uniform_group_instructions;
|
|
}
|
|
};
|
|
|
|
#define _SPIRV_OP(x, ...) \
|
|
typedef SPIRVInstTemplate<SPIRVGroupUniformArithmeticKHRInstBase, \
|
|
Op##x##KHR, __VA_ARGS__> \
|
|
SPIRV##x##KHR;
|
|
_SPIRV_OP(GroupIMul, true, 6, false, 1)
|
|
_SPIRV_OP(GroupFMul, true, 6, false, 1)
|
|
_SPIRV_OP(GroupBitwiseAnd, true, 6, false, 1)
|
|
_SPIRV_OP(GroupBitwiseOr, true, 6, false, 1)
|
|
_SPIRV_OP(GroupBitwiseXor, true, 6, false, 1)
|
|
_SPIRV_OP(GroupLogicalAnd, true, 6, false, 1)
|
|
_SPIRV_OP(GroupLogicalOr, true, 6, false, 1)
|
|
_SPIRV_OP(GroupLogicalXor, true, 6, false, 1)
|
|
#undef _SPIRV_OP
|
|
|
|
class SPIRVMaskedGatherScatterINTELInstBase : public SPIRVInstTemplateBase {
|
|
protected:
|
|
SPIRVCapVec getRequiredCapability() const override {
|
|
return getVec(internal::CapabilityMaskedGatherScatterINTEL);
|
|
}
|
|
std::optional<ExtensionID> getRequiredExtension() const override {
|
|
return ExtensionID::SPV_INTEL_masked_gather_scatter;
|
|
}
|
|
};
|
|
|
|
class SPIRVMaskedGatherINTELInst
|
|
: public SPIRVMaskedGatherScatterINTELInstBase {
|
|
void validate() const override {
|
|
SPIRVInstruction::validate();
|
|
SPIRVErrorLog &SPVErrLog = this->getModule()->getErrorLog();
|
|
std::string InstName = "MaskedGatherINTEL";
|
|
|
|
SPIRVType *ResTy = this->getType();
|
|
SPVErrLog.checkError(ResTy->isTypeVector(), SPIRVEC_InvalidInstruction,
|
|
InstName + "\nResult must be a vector type\n");
|
|
SPIRVWord ResCompCount = ResTy->getVectorComponentCount();
|
|
SPIRVType *ResCompTy = ResTy->getVectorComponentType();
|
|
|
|
SPIRVValue *PtrVec =
|
|
const_cast<SPIRVMaskedGatherINTELInst *>(this)->getOperand(0);
|
|
SPIRVType *PtrVecTy = PtrVec->getType();
|
|
SPVErrLog.checkError(
|
|
PtrVecTy->isTypeVectorPointer(), SPIRVEC_InvalidInstruction,
|
|
InstName + "\nPtrVector must be a vector of pointers type\n");
|
|
SPIRVWord PtrVecCompCount = PtrVecTy->getVectorComponentCount();
|
|
SPIRVType *PtrVecCompTy = PtrVecTy->getVectorComponentType();
|
|
SPIRVType *PtrElemTy = PtrVecCompTy->getPointerElementType();
|
|
|
|
SPVErrLog.checkError(
|
|
this->isOperandLiteral(1), SPIRVEC_InvalidInstruction,
|
|
InstName + "\nAlignment must be a constant expression integer\n");
|
|
const uint32_t Align =
|
|
static_cast<SPIRVConstant *>(
|
|
const_cast<SPIRVMaskedGatherINTELInst *>(this)->getOperand(2))
|
|
->getZExtIntValue();
|
|
SPVErrLog.checkError(
|
|
((Align & (Align - 1)) == 0), SPIRVEC_InvalidInstruction,
|
|
InstName + "\nAlignment must be 0 or power-of-two integer\n");
|
|
|
|
SPIRVValue *Mask =
|
|
const_cast<SPIRVMaskedGatherINTELInst *>(this)->getOperand(2);
|
|
SPIRVType *MaskTy = Mask->getType();
|
|
SPVErrLog.checkError(MaskTy->isTypeVector(), SPIRVEC_InvalidInstruction,
|
|
InstName + "\nMask must be a vector type\n");
|
|
SPIRVType *MaskCompTy = MaskTy->getVectorComponentType();
|
|
SPVErrLog.checkError(MaskCompTy->isTypeBool(), SPIRVEC_InvalidInstruction,
|
|
InstName + "\nMask must be a boolean vector type\n");
|
|
SPIRVWord MaskCompCount = MaskTy->getVectorComponentCount();
|
|
|
|
SPIRVValue *FillEmpty =
|
|
const_cast<SPIRVMaskedGatherINTELInst *>(this)->getOperand(3);
|
|
SPIRVType *FillEmptyTy = FillEmpty->getType();
|
|
SPVErrLog.checkError(FillEmptyTy->isTypeVector(),
|
|
SPIRVEC_InvalidInstruction,
|
|
InstName + "\nFillEmpty must be a vector type\n");
|
|
SPIRVWord FillEmptyCompCount = FillEmptyTy->getVectorComponentCount();
|
|
SPIRVType *FillEmptyCompTy = FillEmptyTy->getVectorComponentType();
|
|
|
|
SPVErrLog.checkError(
|
|
ResCompCount == PtrVecCompCount &&
|
|
PtrVecCompCount == FillEmptyCompCount &&
|
|
FillEmptyCompCount == MaskCompCount,
|
|
SPIRVEC_InvalidInstruction,
|
|
InstName + "\nResult, PtrVector, Mask and FillEmpty vectors must have "
|
|
"the same size\n");
|
|
|
|
SPVErrLog.checkError(
|
|
ResCompTy == PtrElemTy && PtrElemTy == FillEmptyCompTy,
|
|
SPIRVEC_InvalidInstruction,
|
|
InstName + "\nComponent Type of Result and FillEmpty vector must be "
|
|
"same as base type of PtrVector the same base type\n");
|
|
}
|
|
};
|
|
|
|
class SPIRVMaskedScatterINTELInst
|
|
: public SPIRVMaskedGatherScatterINTELInstBase {
|
|
void validate() const override {
|
|
SPIRVInstruction::validate();
|
|
SPIRVErrorLog &SPVErrLog = this->getModule()->getErrorLog();
|
|
std::string InstName = "MaskedScatterINTEL";
|
|
|
|
SPIRVValue *InputVec =
|
|
const_cast<SPIRVMaskedScatterINTELInst *>(this)->getOperand(0);
|
|
SPIRVType *InputVecTy = InputVec->getType();
|
|
SPVErrLog.checkError(
|
|
InputVecTy->isTypeVector(), SPIRVEC_InvalidInstruction,
|
|
InstName + "\nInputVector must be a vector of pointers type\n");
|
|
SPIRVWord InputVecCompCount = InputVecTy->getVectorComponentCount();
|
|
SPIRVType *InputVecCompTy = InputVecTy->getVectorComponentType();
|
|
|
|
SPIRVValue *PtrVec =
|
|
const_cast<SPIRVMaskedScatterINTELInst *>(this)->getOperand(1);
|
|
SPIRVType *PtrVecTy = PtrVec->getType();
|
|
SPVErrLog.checkError(
|
|
PtrVecTy->isTypeVectorPointer(), SPIRVEC_InvalidInstruction,
|
|
InstName + "\nPtrVector must be a vector of pointers type\n");
|
|
SPIRVWord PtrVecCompCount = PtrVecTy->getVectorComponentCount();
|
|
SPIRVType *PtrVecCompTy = PtrVecTy->getVectorComponentType();
|
|
SPIRVType *PtrElemTy = PtrVecCompTy->getPointerElementType();
|
|
|
|
SPVErrLog.checkError(
|
|
this->isOperandLiteral(2), SPIRVEC_InvalidInstruction,
|
|
InstName + "\nAlignment must be a constant expression integer\n");
|
|
const uint32_t Align =
|
|
static_cast<SPIRVConstant *>(
|
|
const_cast<SPIRVMaskedScatterINTELInst *>(this)->getOperand(2))
|
|
->getZExtIntValue();
|
|
SPVErrLog.checkError(
|
|
((Align & (Align - 1)) == 0), SPIRVEC_InvalidInstruction,
|
|
InstName + "\nAlignment must be 0 or power-of-two integer\n");
|
|
|
|
SPIRVValue *Mask =
|
|
const_cast<SPIRVMaskedScatterINTELInst *>(this)->getOperand(2);
|
|
SPIRVType *MaskTy = Mask->getType();
|
|
SPVErrLog.checkError(MaskTy->isTypeVector(), SPIRVEC_InvalidInstruction,
|
|
InstName + "\nMask must be a vector type\n");
|
|
SPIRVType *MaskCompTy = MaskTy->getVectorComponentType();
|
|
SPVErrLog.checkError(MaskCompTy->isTypeBool(), SPIRVEC_InvalidInstruction,
|
|
InstName + "\nMask must be a boolean vector type\n");
|
|
SPIRVWord MaskCompCount = MaskTy->getVectorComponentCount();
|
|
|
|
SPVErrLog.checkError(
|
|
InputVecCompCount == PtrVecCompCount &&
|
|
PtrVecCompCount == MaskCompCount,
|
|
SPIRVEC_InvalidInstruction,
|
|
InstName + "\nInputVector, PtrVector and Mask vectors must have "
|
|
"the same size\n");
|
|
|
|
SPVErrLog.checkError(
|
|
InputVecCompTy == PtrElemTy, SPIRVEC_InvalidInstruction,
|
|
InstName + "\nComponent Type of InputVector must be "
|
|
"same as base type of PtrVector the same base type\n");
|
|
}
|
|
};
|
|
|
|
#define _SPIRV_OP(x, ...) \
|
|
typedef SPIRVInstTemplate<SPIRVMaskedGatherScatterINTELInstBase, \
|
|
internal::Op##x##INTEL, __VA_ARGS__> \
|
|
SPIRV##x##INTEL;
|
|
_SPIRV_OP(MaskedGather, true, 7)
|
|
_SPIRV_OP(MaskedScatter, false, 5)
|
|
#undef _SPIRV_OP
|
|
|
|
template <Op OC>
|
|
class SPIRVTensorFloat32RoundingINTELInstBase : public SPIRVUnaryInst<OC> {
|
|
protected:
|
|
SPIRVCapVec getRequiredCapability() const override {
|
|
SPIRVType *ResCompTy = this->getType();
|
|
if (ResCompTy->isTypeCooperativeMatrixKHR())
|
|
return getVec(
|
|
CapabilityTensorFloat32RoundingINTEL,
|
|
internal::CapabilityCooperativeMatrixTF32ComponentTypeINTEL);
|
|
return getVec(CapabilityTensorFloat32RoundingINTEL);
|
|
}
|
|
|
|
std::optional<ExtensionID> getRequiredExtension() const override {
|
|
SPIRVType *ResCompTy = this->getType();
|
|
if (ResCompTy->isTypeCooperativeMatrixKHR())
|
|
this->getModule()->addExtension(ExtensionID::SPV_INTEL_joint_matrix);
|
|
return ExtensionID::SPV_INTEL_tensor_float32_conversion;
|
|
}
|
|
|
|
void validate() const override {
|
|
SPIRVUnaryInst<OC>::validate();
|
|
|
|
SPIRVType *ResCompTy = this->getType();
|
|
SPIRVWord ResCompCount = 1;
|
|
if (ResCompTy->isTypeVector()) {
|
|
ResCompCount = ResCompTy->getVectorComponentCount();
|
|
ResCompTy = ResCompTy->getVectorComponentType();
|
|
}
|
|
|
|
// validate is a const method, whilst getOperand is non-const method
|
|
// because it may call a method of class Module that may modify LiteralMap
|
|
// of Module field. That modification is not impacting validate method for
|
|
// these instructions, so const_cast is safe here.
|
|
using SPVTF32RoundTy = SPIRVTensorFloat32RoundingINTELInstBase<OC>;
|
|
SPIRVValue *Input = const_cast<SPVTF32RoundTy *>(this)->getOperand(0);
|
|
|
|
SPIRVType *InCompTy = Input->getType();
|
|
SPIRVWord InCompCount = 1;
|
|
if (InCompTy->isTypeVector()) {
|
|
InCompCount = InCompTy->getVectorComponentCount();
|
|
InCompTy = InCompTy->getVectorComponentType();
|
|
}
|
|
|
|
auto InstName = OpCodeNameMap::map(OC);
|
|
auto *Module = this->getModule();
|
|
SPIRVErrorLog &SPVErrLog = Module->getErrorLog();
|
|
|
|
// Cooperative matrix type is allowed as input/output of the instruction
|
|
// if SPV_INTEL_joint_matrix is enabled
|
|
if (ResCompTy->isTypeCooperativeMatrixKHR()) {
|
|
SPVErrLog.checkError(
|
|
Module->isAllowedToUseExtension(ExtensionID::SPV_INTEL_joint_matrix),
|
|
SPIRVEC_InvalidInstruction,
|
|
InstName + "\nCan be used with "
|
|
"cooperative matrices only when SPV_INTEL_joint_matrix is "
|
|
"enabled\n");
|
|
assert(InCompTy->isTypeCooperativeMatrixKHR() &&
|
|
"Input must also be a cooperative matrix");
|
|
ResCompTy = static_cast<SPIRVTypeCooperativeMatrixKHR *>(ResCompTy)
|
|
->getCompType();
|
|
InCompTy =
|
|
static_cast<SPIRVTypeCooperativeMatrixKHR *>(InCompTy)->getCompType();
|
|
}
|
|
|
|
SPVErrLog.checkError(
|
|
ResCompTy->isTypeFloat(32), SPIRVEC_InvalidInstruction,
|
|
InstName + "\nResult value must be a scalar or vector of floating-point"
|
|
" 32-bit type\n");
|
|
SPVErrLog.checkError(InCompTy->isTypeFloat(32), SPIRVEC_InvalidInstruction,
|
|
InstName +
|
|
"\nInput value must be a scalar or vector of "
|
|
"floating-point 32-bit type\n");
|
|
SPVErrLog.checkError(
|
|
ResCompCount == InCompCount, SPIRVEC_InvalidInstruction,
|
|
InstName + "\nInput type must have the same number of components as "
|
|
"result type\n");
|
|
}
|
|
};
|
|
|
|
#define _SPIRV_OP(x) \
|
|
typedef SPIRVTensorFloat32RoundingINTELInstBase<Op##x> SPIRV##x;
|
|
_SPIRV_OP(RoundFToTF32INTEL)
|
|
#undef _SPIRV_OP
|
|
|
|
template <Op OC> class SPIRVReadClockKHRInstBase : public SPIRVUnaryInst<OC> {
|
|
protected:
|
|
SPIRVCapVec getRequiredCapability() const override {
|
|
return getVec(CapabilityShaderClockKHR);
|
|
}
|
|
|
|
std::optional<ExtensionID> getRequiredExtension() const override {
|
|
return ExtensionID::SPV_KHR_shader_clock;
|
|
}
|
|
|
|
void validate() const override {
|
|
SPIRVUnaryInst<OC>::validate();
|
|
|
|
SPIRVType *ResCompTy = this->getType();
|
|
SPIRVWord ResCompCount = 1;
|
|
if (ResCompTy->isTypeVector()) {
|
|
ResCompCount = ResCompTy->getVectorComponentCount();
|
|
ResCompTy = ResCompTy->getVectorComponentType();
|
|
}
|
|
auto InstName = OpCodeNameMap::map(OC);
|
|
SPIRVErrorLog &SPVErrLog = this->getModule()->getErrorLog();
|
|
|
|
// check for either 64 bit int type or two element vector of 32 bit int
|
|
// types.
|
|
SPVErrLog.checkError(
|
|
ResCompTy->isTypeInt(64) ||
|
|
(ResCompCount == 2 && ResCompTy->isTypeInt(32)),
|
|
SPIRVEC_InvalidInstruction,
|
|
InstName + "\nResult value must be a scalar of integer"
|
|
" 64-bit type or two element vector of 32-bit type\n");
|
|
}
|
|
};
|
|
#define _SPIRV_OP(x) typedef SPIRVReadClockKHRInstBase<Op##x> SPIRV##x;
|
|
_SPIRV_OP(ReadClockKHR)
|
|
#undef _SPIRV_OP
|
|
|
|
class SPIRVTaskSequenceINTELInstBase : public SPIRVInstTemplateBase {
|
|
public:
|
|
std::optional<ExtensionID> getRequiredExtension() const override {
|
|
return ExtensionID::SPV_INTEL_task_sequence;
|
|
}
|
|
};
|
|
|
|
class SPIRVTaskSequenceINTELInst : public SPIRVTaskSequenceINTELInstBase {
|
|
public:
|
|
SPIRVCapVec getRequiredCapability() const override {
|
|
return getVec(internal::CapabilityTaskSequenceINTEL);
|
|
}
|
|
};
|
|
|
|
class SPIRVTaskSequenceCreateINTELInst : public SPIRVTaskSequenceINTELInst {
|
|
protected:
|
|
void validate() const override {
|
|
SPIRVInstruction::validate();
|
|
std::string InstName = "TaskSequenceCreateINTEL";
|
|
SPIRVErrorLog &SPVErrLog = this->getModule()->getErrorLog();
|
|
|
|
SPIRVType *ResTy = this->getType();
|
|
SPVErrLog.checkError(
|
|
ResTy->isTypeTaskSequenceINTEL(), SPIRVEC_InvalidInstruction,
|
|
InstName + "\nResult must be TaskSequenceINTEL type\n");
|
|
|
|
SPIRVValue *Func =
|
|
const_cast<SPIRVTaskSequenceCreateINTELInst *>(this)->getOperand(0);
|
|
SPVErrLog.checkError(
|
|
Func->getOpCode() == OpFunction, SPIRVEC_InvalidInstruction,
|
|
InstName + "\nFirst argument is expected to be a function.\n");
|
|
|
|
SPIRVConstant *PipelinedConst = static_cast<SPIRVConstant *>(
|
|
const_cast<SPIRVTaskSequenceCreateINTELInst *>(this)->getOperand(1));
|
|
const int Pipelined = PipelinedConst->getZExtIntValue();
|
|
SPVErrLog.checkError(Pipelined >= -1, SPIRVEC_InvalidInstruction,
|
|
InstName + "\nPipeline must be a 32 bit integer with "
|
|
"the value bigger or equal to -1.\n");
|
|
|
|
const int ClusterMode =
|
|
static_cast<SPIRVConstant *>(
|
|
const_cast<SPIRVTaskSequenceCreateINTELInst *>(this)->getOperand(2))
|
|
->getZExtIntValue();
|
|
SPVErrLog.checkError(
|
|
ClusterMode >= -1 && ClusterMode <= 1, SPIRVEC_InvalidInstruction,
|
|
InstName + "\nClusterMode valid values are -1, 0, 1.\n");
|
|
|
|
const int GetCapacity =
|
|
static_cast<SPIRVConstant *>(
|
|
const_cast<SPIRVTaskSequenceCreateINTELInst *>(this)->getOperand(3))
|
|
->getZExtIntValue();
|
|
SPVErrLog.checkError(
|
|
GetCapacity >= 0, SPIRVEC_InvalidInstruction,
|
|
InstName + "\nGetCapacity must be an unsigned 32-bit integer.\n");
|
|
|
|
const int AsyncCapacity =
|
|
static_cast<SPIRVConstant *>(
|
|
const_cast<SPIRVTaskSequenceCreateINTELInst *>(this)->getOperand(4))
|
|
->getZExtIntValue();
|
|
SPVErrLog.checkError(
|
|
AsyncCapacity >= 0, SPIRVEC_InvalidInstruction,
|
|
InstName + "\nAsyncCapacity must be an unsigned 32-bit integer.\n");
|
|
}
|
|
};
|
|
|
|
#define _SPIRV_OP(x, ...) \
|
|
typedef SPIRVInstTemplate<SPIRVTaskSequenceINTELInst, \
|
|
internal::Op##x##INTEL, __VA_ARGS__> \
|
|
SPIRV##x##INTEL;
|
|
_SPIRV_OP(TaskSequenceAsync, false, 2, true)
|
|
_SPIRV_OP(TaskSequenceGet, true, 4, false)
|
|
_SPIRV_OP(TaskSequenceRelease, false, 2, false)
|
|
#undef _SPIRV_OP
|
|
#define _SPIRV_OP(x, ...) \
|
|
typedef SPIRVInstTemplate<SPIRVTaskSequenceCreateINTELInst, \
|
|
internal::Op##x##INTEL, __VA_ARGS__> \
|
|
SPIRV##x##INTEL;
|
|
_SPIRV_OP(TaskSequenceCreate, true, 8, false, 1, 2, 3, 4)
|
|
#undef _SPIRV_OP
|
|
|
|
template <Op OC> class SPIRVBindlessImagesInstBase : public SPIRVUnaryInst<OC> {
|
|
protected:
|
|
SPIRVCapVec getRequiredCapability() const override {
|
|
return getVec(internal::CapabilityBindlessImagesINTEL);
|
|
}
|
|
|
|
std::optional<ExtensionID> getRequiredExtension() const override {
|
|
return ExtensionID::SPV_INTEL_bindless_images;
|
|
}
|
|
|
|
void validate() const override {
|
|
SPIRVUnary::validate();
|
|
|
|
// validate is a const method, whilst getOperand is non-const method
|
|
// because it may call a method of class Module that may modify LiteralMap
|
|
// of Module field. That modification is not impacting validate method for
|
|
// these instructions, so const_cast is safe here.
|
|
using SPVBindlessImagesInst = SPIRVBindlessImagesInstBase<OC>;
|
|
SPIRVValue *Input =
|
|
const_cast<SPVBindlessImagesInst *>(this)->getOperand(0);
|
|
SPIRVType *InCompTy = Input->getType();
|
|
|
|
auto StringAddrMod = [](SPIRVAddressingModelKind Kind) -> std::string {
|
|
if (Kind == AddressingModelPhysical32)
|
|
return std::string("Physical32");
|
|
if (Kind == AddressingModelPhysical64)
|
|
return std::string("Physical64");
|
|
return std::string("AddressingModel: ") + std::to_string(Kind);
|
|
};
|
|
|
|
auto InstName = OpCodeNameMap::map(OC);
|
|
auto AddrMod = this->getModule()->getAddressingModel();
|
|
SPIRVErrorLog &SPVErrLog = this->getModule()->getErrorLog();
|
|
SPVErrLog.checkError(
|
|
(InCompTy->isTypeInt(32) && AddrMod == AddressingModelPhysical32) ||
|
|
(InCompTy->isTypeInt(64) && AddrMod == AddressingModelPhysical64),
|
|
SPIRVEC_InvalidInstruction,
|
|
InstName +
|
|
"\nParameter value must be a 32-bit scalar in case of "
|
|
"Physical32 addressing model or a 64-bit scalar in case of "
|
|
"Physical64 addressing model\n"
|
|
"Type size: " +
|
|
std::to_string(InCompTy->getBitWidth()) +
|
|
"\nAddressing model: " + StringAddrMod(AddrMod) + "\n");
|
|
|
|
SPIRVType *ResTy = this->getType();
|
|
SPVErrLog.checkError(
|
|
(ResTy->isTypeImage() && OC == internal::OpConvertHandleToImageINTEL) ||
|
|
(ResTy->isTypeSampler() &&
|
|
OC == internal::OpConvertHandleToSamplerINTEL) ||
|
|
(ResTy->isTypeSampledImage() &&
|
|
OC == internal::OpConvertHandleToSampledImageINTEL),
|
|
SPIRVEC_InvalidInstruction,
|
|
InstName + "\nIncorrect return type of the instruction must be "
|
|
"image/sampler\n");
|
|
}
|
|
};
|
|
#define _SPIRV_OP(x) \
|
|
typedef SPIRVBindlessImagesInstBase<internal::Op##x> SPIRV##x;
|
|
_SPIRV_OP(ConvertHandleToImageINTEL)
|
|
_SPIRV_OP(ConvertHandleToSamplerINTEL)
|
|
_SPIRV_OP(ConvertHandleToSampledImageINTEL)
|
|
#undef _SPIRV_OP
|
|
|
|
class SPIRVUntypedPrefetchKHR : public SPIRVInstruction {
|
|
public:
|
|
static const Op OC = OpUntypedPrefetchKHR;
|
|
static const SPIRVWord FixedWordCount = 3;
|
|
|
|
SPIRVUntypedPrefetchKHR(SPIRVType *Ty, std::vector<SPIRVWord> &TheArgs,
|
|
SPIRVBasicBlock *BB)
|
|
: SPIRVInstruction(FixedWordCount, OC, BB) {
|
|
setHasNoId();
|
|
setHasNoType();
|
|
PtrTy = TheArgs[0];
|
|
NumBytes = TheArgs[1];
|
|
if (TheArgs.size() > 2)
|
|
RW.push_back(TheArgs[2]);
|
|
if (TheArgs.size() > 3)
|
|
Locality.push_back(TheArgs[3]);
|
|
if (TheArgs.size() > 4)
|
|
CacheTy.push_back(TheArgs[4]);
|
|
assert(BB && "Invalid BB");
|
|
validate();
|
|
}
|
|
|
|
SPIRVUntypedPrefetchKHR()
|
|
: SPIRVInstruction(OC), PtrTy(SPIRVID_INVALID),
|
|
NumBytes(SPIRVID_INVALID) {
|
|
setHasNoId();
|
|
setHasNoType();
|
|
}
|
|
|
|
void validate() const override {
|
|
SPIRVInstruction::validate();
|
|
SPIRVErrorLog &SPVErrLog = this->getModule()->getErrorLog();
|
|
std::string InstName = "OpUntypedPrefetchKHR";
|
|
SPVErrLog.checkError(getValueType(PtrTy)->isTypePointer(),
|
|
SPIRVEC_InvalidInstruction,
|
|
InstName + "\nFirst argument must be a pointer\n");
|
|
SPVErrLog.checkError(
|
|
getValueType(PtrTy)->getPointerStorageClass() ==
|
|
StorageClassCrossWorkgroup,
|
|
SPIRVEC_InvalidInstruction,
|
|
InstName + "\nFirst argument must be a pointer in CrossWorkgroup "
|
|
"storage class\n");
|
|
SPVErrLog.checkError(
|
|
getValueType(NumBytes)->isTypeInt(), SPIRVEC_InvalidInstruction,
|
|
InstName + "\nSecond argument (Num Bytes) must be an integer\n");
|
|
SPVErrLog.checkError(
|
|
RW.empty() || (RW.size() == 1 && getValueType(RW[0])->isTypeInt()),
|
|
SPIRVEC_InvalidInstruction,
|
|
InstName + "\nThird argument (RW) must be an integer\n");
|
|
SPVErrLog.checkError(
|
|
Locality.empty() ||
|
|
(Locality.size() == 1 && getValueType(Locality[0])->isTypeInt()),
|
|
SPIRVEC_InvalidInstruction,
|
|
InstName + "\nFourth argument (Locality) must be an integer\n");
|
|
SPVErrLog.checkError(
|
|
CacheTy.empty() ||
|
|
(CacheTy.size() == 1 && getValueType(CacheTy[0])->isTypeInt()),
|
|
SPIRVEC_InvalidInstruction,
|
|
InstName + "\nFifth argument (Cache Type) must be an integer\n");
|
|
}
|
|
|
|
void setWordCount(SPIRVWord TheWordCount) override {
|
|
SPIRVEntry::setWordCount(TheWordCount);
|
|
if (TheWordCount > 3)
|
|
RW.resize(1);
|
|
if (TheWordCount > 4)
|
|
Locality.resize(1);
|
|
if (TheWordCount > 5)
|
|
CacheTy.resize(1);
|
|
}
|
|
const std::vector<SPIRVWord> getArguments() const {
|
|
std::vector<SPIRVWord> Args;
|
|
Args.push_back(PtrTy);
|
|
Args.push_back(NumBytes);
|
|
if (!RW.empty())
|
|
Args.push_back(RW[0]);
|
|
if (!Locality.empty())
|
|
Args.push_back(Locality[0]);
|
|
if (!CacheTy.empty())
|
|
Args.push_back(CacheTy[0]);
|
|
return Args;
|
|
}
|
|
|
|
SPIRVCapVec getRequiredCapability() const override {
|
|
return getVec(CapabilityUntypedPointersKHR);
|
|
}
|
|
std::optional<ExtensionID> getRequiredExtension() const override {
|
|
return ExtensionID::SPV_KHR_untyped_pointers;
|
|
}
|
|
_SPIRV_DEF_ENCDEC5(PtrTy, NumBytes, RW, Locality, CacheTy)
|
|
protected:
|
|
SPIRVId PtrTy;
|
|
SPIRVId NumBytes;
|
|
std::vector<SPIRVId> RW;
|
|
std::vector<SPIRVId> Locality;
|
|
std::vector<SPIRVId> CacheTy;
|
|
};
|
|
|
|
class SPIRVSubgroupBlockPrefetchINTELInst : public SPIRVInstTemplateBase {
|
|
public:
|
|
std::optional<ExtensionID> getRequiredExtension() const override {
|
|
return ExtensionID::SPV_INTEL_subgroup_buffer_prefetch;
|
|
}
|
|
SPIRVCapVec getRequiredCapability() const override {
|
|
return getVec(CapabilitySubgroupBufferPrefetchINTEL);
|
|
}
|
|
// Operand 2, if present, is the Memory Operands bitmask.
|
|
bool isOperandLiteral(unsigned I) const override { return I == 2; }
|
|
|
|
protected:
|
|
void validate() const override {
|
|
SPIRVInstruction::validate();
|
|
if (getValue(Ops[0])->isForward())
|
|
return;
|
|
SPIRVErrorLog &SPVErrLog = getModule()->getErrorLog();
|
|
SPIRVType *PtrType = getValueType(Ops[0]);
|
|
std::string InstName = "OpSubgroupBlockPrefetchINTEL";
|
|
SPVErrLog.checkError(
|
|
PtrType->isTypePointer() || PtrType->isTypeUntypedPointerKHR(),
|
|
SPIRVEC_InvalidInstruction, InstName + "\nPtr must be a pointer\n");
|
|
SPVErrLog.checkError(
|
|
PtrType->getPointerStorageClass() == StorageClassCrossWorkgroup,
|
|
SPIRVEC_InvalidInstruction,
|
|
InstName + "\nPtr must be in CrossWorkgroup storage class\n");
|
|
if (!PtrType->isTypeUntypedPointerKHR())
|
|
SPVErrLog.checkError(PtrType->getPointerElementType()->isTypeInt(),
|
|
SPIRVEC_InvalidInstruction,
|
|
InstName +
|
|
"\nPtr must point to a scalar integer type\n");
|
|
SPVErrLog.checkError(
|
|
getValueType(Ops[1])->isTypeInt(32), SPIRVEC_InvalidInstruction,
|
|
InstName + "\nNumBytes must be a 32-bit integer scalar\n");
|
|
}
|
|
};
|
|
typedef SPIRVInstTemplate<SPIRVSubgroupBlockPrefetchINTELInst,
|
|
OpSubgroupBlockPrefetchINTEL, false, 3, true>
|
|
SPIRVSubgroupBlockPrefetchINTEL;
|
|
|
|
class SPIRVSubgroup2DBlockIOINTELInst : public SPIRVInstTemplateBase {
|
|
public:
|
|
std::optional<ExtensionID> getRequiredExtension() const override {
|
|
return ExtensionID::SPV_INTEL_2d_block_io;
|
|
}
|
|
SPIRVCapVec getRequiredCapability() const override {
|
|
return getVec(CapabilitySubgroup2DBlockIOINTEL);
|
|
}
|
|
};
|
|
|
|
class SPIRVSubgroup2DBlockLoadTransposeINTELInst
|
|
: public SPIRVSubgroup2DBlockIOINTELInst {
|
|
SPIRVCapVec getRequiredCapability() const override {
|
|
return getVec(CapabilitySubgroup2DBlockTransposeINTEL);
|
|
}
|
|
};
|
|
|
|
class SPIRVSubgroup2DBlockLoadTransformINTELInst
|
|
: public SPIRVSubgroup2DBlockIOINTELInst {
|
|
SPIRVCapVec getRequiredCapability() const override {
|
|
return getVec(CapabilitySubgroup2DBlockTransformINTEL);
|
|
}
|
|
};
|
|
|
|
#define _SPIRV_OP(x, ...) \
|
|
typedef SPIRVInstTemplate<SPIRVSubgroup2DBlockIOINTELInst, Op##x##INTEL, \
|
|
__VA_ARGS__> \
|
|
SPIRV##x##INTEL;
|
|
_SPIRV_OP(Subgroup2DBlockLoad, false, 11)
|
|
_SPIRV_OP(Subgroup2DBlockPrefetch, false, 10)
|
|
_SPIRV_OP(Subgroup2DBlockStore, false, 11)
|
|
#undef _SPIRV_OP
|
|
#define _SPIRV_OP(x, ...) \
|
|
typedef SPIRVInstTemplate<SPIRVSubgroup2DBlockLoadTransposeINTELInst, \
|
|
Op##x##INTEL, __VA_ARGS__> \
|
|
SPIRV##x##INTEL;
|
|
_SPIRV_OP(Subgroup2DBlockLoadTranspose, false, 11)
|
|
#undef _SPIRV_OP
|
|
#define _SPIRV_OP(x, ...) \
|
|
typedef SPIRVInstTemplate<SPIRVSubgroup2DBlockLoadTransformINTELInst, \
|
|
Op##x##INTEL, __VA_ARGS__> \
|
|
SPIRV##x##INTEL;
|
|
_SPIRV_OP(Subgroup2DBlockLoadTransform, false, 11)
|
|
#undef _SPIRV_OP
|
|
|
|
class SPIRVSubgroupMatrixMultiplyAccumulateINTELInst
|
|
: public SPIRVInstTemplateBase {
|
|
public:
|
|
std::optional<ExtensionID> getRequiredExtension() const override {
|
|
return ExtensionID::SPV_INTEL_subgroup_matrix_multiply_accumulate;
|
|
}
|
|
|
|
protected:
|
|
void validate() const override {
|
|
SPIRVInstTemplateBase::validate();
|
|
|
|
// Check if FP4 or FP8 matrix operands are used
|
|
// Operands parameter is the last operand (index 4)
|
|
auto *NonConstThis =
|
|
const_cast<SPIRVSubgroupMatrixMultiplyAccumulateINTELInst *>(this);
|
|
if (NonConstThis->getOperands().size() > 4) {
|
|
const SPIRVConstant *OperandsConst =
|
|
static_cast<const SPIRVConstant *>(NonConstThis->getOperand(4));
|
|
uint64_t OperandsMask = OperandsConst->getZExtIntValue();
|
|
|
|
// FP4 operand bits
|
|
constexpr uint64_t FP4Mask =
|
|
spv::internal::
|
|
IMatrixMultiplyAccumulateOperandsMatrixAPackedFloat4E2M1INTELMask |
|
|
spv::internal::
|
|
IMatrixMultiplyAccumulateOperandsMatrixBPackedFloat4E2M1INTELMask;
|
|
|
|
// FP8 operand bits
|
|
constexpr uint64_t FP8Mask =
|
|
spv::internal::
|
|
IMatrixMultiplyAccumulateOperandsMatrixAPackedFloat8E4M3INTELMask |
|
|
spv::internal::
|
|
IMatrixMultiplyAccumulateOperandsMatrixBPackedFloat8E4M3INTELMask |
|
|
spv::internal::
|
|
IMatrixMultiplyAccumulateOperandsMatrixAPackedFloat8E5M2INTELMask |
|
|
spv::internal::
|
|
IMatrixMultiplyAccumulateOperandsMatrixBPackedFloat8E5M2INTELMask;
|
|
|
|
if ((OperandsMask & FP4Mask) != 0) {
|
|
getModule()->getErrorLog().checkError(
|
|
getModule()->isAllowedToUseExtension(
|
|
ExtensionID::
|
|
SPV_INTEL_subgroup_matrix_multiply_accumulate_float4),
|
|
SPIRVEC_RequiresExtension,
|
|
"SPV_INTEL_subgroup_matrix_multiply_accumulate_float4\n"
|
|
"SubgroupMatrixMultiplyAccumulateINTEL with FP4 operand flags "
|
|
"requires this extension");
|
|
getModule()->addExtension(
|
|
ExtensionID::SPV_INTEL_subgroup_matrix_multiply_accumulate_float4);
|
|
}
|
|
|
|
if ((OperandsMask & FP8Mask) != 0) {
|
|
getModule()->getErrorLog().checkError(
|
|
getModule()->isAllowedToUseExtension(
|
|
ExtensionID::
|
|
SPV_INTEL_subgroup_matrix_multiply_accumulate_float8),
|
|
SPIRVEC_RequiresExtension,
|
|
"SPV_INTEL_subgroup_matrix_multiply_accumulate_float8\n"
|
|
"SubgroupMatrixMultiplyAccumulateINTEL with FP8 operand flags "
|
|
"requires this extension");
|
|
getModule()->addExtension(
|
|
ExtensionID::SPV_INTEL_subgroup_matrix_multiply_accumulate_float8);
|
|
}
|
|
}
|
|
}
|
|
|
|
SPIRVCapVec getRequiredCapability() const override {
|
|
return getVec(CapabilitySubgroupMatrixMultiplyAccumulateINTEL);
|
|
}
|
|
};
|
|
|
|
#define _SPIRV_OP(x, ...) \
|
|
typedef SPIRVInstTemplate<SPIRVSubgroupMatrixMultiplyAccumulateINTELInst, \
|
|
Op##x##INTEL, __VA_ARGS__> \
|
|
SPIRV##x##INTEL;
|
|
_SPIRV_OP(SubgroupMatrixMultiplyAccumulate, true, 7, true, 4)
|
|
#undef _SPIRV_OP
|
|
|
|
class SPIRVTernaryBitwiseFunctionINTELInst : public SPIRVInstTemplateBase {
|
|
public:
|
|
void validate() const override {
|
|
SPIRVInstruction::validate();
|
|
SPIRVErrorLog &SPVErrLog = this->getModule()->getErrorLog();
|
|
std::string InstName = "BitwiseFunctionINTEL";
|
|
|
|
const SPIRVType *ResTy = this->getType();
|
|
SPVErrLog.checkError(
|
|
ResTy->isTypeInt() || (ResTy->isTypeVector() &&
|
|
ResTy->getVectorComponentType()->isTypeInt()),
|
|
SPIRVEC_InvalidInstruction,
|
|
InstName + "\nResult type must be an integer scalar or vector.\n");
|
|
|
|
auto CommonArgCheck = [this, ResTy, &InstName,
|
|
&SPVErrLog](size_t ArgI, const char *ArgPlacement) {
|
|
SPIRVValue *Arg =
|
|
const_cast<SPIRVTernaryBitwiseFunctionINTELInst *>(this)->getOperand(
|
|
ArgI);
|
|
SPVErrLog.checkError(
|
|
Arg->getType() == ResTy, SPIRVEC_InvalidInstruction,
|
|
InstName + "\n" + ArgPlacement +
|
|
" argument must be the same as the result type.\n");
|
|
};
|
|
|
|
CommonArgCheck(0, "First");
|
|
CommonArgCheck(1, "Second");
|
|
CommonArgCheck(2, "Third");
|
|
|
|
SPIRVValue *LUTIndexArg =
|
|
const_cast<SPIRVTernaryBitwiseFunctionINTELInst *>(this)->getOperand(3);
|
|
const SPIRVType *LUTIndexArgTy = LUTIndexArg->getType();
|
|
SPVErrLog.checkError(
|
|
LUTIndexArgTy->isTypeInt(32), SPIRVEC_InvalidInstruction,
|
|
InstName + "\nFourth argument must be a 32-bit integer scalar.\n");
|
|
SPVErrLog.checkError(
|
|
isConstantOpCode(LUTIndexArg->getOpCode()), SPIRVEC_InvalidInstruction,
|
|
InstName + "\nFourth argument must be constant instruction.\n");
|
|
}
|
|
|
|
std::optional<ExtensionID> getRequiredExtension() const override {
|
|
return ExtensionID::SPV_INTEL_ternary_bitwise_function;
|
|
}
|
|
SPIRVCapVec getRequiredCapability() const override {
|
|
return getVec(CapabilityTernaryBitwiseFunctionINTEL);
|
|
}
|
|
};
|
|
|
|
#define _SPIRV_OP(x, ...) \
|
|
typedef SPIRVInstTemplate<SPIRVTernaryBitwiseFunctionINTELInst, \
|
|
Op##x##INTEL, __VA_ARGS__> \
|
|
SPIRV##x##INTEL;
|
|
_SPIRV_OP(BitwiseFunction, true, 7)
|
|
#undef _SPIRV_OP
|
|
|
|
class SPIRVPredicatedIOINTELInst : public SPIRVInstTemplateBase {
|
|
public:
|
|
std::optional<ExtensionID> getRequiredExtension() const override {
|
|
return ExtensionID::SPV_INTEL_predicated_io;
|
|
}
|
|
SPIRVCapVec getRequiredCapability() const override {
|
|
return getVec(CapabilityPredicatedIOINTEL);
|
|
}
|
|
};
|
|
|
|
#define _SPIRV_OP(x, ...) \
|
|
typedef SPIRVInstTemplate<SPIRVPredicatedIOINTELInst, Op##x##INTEL, \
|
|
__VA_ARGS__> \
|
|
SPIRV##x##INTEL;
|
|
_SPIRV_OP(PredicatedLoad, true, 6, true)
|
|
_SPIRV_OP(PredicatedStore, false, 4, true)
|
|
#undef _SPIRV_OP
|
|
|
|
template <Op OC> class SPIRVFSigmoidINTELInstBase : public SPIRVUnaryInst<OC> {
|
|
public:
|
|
SPIRVCapVec getRequiredCapability() const override {
|
|
return getVec(internal::CapabilitySigmoidINTEL);
|
|
}
|
|
|
|
std::optional<ExtensionID> getRequiredExtension() const override {
|
|
return ExtensionID::SPV_INTEL_sigmoid;
|
|
}
|
|
|
|
void validate() const override {
|
|
SPIRVUnaryInst<OC>::validate();
|
|
|
|
SPIRVType *ResCompTy = this->getType();
|
|
SPIRVWord ResCompCount = 1;
|
|
if (ResCompTy->isTypeVector()) {
|
|
ResCompCount = ResCompTy->getVectorComponentCount();
|
|
ResCompTy = ResCompTy->getVectorComponentType();
|
|
}
|
|
|
|
// validate is a const method, whilst getOperand is non-const method
|
|
// because it may call a method of class Module that may modify LiteralMap
|
|
// of Module field. That modification is not impacting validate method for
|
|
// these instructions, so const_cast is safe here.
|
|
using SPVFSigmoidTy = SPIRVFSigmoidINTELInstBase<OC>;
|
|
const SPIRVValue *Input = const_cast<SPVFSigmoidTy *>(this)->getOperand(0);
|
|
|
|
SPIRVType *InCompTy = Input->getType();
|
|
SPIRVWord InCompCount = 1;
|
|
if (InCompTy->isTypeVector()) {
|
|
InCompCount = InCompTy->getVectorComponentCount();
|
|
InCompTy = InCompTy->getVectorComponentType();
|
|
}
|
|
|
|
auto InstName = OpCodeNameMap::map(OC);
|
|
SPIRVErrorLog &SPVErrLog = this->getModule()->getErrorLog();
|
|
|
|
SPVErrLog.checkError(
|
|
ResCompTy->isTypeFloat(16) || ResCompTy->isTypeFloat(32) ||
|
|
ResCompTy->isTypeFloat(16, FPEncodingBFloat16KHR),
|
|
SPIRVEC_InvalidInstruction,
|
|
InstName + "\nResult value must be a scalar or vector of floating-point"
|
|
" 16-bit or 32-bit type\n");
|
|
SPVErrLog.checkError(
|
|
ResCompTy == InCompTy, SPIRVEC_InvalidInstruction,
|
|
InstName +
|
|
"\nInput type must have the same component type as result type\n");
|
|
SPVErrLog.checkError(
|
|
ResCompCount == InCompCount, SPIRVEC_InvalidInstruction,
|
|
InstName + "\nInput type must have the same number of components as "
|
|
"result type\n");
|
|
}
|
|
};
|
|
|
|
#define _SPIRV_OP(x) \
|
|
typedef SPIRVFSigmoidINTELInstBase<internal::Op##x> SPIRV##x;
|
|
_SPIRV_OP(FSigmoidINTEL)
|
|
#undef _SPIRV_OP
|
|
|
|
class SPIRVFPConversionINTELInstBase : public SPIRVInstTemplateBase {
|
|
public:
|
|
SPIRVCapVec getRequiredCapability() const override {
|
|
return getVec(internal::CapabilityFloatConversionsINTEL);
|
|
}
|
|
|
|
std::optional<ExtensionID> getRequiredExtension() const override {
|
|
return ExtensionID::SPV_INTEL_fp_conversions;
|
|
}
|
|
};
|
|
#define _SPIRV_OP(x, ...) \
|
|
typedef SPIRVInstTemplate<SPIRVFPConversionINTELInstBase, \
|
|
internal::Op##x##INTEL, __VA_ARGS__> \
|
|
SPIRV##x##INTEL;
|
|
_SPIRV_OP(ClampConvertFToF, true, 4, false)
|
|
_SPIRV_OP(ClampConvertFToS, true, 4, false)
|
|
_SPIRV_OP(StochasticRoundFToF, true, 5, true)
|
|
_SPIRV_OP(ClampStochasticRoundFToF, true, 5, true)
|
|
_SPIRV_OP(ClampStochasticRoundFToS, true, 5, true)
|
|
#undef _SPIRV_OP
|
|
|
|
class SPIRVFmaKHRInstBase : public SPIRVInstTemplateBase {
|
|
public:
|
|
SPIRVCapVec getRequiredCapability() const override {
|
|
return getVec(CapabilityFMAKHR);
|
|
}
|
|
|
|
std::optional<ExtensionID> getRequiredExtension() const override {
|
|
return ExtensionID::SPV_KHR_fma;
|
|
}
|
|
};
|
|
|
|
typedef SPIRVInstTemplate<SPIRVFmaKHRInstBase, OpFmaKHR, true, 6, false>
|
|
SPIRVFmaKHR;
|
|
|
|
class SPIRVFreezeKHRInstBase : public SPIRVInstTemplateBase {
|
|
public:
|
|
SPIRVCapVec getRequiredCapability() const override {
|
|
return getVec(CapabilityPoisonFreezeKHR);
|
|
}
|
|
|
|
std::optional<ExtensionID> getRequiredExtension() const override {
|
|
return ExtensionID::SPV_KHR_poison_freeze;
|
|
}
|
|
};
|
|
|
|
typedef SPIRVInstTemplate<SPIRVFreezeKHRInstBase, OpFreezeKHR, true, 4, false>
|
|
SPIRVFreezeKHR;
|
|
|
|
} // namespace SPIRV
|
|
#endif // SPIRV_LIBSPIRV_SPIRVINSTRUCTION_H
|