derek.holloway
7502bf64a4
CodeQL / Analyze (actions) (push) Canceled after 0s
CodeQL / Analyze (python) (push) Canceled after 0s
pre-commit / pre-commit (push) Canceled after 0s
Multi-Arch CI / setup (push) Canceled after 0s
Publish build_manylinux_x86_64 images / publish_build_manylinux_x86_64 (push) Canceled after 0s
Setup (Multi-Arch) / setup (push) Canceled after 0s
Multi-Arch CI / Linux::${{ fromJSON(needs.setup.outputs.linux_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubi10 image / publish_no_rocm_image_ubi10 (push) Canceled after 0s
Multi-Arch CI / Windows::${{ fromJSON(needs.setup.outputs.windows_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Multi-Arch CI / Manifest Diff (push) Canceled after 0s
TheRock Manifest Diff Report / Generate Manifest Diff Report (push) Canceled after 0s
Multi-Arch CI / CI Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_ocl_rt images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_rocgdb images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Unit Tests / Unit Tests :: ubuntu-24.04 (push) Canceled after 0s
Unit Tests / Unit Tests :: windows-2022 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_media (push) Canceled after 0s
Unit Tests / Unit Tests Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_openmpi (push) Canceled after 0s
1068 lines
51 KiB
C++
1068 lines
51 KiB
C++
/* ************************************************************************
|
|
* Copyright (C) 2025-2026 Advanced Micro Devices, Inc.
|
|
*
|
|
* Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
* of this software and associated documentation files (the "Software"), to deal
|
|
* in the Software without restriction, including without limitation the rights
|
|
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
* copies of the Software, and to permit persons to whom the Software is
|
|
* furnished to do so, subject to the following conditions:
|
|
*
|
|
* The above copyright notice and this permission notice shall be included in
|
|
* all copies or substantial portions of the Software.
|
|
*
|
|
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
|
* THE SOFTWARE.
|
|
*
|
|
* ************************************************************************ */
|
|
#include <gtest/gtest.h>
|
|
|
|
#include "TestHelpers.hpp"
|
|
#include "stinkytofu/analysis/AnalysisRegistration.hpp"
|
|
#include "stinkytofu/core/PassManager.hpp"
|
|
#include "stinkytofu/ir/asm/StinkyModifiers.hpp"
|
|
#include "stinkytofu/support/Casting.hpp"
|
|
#include "stinkytofu/transforms/asm/StinkyDAGSchedulerPass.hpp"
|
|
|
|
using namespace stinkytofu;
|
|
using namespace stinkytofu::test;
|
|
|
|
static int countStinkyInstructions(const BasicBlock& bb) {
|
|
int count = 0;
|
|
for (const IRBase& ir : bb) {
|
|
if (ir.getType() == IRBase::IRType::StinkyTofu) count++;
|
|
}
|
|
return count;
|
|
}
|
|
|
|
class DAGSchedulerPassTest : public ::testing::Test {
|
|
protected:
|
|
GfxArchID arch = GfxArchID::Gfx1250;
|
|
GemmTileConfig config;
|
|
std::unique_ptr<Function> func;
|
|
BasicBlock* bb = nullptr;
|
|
std::unique_ptr<Pass> pass;
|
|
AnalysisManager am;
|
|
|
|
void SetUp() override {
|
|
config.arch[0] = 12;
|
|
config.arch[1] = 5;
|
|
config.arch[2] = 0;
|
|
func = std::make_unique<Function>("dag_sched_test");
|
|
setFunctionArch(*func, arch);
|
|
bb = func->createBasicBlock("entry");
|
|
pass = createStinkyDAGSchedulerPass();
|
|
registerAllAnalyses(am);
|
|
}
|
|
|
|
void TearDown() override {
|
|
pass.reset();
|
|
func.reset();
|
|
bb = nullptr;
|
|
}
|
|
|
|
void runPass() {
|
|
PassContext ctx;
|
|
ctx.setGemmTileConfig(config);
|
|
pass->run(*func, ctx, am);
|
|
}
|
|
|
|
void runPassWithUnrollGemm() {
|
|
PassContext ctx;
|
|
ctx.setGemmTileConfig(config);
|
|
PassFeatureConfig pfc;
|
|
pfc.loopConfig.unrollGemm = true;
|
|
ctx.setPassFeatureConfig(pfc);
|
|
pass->run(*func, ctx, am);
|
|
}
|
|
|
|
// Run with the tensor_load_to_lds credit-pool throttle enabled.
|
|
// distributeGlobalRead routes tensor loads into globalReadQueue; depth/latency
|
|
// configure the in-flight credit pool.
|
|
void runPassWithGlobalReadThrottle(int depth, int drainLatency) {
|
|
PassContext ctx;
|
|
ctx.setGemmTileConfig(config);
|
|
PassFeatureConfig pfc;
|
|
pfc.loopConfig.unrollGemm = true;
|
|
pfc.dagFeatures.distributeGlobalRead = true;
|
|
pfc.dagFeatures.globalReadQueueDepth = depth;
|
|
pfc.dagFeatures.globalReadDrainLatency = drainLatency;
|
|
ctx.setPassFeatureConfig(pfc);
|
|
pass->run(*func, ctx, am);
|
|
}
|
|
|
|
// Run with the ds_read in-flight credit-pool throttle enabled. perWmma is held
|
|
// generously high by default so the separate per-WMMA-window ds cap never binds,
|
|
// isolating queueDepth/drainLatency as the only active constraint (mirrors how
|
|
// runPassWithGlobalReadThrottle isolates globalReadQueueDepth/globalReadDrainLatency).
|
|
void runPassWithDsReadThrottle(int queueDepth, int drainLatency, int perWmma = 100) {
|
|
PassContext ctx;
|
|
ctx.setGemmTileConfig(config);
|
|
PassFeatureConfig pfc;
|
|
pfc.loopConfig.unrollGemm = true;
|
|
pfc.dagFeatures.dsReadQueueDepth = queueDepth;
|
|
pfc.dagFeatures.dsReadDrainLatency = drainLatency;
|
|
pfc.dagFeatures.dsReadPerWmma = perWmma;
|
|
ctx.setPassFeatureConfig(pfc);
|
|
pass->run(*func, ctx, am);
|
|
}
|
|
|
|
// Linearized mnemonic order of a block (skips PHIs and non-Stinky IR).
|
|
static std::vector<std::string> mnemonicSequence(const BasicBlock& block) {
|
|
std::vector<std::string> seq;
|
|
for (const IRBase& ir : block) {
|
|
if (ir.getType() != IRBase::IRType::StinkyTofu) continue;
|
|
const auto* inst = cast<StinkyInstruction>(&ir);
|
|
const HwInstDesc* hw = inst->getHwInstDesc();
|
|
if (!hw || !hw->mnemonic) continue;
|
|
seq.push_back(hw->mnemonic);
|
|
}
|
|
return seq;
|
|
}
|
|
|
|
// Largest run of consecutive tensor_load_to_lds in a mnemonic sequence.
|
|
static int maxConsecutiveTensorLoads(const std::vector<std::string>& seq) {
|
|
int run = 0, best = 0;
|
|
for (const std::string& m : seq) {
|
|
if (m == "tensor_load_to_lds") {
|
|
run++;
|
|
best = std::max(best, run);
|
|
} else {
|
|
run = 0;
|
|
}
|
|
}
|
|
return best;
|
|
}
|
|
|
|
// Largest run of consecutive ds_load_b128 in a mnemonic sequence
|
|
// (gfx1250's actual mnemonic for this op; see Gfx1250Instructions.def).
|
|
static int maxConsecutiveDsReads(const std::vector<std::string>& seq) {
|
|
int run = 0, best = 0;
|
|
for (const std::string& m : seq) {
|
|
if (m == "ds_load_b128") {
|
|
run++;
|
|
best = std::max(best, run);
|
|
} else {
|
|
run = 0;
|
|
}
|
|
}
|
|
return best;
|
|
}
|
|
|
|
// Build a single-BB self-loop so the scheduler uses the loop-aware CDNA5 path.
|
|
// Returns the loop body BB with the branch already appended.
|
|
BasicBlock* buildLoopBB(const char* label = "loop_body") {
|
|
BasicBlock* body = func->createBasicBlock(label);
|
|
body->addSuccessor(body);
|
|
return body;
|
|
}
|
|
|
|
static StinkyInstruction* createSCbranchInBlock(BasicBlock* bb, GfxArchID arch) {
|
|
AsmIRBuilder builder(*bb, arch);
|
|
return builder.create(getMCIDByUOp(GFX::s_cbranch_scc0, arch));
|
|
}
|
|
|
|
StinkyInstruction* createWmmaF32_16x16x16_bf16_in(BasicBlock* targetBB, int destStart,
|
|
int src0Start) {
|
|
AsmIRBuilder builder(*targetBB, arch);
|
|
const HwInstDesc* desc = getMCIDByUOp(GFX::v_wmma_f32_16x16x16_bf16, arch);
|
|
if (!desc) return nullptr;
|
|
StinkyInstruction* inst = builder.create(desc);
|
|
inst->addDestReg(StinkyRegister("v", destStart, 8));
|
|
inst->addSrcReg(StinkyRegister("v", src0Start, 8));
|
|
inst->addSrcReg(StinkyRegister("v", src0Start, 8));
|
|
inst->addSrcReg(StinkyRegister("v", destStart, 8));
|
|
return inst;
|
|
}
|
|
|
|
StinkyInstruction* createWmmaF32_16x16x16_bf16(int destStart, int src0Start) {
|
|
return createWmmaF32_16x16x16_bf16_in(bb, destStart, src0Start);
|
|
}
|
|
|
|
// v_wmma_scale_f32_16x16x128_f8f6f4 with F8 (FP8) input matrix formats. Its
|
|
// cost is {issue=1, latency=8}, so the co-issue latency window stays open
|
|
// right after issue, which is what exercises the co-exec hazard gate.
|
|
// src VGPRs: v[src0Start:src0Start+8) (src0/src1) and v[destStart:destStart+8) (acc).
|
|
StinkyInstruction* createWmmaScaleF8_in(BasicBlock* targetBB, int destStart, int src0Start) {
|
|
AsmIRBuilder builder(*targetBB, arch);
|
|
const HwInstDesc* desc = getMCIDByUOp(GFX::v_wmma_scale_f32_16x16x128_f8f6f4, arch);
|
|
if (!desc) return nullptr;
|
|
StinkyInstruction* inst = builder.create(desc);
|
|
inst->addDestReg(StinkyRegister("v", destStart, 8));
|
|
inst->addSrcReg(StinkyRegister("v", src0Start, 8));
|
|
inst->addSrcReg(StinkyRegister("v", src0Start, 8));
|
|
inst->addSrcReg(StinkyRegister("v", destStart, 8));
|
|
MatrixFmtModifiers fmtMod;
|
|
fmtMod.fmtA = MatrixFmt::FP8;
|
|
fmtMod.fmtB = MatrixFmt::FP8;
|
|
inst->addModifier(fmtMod);
|
|
return inst;
|
|
}
|
|
|
|
StinkyInstruction* createWmmaScaleF8(int destStart, int src0Start) {
|
|
return createWmmaScaleF8_in(bb, destStart, src0Start);
|
|
}
|
|
|
|
StinkyInstruction* createMovableDsLoad(int destReg, int addrReg, int ldsToken) {
|
|
StinkyInstruction* inst = createDsReadB128InBlock(bb, arch, destReg, addrReg);
|
|
inst->addSrcReg(StinkyRegister(RegType::LDS, ldsToken, 1));
|
|
return inst;
|
|
}
|
|
|
|
// A tensor_load_to_lds with an LDS pseudo dest-reg so the DAG scheduler treats
|
|
// it as movable (without LDS pseudo-regs hasSideEffect() makes it a region
|
|
// boundary and it never enters globalReadQueue). Mirrors createMovableDsLoad.
|
|
StinkyInstruction* createMovableTensorLoad(BasicBlock* targetBB, int src0Reg, int src1Reg,
|
|
int ldsToken) {
|
|
StinkyInstruction* inst = createTensorLoadInBlock(targetBB, arch, src0Reg, src1Reg);
|
|
inst->addDestReg(StinkyRegister(RegType::LDS, ldsToken, 1));
|
|
return inst;
|
|
}
|
|
|
|
StinkyInstruction* createExecNarrow(int srcSgpr) {
|
|
AsmIRBuilder builder(*bb, arch);
|
|
StinkyInstruction* inst = builder.create(getMCIDByUOp(GFX::s_mov_b32, arch));
|
|
inst->addDestReg(StinkyRegister::getEXECRegister(32));
|
|
inst->addSrcReg(StinkyRegister("s", srcSgpr, 1));
|
|
return inst;
|
|
}
|
|
|
|
StinkyInstruction* createExecReset() {
|
|
AsmIRBuilder builder(*bb, arch);
|
|
StinkyInstruction* inst = builder.create(getMCIDByUOp(GFX::s_mov_b32, arch));
|
|
inst->addDestReg(StinkyRegister::getEXECRegister(32));
|
|
inst->addSrcReg(StinkyRegister(-1));
|
|
return inst;
|
|
}
|
|
};
|
|
|
|
// Integration check: collapse + schedule + expand together, through the real pass.
|
|
// See ExecMaskGroupingTest.cpp for isolated collapse/expand tests, and the
|
|
// ExecMaskGroup_* tests below for whether the scheduler treats a group as atomic.
|
|
TEST_F(DAGSchedulerPassTest, ExecMaskedRegion_PreservesSpanAndOrder) {
|
|
createVAddInBlock(bb, arch, 40, 41, 42);
|
|
createExecNarrow(10);
|
|
createVAddInBlock(bb, arch, 0, 1, 2);
|
|
createVAddInBlock(bb, arch, 3, 4, 5);
|
|
createVAddInBlock(bb, arch, 6, 7, 8);
|
|
createExecReset();
|
|
createVAddInBlock(bb, arch, 50, 51, 52);
|
|
|
|
const int n = countStinkyInstructions(*bb);
|
|
runPass();
|
|
|
|
EXPECT_EQ(countStinkyInstructions(*bb), n);
|
|
for (const IRBase& ir : *bb) {
|
|
if (ir.getType() != IRBase::IRType::StinkyTofu) continue;
|
|
EXPECT_NE(cast<StinkyInstruction>(&ir)->getUnifiedOpcode(), GFX::EXEC_GROUP);
|
|
}
|
|
|
|
std::vector<int> destSeq;
|
|
std::vector<bool> isExecWriteSeq;
|
|
for (const IRBase& ir : *bb) {
|
|
if (ir.getType() != IRBase::IRType::StinkyTofu) continue;
|
|
const auto* inst = cast<StinkyInstruction>(&ir);
|
|
if (inst->getUnifiedOpcode() != GFX::s_mov_b32 &&
|
|
inst->getUnifiedOpcode() != GFX::v_add_f32)
|
|
continue;
|
|
ASSERT_FALSE(inst->getDestRegs().empty());
|
|
destSeq.push_back(static_cast<int>(inst->getDestReg(0).reg.idx));
|
|
isExecWriteSeq.push_back(inst->getDestReg(0).reg.type == RegType::EXEC_LO);
|
|
}
|
|
|
|
ASSERT_EQ(destSeq.size(), 7u);
|
|
EXPECT_EQ(destSeq[0], 40);
|
|
EXPECT_TRUE(isExecWriteSeq[1]);
|
|
EXPECT_EQ(destSeq[2], 0);
|
|
EXPECT_EQ(destSeq[3], 3);
|
|
EXPECT_EQ(destSeq[4], 6);
|
|
EXPECT_TRUE(isExecWriteSeq[5]);
|
|
EXPECT_EQ(destSeq[6], 50);
|
|
}
|
|
|
|
// Layer 2: does the scheduler treat a hand-built ExecMaskGroup (bypassing
|
|
// collapseExecMaskedRegions() entirely) as a single atomic node?
|
|
|
|
// runPass() always runs collapseExecMaskedRegions()/expandExecMaskedGroups() around
|
|
// scheduling (see StinkyDAGSchedulerPass::run()'s scheduleBlock lambda), so any node
|
|
// with GFX::EXEC_GROUP -- hand-built or not -- gets unzipped via its ExecGroupData at
|
|
// the end. So a hand-built group under a real pass run needs a real (if minimal)
|
|
// ExecGroupData child to unzip into; that child's own registers are irrelevant here
|
|
// since ordering is driven by the group's own declared src/dest, set explicitly below.
|
|
TEST_F(DAGSchedulerPassTest, ExecMaskGroup_TreatedAsSingleAtomicNode) {
|
|
createVAddInBlock(bb, arch, 20, 21, 22);
|
|
StinkyInstruction* consumer = createVAddInBlock(bb, arch, 40, 30, 31);
|
|
|
|
StinkyInstruction* child = createVAddInBlock(bb, arch, 60, 61, 62);
|
|
bb->removeIR(child);
|
|
|
|
AsmIRBuilder builder(*bb, arch);
|
|
StinkyInstruction* group = builder.createExecMaskGroup(consumer);
|
|
group->addSrcReg(StinkyRegister("v", 20, 1));
|
|
group->addDestReg(StinkyRegister("v", 30, 1));
|
|
group->issueCycles = 4;
|
|
group->latencyCycles = 4;
|
|
group->addModifier<ExecGroupData>(ExecGroupData{{child}});
|
|
|
|
runPass();
|
|
|
|
std::vector<int> destSeq;
|
|
for (const IRBase& ir : *bb) {
|
|
if (ir.getType() != IRBase::IRType::StinkyTofu) continue;
|
|
const auto* inst = cast<StinkyInstruction>(&ir);
|
|
ASSERT_FALSE(inst->getDestRegs().empty());
|
|
destSeq.push_back(static_cast<int>(inst->getDestReg(0).reg.idx));
|
|
}
|
|
ASSERT_EQ(destSeq.size(), 3u);
|
|
EXPECT_EQ(destSeq[0], 20); // producer
|
|
EXPECT_EQ(destSeq[1], 60); // group's child, unzipped in its place
|
|
EXPECT_EQ(destSeq[2], 40); // consumer
|
|
}
|
|
|
|
TEST_F(DAGSchedulerPassTest, ExecMaskGroup_NotMisclassified) {
|
|
StinkyInstruction* anchor = createVAddInBlock(bb, arch, 0, 1, 2);
|
|
|
|
AsmIRBuilder builder(*bb, arch);
|
|
StinkyInstruction* group = builder.createExecMaskGroup(anchor);
|
|
group->addModifier<ExecGroupData>(ExecGroupData{{}});
|
|
|
|
EXPECT_TRUE(isExecMaskGroup(*group));
|
|
EXPECT_FALSE(isMatrixInstruction(*group));
|
|
EXPECT_FALSE(isDSRead(*group));
|
|
EXPECT_FALSE(isDSWrite(*group));
|
|
EXPECT_FALSE(isBarrier(*group));
|
|
EXPECT_FALSE(isVectorALU(*group));
|
|
EXPECT_FALSE(isTensorLoad(*group));
|
|
EXPECT_FALSE(hasSideEffect(*group));
|
|
|
|
runPassWithUnrollGemm();
|
|
// The group has no children to unzip into, so it's simply dropped; only the
|
|
// anchor v_add remains.
|
|
EXPECT_EQ(countStinkyInstructions(*bb), 1);
|
|
}
|
|
|
|
TEST_F(DAGSchedulerPassTest, ExecMaskGroup_InheritsSideEffectFromChildren) {
|
|
StinkyInstruction* sideEffecting = createDSWriteInBlock(bb, arch, 0, 1); // no MemTokenData
|
|
bb->removeIR(sideEffecting);
|
|
|
|
StinkyInstruction* anchor = createVAddInBlock(bb, arch, 0, 1, 2);
|
|
AsmIRBuilder builder(*bb, arch);
|
|
StinkyInstruction* group = builder.createExecMaskGroup(anchor);
|
|
group->addModifier<ExecGroupData>(ExecGroupData{{sideEffecting}});
|
|
|
|
EXPECT_TRUE(hasSideEffect(*group));
|
|
}
|
|
|
|
// Empty block: pass should not crash
|
|
TEST_F(DAGSchedulerPassTest, EmptyBlock_DoesNotCrash) {
|
|
runPass();
|
|
EXPECT_EQ(countStinkyInstructions(*bb), 0);
|
|
}
|
|
|
|
// Single instruction: pass should not crash
|
|
TEST_F(DAGSchedulerPassTest, SingleInstruction_DoesNotCrash) {
|
|
createVAddInBlock(bb, arch, 0, 1, 2);
|
|
int n = countStinkyInstructions(*bb);
|
|
runPass();
|
|
EXPECT_EQ(countStinkyInstructions(*bb), n);
|
|
}
|
|
|
|
// A few independent instructions: pass should not crash, count unchanged
|
|
TEST_F(DAGSchedulerPassTest, IndependentInstructions_DoesNotCrash) {
|
|
createVAddInBlock(bb, arch, 0, 1, 2);
|
|
createVAddInBlock(bb, arch, 3, 4, 5);
|
|
createVAddInBlock(bb, arch, 6, 7, 8);
|
|
int n = countStinkyInstructions(*bb);
|
|
runPass();
|
|
EXPECT_EQ(countStinkyInstructions(*bb), n);
|
|
}
|
|
|
|
// Chain of dependencies: pass should not crash, count unchanged
|
|
TEST_F(DAGSchedulerPassTest, DependentInstructions_DoesNotCrash) {
|
|
createVAddInBlock(bb, arch, 0, 1, 2); // v0 = v1 + v2
|
|
createVAddInBlock(bb, arch, 3, 0, 4); // v3 = v0 + v4
|
|
createVAddInBlock(bb, arch, 5, 3, 6); // v5 = v3 + v6
|
|
int n = countStinkyInstructions(*bb);
|
|
runPass();
|
|
EXPECT_EQ(countStinkyInstructions(*bb), n);
|
|
}
|
|
|
|
// DS reads + WMMAs: scheduler must not issue WMMAs back-to-back when other instructions exist.
|
|
// With real ds_load latency, WMMAs are not latency-free until ds_reads are issued and latency
|
|
// elapses, so we get: 4 ds_load, then 2 wmma. The rule "lastPickedWasWMMA => prefer other"
|
|
// ensures that when both WMMA and other are ready we interleave (no consecutive WMMAs).
|
|
TEST_F(DAGSchedulerPassTest, DSReadAndWMMA_NoConsecutiveWMMA) {
|
|
const int addrReg = 24;
|
|
createDsReadB128InBlock(bb, arch, 8, addrReg); // v[8:11]
|
|
createDsReadB128InBlock(bb, arch, 12, addrReg); // v[12:15]
|
|
createDsReadB128InBlock(bb, arch, 16, addrReg); // v[16:19]
|
|
createDsReadB128InBlock(bb, arch, 20, addrReg); // v[20:23]
|
|
StinkyInstruction* wmma1 = createWmmaF32_16x16x16_bf16(0, 8); // v[0:7] v[8:15] v[8:15] v[0:7]
|
|
StinkyInstruction* wmma2 =
|
|
createWmmaF32_16x16x16_bf16(0, 16); // v[0:7] v[16:23] v[16:23] v[0:7]
|
|
ASSERT_NE(wmma1, nullptr);
|
|
ASSERT_NE(wmma2, nullptr);
|
|
|
|
runPassWithUnrollGemm();
|
|
|
|
// With real latency, all 4 ds_load are issued first, then 2 wmma. No two WMMAs in a row.
|
|
std::vector<std::pair<std::string, int>> sequence;
|
|
for (const IRBase& ir : *bb) {
|
|
if (ir.getType() != IRBase::IRType::StinkyTofu) continue;
|
|
const StinkyInstruction* inst = cast<StinkyInstruction>(&ir);
|
|
const char* mnem = inst->getHwInstDesc() ? inst->getHwInstDesc()->mnemonic : nullptr;
|
|
if (!mnem) continue;
|
|
if (std::string(mnem) == "ds_load_b128") {
|
|
if (!inst->getDestRegs().empty() && inst->getDestRegs()[0].isRegister())
|
|
sequence.push_back(
|
|
{"ds_load_b128", static_cast<int>(inst->getDestRegs()[0].reg.idx)});
|
|
} else if (std::string(mnem) == "v_wmma_f32_16x16x16_bf16") {
|
|
if (!inst->getSrcRegs().empty() && inst->getSrcRegs()[0].isRegister())
|
|
sequence.push_back(
|
|
{"v_wmma_f32_16x16x16_bf16", static_cast<int>(inst->getSrcRegs()[0].reg.idx)});
|
|
}
|
|
}
|
|
|
|
ASSERT_EQ(sequence.size(), 6u) << "Expected 6 instructions (4 ds_load + 2 wmma)";
|
|
// All 4 ds_load first (real latency: WMMAs not ready until latency elapses)
|
|
EXPECT_EQ(sequence[0].first, "ds_load_b128");
|
|
EXPECT_EQ(sequence[0].second, 8);
|
|
EXPECT_EQ(sequence[1].first, "ds_load_b128");
|
|
EXPECT_EQ(sequence[1].second, 12);
|
|
EXPECT_EQ(sequence[2].first, "ds_load_b128");
|
|
EXPECT_EQ(sequence[2].second, 16);
|
|
EXPECT_EQ(sequence[3].first, "ds_load_b128");
|
|
EXPECT_EQ(sequence[3].second, 20);
|
|
// Then 2 wmma; rule ensures we never issue two WMMAs in a row when other work exists
|
|
EXPECT_EQ(sequence[4].first, "v_wmma_f32_16x16x16_bf16");
|
|
EXPECT_EQ(sequence[4].second, 8);
|
|
EXPECT_EQ(sequence[5].first, "v_wmma_f32_16x16x16_bf16");
|
|
EXPECT_EQ(sequence[5].second, 16);
|
|
// When other instructions exist, scheduler prefers them after a WMMA (no back-to-back WMMA).
|
|
// Here with real latency only WMMAs are left at the end so they are issued consecutively.
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Property: when all independent, WMMA fires first (Phase B), then ds_loads
|
|
// and VALU fill the WMMA latency window.
|
|
// Within that window, ds_load has priority over VALU.
|
|
// ---------------------------------------------------------------------------
|
|
TEST_F(DAGSchedulerPassTest, IndependentWMMAFirst_ThenDsThenVALU) {
|
|
const int addrReg = 80;
|
|
// 3 independent ds_loads (LDS pseudo-reg makes them movable in DAG)
|
|
createMovableDsLoad(0, addrReg, 1);
|
|
createMovableDsLoad(4, addrReg, 2);
|
|
createMovableDsLoad(8, addrReg, 3);
|
|
// 3 independent VALUs
|
|
createVAddInBlock(bb, arch, 40, 41, 42);
|
|
createVAddInBlock(bb, arch, 43, 44, 45);
|
|
createVAddInBlock(bb, arch, 46, 47, 48);
|
|
// 1 independent WMMA
|
|
createWmmaF32_16x16x16_bf16(12, 50);
|
|
|
|
runPassWithUnrollGemm();
|
|
|
|
int firstWmmaPos = -1;
|
|
int firstDsPos = -1;
|
|
int firstValuPos = -1;
|
|
int pos = 0;
|
|
for (const IRBase& ir : *bb) {
|
|
if (ir.getType() != IRBase::IRType::StinkyTofu) continue;
|
|
const auto* inst = cast<StinkyInstruction>(&ir);
|
|
const HwInstDesc* hw = inst->getHwInstDesc();
|
|
if (!hw || !hw->mnemonic) continue;
|
|
std::string_view mnem(hw->mnemonic);
|
|
if (mnem.find("wmma") != std::string_view::npos) {
|
|
if (firstWmmaPos < 0) firstWmmaPos = pos;
|
|
} else if (mnem.find("ds_load") != std::string_view::npos) {
|
|
if (firstDsPos < 0) firstDsPos = pos;
|
|
} else if (mnem.find("v_add") != std::string_view::npos) {
|
|
if (firstValuPos < 0) firstValuPos = pos;
|
|
}
|
|
pos++;
|
|
}
|
|
|
|
ASSERT_GE(firstWmmaPos, 0) << "No WMMA found";
|
|
ASSERT_GE(firstDsPos, 0) << "No ds_load found";
|
|
ASSERT_GE(firstValuPos, 0) << "No VALU found";
|
|
EXPECT_LT(firstWmmaPos, firstDsPos) << "WMMA should fire before ds_load (Phase B)";
|
|
EXPECT_LT(firstDsPos, firstValuPos)
|
|
<< "DS loads should be prioritized before VALU during WMMA latency";
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Co-execution hazard (regression test for destOverlapsActiveWmmaSrc):
|
|
// a ds_load whose dest VGPRs overlap the in-flight WMMA's src VGPRs must NOT be
|
|
// issued inside that WMMA's latency window, because the load could clobber a
|
|
// source register the WMMA is still reading.
|
|
//
|
|
// Setup: WMMA #0 reads v[50:58); the ds_load writes v[52:56) (overlap). Four
|
|
// more independent WMMAs (disjoint registers) are available. While WMMA #0 is
|
|
// in flight the ds_load is held back by the hazard gate, so the scheduler
|
|
// issues the next independent WMMA (D#100) first and only then the ds_load,
|
|
// once a WMMA whose sources it does not touch is the active one:
|
|
//
|
|
// wmma D#12 -> wmma D#100 -> ds_load D#52 -> wmma D#108/116/124
|
|
//
|
|
// Without the hazard gate the ds_load would issue right after WMMA #0
|
|
// (wmma D#12 -> ds_load -> wmma D#100 -> ...), clobbering v[52:56) mid-read.
|
|
// ---------------------------------------------------------------------------
|
|
TEST_F(DAGSchedulerPassTest, WmmaSrcOverlap_HazardDsLoadDeferredPastWindow) {
|
|
const int addrReg = 80;
|
|
// F8 MX WMMA fires first (Phase B). Its src VGPRs are v[50:58) (src0/src1)
|
|
// and v[12:20) (acc); see createWmmaScaleF8. cost latency=8 keeps the
|
|
// co-issue window open so the hazard gate is exercised.
|
|
createWmmaScaleF8(/*destStart=*/12, /*src0Start=*/50);
|
|
// ds_load dest v[52:56) overlaps the WMMA's src0 v[50:58): co-exec hazard.
|
|
createMovableDsLoad(/*destReg=*/52, addrReg, /*ldsToken=*/1);
|
|
// Independent WMMAs (registers disjoint from the hazard pair and from each
|
|
// other) to fill the latency window ahead of the deferred ds_load.
|
|
for (int i = 0; i < 4; i++)
|
|
createWmmaScaleF8(/*destStart=*/100 + i * 8, /*src0Start=*/200 + i * 8);
|
|
|
|
int beforeCount = countStinkyInstructions(*bb);
|
|
runPassWithUnrollGemm();
|
|
EXPECT_EQ(countStinkyInstructions(*bb), beforeCount)
|
|
<< "hazard deferral must not drop instructions";
|
|
|
|
// Collect (mnemonic-kind, first-dest-vgpr) in scheduled order.
|
|
std::vector<std::pair<std::string, int>> seq;
|
|
for (const IRBase& ir : *bb) {
|
|
if (ir.getType() != IRBase::IRType::StinkyTofu) continue;
|
|
const auto* inst = cast<StinkyInstruction>(&ir);
|
|
const HwInstDesc* hw = inst->getHwInstDesc();
|
|
if (!hw || !hw->mnemonic) continue;
|
|
std::string_view mnem(hw->mnemonic);
|
|
std::string kind = mnem.find("wmma") != std::string_view::npos ? "wmma"
|
|
: mnem.find("ds_load") != std::string_view::npos ? "ds"
|
|
: std::string(mnem);
|
|
int dst = (!inst->getDestRegs().empty() && inst->getDestRegs()[0].isRegister())
|
|
? static_cast<int>(inst->getDestRegs()[0].reg.idx)
|
|
: -1;
|
|
seq.push_back({kind, dst});
|
|
}
|
|
|
|
const std::vector<std::pair<std::string, int>> expected = {
|
|
{"wmma", 12}, {"wmma", 100}, {"ds", 52}, {"wmma", 108}, {"wmma", 116}, {"wmma", 124},
|
|
};
|
|
EXPECT_EQ(seq, expected)
|
|
<< "hazardous ds_load must be deferred until an independent WMMA (D#100) has "
|
|
"issued; it must not co-issue inside WMMA D#12's latency window";
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Co-execution hazard, VALU variant (regression test for destOverlapsActiveWmmaSrc
|
|
// on the VALU path): a VALU whose dest VGPR overlaps the in-flight WMMA's src VGPRs
|
|
// must NOT be issued inside that WMMA's latency window, because it could clobber a
|
|
// source register the WMMA is still reading.
|
|
//
|
|
// Unlike the ds_load variant, a VALU only becomes co-issue pickable at the positions
|
|
// set in the WMMA's co-issue window (MXWMMA_SCALE = 0x00C0, i.e. positions 6/7). Right
|
|
// after issue the position is 1, where isValuPickable() is already false, so extra
|
|
// non-hazardous fillers are needed to advance the co-issue timeline into a pickable
|
|
// position while WMMA #0 is still in flight. That is exactly the moment the hazard gate
|
|
// must fire:
|
|
// - 3 non-hazardous ds_loads (v[300:], v[320:], v[340:]) fill the per-WMMA DS cap and
|
|
// advance positions 1 -> 4.
|
|
// - 3 independent scalar ops advance positions 4 -> 7; at position 6 the VALU becomes
|
|
// co-issue pickable while WMMA #0 (v[50:58)) is still the active window.
|
|
//
|
|
// With the gate the hazardous VALU (dst v52) is skipped at position 6 (inside WMMA #0's
|
|
// window) and deferred until that window closes; it then issues right after D#100 opens a
|
|
// non-overlapping window. Without the gate it would co-issue at position 6, clobbering v52.
|
|
// ---------------------------------------------------------------------------
|
|
TEST_F(DAGSchedulerPassTest, WmmaSrcOverlap_HazardValuDeferredPastWindow) {
|
|
const int addrReg = 400;
|
|
auto createScalarOp = [&](int dst, int src0, int src1) {
|
|
AsmIRBuilder builder(*bb, arch);
|
|
StinkyInstruction* inst = builder.create(getMCIDByUOp(GFX::s_add_u32, arch));
|
|
inst->addDestReg(StinkyRegister("s", dst, 1));
|
|
inst->addSrcReg(StinkyRegister("s", src0, 1));
|
|
inst->addSrcReg(StinkyRegister("s", src1, 1));
|
|
return inst;
|
|
};
|
|
// F8 MX WMMA fires first (Phase B). Its src VGPRs are v[50:58) (src0/src1) and
|
|
// v[12:20) (acc); latency=8 keeps the co-issue window open so the gate is exercised.
|
|
createWmmaScaleF8(/*destStart=*/12, /*src0Start=*/50);
|
|
// Hazardous VALU: dst v52 overlaps the WMMA's src0 v[50:58): co-exec hazard.
|
|
createVAddInBlock(bb, arch, /*destReg=*/52, /*src0Reg=*/60, /*src1Reg=*/61);
|
|
// Non-hazardous fillers to advance the co-issue timeline into a VALU-pickable
|
|
// position (6) while WMMA #0 is still in flight.
|
|
createMovableDsLoad(/*destReg=*/300, addrReg, /*ldsToken=*/1);
|
|
createMovableDsLoad(/*destReg=*/320, addrReg, /*ldsToken=*/2);
|
|
createMovableDsLoad(/*destReg=*/340, addrReg, /*ldsToken=*/3);
|
|
createScalarOp(/*dst=*/10, 11, 12);
|
|
createScalarOp(/*dst=*/13, 14, 15);
|
|
createScalarOp(/*dst=*/16, 17, 18);
|
|
// Independent WMMAs (registers disjoint from the hazard pair and each other) to fill
|
|
// the latency windows ahead of the deferred VALU.
|
|
for (int i = 0; i < 4; i++)
|
|
createWmmaScaleF8(/*destStart=*/100 + i * 16, /*src0Start=*/200 + i * 16);
|
|
|
|
int beforeCount = countStinkyInstructions(*bb);
|
|
runPassWithUnrollGemm();
|
|
EXPECT_EQ(countStinkyInstructions(*bb), beforeCount)
|
|
<< "hazard deferral must not drop instructions";
|
|
|
|
// Collect (mnemonic-kind, first-dest-reg) in scheduled order.
|
|
std::vector<std::pair<std::string, int>> seq;
|
|
for (const IRBase& ir : *bb) {
|
|
if (ir.getType() != IRBase::IRType::StinkyTofu) continue;
|
|
const auto* inst = cast<StinkyInstruction>(&ir);
|
|
const HwInstDesc* hw = inst->getHwInstDesc();
|
|
if (!hw || !hw->mnemonic) continue;
|
|
std::string_view mnem(hw->mnemonic);
|
|
std::string kind = mnem.find("wmma") != std::string_view::npos ? "wmma"
|
|
: mnem.find("ds_load") != std::string_view::npos ? "ds"
|
|
: mnem.rfind("v_", 0) == 0 ? "valu"
|
|
: mnem.rfind("s_", 0) == 0 ? "s"
|
|
: std::string(mnem);
|
|
int dst = (!inst->getDestRegs().empty() && inst->getDestRegs()[0].isRegister())
|
|
? static_cast<int>(inst->getDestRegs()[0].reg.idx)
|
|
: -1;
|
|
seq.push_back({kind, dst});
|
|
}
|
|
|
|
// Hazardous VALU (dst v52) is deferred past WMMA D#12's window, then issues right after
|
|
// the first independent WMMA (D#100), whose window no longer overlaps v52.
|
|
const std::vector<std::pair<std::string, int>> expected = {
|
|
{"wmma", 12}, {"ds", 300}, {"ds", 320}, {"ds", 340}, {"s", 10}, {"s", 13},
|
|
{"s", 16}, {"wmma", 100}, {"valu", 52}, {"wmma", 116}, {"wmma", 132}, {"wmma", 148},
|
|
};
|
|
EXPECT_EQ(seq, expected)
|
|
<< "hazardous VALU (dst v52) must not co-issue inside WMMA D#12's latency window; "
|
|
"it must be deferred until that window closes, then issue once a non-overlapping "
|
|
"WMMA window is open";
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Hidden-stall window fill (pickFreeBest allowHiddenStall path): a SALU that is
|
|
// only blocked by a src RAW hazard whose remaining wait fits under the active
|
|
// WMMA's latency shadow may be co-issued *inside* that window — the stall we pay
|
|
// waiting for its src is hidden by the in-flight WMMA, so it costs no extra
|
|
// cycles. It must therefore be preferred over starting the next independent WMMA.
|
|
//
|
|
// Setup (region, not a loop, so no loop-head deferral):
|
|
// - WMMA #0 (v[12:20)) fires first (Phase B) and opens an 8-cycle window.
|
|
// - A chain of inter-dependent SALUs a0 -> a1 -> a2 -> a3, each writing s(100+i)
|
|
// with latency=2 > issue=1 so issuing it stamps a 1-cycle data-ready latency
|
|
// on its dest (the src RAW gate for the next link). a0 is free; a1..a3 are
|
|
// each RAW-blocked for 1 cycle, which fits under WMMA #0's remaining latency
|
|
// shadow, so every link is a valid hidden-stall fill. The chain is strict, so
|
|
// only one link is ready at a time — their relative order is forced by the
|
|
// DAG; the test is purely about whether each link lands inside the window.
|
|
// - WMMA #1 (v[200:208)) is independent and ready.
|
|
//
|
|
// Expected (new behavior): wmma#0, a0, a1, a2, a3, wmma#1
|
|
// Every chain link is co-issued inside wmma#0's window, ahead of wmma#1.
|
|
// Old behavior would issue wmma#1 as soon as a0's consumer was RAW-blocked
|
|
// (wmma#0, a0, wmma#1, a1, a2, a3), because a RAW-blocked SALU was never pickable
|
|
// inside the window.
|
|
// ---------------------------------------------------------------------------
|
|
TEST_F(DAGSchedulerPassTest, HiddenStallSaluFillsWmmaWindowBeforeNextWmma) {
|
|
auto createScalarAdd = [&](int dst, int src0, int src1) {
|
|
AsmIRBuilder builder(*bb, arch);
|
|
StinkyInstruction* inst = builder.create(getMCIDByUOp(GFX::s_add_u32, arch));
|
|
inst->addDestReg(StinkyRegister("s", dst, 1));
|
|
inst->addSrcReg(StinkyRegister("s", src0, 1));
|
|
inst->addSrcReg(StinkyRegister("s", src1, 1));
|
|
return inst;
|
|
};
|
|
|
|
// WMMA #0 fires first (Phase B), latency=8 keeps its co-issue window open.
|
|
createWmmaScaleF8(/*destStart=*/12, /*src0Start=*/50);
|
|
// Chain a0 -> a1 -> a2 -> a3: a_i writes s(100+i), a_(i+1) reads it (RAW).
|
|
// Each 1-cycle wait fits the shrinking window (positions 2,4,6,8), so all four
|
|
// are hidden-stall filled inside WMMA #0's window.
|
|
const int kChain = 4;
|
|
for (int i = 0; i < kChain; i++) {
|
|
const int src0 = (i == 0) ? 0 : (100 + i - 1); // previous link's dest
|
|
StinkyInstruction* a = createScalarAdd(/*dst=*/100 + i, src0, /*src1=*/1);
|
|
a->issueCycles = 1;
|
|
a->latencyCycles = 2;
|
|
}
|
|
// WMMA #1: independent (disjoint regs) and ready.
|
|
createWmmaScaleF8(/*destStart=*/200, /*src0Start=*/220);
|
|
|
|
int beforeCount = countStinkyInstructions(*bb);
|
|
runPassWithUnrollGemm();
|
|
EXPECT_EQ(countStinkyInstructions(*bb), beforeCount)
|
|
<< "hidden-stall fill must not drop instructions";
|
|
|
|
std::vector<std::pair<std::string, int>> seq;
|
|
for (const IRBase& ir : *bb) {
|
|
if (ir.getType() != IRBase::IRType::StinkyTofu) continue;
|
|
const auto* inst = cast<StinkyInstruction>(&ir);
|
|
const HwInstDesc* hw = inst->getHwInstDesc();
|
|
if (!hw || !hw->mnemonic) continue;
|
|
std::string_view mnem(hw->mnemonic);
|
|
std::string kind = mnem.find("wmma") != std::string_view::npos ? "wmma"
|
|
: mnem.rfind("s_", 0) == 0 ? "s"
|
|
: std::string(mnem);
|
|
int dst = (!inst->getDestRegs().empty() && inst->getDestRegs()[0].isRegister())
|
|
? static_cast<int>(inst->getDestRegs()[0].reg.idx)
|
|
: -1;
|
|
seq.push_back({kind, dst});
|
|
}
|
|
|
|
const std::vector<std::pair<std::string, int>> expected = {
|
|
{"wmma", 12}, {"s", 100}, {"s", 101}, {"s", 102}, {"s", 103}, {"wmma", 200},
|
|
};
|
|
EXPECT_EQ(seq, expected)
|
|
<< "every link of the RAW-dependent SALU chain must be co-issued inside WMMA #0's latency "
|
|
"window (each 1-cycle wait hidden by the in-flight WMMA), ahead of the independent "
|
|
"WMMA #1";
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Property: per-WMMA-window DS cap — after a WMMA fires,
|
|
// at most floor((latency - issue) / 2) = 3 ds_loads can issue in its window
|
|
// because back-to-back ds_load issue cost doubles.
|
|
// ---------------------------------------------------------------------------
|
|
TEST_F(DAGSchedulerPassTest, DSWindowCap_VALUInterleaveAfter3) {
|
|
const int addrReg = 80;
|
|
// 5 independent ds_loads (LDS pseudo-reg makes them movable in DAG)
|
|
createMovableDsLoad(0, addrReg, 1);
|
|
createMovableDsLoad(4, addrReg, 2);
|
|
createMovableDsLoad(8, addrReg, 3);
|
|
createMovableDsLoad(12, addrReg, 4);
|
|
createMovableDsLoad(16, addrReg, 5);
|
|
// 2 independent VALUs
|
|
createVAddInBlock(bb, arch, 60, 61, 62);
|
|
createVAddInBlock(bb, arch, 63, 64, 65);
|
|
// 2 independent WMMAs (fire first, create co-issue window)
|
|
createWmmaF32_16x16x16_bf16(20, 28);
|
|
createWmmaF32_16x16x16_bf16(36, 44);
|
|
|
|
runPassWithUnrollGemm();
|
|
|
|
int consecutiveDs = 0;
|
|
int maxConsecutiveDs = 0;
|
|
for (const IRBase& ir : *bb) {
|
|
if (ir.getType() != IRBase::IRType::StinkyTofu) continue;
|
|
const auto* inst = cast<StinkyInstruction>(&ir);
|
|
const HwInstDesc* hw = inst->getHwInstDesc();
|
|
if (!hw || !hw->mnemonic) continue;
|
|
std::string_view mnem(hw->mnemonic);
|
|
if (mnem.find("ds_load") != std::string_view::npos) {
|
|
consecutiveDs++;
|
|
maxConsecutiveDs = std::max(maxConsecutiveDs, consecutiveDs);
|
|
} else {
|
|
consecutiveDs = 0;
|
|
}
|
|
}
|
|
|
|
EXPECT_LE(maxConsecutiveDs, 3) << "DS window cap violated: found " << maxConsecutiveDs
|
|
<< " consecutive ds_loads (max 3 per WMMA window)";
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Property: all original instructions are preserved.
|
|
// ---------------------------------------------------------------------------
|
|
TEST_F(DAGSchedulerPassTest, DSWindowCap_InstructionCountPreserved) {
|
|
const int addrReg = 100;
|
|
// 6 independent ds_loads (LDS pseudo-reg makes them movable in DAG)
|
|
for (int i = 0; i < 6; i++) createMovableDsLoad(i * 4, addrReg, i + 1);
|
|
// 4 independent VALUs
|
|
for (int i = 0; i < 4; i++) createVAddInBlock(bb, arch, 30 + i, 40 + i, 50 + i);
|
|
// 3 independent WMMAs
|
|
for (int i = 0; i < 3; i++) createWmmaF32_16x16x16_bf16(60 + i * 8, 84 + i * 8);
|
|
|
|
int beforeCount = countStinkyInstructions(*bb);
|
|
runPassWithUnrollGemm();
|
|
int afterCount = countStinkyInstructions(*bb);
|
|
|
|
EXPECT_EQ(afterCount, beforeCount) << "Scheduler must preserve instruction count";
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// tensor_load_to_lds bounded in-flight credit pool (dagFeatures.globalReadQueueDepth
|
|
// / globalReadDrainLatency). The throttle is loop-only (uses the CDNA5 loop path),
|
|
// so all of these build a self-loop body with 4 movable tensor_loads + 10 VALU.
|
|
//
|
|
// Model: each issued tensor_load holds a credit for `drainLatency` cycles, decayed
|
|
// once per issued instruction (~1 cycle/pick here). With depth D, at most D credits
|
|
// may be in flight, so once D loads have issued the scheduler must interleave VALU
|
|
// until a credit drains before issuing the next load.
|
|
//
|
|
// NOTE on the chosen numbers: a credit must survive long enough to still be in
|
|
// flight when the next load wants to issue, so the throttle only engages when
|
|
// drainLatency > depth. At 1 issue/cycle a credit that drains in <= depth cycles
|
|
// frees before the cap is reached, so the hardware sustains D loads/cycle with no
|
|
// stall. We therefore use drainLatency comfortably above depth and assert the
|
|
// back-to-back run equals exactly depth (D loads, then forced interleave).
|
|
// ---------------------------------------------------------------------------
|
|
|
|
// Depth 2: exactly 2 tensor_loads issue back-to-back, then VALU must interleave.
|
|
TEST_F(DAGSchedulerPassTest, GlobalReadThrottle_Depth2_RespectsQueueDepth) {
|
|
// Self-loop on the entry block so it is in RPO (scheduled) AND detected as a
|
|
// loop (the cross-BB credit carry is loop-only). buildLoopBB makes an
|
|
// unreachable second block, so reuse the entry block here.
|
|
BasicBlock* body = bb;
|
|
body->addSuccessor(body);
|
|
for (int i = 0; i < 4; i++)
|
|
createMovableTensorLoad(body, /*s0=*/i * 12, /*s1=*/i * 12 + 4, /*ldsToken=*/i + 1);
|
|
for (int i = 0; i < 30; i++) createVAddInBlock(body, arch, 40 + i, 80 + i, 100 + i);
|
|
|
|
runPassWithGlobalReadThrottle(/*depth=*/2, /*drainLatency=*/8);
|
|
|
|
std::vector<std::string> seq = mnemonicSequence(*body);
|
|
EXPECT_EQ(maxConsecutiveTensorLoads(seq), 2)
|
|
<< "depth=2: at most 2 tensor_loads in flight before an interleave is forced";
|
|
}
|
|
|
|
// Depth 3: exactly 3 tensor_loads issue back-to-back, then VALU must interleave.
|
|
TEST_F(DAGSchedulerPassTest, GlobalReadThrottle_Depth3_RespectsQueueDepth) {
|
|
// Self-loop on the entry block so it is in RPO (scheduled) AND detected as a
|
|
// loop (the cross-BB credit carry is loop-only). buildLoopBB makes an
|
|
// unreachable second block, so reuse the entry block here.
|
|
BasicBlock* body = bb;
|
|
body->addSuccessor(body);
|
|
for (int i = 0; i < 4; i++)
|
|
createMovableTensorLoad(body, /*s0=*/i * 12, /*s1=*/i * 12 + 4, /*ldsToken=*/i + 1);
|
|
for (int i = 0; i < 30; i++) createVAddInBlock(body, arch, 40 + i, 80 + i, 100 + i);
|
|
|
|
runPassWithGlobalReadThrottle(/*depth=*/3, /*drainLatency=*/8);
|
|
|
|
std::vector<std::string> seq = mnemonicSequence(*body);
|
|
EXPECT_EQ(maxConsecutiveTensorLoads(seq), 3)
|
|
<< "depth=3: at most 3 tensor_loads in flight before an interleave is forced";
|
|
}
|
|
|
|
// Depth 1: degenerate cap — every tensor_load must be separated by other work.
|
|
TEST_F(DAGSchedulerPassTest, GlobalReadThrottle_Depth1_SeparatesEveryLoad) {
|
|
// Self-loop on the entry block so it is in RPO (scheduled) AND detected as a
|
|
// loop (the cross-BB credit carry is loop-only). buildLoopBB makes an
|
|
// unreachable second block, so reuse the entry block here.
|
|
BasicBlock* body = bb;
|
|
body->addSuccessor(body);
|
|
for (int i = 0; i < 4; i++)
|
|
createMovableTensorLoad(body, /*s0=*/i * 12, /*s1=*/i * 12 + 4, /*ldsToken=*/i + 1);
|
|
for (int i = 0; i < 30; i++) createVAddInBlock(body, arch, 40 + i, 80 + i, 100 + i);
|
|
|
|
runPassWithGlobalReadThrottle(/*depth=*/1, /*drainLatency=*/8);
|
|
|
|
std::vector<std::string> seq = mnemonicSequence(*body);
|
|
EXPECT_EQ(maxConsecutiveTensorLoads(seq), 1) << "depth=1: no two tensor_loads may be adjacent";
|
|
}
|
|
|
|
// All instructions are preserved regardless of throttle (count invariant).
|
|
TEST_F(DAGSchedulerPassTest, GlobalReadThrottle_PreservesInstructionCount) {
|
|
// Self-loop on the entry block so it is in RPO (scheduled) AND detected as a
|
|
// loop (the cross-BB credit carry is loop-only). buildLoopBB makes an
|
|
// unreachable second block, so reuse the entry block here.
|
|
BasicBlock* body = bb;
|
|
body->addSuccessor(body);
|
|
for (int i = 0; i < 4; i++)
|
|
createMovableTensorLoad(body, /*s0=*/i * 12, /*s1=*/i * 12 + 4, /*ldsToken=*/i + 1);
|
|
for (int i = 0; i < 30; i++) createVAddInBlock(body, arch, 40 + i, 80 + i, 100 + i);
|
|
|
|
int beforeCount = countStinkyInstructions(*body);
|
|
runPassWithGlobalReadThrottle(/*depth=*/2, /*drainLatency=*/8);
|
|
EXPECT_EQ(countStinkyInstructions(*body), beforeCount) << "throttle must not drop instructions";
|
|
}
|
|
|
|
// Throttle off (depth=0): feature is opt-in and does not perturb the default path.
|
|
TEST_F(DAGSchedulerPassTest, GlobalReadThrottle_Disabled_PreservesAll) {
|
|
// Self-loop on the entry block so it is in RPO (scheduled) AND detected as a
|
|
// loop (the cross-BB credit carry is loop-only). buildLoopBB makes an
|
|
// unreachable second block, so reuse the entry block here.
|
|
BasicBlock* body = bb;
|
|
body->addSuccessor(body);
|
|
for (int i = 0; i < 4; i++)
|
|
createMovableTensorLoad(body, /*s0=*/i * 12, /*s1=*/i * 12 + 4, /*ldsToken=*/i + 1);
|
|
for (int i = 0; i < 30; i++) createVAddInBlock(body, arch, 40 + i, 80 + i, 100 + i);
|
|
|
|
int beforeCount = countStinkyInstructions(*body);
|
|
runPassWithGlobalReadThrottle(/*depth=*/0, /*drainLatency=*/0);
|
|
EXPECT_EQ(countStinkyInstructions(*body), beforeCount);
|
|
}
|
|
|
|
// SGPR->tensor_load hazard: a SALU that writes an SGPR a tensor_load reads must be
|
|
// separated from that tensor_load by the fixed hardware gap (kCdna5HazardRules'
|
|
// SaluSgprToMemAddr entry, 8 cycles). Mirrors the real case (wmma/ds fill around the
|
|
// SALU): the scheduler hoists the SALU and/or holds the tensor_load so >= 8 cycles of
|
|
// work sit between them. We assert the cycle invariant, not an exact order. A WMMA
|
|
// counts as its latencyCycles (the co-issue window it opens, 8 here), other ops as
|
|
// issueCycles.
|
|
TEST_F(DAGSchedulerPassTest, SgprToTensorLoadHazard_AtLeast8CycleGap) {
|
|
BasicBlock* body = bb;
|
|
body->addSuccessor(body);
|
|
|
|
// Movable ds_loads (LDS token -> stay in one region, no side-effect boundary) as the
|
|
// fill work, a SALU writing s0, and a tensor_load reading s[0:4) so s0 is the hazard
|
|
// register. Enough ds fill (>= hazard) so the gap is filled by real work, observable
|
|
// in the emitted order rather than an invisible stall.
|
|
for (int i = 0; i < 12; i++)
|
|
createMovableDsLoad(/*destReg=*/8 + i * 4, /*addrReg=*/60, /*ldsToken=*/i + 2);
|
|
|
|
AsmIRBuilder builder(*body, arch);
|
|
StinkyInstruction* salu = builder.create(getMCIDByUOp(GFX::s_mov_b32, arch));
|
|
salu->addDestReg(StinkyRegister("s", 0, 1));
|
|
salu->addSrcReg(StinkyRegister(0));
|
|
|
|
createMovableTensorLoad(body, /*s0=*/0, /*s1=*/4, /*ldsToken=*/1);
|
|
|
|
runPassWithGlobalReadThrottle(/*depth=*/4, /*drainLatency=*/8);
|
|
|
|
// Locate the SALU and the tensor_load in the scheduled order, and total the cycles
|
|
// of the work between them (WMMA -> latency window, else issue cycles).
|
|
int saluPos = -1, tensorPos = -1, idx = 0;
|
|
std::vector<int> cyclesAt;
|
|
for (const IRBase& ir : *body) {
|
|
if (ir.getType() != IRBase::IRType::StinkyTofu) continue;
|
|
const auto* inst = cast<StinkyInstruction>(&ir);
|
|
cyclesAt.push_back(isMatrixInstruction(*inst) ? inst->latencyCycles : inst->issueCycles);
|
|
if (inst == salu) saluPos = idx;
|
|
if (isTensorLoad(*inst)) tensorPos = idx;
|
|
idx++;
|
|
}
|
|
ASSERT_GE(saluPos, 0);
|
|
ASSERT_GE(tensorPos, 0);
|
|
ASSERT_LT(saluPos, tensorPos) << "SALU must be scheduled before the tensor_load it feeds";
|
|
|
|
int gap = 0;
|
|
for (int i = saluPos + 1; i < tensorPos; i++) gap += cyclesAt[i];
|
|
EXPECT_GE(gap, 8) << "tensor_load must be >= 8 cycles after the SALU writing its SGPR";
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// dsReadQueueDepth / dsReadDrainLatency / dsReadPerWmma: same in-flight
|
|
// credit-pool mechanism as globalReadQueueDepth/globalReadDrainLatency, but
|
|
// gating ds_read_b128 instead of tensor_load_to_lds. Unlike global-read
|
|
// throttling, the ds_read gate additionally requires a WMMA to have been
|
|
// picked at least once (it seeds maxDsPerWmmaWindow_), so each test below
|
|
// includes one WMMA read (with dest/src registers disjoint from the ds_reads,
|
|
// so its DS-latency gate is trivially satisfied and it issues first).
|
|
// ---------------------------------------------------------------------------
|
|
|
|
// Depth 2: exactly 2 ds_reads issue back-to-back, then VALU must interleave.
|
|
TEST_F(DAGSchedulerPassTest, DsReadThrottle_Depth2_RespectsQueueDepth) {
|
|
BasicBlock* body = bb;
|
|
body->addSuccessor(body);
|
|
createWmmaF32_16x16x16_bf16_in(body, /*destStart=*/200, /*src0Start=*/204);
|
|
for (int i = 0; i < 4; i++)
|
|
createMovableDsLoad(/*destReg=*/i * 4, /*addrReg=*/300 + i * 4, /*ldsToken=*/i + 1);
|
|
for (int i = 0; i < 30; i++) createVAddInBlock(body, arch, 40 + i, 80 + i, 100 + i);
|
|
|
|
runPassWithDsReadThrottle(/*queueDepth=*/2, /*drainLatency=*/8);
|
|
|
|
std::vector<std::string> seq = mnemonicSequence(*body);
|
|
EXPECT_EQ(maxConsecutiveDsReads(seq), 2)
|
|
<< "depth=2: at most 2 ds_reads in flight before an interleave is forced";
|
|
}
|
|
|
|
// Depth 1: degenerate cap — every ds_read must be separated by other work.
|
|
TEST_F(DAGSchedulerPassTest, DsReadThrottle_Depth1_SeparatesEveryLoad) {
|
|
BasicBlock* body = bb;
|
|
body->addSuccessor(body);
|
|
createWmmaF32_16x16x16_bf16_in(body, /*destStart=*/200, /*src0Start=*/204);
|
|
for (int i = 0; i < 4; i++)
|
|
createMovableDsLoad(/*destReg=*/i * 4, /*addrReg=*/300 + i * 4, /*ldsToken=*/i + 1);
|
|
for (int i = 0; i < 30; i++) createVAddInBlock(body, arch, 40 + i, 80 + i, 100 + i);
|
|
|
|
runPassWithDsReadThrottle(/*queueDepth=*/1, /*drainLatency=*/8);
|
|
|
|
std::vector<std::string> seq = mnemonicSequence(*body);
|
|
EXPECT_EQ(maxConsecutiveDsReads(seq), 1) << "depth=1: no two ds_reads may be adjacent";
|
|
}
|
|
|
|
// NOTE: the former DsReadThrottle_PerWmmaCap_RespectsCap test isolated the
|
|
// per-WMMA-window cap with a single WMMA. That predated the ds_load in-flight
|
|
// queue; now the cap only binds while a WMMA window is active (covered by
|
|
// DSWindowCap_VALUInterleaveAfter3, which keeps two WMMAs pending), and the
|
|
// no-WMMA case is bounded by the in-flight queue (covered by
|
|
// DsReadThrottle_Depth2_RespectsQueueDepth). No standalone single-WMMA cap test
|
|
// is kept — it would assert behavior the cap no longer has.
|
|
|
|
// Regression (see image(2).png bug): with no WMMA to issue and a chain of
|
|
// ds_loads each consumed by a VALU (RAW), the scheduler must NOT interleave
|
|
// ds,ds,valu,ds,ds,valu — that pattern forces an s_wait_dscnt per pair and
|
|
// tanks the kernel. Loads have their own in-flight queue, so they should drain
|
|
// (up to queue depth) before the consumer VALUs run: the consumers RAW-depend on
|
|
// the loads and are hazard-deferred until the load latency clears. Assert the
|
|
// loads front-load ahead of every consumer VALU.
|
|
TEST_F(DAGSchedulerPassTest, DsReadThrottle_NoWmma_LoadsDrainBeforeConsumerValu) {
|
|
BasicBlock* body = bb;
|
|
body->addSuccessor(body);
|
|
// 6 ds_loads (movable via LDS token) on the same address register; each VALU
|
|
// consumes the matching load's dest (RAW), mirroring the image's
|
|
// ds_load_u8 -> v_lshl_or_b32 dependency chain. No WMMA in the region.
|
|
for (int i = 0; i < 6; i++)
|
|
createMovableDsLoad(/*destReg=*/i * 4, /*addrReg=*/200, /*ldsToken=*/i + 1);
|
|
for (int i = 0; i < 6; i++)
|
|
createVAddInBlock(body, arch, /*dst=*/100 + i, /*src0=*/i * 4, /*src1=*/i * 4 + 1);
|
|
|
|
// Queue depth 6 so all loads can be in flight at once; perWmma irrelevant (no WMMA).
|
|
runPassWithDsReadThrottle(/*queueDepth=*/6, /*drainLatency=*/8, /*perWmma=*/100);
|
|
|
|
std::vector<std::string> seq = mnemonicSequence(*body);
|
|
// Every ds_load must precede every v_add: find the last load and first valu.
|
|
int lastLoad = -1, firstValu = -1;
|
|
for (int i = 0; i < (int)seq.size(); i++) {
|
|
if (seq[i] == "ds_load_b128") lastLoad = i;
|
|
if (seq[i] == "v_add_f32" && firstValu < 0) firstValu = i;
|
|
}
|
|
ASSERT_GE(lastLoad, 0);
|
|
ASSERT_GE(firstValu, 0);
|
|
EXPECT_LT(lastLoad, firstValu)
|
|
<< "no-WMMA: all ds_loads must drain before consumer VALUs (no ds,valu,ds interleave)";
|
|
}
|
|
|
|
// Type-A WAR via elapse-time ordering (replaces the old dsAddrReadLatencyCounters):
|
|
// a VALU that overwrites the ds_load's address reg must be deferred behind other
|
|
// independent VALUs, because that reg was just touched (small elapse) — even though
|
|
// the overwrite is EARLIEST in program order (smallest DAG id, which plain
|
|
// pop()-by-id would pick first). This proves the read->write gap comes from elapse
|
|
// ordering, not a hard counter.
|
|
TEST_F(DAGSchedulerPassTest, WarOverwriteOfDsAddrDeferredByElapse) {
|
|
BasicBlock* body = bb;
|
|
body->addSuccessor(body);
|
|
// ds_load reads address v200 (single ds_load so no load-drain effects dominate).
|
|
createMovableDsLoad(/*destReg=*/8, /*addrReg=*/200, /*ldsToken=*/1);
|
|
// Overwrite of v200 — created FIRST after the load, so it has the smallest DAG id
|
|
// among the VALUs. Independent VALUs (disjoint regs) created after it.
|
|
StinkyInstruction* overwrite = createVAddInBlock(body, arch, /*dst=*/200, /*src0=*/101,
|
|
/*src1=*/102);
|
|
for (int i = 0; i < 3; i++)
|
|
createVAddInBlock(body, arch, /*dst=*/50 + i, /*src0=*/60 + i, /*src1=*/70 + i);
|
|
|
|
runPassWithUnrollGemm();
|
|
|
|
// Find the scheduled position of the overwrite vs. the independent VALUs.
|
|
int overwritePos = -1, firstIndependentPos = -1, idx = 0;
|
|
for (const IRBase& ir : *body) {
|
|
if (ir.getType() != IRBase::IRType::StinkyTofu) continue;
|
|
const auto* inst = cast<StinkyInstruction>(&ir);
|
|
if (inst == overwrite)
|
|
overwritePos = idx;
|
|
else if (inst->getUnifiedOpcode() == GFX::v_add_f32 && firstIndependentPos < 0)
|
|
firstIndependentPos = idx;
|
|
idx++;
|
|
}
|
|
ASSERT_GE(overwritePos, 0);
|
|
ASSERT_GE(firstIndependentPos, 0);
|
|
EXPECT_GT(overwritePos, firstIndependentPos)
|
|
<< "WAR overwrite of the ds_load address must be deferred behind independent VALUs "
|
|
"by elapse-time ordering, despite having the smallest DAG id";
|
|
}
|
|
|
|
// All instructions are preserved regardless of throttle (count invariant).
|
|
TEST_F(DAGSchedulerPassTest, DsReadThrottle_PreservesInstructionCount) {
|
|
BasicBlock* body = bb;
|
|
body->addSuccessor(body);
|
|
createWmmaF32_16x16x16_bf16_in(body, /*destStart=*/200, /*src0Start=*/204);
|
|
for (int i = 0; i < 4; i++)
|
|
createMovableDsLoad(/*destReg=*/i * 4, /*addrReg=*/300 + i * 4, /*ldsToken=*/i + 1);
|
|
for (int i = 0; i < 30; i++) createVAddInBlock(body, arch, 40 + i, 80 + i, 100 + i);
|
|
|
|
int beforeCount = countStinkyInstructions(*body);
|
|
runPassWithDsReadThrottle(/*queueDepth=*/2, /*drainLatency=*/8);
|
|
EXPECT_EQ(countStinkyInstructions(*body), beforeCount) << "throttle must not drop instructions";
|
|
}
|