TheRock/rocm-libraries/shared/stinkytofu/tests/unit/asm/DAGSchedulerPassTest.cpp

/* ************************************************************************
 * Copyright (C) 2025-2026 Advanced Micro Devices, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 *
 * ************************************************************************ */
#include <gtest/gtest.h>

#include "TestHelpers.hpp"
#include "stinkytofu/analysis/AnalysisRegistration.hpp"
#include "stinkytofu/core/PassManager.hpp"
#include "stinkytofu/ir/asm/StinkyModifiers.hpp"
#include "stinkytofu/support/Casting.hpp"
#include "stinkytofu/transforms/asm/StinkyDAGSchedulerPass.hpp"

using namespace stinkytofu;
using namespace stinkytofu::test;

static int countStinkyInstructions(const BasicBlock& bb) {
    int count = 0;
    for (const IRBase& ir : bb) {
        if (ir.getType() == IRBase::IRType::StinkyTofu) count++;
    }
    return count;
}

class DAGSchedulerPassTest : public ::testing::Test {
   protected:
    GfxArchID arch = GfxArchID::Gfx1250;
    GemmTileConfig config;
    std::unique_ptr<Function> func;
    BasicBlock* bb = nullptr;
    std::unique_ptr<Pass> pass;
    AnalysisManager am;

    void SetUp() override {
        config.arch[0] = 12;
        config.arch[1] = 5;
        config.arch[2] = 0;
        func = std::make_unique<Function>("dag_sched_test");
        setFunctionArch(*func, arch);
        bb = func->createBasicBlock("entry");
        pass = createStinkyDAGSchedulerPass();
        registerAllAnalyses(am);
    }

    void TearDown() override {
        pass.reset();
        func.reset();
        bb = nullptr;
    }

    void runPass() {
        PassContext ctx;
        ctx.setGemmTileConfig(config);
        pass->run(*func, ctx, am);
    }

    void runPassWithUnrollGemm() {
        PassContext ctx;
        ctx.setGemmTileConfig(config);
        PassFeatureConfig pfc;
        pfc.loopConfig.unrollGemm = true;
        ctx.setPassFeatureConfig(pfc);
        pass->run(*func, ctx, am);
    }

    // Run with the tensor_load_to_lds credit-pool throttle enabled.
    // distributeGlobalRead routes tensor loads into globalReadQueue; depth/latency
    // configure the in-flight credit pool.
    void runPassWithGlobalReadThrottle(int depth, int drainLatency) {
        PassContext ctx;
        ctx.setGemmTileConfig(config);
        PassFeatureConfig pfc;
        pfc.loopConfig.unrollGemm = true;
        pfc.dagFeatures.distributeGlobalRead = true;
        pfc.dagFeatures.globalReadQueueDepth = depth;
        pfc.dagFeatures.globalReadDrainLatency = drainLatency;
        ctx.setPassFeatureConfig(pfc);
        pass->run(*func, ctx, am);
    }

    // Run with the ds_read in-flight credit-pool throttle enabled. perWmma is held
    // generously high by default so the separate per-WMMA-window ds cap never binds,
    // isolating queueDepth/drainLatency as the only active constraint (mirrors how
    // runPassWithGlobalReadThrottle isolates globalReadQueueDepth/globalReadDrainLatency).
    void runPassWithDsReadThrottle(int queueDepth, int drainLatency, int perWmma = 100) {
        PassContext ctx;
        ctx.setGemmTileConfig(config);
        PassFeatureConfig pfc;
        pfc.loopConfig.unrollGemm = true;
        pfc.dagFeatures.dsReadQueueDepth = queueDepth;
        pfc.dagFeatures.dsReadDrainLatency = drainLatency;
        pfc.dagFeatures.dsReadPerWmma = perWmma;
        ctx.setPassFeatureConfig(pfc);
        pass->run(*func, ctx, am);
    }

    // Linearized mnemonic order of a block (skips PHIs and non-Stinky IR).
    static std::vector<std::string> mnemonicSequence(const BasicBlock& block) {
        std::vector<std::string> seq;
        for (const IRBase& ir : block) {
            if (ir.getType() != IRBase::IRType::StinkyTofu) continue;
            const auto* inst = cast<StinkyInstruction>(&ir);
            const HwInstDesc* hw = inst->getHwInstDesc();
            if (!hw || !hw->mnemonic) continue;
            seq.push_back(hw->mnemonic);
        }
        return seq;
    }

    // Largest run of consecutive tensor_load_to_lds in a mnemonic sequence.
    static int maxConsecutiveTensorLoads(const std::vector<std::string>& seq) {
        int run = 0, best = 0;
        for (const std::string& m : seq) {
            if (m == "tensor_load_to_lds") {
                run++;
                best = std::max(best, run);
            } else {
                run = 0;
            }
        }
        return best;
    }

    // Largest run of consecutive ds_load_b128 in a mnemonic sequence
    // (gfx1250's actual mnemonic for this op; see Gfx1250Instructions.def).
    static int maxConsecutiveDsReads(const std::vector<std::string>& seq) {
        int run = 0, best = 0;
        for (const std::string& m : seq) {
            if (m == "ds_load_b128") {
                run++;
                best = std::max(best, run);
            } else {
                run = 0;
            }
        }
        return best;
    }

    // Build a single-BB self-loop so the scheduler uses the loop-aware CDNA5 path.
    // Returns the loop body BB with the branch already appended.
    BasicBlock* buildLoopBB(const char* label = "loop_body") {
        BasicBlock* body = func->createBasicBlock(label);
        body->addSuccessor(body);
        return body;
    }

    static StinkyInstruction* createSCbranchInBlock(BasicBlock* bb, GfxArchID arch) {
        AsmIRBuilder builder(*bb, arch);
        return builder.create(getMCIDByUOp(GFX::s_cbranch_scc0, arch));
    }

    StinkyInstruction* createWmmaF32_16x16x16_bf16_in(BasicBlock* targetBB, int destStart,
                                                      int src0Start) {
        AsmIRBuilder builder(*targetBB, arch);
        const HwInstDesc* desc = getMCIDByUOp(GFX::v_wmma_f32_16x16x16_bf16, arch);
        if (!desc) return nullptr;
        StinkyInstruction* inst = builder.create(desc);
        inst->addDestReg(StinkyRegister("v", destStart, 8));
        inst->addSrcReg(StinkyRegister("v", src0Start, 8));
        inst->addSrcReg(StinkyRegister("v", src0Start, 8));
        inst->addSrcReg(StinkyRegister("v", destStart, 8));
        return inst;
    }

    StinkyInstruction* createWmmaF32_16x16x16_bf16(int destStart, int src0Start) {
        return createWmmaF32_16x16x16_bf16_in(bb, destStart, src0Start);
    }

    // v_wmma_scale_f32_16x16x128_f8f6f4 with F8 (FP8) input matrix formats. Its
    // cost is {issue=1, latency=8}, so the co-issue latency window stays open
    // right after issue, which is what exercises the co-exec hazard gate.
    // src VGPRs: v[src0Start:src0Start+8) (src0/src1) and v[destStart:destStart+8) (acc).
    StinkyInstruction* createWmmaScaleF8_in(BasicBlock* targetBB, int destStart, int src0Start) {
        AsmIRBuilder builder(*targetBB, arch);
        const HwInstDesc* desc = getMCIDByUOp(GFX::v_wmma_scale_f32_16x16x128_f8f6f4, arch);
        if (!desc) return nullptr;
        StinkyInstruction* inst = builder.create(desc);
        inst->addDestReg(StinkyRegister("v", destStart, 8));
        inst->addSrcReg(StinkyRegister("v", src0Start, 8));
        inst->addSrcReg(StinkyRegister("v", src0Start, 8));
        inst->addSrcReg(StinkyRegister("v", destStart, 8));
        MatrixFmtModifiers fmtMod;
        fmtMod.fmtA = MatrixFmt::FP8;
        fmtMod.fmtB = MatrixFmt::FP8;
        inst->addModifier(fmtMod);
        return inst;
    }

    StinkyInstruction* createWmmaScaleF8(int destStart, int src0Start) {
        return createWmmaScaleF8_in(bb, destStart, src0Start);
    }

    StinkyInstruction* createMovableDsLoad(int destReg, int addrReg, int ldsToken) {
        StinkyInstruction* inst = createDsReadB128InBlock(bb, arch, destReg, addrReg);
        inst->addSrcReg(StinkyRegister(RegType::LDS, ldsToken, 1));
        return inst;
    }

    // A tensor_load_to_lds with an LDS pseudo dest-reg so the DAG scheduler treats
    // it as movable (without LDS pseudo-regs hasSideEffect() makes it a region
    // boundary and it never enters globalReadQueue). Mirrors createMovableDsLoad.
    StinkyInstruction* createMovableTensorLoad(BasicBlock* targetBB, int src0Reg, int src1Reg,
                                               int ldsToken) {
        StinkyInstruction* inst = createTensorLoadInBlock(targetBB, arch, src0Reg, src1Reg);
        inst->addDestReg(StinkyRegister(RegType::LDS, ldsToken, 1));
        return inst;
    }

    StinkyInstruction* createExecNarrow(int srcSgpr) {
        AsmIRBuilder builder(*bb, arch);
        StinkyInstruction* inst = builder.create(getMCIDByUOp(GFX::s_mov_b32, arch));
        inst->addDestReg(StinkyRegister::getEXECRegister(32));
        inst->addSrcReg(StinkyRegister("s", srcSgpr, 1));
        return inst;
    }

    StinkyInstruction* createExecReset() {
        AsmIRBuilder builder(*bb, arch);
        StinkyInstruction* inst = builder.create(getMCIDByUOp(GFX::s_mov_b32, arch));
        inst->addDestReg(StinkyRegister::getEXECRegister(32));
        inst->addSrcReg(StinkyRegister(-1));
        return inst;
    }
};

// Integration check: collapse + schedule + expand together, through the real pass.
// See ExecMaskGroupingTest.cpp for isolated collapse/expand tests, and the
// ExecMaskGroup_* tests below for whether the scheduler treats a group as atomic.
TEST_F(DAGSchedulerPassTest, ExecMaskedRegion_PreservesSpanAndOrder) {
    createVAddInBlock(bb, arch, 40, 41, 42);
    createExecNarrow(10);
    createVAddInBlock(bb, arch, 0, 1, 2);
    createVAddInBlock(bb, arch, 3, 4, 5);
    createVAddInBlock(bb, arch, 6, 7, 8);
    createExecReset();
    createVAddInBlock(bb, arch, 50, 51, 52);

    const int n = countStinkyInstructions(*bb);
    runPass();

    EXPECT_EQ(countStinkyInstructions(*bb), n);
    for (const IRBase& ir : *bb) {
        if (ir.getType() != IRBase::IRType::StinkyTofu) continue;
        EXPECT_NE(cast<StinkyInstruction>(&ir)->getUnifiedOpcode(), GFX::EXEC_GROUP);
    }

    std::vector<int> destSeq;
    std::vector<bool> isExecWriteSeq;
    for (const IRBase& ir : *bb) {
        if (ir.getType() != IRBase::IRType::StinkyTofu) continue;
        const auto* inst = cast<StinkyInstruction>(&ir);
        if (inst->getUnifiedOpcode() != GFX::s_mov_b32 &&
            inst->getUnifiedOpcode() != GFX::v_add_f32)
            continue;
        ASSERT_FALSE(inst->getDestRegs().empty());
        destSeq.push_back(static_cast<int>(inst->getDestReg(0).reg.idx));
        isExecWriteSeq.push_back(inst->getDestReg(0).reg.type == RegType::EXEC_LO);
    }

    ASSERT_EQ(destSeq.size(), 7u);
    EXPECT_EQ(destSeq[0], 40);
    EXPECT_TRUE(isExecWriteSeq[1]);
    EXPECT_EQ(destSeq[2], 0);
    EXPECT_EQ(destSeq[3], 3);
    EXPECT_EQ(destSeq[4], 6);
    EXPECT_TRUE(isExecWriteSeq[5]);
    EXPECT_EQ(destSeq[6], 50);
}

// Layer 2: does the scheduler treat a hand-built ExecMaskGroup (bypassing
// collapseExecMaskedRegions() entirely) as a single atomic node?

// runPass() always runs collapseExecMaskedRegions()/expandExecMaskedGroups() around
// scheduling (see StinkyDAGSchedulerPass::run()'s scheduleBlock lambda), so any node
// with GFX::EXEC_GROUP -- hand-built or not -- gets unzipped via its ExecGroupData at
// the end. So a hand-built group under a real pass run needs a real (if minimal)
// ExecGroupData child to unzip into; that child's own registers are irrelevant here
// since ordering is driven by the group's own declared src/dest, set explicitly below.
TEST_F(DAGSchedulerPassTest, ExecMaskGroup_TreatedAsSingleAtomicNode) {
    createVAddInBlock(bb, arch, 20, 21, 22);
    StinkyInstruction* consumer = createVAddInBlock(bb, arch, 40, 30, 31);

    StinkyInstruction* child = createVAddInBlock(bb, arch, 60, 61, 62);
    bb->removeIR(child);

    AsmIRBuilder builder(*bb, arch);
    StinkyInstruction* group = builder.createExecMaskGroup(consumer);
    group->addSrcReg(StinkyRegister("v", 20, 1));
    group->addDestReg(StinkyRegister("v", 30, 1));
    group->issueCycles = 4;
    group->latencyCycles = 4;
    group->addModifier<ExecGroupData>(ExecGroupData{{child}});

    runPass();

    std::vector<int> destSeq;
    for (const IRBase& ir : *bb) {
        if (ir.getType() != IRBase::IRType::StinkyTofu) continue;
        const auto* inst = cast<StinkyInstruction>(&ir);
        ASSERT_FALSE(inst->getDestRegs().empty());
        destSeq.push_back(static_cast<int>(inst->getDestReg(0).reg.idx));
    }
    ASSERT_EQ(destSeq.size(), 3u);
    EXPECT_EQ(destSeq[0], 20);  // producer
    EXPECT_EQ(destSeq[1], 60);  // group's child, unzipped in its place
    EXPECT_EQ(destSeq[2], 40);  // consumer
}

TEST_F(DAGSchedulerPassTest, ExecMaskGroup_NotMisclassified) {
    StinkyInstruction* anchor = createVAddInBlock(bb, arch, 0, 1, 2);

    AsmIRBuilder builder(*bb, arch);
    StinkyInstruction* group = builder.createExecMaskGroup(anchor);
    group->addModifier<ExecGroupData>(ExecGroupData{{}});

    EXPECT_TRUE(isExecMaskGroup(*group));
    EXPECT_FALSE(isMatrixInstruction(*group));
    EXPECT_FALSE(isDSRead(*group));
    EXPECT_FALSE(isDSWrite(*group));
    EXPECT_FALSE(isBarrier(*group));
    EXPECT_FALSE(isVectorALU(*group));
    EXPECT_FALSE(isTensorLoad(*group));
    EXPECT_FALSE(hasSideEffect(*group));

    runPassWithUnrollGemm();
    // The group has no children to unzip into, so it's simply dropped; only the
    // anchor v_add remains.
    EXPECT_EQ(countStinkyInstructions(*bb), 1);
}

TEST_F(DAGSchedulerPassTest, ExecMaskGroup_InheritsSideEffectFromChildren) {
    StinkyInstruction* sideEffecting = createDSWriteInBlock(bb, arch, 0, 1);  // no MemTokenData
    bb->removeIR(sideEffecting);

    StinkyInstruction* anchor = createVAddInBlock(bb, arch, 0, 1, 2);
    AsmIRBuilder builder(*bb, arch);
    StinkyInstruction* group = builder.createExecMaskGroup(anchor);
    group->addModifier<ExecGroupData>(ExecGroupData{{sideEffecting}});

    EXPECT_TRUE(hasSideEffect(*group));
}

// Empty block: pass should not crash
TEST_F(DAGSchedulerPassTest, EmptyBlock_DoesNotCrash) {
    runPass();
    EXPECT_EQ(countStinkyInstructions(*bb), 0);
}

// Single instruction: pass should not crash
TEST_F(DAGSchedulerPassTest, SingleInstruction_DoesNotCrash) {
    createVAddInBlock(bb, arch, 0, 1, 2);
    int n = countStinkyInstructions(*bb);
    runPass();
    EXPECT_EQ(countStinkyInstructions(*bb), n);
}

// A few independent instructions: pass should not crash, count unchanged
TEST_F(DAGSchedulerPassTest, IndependentInstructions_DoesNotCrash) {
    createVAddInBlock(bb, arch, 0, 1, 2);
    createVAddInBlock(bb, arch, 3, 4, 5);
    createVAddInBlock(bb, arch, 6, 7, 8);
    int n = countStinkyInstructions(*bb);
    runPass();
    EXPECT_EQ(countStinkyInstructions(*bb), n);
}

// Chain of dependencies: pass should not crash, count unchanged
TEST_F(DAGSchedulerPassTest, DependentInstructions_DoesNotCrash) {
    createVAddInBlock(bb, arch, 0, 1, 2);  // v0 = v1 + v2
    createVAddInBlock(bb, arch, 3, 0, 4);  // v3 = v0 + v4
    createVAddInBlock(bb, arch, 5, 3, 6);  // v5 = v3 + v6
    int n = countStinkyInstructions(*bb);
    runPass();
    EXPECT_EQ(countStinkyInstructions(*bb), n);
}

// DS reads + WMMAs: scheduler must not issue WMMAs back-to-back when other instructions exist.
// With real ds_load latency, WMMAs are not latency-free until ds_reads are issued and latency
// elapses, so we get: 4 ds_load, then 2 wmma. The rule "lastPickedWasWMMA => prefer other"
// ensures that when both WMMA and other are ready we interleave (no consecutive WMMAs).
TEST_F(DAGSchedulerPassTest, DSReadAndWMMA_NoConsecutiveWMMA) {
    const int addrReg = 24;
    createDsReadB128InBlock(bb, arch, 8, addrReg);                 // v[8:11]
    createDsReadB128InBlock(bb, arch, 12, addrReg);                // v[12:15]
    createDsReadB128InBlock(bb, arch, 16, addrReg);                // v[16:19]
    createDsReadB128InBlock(bb, arch, 20, addrReg);                // v[20:23]
    StinkyInstruction* wmma1 = createWmmaF32_16x16x16_bf16(0, 8);  // v[0:7] v[8:15] v[8:15] v[0:7]
    StinkyInstruction* wmma2 =
        createWmmaF32_16x16x16_bf16(0, 16);  // v[0:7] v[16:23] v[16:23] v[0:7]
    ASSERT_NE(wmma1, nullptr);
    ASSERT_NE(wmma2, nullptr);

    runPassWithUnrollGemm();

    // With real latency, all 4 ds_load are issued first, then 2 wmma. No two WMMAs in a row.
    std::vector<std::pair<std::string, int>> sequence;
    for (const IRBase& ir : *bb) {
        if (ir.getType() != IRBase::IRType::StinkyTofu) continue;
        const StinkyInstruction* inst = cast<StinkyInstruction>(&ir);
        const char* mnem = inst->getHwInstDesc() ? inst->getHwInstDesc()->mnemonic : nullptr;
        if (!mnem) continue;
        if (std::string(mnem) == "ds_load_b128") {
            if (!inst->getDestRegs().empty() && inst->getDestRegs()[0].isRegister())
                sequence.push_back(
                    {"ds_load_b128", static_cast<int>(inst->getDestRegs()[0].reg.idx)});
        } else if (std::string(mnem) == "v_wmma_f32_16x16x16_bf16") {
            if (!inst->getSrcRegs().empty() && inst->getSrcRegs()[0].isRegister())
                sequence.push_back(
                    {"v_wmma_f32_16x16x16_bf16", static_cast<int>(inst->getSrcRegs()[0].reg.idx)});
        }
    }

    ASSERT_EQ(sequence.size(), 6u) << "Expected 6 instructions (4 ds_load + 2 wmma)";
    // All 4 ds_load first (real latency: WMMAs not ready until latency elapses)
    EXPECT_EQ(sequence[0].first, "ds_load_b128");
    EXPECT_EQ(sequence[0].second, 8);
    EXPECT_EQ(sequence[1].first, "ds_load_b128");
    EXPECT_EQ(sequence[1].second, 12);
    EXPECT_EQ(sequence[2].first, "ds_load_b128");
    EXPECT_EQ(sequence[2].second, 16);
    EXPECT_EQ(sequence[3].first, "ds_load_b128");
    EXPECT_EQ(sequence[3].second, 20);
    // Then 2 wmma; rule ensures we never issue two WMMAs in a row when other work exists
    EXPECT_EQ(sequence[4].first, "v_wmma_f32_16x16x16_bf16");
    EXPECT_EQ(sequence[4].second, 8);
    EXPECT_EQ(sequence[5].first, "v_wmma_f32_16x16x16_bf16");
    EXPECT_EQ(sequence[5].second, 16);
    // When other instructions exist, scheduler prefers them after a WMMA (no back-to-back WMMA).
    // Here with real latency only WMMAs are left at the end so they are issued consecutively.
}

// ---------------------------------------------------------------------------
// Property: when all independent, WMMA fires first (Phase B), then ds_loads
// and VALU fill the WMMA latency window.
// Within that window, ds_load has priority over VALU.
// ---------------------------------------------------------------------------
TEST_F(DAGSchedulerPassTest, IndependentWMMAFirst_ThenDsThenVALU) {
    const int addrReg = 80;
    // 3 independent ds_loads (LDS pseudo-reg makes them movable in DAG)
    createMovableDsLoad(0, addrReg, 1);
    createMovableDsLoad(4, addrReg, 2);
    createMovableDsLoad(8, addrReg, 3);
    // 3 independent VALUs
    createVAddInBlock(bb, arch, 40, 41, 42);
    createVAddInBlock(bb, arch, 43, 44, 45);
    createVAddInBlock(bb, arch, 46, 47, 48);
    // 1 independent WMMA
    createWmmaF32_16x16x16_bf16(12, 50);

    runPassWithUnrollGemm();

    int firstWmmaPos = -1;
    int firstDsPos = -1;
    int firstValuPos = -1;
    int pos = 0;
    for (const IRBase& ir : *bb) {
        if (ir.getType() != IRBase::IRType::StinkyTofu) continue;
        const auto* inst = cast<StinkyInstruction>(&ir);
        const HwInstDesc* hw = inst->getHwInstDesc();
        if (!hw || !hw->mnemonic) continue;
        std::string_view mnem(hw->mnemonic);
        if (mnem.find("wmma") != std::string_view::npos) {
            if (firstWmmaPos < 0) firstWmmaPos = pos;
        } else if (mnem.find("ds_load") != std::string_view::npos) {
            if (firstDsPos < 0) firstDsPos = pos;
        } else if (mnem.find("v_add") != std::string_view::npos) {
            if (firstValuPos < 0) firstValuPos = pos;
        }
        pos++;
    }

    ASSERT_GE(firstWmmaPos, 0) << "No WMMA found";
    ASSERT_GE(firstDsPos, 0) << "No ds_load found";
    ASSERT_GE(firstValuPos, 0) << "No VALU found";
    EXPECT_LT(firstWmmaPos, firstDsPos) << "WMMA should fire before ds_load (Phase B)";
    EXPECT_LT(firstDsPos, firstValuPos)
        << "DS loads should be prioritized before VALU during WMMA latency";
}

// ---------------------------------------------------------------------------
// Co-execution hazard (regression test for destOverlapsActiveWmmaSrc):
// a ds_load whose dest VGPRs overlap the in-flight WMMA's src VGPRs must NOT be
// issued inside that WMMA's latency window, because the load could clobber a
// source register the WMMA is still reading.
//
// Setup: WMMA #0 reads v[50:58); the ds_load writes v[52:56) (overlap). Four
// more independent WMMAs (disjoint registers) are available. While WMMA #0 is
// in flight the ds_load is held back by the hazard gate, so the scheduler
// issues the next independent WMMA (D#100) first and only then the ds_load,
// once a WMMA whose sources it does not touch is the active one:
//
//   wmma D#12  ->  wmma D#100  ->  ds_load D#52  ->  wmma D#108/116/124
//
// Without the hazard gate the ds_load would issue right after WMMA #0
// (wmma D#12 -> ds_load -> wmma D#100 -> ...), clobbering v[52:56) mid-read.
// ---------------------------------------------------------------------------
TEST_F(DAGSchedulerPassTest, WmmaSrcOverlap_HazardDsLoadDeferredPastWindow) {
    const int addrReg = 80;
    // F8 MX WMMA fires first (Phase B). Its src VGPRs are v[50:58) (src0/src1)
    // and v[12:20) (acc); see createWmmaScaleF8. cost latency=8 keeps the
    // co-issue window open so the hazard gate is exercised.
    createWmmaScaleF8(/*destStart=*/12, /*src0Start=*/50);
    // ds_load dest v[52:56) overlaps the WMMA's src0 v[50:58): co-exec hazard.
    createMovableDsLoad(/*destReg=*/52, addrReg, /*ldsToken=*/1);
    // Independent WMMAs (registers disjoint from the hazard pair and from each
    // other) to fill the latency window ahead of the deferred ds_load.
    for (int i = 0; i < 4; i++)
        createWmmaScaleF8(/*destStart=*/100 + i * 8, /*src0Start=*/200 + i * 8);

    int beforeCount = countStinkyInstructions(*bb);
    runPassWithUnrollGemm();
    EXPECT_EQ(countStinkyInstructions(*bb), beforeCount)
        << "hazard deferral must not drop instructions";

    // Collect (mnemonic-kind, first-dest-vgpr) in scheduled order.
    std::vector<std::pair<std::string, int>> seq;
    for (const IRBase& ir : *bb) {
        if (ir.getType() != IRBase::IRType::StinkyTofu) continue;
        const auto* inst = cast<StinkyInstruction>(&ir);
        const HwInstDesc* hw = inst->getHwInstDesc();
        if (!hw || !hw->mnemonic) continue;
        std::string_view mnem(hw->mnemonic);
        std::string kind = mnem.find("wmma") != std::string_view::npos      ? "wmma"
                           : mnem.find("ds_load") != std::string_view::npos ? "ds"
                                                                            : std::string(mnem);
        int dst = (!inst->getDestRegs().empty() && inst->getDestRegs()[0].isRegister())
                      ? static_cast<int>(inst->getDestRegs()[0].reg.idx)
                      : -1;
        seq.push_back({kind, dst});
    }

    const std::vector<std::pair<std::string, int>> expected = {
        {"wmma", 12}, {"wmma", 100}, {"ds", 52}, {"wmma", 108}, {"wmma", 116}, {"wmma", 124},
    };
    EXPECT_EQ(seq, expected)
        << "hazardous ds_load must be deferred until an independent WMMA (D#100) has "
           "issued; it must not co-issue inside WMMA D#12's latency window";
}

// ---------------------------------------------------------------------------
// Co-execution hazard, VALU variant (regression test for destOverlapsActiveWmmaSrc
// on the VALU path): a VALU whose dest VGPR overlaps the in-flight WMMA's src VGPRs
// must NOT be issued inside that WMMA's latency window, because it could clobber a
// source register the WMMA is still reading.
//
// Unlike the ds_load variant, a VALU only becomes co-issue pickable at the positions
// set in the WMMA's co-issue window (MXWMMA_SCALE = 0x00C0, i.e. positions 6/7). Right
// after issue the position is 1, where isValuPickable() is already false, so extra
// non-hazardous fillers are needed to advance the co-issue timeline into a pickable
// position while WMMA #0 is still in flight. That is exactly the moment the hazard gate
// must fire:
//   - 3 non-hazardous ds_loads (v[300:], v[320:], v[340:]) fill the per-WMMA DS cap and
//     advance positions 1 -> 4.
//   - 3 independent scalar ops advance positions 4 -> 7; at position 6 the VALU becomes
//     co-issue pickable while WMMA #0 (v[50:58)) is still the active window.
//
// With the gate the hazardous VALU (dst v52) is skipped at position 6 (inside WMMA #0's
// window) and deferred until that window closes; it then issues right after D#100 opens a
// non-overlapping window. Without the gate it would co-issue at position 6, clobbering v52.
// ---------------------------------------------------------------------------
TEST_F(DAGSchedulerPassTest, WmmaSrcOverlap_HazardValuDeferredPastWindow) {
    const int addrReg = 400;
    auto createScalarOp = [&](int dst, int src0, int src1) {
        AsmIRBuilder builder(*bb, arch);
        StinkyInstruction* inst = builder.create(getMCIDByUOp(GFX::s_add_u32, arch));
        inst->addDestReg(StinkyRegister("s", dst, 1));
        inst->addSrcReg(StinkyRegister("s", src0, 1));
        inst->addSrcReg(StinkyRegister("s", src1, 1));
        return inst;
    };
    // F8 MX WMMA fires first (Phase B). Its src VGPRs are v[50:58) (src0/src1) and
    // v[12:20) (acc); latency=8 keeps the co-issue window open so the gate is exercised.
    createWmmaScaleF8(/*destStart=*/12, /*src0Start=*/50);
    // Hazardous VALU: dst v52 overlaps the WMMA's src0 v[50:58): co-exec hazard.
    createVAddInBlock(bb, arch, /*destReg=*/52, /*src0Reg=*/60, /*src1Reg=*/61);
    // Non-hazardous fillers to advance the co-issue timeline into a VALU-pickable
    // position (6) while WMMA #0 is still in flight.
    createMovableDsLoad(/*destReg=*/300, addrReg, /*ldsToken=*/1);
    createMovableDsLoad(/*destReg=*/320, addrReg, /*ldsToken=*/2);
    createMovableDsLoad(/*destReg=*/340, addrReg, /*ldsToken=*/3);
    createScalarOp(/*dst=*/10, 11, 12);
    createScalarOp(/*dst=*/13, 14, 15);
    createScalarOp(/*dst=*/16, 17, 18);
    // Independent WMMAs (registers disjoint from the hazard pair and each other) to fill
    // the latency windows ahead of the deferred VALU.
    for (int i = 0; i < 4; i++)
        createWmmaScaleF8(/*destStart=*/100 + i * 16, /*src0Start=*/200 + i * 16);

    int beforeCount = countStinkyInstructions(*bb);
    runPassWithUnrollGemm();
    EXPECT_EQ(countStinkyInstructions(*bb), beforeCount)
        << "hazard deferral must not drop instructions";

    // Collect (mnemonic-kind, first-dest-reg) in scheduled order.
    std::vector<std::pair<std::string, int>> seq;
    for (const IRBase& ir : *bb) {
        if (ir.getType() != IRBase::IRType::StinkyTofu) continue;
        const auto* inst = cast<StinkyInstruction>(&ir);
        const HwInstDesc* hw = inst->getHwInstDesc();
        if (!hw || !hw->mnemonic) continue;
        std::string_view mnem(hw->mnemonic);
        std::string kind = mnem.find("wmma") != std::string_view::npos      ? "wmma"
                           : mnem.find("ds_load") != std::string_view::npos ? "ds"
                           : mnem.rfind("v_", 0) == 0                       ? "valu"
                           : mnem.rfind("s_", 0) == 0                       ? "s"
                                                                            : std::string(mnem);
        int dst = (!inst->getDestRegs().empty() && inst->getDestRegs()[0].isRegister())
                      ? static_cast<int>(inst->getDestRegs()[0].reg.idx)
                      : -1;
        seq.push_back({kind, dst});
    }

    // Hazardous VALU (dst v52) is deferred past WMMA D#12's window, then issues right after
    // the first independent WMMA (D#100), whose window no longer overlaps v52.
    const std::vector<std::pair<std::string, int>> expected = {
        {"wmma", 12}, {"ds", 300},   {"ds", 320},  {"ds", 340},   {"s", 10},     {"s", 13},
        {"s", 16},    {"wmma", 100}, {"valu", 52}, {"wmma", 116}, {"wmma", 132}, {"wmma", 148},
    };
    EXPECT_EQ(seq, expected)
        << "hazardous VALU (dst v52) must not co-issue inside WMMA D#12's latency window; "
           "it must be deferred until that window closes, then issue once a non-overlapping "
           "WMMA window is open";
}

// ---------------------------------------------------------------------------
// Hidden-stall window fill (pickFreeBest allowHiddenStall path): a SALU that is
// only blocked by a src RAW hazard whose remaining wait fits under the active
// WMMA's latency shadow may be co-issued *inside* that window — the stall we pay
// waiting for its src is hidden by the in-flight WMMA, so it costs no extra
// cycles. It must therefore be preferred over starting the next independent WMMA.
//
// Setup (region, not a loop, so no loop-head deferral):
//   - WMMA #0 (v[12:20)) fires first (Phase B) and opens an 8-cycle window.
//   - A chain of inter-dependent SALUs a0 -> a1 -> a2 -> a3, each writing s(100+i)
//     with latency=2 > issue=1 so issuing it stamps a 1-cycle data-ready latency
//     on its dest (the src RAW gate for the next link). a0 is free; a1..a3 are
//     each RAW-blocked for 1 cycle, which fits under WMMA #0's remaining latency
//     shadow, so every link is a valid hidden-stall fill. The chain is strict, so
//     only one link is ready at a time — their relative order is forced by the
//     DAG; the test is purely about whether each link lands inside the window.
//   - WMMA #1 (v[200:208)) is independent and ready.
//
// Expected (new behavior):  wmma#0, a0, a1, a2, a3, wmma#1
//   Every chain link is co-issued inside wmma#0's window, ahead of wmma#1.
// Old behavior would issue wmma#1 as soon as a0's consumer was RAW-blocked
// (wmma#0, a0, wmma#1, a1, a2, a3), because a RAW-blocked SALU was never pickable
// inside the window.
// ---------------------------------------------------------------------------
TEST_F(DAGSchedulerPassTest, HiddenStallSaluFillsWmmaWindowBeforeNextWmma) {
    auto createScalarAdd = [&](int dst, int src0, int src1) {
        AsmIRBuilder builder(*bb, arch);
        StinkyInstruction* inst = builder.create(getMCIDByUOp(GFX::s_add_u32, arch));
        inst->addDestReg(StinkyRegister("s", dst, 1));
        inst->addSrcReg(StinkyRegister("s", src0, 1));
        inst->addSrcReg(StinkyRegister("s", src1, 1));
        return inst;
    };

    // WMMA #0 fires first (Phase B), latency=8 keeps its co-issue window open.
    createWmmaScaleF8(/*destStart=*/12, /*src0Start=*/50);
    // Chain a0 -> a1 -> a2 -> a3: a_i writes s(100+i), a_(i+1) reads it (RAW).
    // Each 1-cycle wait fits the shrinking window (positions 2,4,6,8), so all four
    // are hidden-stall filled inside WMMA #0's window.
    const int kChain = 4;
    for (int i = 0; i < kChain; i++) {
        const int src0 = (i == 0) ? 0 : (100 + i - 1);  // previous link's dest
        StinkyInstruction* a = createScalarAdd(/*dst=*/100 + i, src0, /*src1=*/1);
        a->issueCycles = 1;
        a->latencyCycles = 2;
    }
    // WMMA #1: independent (disjoint regs) and ready.
    createWmmaScaleF8(/*destStart=*/200, /*src0Start=*/220);

    int beforeCount = countStinkyInstructions(*bb);
    runPassWithUnrollGemm();
    EXPECT_EQ(countStinkyInstructions(*bb), beforeCount)
        << "hidden-stall fill must not drop instructions";

    std::vector<std::pair<std::string, int>> seq;
    for (const IRBase& ir : *bb) {
        if (ir.getType() != IRBase::IRType::StinkyTofu) continue;
        const auto* inst = cast<StinkyInstruction>(&ir);
        const HwInstDesc* hw = inst->getHwInstDesc();
        if (!hw || !hw->mnemonic) continue;
        std::string_view mnem(hw->mnemonic);
        std::string kind = mnem.find("wmma") != std::string_view::npos ? "wmma"
                           : mnem.rfind("s_", 0) == 0                  ? "s"
                                                                       : std::string(mnem);
        int dst = (!inst->getDestRegs().empty() && inst->getDestRegs()[0].isRegister())
                      ? static_cast<int>(inst->getDestRegs()[0].reg.idx)
                      : -1;
        seq.push_back({kind, dst});
    }

    const std::vector<std::pair<std::string, int>> expected = {
        {"wmma", 12}, {"s", 100}, {"s", 101}, {"s", 102}, {"s", 103}, {"wmma", 200},
    };
    EXPECT_EQ(seq, expected)
        << "every link of the RAW-dependent SALU chain must be co-issued inside WMMA #0's latency "
           "window (each 1-cycle wait hidden by the in-flight WMMA), ahead of the independent "
           "WMMA #1";
}

// ---------------------------------------------------------------------------
// Property: per-WMMA-window DS cap — after a WMMA fires,
// at most floor((latency - issue) / 2) = 3 ds_loads can issue in its window
// because back-to-back ds_load issue cost doubles.
// ---------------------------------------------------------------------------
TEST_F(DAGSchedulerPassTest, DSWindowCap_VALUInterleaveAfter3) {
    const int addrReg = 80;
    // 5 independent ds_loads (LDS pseudo-reg makes them movable in DAG)
    createMovableDsLoad(0, addrReg, 1);
    createMovableDsLoad(4, addrReg, 2);
    createMovableDsLoad(8, addrReg, 3);
    createMovableDsLoad(12, addrReg, 4);
    createMovableDsLoad(16, addrReg, 5);
    // 2 independent VALUs
    createVAddInBlock(bb, arch, 60, 61, 62);
    createVAddInBlock(bb, arch, 63, 64, 65);
    // 2 independent WMMAs (fire first, create co-issue window)
    createWmmaF32_16x16x16_bf16(20, 28);
    createWmmaF32_16x16x16_bf16(36, 44);

    runPassWithUnrollGemm();

    int consecutiveDs = 0;
    int maxConsecutiveDs = 0;
    for (const IRBase& ir : *bb) {
        if (ir.getType() != IRBase::IRType::StinkyTofu) continue;
        const auto* inst = cast<StinkyInstruction>(&ir);
        const HwInstDesc* hw = inst->getHwInstDesc();
        if (!hw || !hw->mnemonic) continue;
        std::string_view mnem(hw->mnemonic);
        if (mnem.find("ds_load") != std::string_view::npos) {
            consecutiveDs++;
            maxConsecutiveDs = std::max(maxConsecutiveDs, consecutiveDs);
        } else {
            consecutiveDs = 0;
        }
    }

    EXPECT_LE(maxConsecutiveDs, 3) << "DS window cap violated: found " << maxConsecutiveDs
                                   << " consecutive ds_loads (max 3 per WMMA window)";
}

// ---------------------------------------------------------------------------
// Property: all original instructions are preserved.
// ---------------------------------------------------------------------------
TEST_F(DAGSchedulerPassTest, DSWindowCap_InstructionCountPreserved) {
    const int addrReg = 100;
    // 6 independent ds_loads (LDS pseudo-reg makes them movable in DAG)
    for (int i = 0; i < 6; i++) createMovableDsLoad(i * 4, addrReg, i + 1);
    // 4 independent VALUs
    for (int i = 0; i < 4; i++) createVAddInBlock(bb, arch, 30 + i, 40 + i, 50 + i);
    // 3 independent WMMAs
    for (int i = 0; i < 3; i++) createWmmaF32_16x16x16_bf16(60 + i * 8, 84 + i * 8);

    int beforeCount = countStinkyInstructions(*bb);
    runPassWithUnrollGemm();
    int afterCount = countStinkyInstructions(*bb);

    EXPECT_EQ(afterCount, beforeCount) << "Scheduler must preserve instruction count";
}

// ---------------------------------------------------------------------------
// tensor_load_to_lds bounded in-flight credit pool (dagFeatures.globalReadQueueDepth
// / globalReadDrainLatency). The throttle is loop-only (uses the CDNA5 loop path),
// so all of these build a self-loop body with 4 movable tensor_loads + 10 VALU.
//
// Model: each issued tensor_load holds a credit for `drainLatency` cycles, decayed
// once per issued instruction (~1 cycle/pick here). With depth D, at most D credits
// may be in flight, so once D loads have issued the scheduler must interleave VALU
// until a credit drains before issuing the next load.
//
// NOTE on the chosen numbers: a credit must survive long enough to still be in
// flight when the next load wants to issue, so the throttle only engages when
// drainLatency > depth. At 1 issue/cycle a credit that drains in <= depth cycles
// frees before the cap is reached, so the hardware sustains D loads/cycle with no
// stall. We therefore use drainLatency comfortably above depth and assert the
// back-to-back run equals exactly depth (D loads, then forced interleave).
// ---------------------------------------------------------------------------

// Depth 2: exactly 2 tensor_loads issue back-to-back, then VALU must interleave.
TEST_F(DAGSchedulerPassTest, GlobalReadThrottle_Depth2_RespectsQueueDepth) {
    // Self-loop on the entry block so it is in RPO (scheduled) AND detected as a
    // loop (the cross-BB credit carry is loop-only). buildLoopBB makes an
    // unreachable second block, so reuse the entry block here.
    BasicBlock* body = bb;
    body->addSuccessor(body);
    for (int i = 0; i < 4; i++)
        createMovableTensorLoad(body, /*s0=*/i * 12, /*s1=*/i * 12 + 4, /*ldsToken=*/i + 1);
    for (int i = 0; i < 30; i++) createVAddInBlock(body, arch, 40 + i, 80 + i, 100 + i);

    runPassWithGlobalReadThrottle(/*depth=*/2, /*drainLatency=*/8);

    std::vector<std::string> seq = mnemonicSequence(*body);
    EXPECT_EQ(maxConsecutiveTensorLoads(seq), 2)
        << "depth=2: at most 2 tensor_loads in flight before an interleave is forced";
}

// Depth 3: exactly 3 tensor_loads issue back-to-back, then VALU must interleave.
TEST_F(DAGSchedulerPassTest, GlobalReadThrottle_Depth3_RespectsQueueDepth) {
    // Self-loop on the entry block so it is in RPO (scheduled) AND detected as a
    // loop (the cross-BB credit carry is loop-only). buildLoopBB makes an
    // unreachable second block, so reuse the entry block here.
    BasicBlock* body = bb;
    body->addSuccessor(body);
    for (int i = 0; i < 4; i++)
        createMovableTensorLoad(body, /*s0=*/i * 12, /*s1=*/i * 12 + 4, /*ldsToken=*/i + 1);
    for (int i = 0; i < 30; i++) createVAddInBlock(body, arch, 40 + i, 80 + i, 100 + i);

    runPassWithGlobalReadThrottle(/*depth=*/3, /*drainLatency=*/8);

    std::vector<std::string> seq = mnemonicSequence(*body);
    EXPECT_EQ(maxConsecutiveTensorLoads(seq), 3)
        << "depth=3: at most 3 tensor_loads in flight before an interleave is forced";
}

// Depth 1: degenerate cap — every tensor_load must be separated by other work.
TEST_F(DAGSchedulerPassTest, GlobalReadThrottle_Depth1_SeparatesEveryLoad) {
    // Self-loop on the entry block so it is in RPO (scheduled) AND detected as a
    // loop (the cross-BB credit carry is loop-only). buildLoopBB makes an
    // unreachable second block, so reuse the entry block here.
    BasicBlock* body = bb;
    body->addSuccessor(body);
    for (int i = 0; i < 4; i++)
        createMovableTensorLoad(body, /*s0=*/i * 12, /*s1=*/i * 12 + 4, /*ldsToken=*/i + 1);
    for (int i = 0; i < 30; i++) createVAddInBlock(body, arch, 40 + i, 80 + i, 100 + i);

    runPassWithGlobalReadThrottle(/*depth=*/1, /*drainLatency=*/8);

    std::vector<std::string> seq = mnemonicSequence(*body);
    EXPECT_EQ(maxConsecutiveTensorLoads(seq), 1) << "depth=1: no two tensor_loads may be adjacent";
}

// All instructions are preserved regardless of throttle (count invariant).
TEST_F(DAGSchedulerPassTest, GlobalReadThrottle_PreservesInstructionCount) {
    // Self-loop on the entry block so it is in RPO (scheduled) AND detected as a
    // loop (the cross-BB credit carry is loop-only). buildLoopBB makes an
    // unreachable second block, so reuse the entry block here.
    BasicBlock* body = bb;
    body->addSuccessor(body);
    for (int i = 0; i < 4; i++)
        createMovableTensorLoad(body, /*s0=*/i * 12, /*s1=*/i * 12 + 4, /*ldsToken=*/i + 1);
    for (int i = 0; i < 30; i++) createVAddInBlock(body, arch, 40 + i, 80 + i, 100 + i);

    int beforeCount = countStinkyInstructions(*body);
    runPassWithGlobalReadThrottle(/*depth=*/2, /*drainLatency=*/8);
    EXPECT_EQ(countStinkyInstructions(*body), beforeCount) << "throttle must not drop instructions";
}

// Throttle off (depth=0): feature is opt-in and does not perturb the default path.
TEST_F(DAGSchedulerPassTest, GlobalReadThrottle_Disabled_PreservesAll) {
    // Self-loop on the entry block so it is in RPO (scheduled) AND detected as a
    // loop (the cross-BB credit carry is loop-only). buildLoopBB makes an
    // unreachable second block, so reuse the entry block here.
    BasicBlock* body = bb;
    body->addSuccessor(body);
    for (int i = 0; i < 4; i++)
        createMovableTensorLoad(body, /*s0=*/i * 12, /*s1=*/i * 12 + 4, /*ldsToken=*/i + 1);
    for (int i = 0; i < 30; i++) createVAddInBlock(body, arch, 40 + i, 80 + i, 100 + i);

    int beforeCount = countStinkyInstructions(*body);
    runPassWithGlobalReadThrottle(/*depth=*/0, /*drainLatency=*/0);
    EXPECT_EQ(countStinkyInstructions(*body), beforeCount);
}

// SGPR->tensor_load hazard: a SALU that writes an SGPR a tensor_load reads must be
// separated from that tensor_load by the fixed hardware gap (kCdna5HazardRules'
// SaluSgprToMemAddr entry, 8 cycles). Mirrors the real case (wmma/ds fill around the
// SALU): the scheduler hoists the SALU and/or holds the tensor_load so >= 8 cycles of
// work sit between them. We assert the cycle invariant, not an exact order. A WMMA
// counts as its latencyCycles (the co-issue window it opens, 8 here), other ops as
// issueCycles.
TEST_F(DAGSchedulerPassTest, SgprToTensorLoadHazard_AtLeast8CycleGap) {
    BasicBlock* body = bb;
    body->addSuccessor(body);

    // Movable ds_loads (LDS token -> stay in one region, no side-effect boundary) as the
    // fill work, a SALU writing s0, and a tensor_load reading s[0:4) so s0 is the hazard
    // register. Enough ds fill (>= hazard) so the gap is filled by real work, observable
    // in the emitted order rather than an invisible stall.
    for (int i = 0; i < 12; i++)
        createMovableDsLoad(/*destReg=*/8 + i * 4, /*addrReg=*/60, /*ldsToken=*/i + 2);

    AsmIRBuilder builder(*body, arch);
    StinkyInstruction* salu = builder.create(getMCIDByUOp(GFX::s_mov_b32, arch));
    salu->addDestReg(StinkyRegister("s", 0, 1));
    salu->addSrcReg(StinkyRegister(0));

    createMovableTensorLoad(body, /*s0=*/0, /*s1=*/4, /*ldsToken=*/1);

    runPassWithGlobalReadThrottle(/*depth=*/4, /*drainLatency=*/8);

    // Locate the SALU and the tensor_load in the scheduled order, and total the cycles
    // of the work between them (WMMA -> latency window, else issue cycles).
    int saluPos = -1, tensorPos = -1, idx = 0;
    std::vector<int> cyclesAt;
    for (const IRBase& ir : *body) {
        if (ir.getType() != IRBase::IRType::StinkyTofu) continue;
        const auto* inst = cast<StinkyInstruction>(&ir);
        cyclesAt.push_back(isMatrixInstruction(*inst) ? inst->latencyCycles : inst->issueCycles);
        if (inst == salu) saluPos = idx;
        if (isTensorLoad(*inst)) tensorPos = idx;
        idx++;
    }
    ASSERT_GE(saluPos, 0);
    ASSERT_GE(tensorPos, 0);
    ASSERT_LT(saluPos, tensorPos) << "SALU must be scheduled before the tensor_load it feeds";

    int gap = 0;
    for (int i = saluPos + 1; i < tensorPos; i++) gap += cyclesAt[i];
    EXPECT_GE(gap, 8) << "tensor_load must be >= 8 cycles after the SALU writing its SGPR";
}

// ---------------------------------------------------------------------------
// dsReadQueueDepth / dsReadDrainLatency / dsReadPerWmma: same in-flight
// credit-pool mechanism as globalReadQueueDepth/globalReadDrainLatency, but
// gating ds_read_b128 instead of tensor_load_to_lds. Unlike global-read
// throttling, the ds_read gate additionally requires a WMMA to have been
// picked at least once (it seeds maxDsPerWmmaWindow_), so each test below
// includes one WMMA read (with dest/src registers disjoint from the ds_reads,
// so its DS-latency gate is trivially satisfied and it issues first).
// ---------------------------------------------------------------------------

// Depth 2: exactly 2 ds_reads issue back-to-back, then VALU must interleave.
TEST_F(DAGSchedulerPassTest, DsReadThrottle_Depth2_RespectsQueueDepth) {
    BasicBlock* body = bb;
    body->addSuccessor(body);
    createWmmaF32_16x16x16_bf16_in(body, /*destStart=*/200, /*src0Start=*/204);
    for (int i = 0; i < 4; i++)
        createMovableDsLoad(/*destReg=*/i * 4, /*addrReg=*/300 + i * 4, /*ldsToken=*/i + 1);
    for (int i = 0; i < 30; i++) createVAddInBlock(body, arch, 40 + i, 80 + i, 100 + i);

    runPassWithDsReadThrottle(/*queueDepth=*/2, /*drainLatency=*/8);

    std::vector<std::string> seq = mnemonicSequence(*body);
    EXPECT_EQ(maxConsecutiveDsReads(seq), 2)
        << "depth=2: at most 2 ds_reads in flight before an interleave is forced";
}

// Depth 1: degenerate cap — every ds_read must be separated by other work.
TEST_F(DAGSchedulerPassTest, DsReadThrottle_Depth1_SeparatesEveryLoad) {
    BasicBlock* body = bb;
    body->addSuccessor(body);
    createWmmaF32_16x16x16_bf16_in(body, /*destStart=*/200, /*src0Start=*/204);
    for (int i = 0; i < 4; i++)
        createMovableDsLoad(/*destReg=*/i * 4, /*addrReg=*/300 + i * 4, /*ldsToken=*/i + 1);
    for (int i = 0; i < 30; i++) createVAddInBlock(body, arch, 40 + i, 80 + i, 100 + i);

    runPassWithDsReadThrottle(/*queueDepth=*/1, /*drainLatency=*/8);

    std::vector<std::string> seq = mnemonicSequence(*body);
    EXPECT_EQ(maxConsecutiveDsReads(seq), 1) << "depth=1: no two ds_reads may be adjacent";
}

// NOTE: the former DsReadThrottle_PerWmmaCap_RespectsCap test isolated the
// per-WMMA-window cap with a single WMMA. That predated the ds_load in-flight
// queue; now the cap only binds while a WMMA window is active (covered by
// DSWindowCap_VALUInterleaveAfter3, which keeps two WMMAs pending), and the
// no-WMMA case is bounded by the in-flight queue (covered by
// DsReadThrottle_Depth2_RespectsQueueDepth). No standalone single-WMMA cap test
// is kept — it would assert behavior the cap no longer has.

// Regression (see image(2).png bug): with no WMMA to issue and a chain of
// ds_loads each consumed by a VALU (RAW), the scheduler must NOT interleave
// ds,ds,valu,ds,ds,valu — that pattern forces an s_wait_dscnt per pair and
// tanks the kernel. Loads have their own in-flight queue, so they should drain
// (up to queue depth) before the consumer VALUs run: the consumers RAW-depend on
// the loads and are hazard-deferred until the load latency clears. Assert the
// loads front-load ahead of every consumer VALU.
TEST_F(DAGSchedulerPassTest, DsReadThrottle_NoWmma_LoadsDrainBeforeConsumerValu) {
    BasicBlock* body = bb;
    body->addSuccessor(body);
    // 6 ds_loads (movable via LDS token) on the same address register; each VALU
    // consumes the matching load's dest (RAW), mirroring the image's
    // ds_load_u8 -> v_lshl_or_b32 dependency chain. No WMMA in the region.
    for (int i = 0; i < 6; i++)
        createMovableDsLoad(/*destReg=*/i * 4, /*addrReg=*/200, /*ldsToken=*/i + 1);
    for (int i = 0; i < 6; i++)
        createVAddInBlock(body, arch, /*dst=*/100 + i, /*src0=*/i * 4, /*src1=*/i * 4 + 1);

    // Queue depth 6 so all loads can be in flight at once; perWmma irrelevant (no WMMA).
    runPassWithDsReadThrottle(/*queueDepth=*/6, /*drainLatency=*/8, /*perWmma=*/100);

    std::vector<std::string> seq = mnemonicSequence(*body);
    // Every ds_load must precede every v_add: find the last load and first valu.
    int lastLoad = -1, firstValu = -1;
    for (int i = 0; i < (int)seq.size(); i++) {
        if (seq[i] == "ds_load_b128") lastLoad = i;
        if (seq[i] == "v_add_f32" && firstValu < 0) firstValu = i;
    }
    ASSERT_GE(lastLoad, 0);
    ASSERT_GE(firstValu, 0);
    EXPECT_LT(lastLoad, firstValu)
        << "no-WMMA: all ds_loads must drain before consumer VALUs (no ds,valu,ds interleave)";
}

// Type-A WAR via elapse-time ordering (replaces the old dsAddrReadLatencyCounters):
// a VALU that overwrites the ds_load's address reg must be deferred behind other
// independent VALUs, because that reg was just touched (small elapse) — even though
// the overwrite is EARLIEST in program order (smallest DAG id, which plain
// pop()-by-id would pick first). This proves the read->write gap comes from elapse
// ordering, not a hard counter.
TEST_F(DAGSchedulerPassTest, WarOverwriteOfDsAddrDeferredByElapse) {
    BasicBlock* body = bb;
    body->addSuccessor(body);
    // ds_load reads address v200 (single ds_load so no load-drain effects dominate).
    createMovableDsLoad(/*destReg=*/8, /*addrReg=*/200, /*ldsToken=*/1);
    // Overwrite of v200 — created FIRST after the load, so it has the smallest DAG id
    // among the VALUs. Independent VALUs (disjoint regs) created after it.
    StinkyInstruction* overwrite = createVAddInBlock(body, arch, /*dst=*/200, /*src0=*/101,
                                                     /*src1=*/102);
    for (int i = 0; i < 3; i++)
        createVAddInBlock(body, arch, /*dst=*/50 + i, /*src0=*/60 + i, /*src1=*/70 + i);

    runPassWithUnrollGemm();

    // Find the scheduled position of the overwrite vs. the independent VALUs.
    int overwritePos = -1, firstIndependentPos = -1, idx = 0;
    for (const IRBase& ir : *body) {
        if (ir.getType() != IRBase::IRType::StinkyTofu) continue;
        const auto* inst = cast<StinkyInstruction>(&ir);
        if (inst == overwrite)
            overwritePos = idx;
        else if (inst->getUnifiedOpcode() == GFX::v_add_f32 && firstIndependentPos < 0)
            firstIndependentPos = idx;
        idx++;
    }
    ASSERT_GE(overwritePos, 0);
    ASSERT_GE(firstIndependentPos, 0);
    EXPECT_GT(overwritePos, firstIndependentPos)
        << "WAR overwrite of the ds_load address must be deferred behind independent VALUs "
           "by elapse-time ordering, despite having the smallest DAG id";
}

// All instructions are preserved regardless of throttle (count invariant).
TEST_F(DAGSchedulerPassTest, DsReadThrottle_PreservesInstructionCount) {
    BasicBlock* body = bb;
    body->addSuccessor(body);
    createWmmaF32_16x16x16_bf16_in(body, /*destStart=*/200, /*src0Start=*/204);
    for (int i = 0; i < 4; i++)
        createMovableDsLoad(/*destReg=*/i * 4, /*addrReg=*/300 + i * 4, /*ldsToken=*/i + 1);
    for (int i = 0; i < 30; i++) createVAddInBlock(body, arch, 40 + i, 80 + i, 100 + i);

    int beforeCount = countStinkyInstructions(*body);
    runPassWithDsReadThrottle(/*queueDepth=*/2, /*drainLatency=*/8);
    EXPECT_EQ(countStinkyInstructions(*body), beforeCount) << "throttle must not drop instructions";
}