// Copyright (c) 2026 Advanced Micro Devices, Inc.
// SPDX-License-Identifier: MIT

#include "aql_queue.h"
#include "halt_snapshot_plugin.h"

#include "embedded_schema.h"
#include "rocjitsu/code/amdgpu_code_object.h"
#include "rocjitsu/code/amdgpu_elf.h"
#include "rocjitsu/config/config_loader.h"
#include "rocjitsu/isa/arch/amdgpu/gfx1250/builders.h"
#include "rocjitsu/isa/arch/amdgpu/gfx1250/execution_backend.h"
#include "rocjitsu/isa/arch/amdgpu/gfx1250/opcodes.h"
#include "rocjitsu/isa/arch/amdgpu/gfx1250/operand.h"
#include "rocjitsu/isa/arch/amdgpu/gfx1250/vds.h"
#include "rocjitsu/isa/arch/amdgpu/gfx1250/vglobal.h"
#include "rocjitsu/isa/arch/amdgpu/gfx1250/vimage.h"
#include "rocjitsu/isa/arch/amdgpu/gfx1250/vop1.h"
#include "rocjitsu/isa/arch/amdgpu/gfx1250/vop2.h"
#include "rocjitsu/isa/arch/amdgpu/gfx1250/vop3.h"
#include "rocjitsu/isa/decoder.h"
#include "rocjitsu/isa/instruction.h"
#include "rocjitsu/isa/target_registry.h"
#include "rocjitsu/vm/amdgpu/cluster_lds_multicast.h"
#include "rocjitsu/vm/amdgpu/compute_unit.h"
#include "rocjitsu/vm/amdgpu/dispatch_entry.h"
#include "rocjitsu/vm/amdgpu/gpu_memory.h"
#include "rocjitsu/vm/amdgpu/iod.h"
#include "rocjitsu/vm/amdgpu/lds_barrier_cell.h"
#include "rocjitsu/vm/amdgpu/mem_state.h"
#include "rocjitsu/vm/amdgpu/memory_pipeline.h"
#include "rocjitsu/vm/amdgpu/memory_side_cache.h"
#include "rocjitsu/vm/amdgpu/vgpr_msb.h"
#include "rocjitsu/vm/soc.h"
#include "util/except.h"

#include "simdojo/sim/simulation.h"

#include "rocjitsu/base/rj_compiler.h"
RJ_DIAGNOSTIC_PUSH
RJ_DIAGNOSTIC_IGNORE_PEDANTIC
#include "hsa/AMDHSAKernelDescriptor.h"
RJ_DIAGNOSTIC_POP

#include <gtest/gtest.h>

#include "rocjitsu/vm/amdgpu/register_access.h"
#include <algorithm>
#include <array>
#include <atomic>
#include <barrier>
#include <bit>
#include <chrono>
#include <cmath>
#include <cstdint>
#include <cstring>
#include <memory>
#include <optional>
#include <set>
#include <shared_mutex>
#include <stdexcept>
#include <string>
#include <string_view>
#include <thread>
#include <utility>
#include <vector>

namespace rocjitsu::amdgpu {

class WriterPreferredAccessGateTestAccess {
public:
  static bool has_waiting_writer(WriterPreferredAccessGate &gate) {
    std::lock_guard lock(gate.mutex_);
    return gate.waiting_writers_ != 0;
  }
};

} // namespace rocjitsu::amdgpu

namespace {

using namespace rocjitsu;

const std::string kGfx1250ConfigPath = std::string(CONFIG_DIR) + "/gfx1250.json";

constexpr uint32_t S_ENDPGM_GFX12 = 0xBFB00000u;
constexpr uint32_t S_WAIT_KMCNT_0_GFX12 = 0xBFC70000u;
constexpr uint32_t S_SET_VGPR_MSB = 0xBF860000u;
// LLVM references for these gfx1250 register capacities:
// - llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp:
//   getAddressableNumSGPRs() returns 106 ordinary SGPRs for GFX10+.
// - llvm/lib/Target/AMDGPU/SIRegisterInfo.td:
//   VCC, TTMP0-15, null, M0, and EXEC occupy scalar selector values 106-127.
// - llvm/lib/Target/AMDGPU/AMDGPU.td:
//   FeatureISAVersion12_50_Common includes Feature1024AddressableVGPRs.
// - llvm/test/MC/AMDGPU/hsa-gfx1250-v4.s:
//   max_vgprs accepts .amdhsa_next_free_vgpr 1024 for Wave32 gfx1250.
// - llvm/lib/Target/AMDGPU/AMDGPULowerVGPREncoding.cpp:
//   high VGPRs still use the encoded v0-v255 operand window; s_set_vgpr_msb
//   supplies two high address bits per operand role.
// - llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp:
//   GFX12.5 reports four SIMDs per CU, 20 waves per SIMD, and 160 KiB of
//   addressable local memory.
constexpr uint32_t kGfx1250ScalarSlots = 128;
constexpr uint32_t kGfx1250Wave32VgprAllocation = 1024;
constexpr uint32_t kGfx1250VgprEncodingGranule = 16;
constexpr uint32_t kGfx1250SimdsPerCu = 4;
constexpr uint32_t kGfx1250MaxWavesPerSimd = 20;
constexpr uint32_t kGfx1250WaveSlotsPerCu = kGfx1250SimdsPerCu * kGfx1250MaxWavesPerSimd;
constexpr uint32_t kGfx1250LdsSizeKb = 160;
constexpr uint32_t kSdmaOpCopy = 1;
constexpr uint32_t kSdmaOpFence = 5;
constexpr uint32_t kSdmaOpPollRegmem = 8;
constexpr uint32_t kSdmaOpConstFill = 11;
constexpr uint32_t kSdmaOpTimestamp = 13;
constexpr uint32_t kSdmaOpGcr = 17;
constexpr uint32_t kSdmaSubopCopyLinear = 0;
constexpr uint32_t kSdmaSubopFence64 = 2;
constexpr uint32_t kSdmaSubopPollMem64 = 5;

class TestMemoryInstruction : public Instruction {
public:
  explicit TestMemoryInstruction(std::unique_ptr<DynamicInstState> state)
      : Instruction("test_mem", nullptr) {
    flags_ |= MEMORY_OP;
    set_data(std::move(state));
  }
};

class DeferredClusterLdsMulticastEngine : public amdgpu::ClusterLdsMulticastEngine {
public:
  amdgpu::ClusterLdsMulticastResult
  submit(amdgpu::ClusterLdsMulticastTransaction submitted,
         amdgpu::ClusterLdsMulticastCompletion submitted_completion) override {
    txn = std::move(submitted);
    completion = std::move(submitted_completion);
    return amdgpu::ClusterLdsMulticastResult::Deferred;
  }

  amdgpu::ClusterLdsMulticastTransaction txn;
  amdgpu::ClusterLdsMulticastCompletion completion;
};

struct Gfx1250Sim {
  // Direct-construction tests intentionally bypass Decoder. Select the same
  // immutable backend that a full gfx1250 decoder would inject while building
  // their generated instruction and operand objects.
  ScopedIsaExecutionBackend execution_backend_scope{&gfx1250::execution_backend()};
  config::LoadedConfig loaded;
  SoC *soc = nullptr;
  amdgpu::GpuMemory *memory = nullptr;
  std::unique_ptr<simdojo::SimulationEngine> engine;
  std::shared_ptr<ExecutionPluginGroup> plugin_group;
  test::HaltSnapshotPlugin *snapshot = nullptr;

  Gfx1250Sim() : loaded(config::load_config(kGfx1250ConfigPath, rocjitsu::kEmbeddedSchema)) {
    build();
  }

  explicit Gfx1250Sim(const std::string &config_json)
      : loaded(config::load_config_from_string(config_json, rocjitsu::kEmbeddedSchema)) {
    build();
  }

  void build() {
    soc = loaded.soc();
    memory = loaded.memory();
    engine = std::make_unique<simdojo::SimulationEngine>(loaded.engine_config);
    engine->topology().set_root(loaded.take_root());
    loaded.wire_links(engine->topology());
    engine->create();
    // Capture every wavefront's final register state at halt. A wave frees its
    // registers at s_endpgm, so tests that run a kernel to completion read the
    // result from the snapshot rather than the (freed) slot.
    plugin_group = test::make_halt_snapshot_group(&snapshot);
    soc->set_plugin_group(plugin_group);
  }

  amdgpu::Xcd *xcd(uint32_t idx = 0) { return soc->xcd(idx); }
  amdgpu::CommandProcessor *cp(uint32_t idx = 0) { return xcd(idx)->command_processor(); }
  amdgpu::ComputeUnitCore *cu(uint32_t idx = 0) {
    return xcd()->shader_engine(0)->compute_unit(idx);
  }

  /// Place a single resident wavefront on CU 0 without running it to s_endpgm, for
  /// tests that drive individual instructions and inspect/modify live wave state.
  amdgpu::Wavefront *dispatch_scratch_wf(uint32_t vgprs = kGfx1250Wave32VgprAllocation) {
    return cu()->dispatch_wf(/*wg_id=*/0, /*pc=*/0, kGfx1250ScalarSlots, vgprs);
  }

  uint64_t write_kernel(uint64_t addr, const uint32_t *words, size_t num_words,
                        uint32_t sgprs = 104, uint32_t vgprs = 32, uint32_t user_sgprs = 2,
                        bool enable_wg_id_x = false, bool enable_wg_id_y = false,
                        bool enable_wg_id_z = false, uint32_t kernel_code_properties = 0,
                        uint32_t kernarg_size = 0, uint32_t kernarg_preload_length = 0,
                        uint32_t kernarg_preload_offset = 0, uint32_t enable_vgpr_workitem_id = 0) {
    using namespace rocr::llvm::amdhsa;
    kernel_descriptor_t kd{};
    kd.kernel_code_entry_byte_offset = sizeof(kernel_descriptor_t);
    kd.kernarg_size = kernarg_size;
    AMDHSA_BITS_SET(kd.compute_pgm_rsrc1, COMPUTE_PGM_RSRC1_GRANULATED_WORKITEM_VGPR_COUNT,
                    ((vgprs / kGfx1250VgprEncodingGranule) - 1));
    AMDHSA_BITS_SET(kd.compute_pgm_rsrc1, COMPUTE_PGM_RSRC1_GRANULATED_WAVEFRONT_SGPR_COUNT,
                    ((sgprs / 8) - 1));
    AMDHSA_BITS_SET(kd.compute_pgm_rsrc2, COMPUTE_PGM_RSRC2_USER_SGPR_COUNT, user_sgprs);
    AMDHSA_BITS_SET(kd.compute_pgm_rsrc2, COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_X,
                    enable_wg_id_x);
    AMDHSA_BITS_SET(kd.compute_pgm_rsrc2, COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Y,
                    enable_wg_id_y);
    AMDHSA_BITS_SET(kd.compute_pgm_rsrc2, COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Z,
                    enable_wg_id_z);
    AMDHSA_BITS_SET(kd.compute_pgm_rsrc2, COMPUTE_PGM_RSRC2_ENABLE_VGPR_WORKITEM_ID,
                    enable_vgpr_workitem_id);
    kd.kernel_code_properties = kernel_code_properties;
    AMDHSA_BITS_SET(kd.kernarg_preload, KERNARG_PRELOAD_SPEC_LENGTH, kernarg_preload_length);
    AMDHSA_BITS_SET(kd.kernarg_preload, KERNARG_PRELOAD_SPEC_OFFSET, kernarg_preload_offset);

    memory->load_image(reinterpret_cast<const uint8_t *>(&kd), sizeof(kd), addr);
    memory->load_image(reinterpret_cast<const uint8_t *>(words), num_words * sizeof(uint32_t),
                       addr + sizeof(kernel_descriptor_t));
    return addr;
  }
};

std::string make_single_se_gfx1250_config(uint32_t num_cus) {
  std::string cu_range = "cu[0:" + std::to_string(num_cus) + "]";
  std::string links;
  for (uint32_t i = 0; i < num_cus; ++i) {
    if (i > 0)
      links += ",";
    links += R"({"src":"xcd0.cp.req_)" + std::to_string(i) + R"(","dst":"xcd0.se0.cu)" +
             std::to_string(i) + R"(.cpl","latency":1,"weight":2})";
    links += R"(,{"src":"xcd0.se0.cu)" + std::to_string(i) + R"(.req","dst":"xcd0.l2.cpl_)" +
             std::to_string(i) + R"(","latency":1,"weight":10})";
  }

  return R"({"max_ticks":10000,"num_threads":1,"vm":{"arch":"gfx1250"},)"
         R"("topology":{"root":{"name":"soc","type":"soc","children":[)"
         R"({"name":"vram","type":"gpu_memory"},)"
         R"({"name":"xcd0","type":"xcd","children":[)"
         R"({"name":"l2","type":"l2_cache"},)"
         R"({"name":"cp","type":"command_processor"},)"
         R"({"name":"se0","type":"shader_engine","children":[)"
         R"({"name":")" +
         cu_range +
         R"(","type":"compute_unit","config":[)"
         R"({"key":"num_wf_slots","value":"80"},)"
         R"({"key":"sgprs_per_wf","value":"128"},)"
         R"({"key":"vgprs_per_wf","value":"1024"},)"
         R"({"key":"lds_size_kb","value":"160"})"
         R"(]}]}]}]},"links":[)" +
         links + R"(]}})";
}

class HostSdmaQueueForTest {
public:
  explicit HostSdmaQueueForTest(Gfx1250Sim &sim) : sim_(sim) {
    sim_.memory->set_passthrough(true);

    amdgpu::HwQueue queue{};
    queue.process_id = kProcessId;
    queue.queue_id = kQueueId;
    queue.ring_base_va = reinterpret_cast<uint64_t>(ring_.data());
    queue.ring_size = static_cast<uint32_t>(ring_.size() * sizeof(uint32_t));
    queue.read_ptr_va = reinterpret_cast<uint64_t>(&read_idx_);
    queue.write_ptr_va = reinterpret_cast<uint64_t>(&write_idx_);
    queue.doorbell_base = doorbells_.data();
    queue.doorbell_offset = 0;
    queue.host_accessible = true;
    queue.is_sdma = true;
    sim_.cp()->register_queue(std::move(queue));
  }

  ~HostSdmaQueueForTest() { sim_.cp()->unregister_queue(kQueueId, kProcessId); }

  uint32_t *ring() { return ring_.data(); }

  void submit(uint32_t dwords) {
    uint64_t write_idx = static_cast<uint64_t>(dwords) * sizeof(uint32_t);
    std::atomic_ref<uint64_t>(write_idx_).store(write_idx, std::memory_order_release);
    std::atomic_ref<uint64_t>(doorbells_[0]).store(write_idx, std::memory_order_release);
    sim_.engine->schedule_event_now(sim_.cp()->doorbell_event());
  }

  uint64_t read_idx() const {
    return std::atomic_ref<const uint64_t>(read_idx_).load(std::memory_order_acquire);
  }

private:
  static constexpr uint32_t kProcessId = 0;
  static constexpr uint32_t kQueueId = 1250;

  Gfx1250Sim &sim_;
  std::array<uint32_t, 64> ring_{};
  alignas(8) uint64_t read_idx_ = 0;
  alignas(8) uint64_t write_idx_ = 0;
  std::array<uint64_t, 1> doorbells_{};
};

class TranslatedSdmaQueueForTest {
public:
  explicit TranslatedSdmaQueueForTest(Gfx1250Sim &sim) : sim_(sim), process_(kProcessId) {
    sim_.memory->register_process(kProcessId, &process_.page_table_, &process_.page_table_mutex_,
                                  process_.page_table_generation());
    process_.map_pages(kRingVa, ring_.data(), ring_.size() * sizeof(ring_[0]));
    process_.map_pages(kQueueStateVa, queue_state_.data(),
                       queue_state_.size() * sizeof(queue_state_[0]));
    process_.map_pages(kSrcVa, src_.data(), src_.size());
    process_.map_pages(kDstVa, dst_.data(), dst_.size());
    process_.map_pages(kSignalVa, signal_.data(), signal_.size() * sizeof(signal_[0]));
    process_.map_pages(kPollVa, poll_.data(), poll_.size() * sizeof(poll_[0]));

    amdgpu::HwQueue queue{};
    queue.process_id = kProcessId;
    queue.queue_id = kQueueId;
    queue.ring_base_va = kRingVa;
    queue.ring_size = static_cast<uint32_t>(ring_.size() * sizeof(ring_[0]));
    queue.read_ptr_va = kQueueStateVa;
    queue.write_ptr_va = kQueueStateVa + sizeof(queue_state_[0]);
    queue.doorbell_base = doorbells_.data();
    queue.doorbell_offset = 0;
    queue.host_accessible = true;
    queue.is_sdma = true;
    sim_.cp()->register_queue(std::move(queue));
  }

  ~TranslatedSdmaQueueForTest() {
    sim_.cp()->unregister_queue(kQueueId, kProcessId);
    sim_.memory->unregister_process(kProcessId);
  }

  uint32_t *ring() { return ring_.data(); }
  uint8_t *src() { return src_.data(); }
  uint8_t *dst() { return dst_.data(); }
  int64_t &signal_value() { return signal_[0]; }
  uint64_t &poll_value() { return poll_[0]; }

  uint64_t src_va() const { return kSrcVa; }
  uint64_t dst_va() const { return kDstVa; }
  uint64_t signal_va() const { return kSignalVa; }
  uint64_t poll_va() const { return kPollVa; }

  void submit(uint32_t dwords) {
    uint64_t write_idx = static_cast<uint64_t>(dwords) * sizeof(uint32_t);
    std::atomic_ref<uint64_t>(queue_state_[1]).store(write_idx, std::memory_order_release);
    std::atomic_ref<uint64_t>(doorbells_[0]).store(write_idx, std::memory_order_release);
    sim_.engine->schedule_event_now(sim_.cp()->doorbell_event());
  }

  uint64_t read_idx() const {
    return std::atomic_ref<const uint64_t>(queue_state_[0]).load(std::memory_order_acquire);
  }

private:
  static constexpr uint32_t kProcessId = 1251;
  static constexpr uint32_t kQueueId = 1251;
  static constexpr uint64_t kRingVa = 0x1000'0000'0000ULL;
  static constexpr uint64_t kQueueStateVa = 0x1000'0000'1000ULL;
  static constexpr uint64_t kSrcVa = 0x1000'0000'2000ULL;
  static constexpr uint64_t kDstVa = 0x1000'0000'3000ULL;
  static constexpr uint64_t kSignalVa = 0x1000'0000'4000ULL;
  static constexpr uint64_t kPollVa = 0x1000'0000'5000ULL;

  Gfx1250Sim &sim_;
  KfdProcess process_;
  alignas(4096) std::array<uint32_t, 1024> ring_{};
  alignas(4096) std::array<uint64_t, 512> queue_state_{};
  alignas(4096) std::array<uint8_t, 4096> src_{};
  alignas(4096) std::array<uint8_t, 4096> dst_{};
  alignas(4096) std::array<int64_t, 512> signal_{};
  alignas(4096) std::array<uint64_t, 512> poll_{};
  std::array<uint64_t, 1> doorbells_{};
};

void write_sdma_qword_va(uint32_t *packet, uint32_t lo_dw, uint32_t hi_dw, uint64_t va) {
  packet[lo_dw] = static_cast<uint32_t>(va) & ~0x7u;
  packet[hi_dw] = static_cast<uint32_t>(va >> 32);
}

void write_sdma_qword_address(uint32_t *packet, uint32_t lo_dw, uint32_t hi_dw, const void *addr) {
  write_sdma_qword_va(packet, lo_dw, hi_dw, reinterpret_cast<uintptr_t>(addr));
}

// Drive the engine until the CU has no resident wavefronts. A wave frees itself at
// s_endpgm, so the kernel is complete once the CU reports idle (after having had
// work). Final wavefront state should be captured via HaltSnapshotPlugin.
void step_until_halted(simdojo::SimulationEngine &engine, amdgpu::ComputeUnitCore &cu,
                       uint32_t max_steps = 10000) {
  bool saw_work = false;
  for (uint32_t i = 0; i < max_steps && engine.step(); ++i) {
    if (cu.has_active_wfs())
      saw_work = true;
    else if (saw_work)
      return;
  }
}

void step_until_xcd_halted(Gfx1250Sim &sim, uint32_t max_steps = 10000) {
  auto any_active = [&]() {
    for (uint32_t se_idx = 0; se_idx < sim.xcd()->num_shader_engines(); ++se_idx) {
      auto *se = sim.xcd()->shader_engine(se_idx);
      for (uint32_t cu_idx = 0; cu_idx < se->num_compute_units(); ++cu_idx)
        if (se->compute_unit(cu_idx)->has_active_wfs())
          return true;
    }
    return false;
  };

  bool saw_work = false;
  for (uint32_t i = 0; i < max_steps && sim.engine->step(); ++i) {
    if (any_active()) {
      saw_work = true;
      continue;
    }
    if (saw_work)
      return;
  }
}

// Runs a one-wave kernel to s_endpgm and returns the wave's final-state snapshot
// (captured at halt, before its registers are freed), or nullptr if THIS dispatch
// produced no new halt. Snapshots accumulate across calls in the fixture plugin, so
// compare the count before/after and only return a pointer when this dispatch
// appended one — otherwise a regression that fails to run/halt would silently return
// a stale snapshot from a previous dispatch.
const test::WavefrontSnapshot *dispatch_one_wave(Gfx1250Sim &sim, const uint32_t *code,
                                                 size_t num_words, uint32_t vgprs = 32) {
  const size_t before = sim.snapshot->snapshots().size();
  uint64_t kernel_object = sim.write_kernel(0x10000, code, num_words, 104, vgprs);
  test::AqlQueue queue(sim.memory, sim.cp());
  queue.dispatch(kernel_object, 32, 32);
  step_until_halted(*sim.engine, *sim.cu());
  if (sim.snapshot->snapshots().size() == before)
    return nullptr;
  return &sim.snapshot->snapshots().back();
}

constexpr uint32_t make_vmov_b32(uint8_t vdst) {
  return 0x7E0002FFu | (static_cast<uint32_t>(vdst) << 17);
}

constexpr std::array<uint32_t, 2> make_vmov_b32_literal(uint8_t vdst, uint32_t literal) {
  return {make_vmov_b32(vdst), literal};
}

constexpr std::array<uint32_t, 2> make_s_load_b32_scaled_imm(uint8_t sdata, uint8_t sbase_pair,
                                                             uint32_t scaled_offset) {
  constexpr uint32_t kSmemEncoding = 0x3Du << 26;
  constexpr uint32_t kSoffsetNull = 0x7Cu;
  return {kSmemEncoding | ((static_cast<uint32_t>(sdata) & 0x7Fu) << 6) |
              (static_cast<uint32_t>(sbase_pair) & 0x3Fu),
          (scaled_offset & 0x00FF'FFFFu) | (1u << 24) | (kSoffsetNull << 25)};
}

constexpr uint16_t vopd_src0_vgpr(uint16_t reg) { return 256 + reg; }

enum class VopdOp : uint16_t {
  FmamkF32 = 2,
  MulF32 = 3,
  MulDx9ZeroF32 = 7,
  MovB32 = 8,
  CndmaskB32 = 9,
  FmaF32 = 19,
};

struct VopdSlot {
  VopdOp op;
  uint16_t src0;
  uint8_t src1;
  uint8_t src2;
  uint8_t dst;
};

constexpr std::array<uint32_t, 3> make_vopd3_pair(VopdSlot x, VopdSlot y, uint8_t negx = 0,
                                                  uint8_t negy = 0) {
  return {
      0xCF000000u | ((static_cast<uint32_t>(x.op) & 0x3Fu) << 18) |
          ((static_cast<uint32_t>(y.op) & 0x3Fu) << 12) | (static_cast<uint32_t>(x.src0) & 0x1FFu),
      (static_cast<uint32_t>(y.src0) & 0x1FFu) | ((static_cast<uint32_t>(negx) & 0x7u) << 9) |
          ((static_cast<uint32_t>(negy) & 0x7u) << 12) | (static_cast<uint32_t>(x.src1) << 16) |
          (static_cast<uint32_t>(x.src2) << 24),
      static_cast<uint32_t>(x.dst) | (static_cast<uint32_t>(y.src1) << 8) |
          (static_cast<uint32_t>(y.src2) << 16) | (static_cast<uint32_t>(y.dst) << 24),
  };
}

template <size_t N>
void append_instruction(std::vector<uint32_t> &code, const std::array<uint32_t, N> &words) {
  code.insert(code.end(), words.begin(), words.end());
}

void append_instruction(std::vector<uint32_t> &code, uint32_t word) { code.push_back(word); }

void write_wave_sgpr(amdgpu::ComputeUnitCore &cu, amdgpu::Wavefront &wf, uint32_t reg,
                     uint32_t value) {
  cu.write_sgpr(wf.sgpr_alloc().base + reg, value);
}

uint32_t read_wave_sgpr(const amdgpu::ComputeUnitCore &cu, const amdgpu::Wavefront &wf,
                        uint32_t reg) {
  return cu.read_sgpr(wf.sgpr_alloc().base + reg);
}

void write_tensor_dma_d0(amdgpu::ComputeUnitCore &cu, amdgpu::Wavefront &wf, uint32_t reg,
                         uint64_t global_addr, uint32_t lds_base = 0) {
  write_wave_sgpr(cu, wf, reg + 0, 1);
  write_wave_sgpr(cu, wf, reg + 1, lds_base);
  write_wave_sgpr(cu, wf, reg + 2, static_cast<uint32_t>(global_addr));
  write_wave_sgpr(cu, wf, reg + 3,
                  static_cast<uint32_t>((global_addr >> 32) & 0x01ffffffu) | 0x80000000u);
}

void write_global_u32(amdgpu::GpuMemory &memory, uint64_t addr, uint32_t value) {
  for (uint32_t byte = 0; byte < 4; ++byte)
    memory.write8(addr + byte, static_cast<uint8_t>(value >> (byte * 8)));
}

uint32_t read_global_u32(amdgpu::GpuMemory &memory, uint64_t addr) {
  uint32_t value = 0;
  for (uint32_t byte = 0; byte < 4; ++byte)
    value |= static_cast<uint32_t>(memory.read8(addr + byte)) << (byte * 8);
  return value;
}

std::unique_ptr<Instruction> decode_gfx1250(const std::array<uint32_t, 3> &words,
                                            std::string_view expected_mnemonic) {
  auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
  if (!decoder) {
    ADD_FAILURE() << "Decoder::create() returned nullptr for gfx1250";
    return nullptr;
  }

  std::unique_ptr<Instruction> inst(decoder->decode(words.data()));
  if (!inst) {
    ADD_FAILURE() << "decode() returned nullptr for gfx1250 instruction";
    return nullptr;
  }

  EXPECT_EQ(inst->mnemonic(), expected_mnemonic);
  EXPECT_EQ(inst->size(), static_cast<int>(words.size() * sizeof(words[0])));
  return inst;
}

template <typename T> void append_bytes(std::vector<uint8_t> &bytes, const T &value) {
  auto *src = reinterpret_cast<const uint8_t *>(&value);
  bytes.insert(bytes.end(), src, src + sizeof(T));
}

size_t align_up(size_t value, size_t alignment) {
  return (value + alignment - 1) & ~(alignment - 1);
}

std::vector<uint8_t> make_minimal_gfx1250_elf() {
  constexpr uint8_t text[] = {0x00, 0x00, 0xB0, 0xBF};
  constexpr char shstrtab[] = "\0.text\0.shstrtab\0";
  constexpr uint32_t text_name = 1;
  constexpr uint32_t shstrtab_name = 7;

  std::vector<uint8_t> image(sizeof(Elf64_Ehdr), 0);
  const size_t text_offset = image.size();
  image.insert(image.end(), std::begin(text), std::end(text));
  const size_t shstrtab_offset = image.size();
  image.insert(image.end(), std::begin(shstrtab), std::end(shstrtab));
  image.resize(align_up(image.size(), alignof(Elf64_Shdr)), 0);
  const size_t shoff = image.size();

  Elf64_Shdr null_shdr{};
  append_bytes(image, null_shdr);

  Elf64_Shdr text_shdr{};
  text_shdr.sh_name = text_name;
  text_shdr.sh_type = SHT_PROGBITS;
  text_shdr.sh_flags = SHF_ALLOC | SHF_EXECINSTR;
  text_shdr.sh_offset = text_offset;
  text_shdr.sh_size = sizeof(text);
  text_shdr.sh_addralign = alignof(uint32_t);
  append_bytes(image, text_shdr);

  Elf64_Shdr shstrtab_shdr{};
  shstrtab_shdr.sh_name = shstrtab_name;
  shstrtab_shdr.sh_type = SHT_STRTAB;
  shstrtab_shdr.sh_offset = shstrtab_offset;
  shstrtab_shdr.sh_size = sizeof(shstrtab);
  shstrtab_shdr.sh_addralign = 1;
  append_bytes(image, shstrtab_shdr);

  Elf64_Ehdr ehdr{};
  std::memcpy(ehdr.e_ident, EI_MAGIC, EI_MAGIC_SIZE);
  ehdr.e_ident[EI_CLASS] = ELFCLASS64;
  ehdr.e_ident[EI_DATA] = 1;
  ehdr.e_ident[EI_VERSION] = 1;
  ehdr.e_ident[EI_OSABI] = ELFOSABI_AMDGPU_HSA;
  ehdr.e_ident[EI_ABIVERSION] = ELFABIVERSION_AMDGPU_HSA_V5;
  ehdr.e_type = ET_REL;
  ehdr.e_machine = EM_AMDGPU;
  ehdr.e_version = 1;
  ehdr.e_shoff = shoff;
  ehdr.e_flags = EF_AMDGPU_MACH_AMDGCN_GFX1250;
  ehdr.e_ehsize = sizeof(Elf64_Ehdr);
  ehdr.e_shentsize = sizeof(Elf64_Shdr);
  ehdr.e_shnum = 3;
  ehdr.e_shstrndx = 2;
  std::memcpy(image.data(), &ehdr, sizeof(ehdr));
  return image;
}

TEST(Gfx1250ConfigTest, ConfigLoadsTopology) {
  auto loaded = config::load_config(kGfx1250ConfigPath, rocjitsu::kEmbeddedSchema);
  auto *soc = loaded.soc();
  ASSERT_NE(soc, nullptr);
  EXPECT_EQ(soc->arch(), ROCJITSU_CODE_ARCH_GFX1250);
  EXPECT_EQ(config::parse_arch("gfx1250"), ROCJITSU_CODE_ARCH_GFX1250);
  EXPECT_STREQ(config::arch_to_string(ROCJITSU_CODE_ARCH_GFX1250), "gfx1250");

  EXPECT_TRUE(loaded.device.present);
  EXPECT_EQ(loaded.device.gfx_target_version, 120500u);
  EXPECT_EQ(loaded.device.simd_count, 1024u);
  EXPECT_EQ(loaded.device.max_waves_per_simd, kGfx1250MaxWavesPerSimd);
  EXPECT_EQ(loaded.device.num_shader_arrays_per_engine, 2u);
  EXPECT_EQ(loaded.device.num_cu_per_sh, 4u);
  EXPECT_EQ(loaded.device.simd_per_cu, kGfx1250SimdsPerCu);
  EXPECT_EQ(loaded.device.wave_front_size, 32u);
  EXPECT_EQ(loaded.device.lds_size_kb, kGfx1250LdsSizeKb);
  EXPECT_EQ(loaded.device.mem_width, 8192u);
  EXPECT_EQ(loaded.device.marketing_name, "gfx1250");

  EXPECT_EQ(soc->num_xcds(), 8u);
  EXPECT_EQ(soc->num_iods(), 2u);
  EXPECT_EQ(soc->xcd(0)->num_shader_engines(), 4u);
  EXPECT_EQ(soc->xcd(0)->shader_engine(0)->num_compute_units(), 8u);
  auto *cu = soc->xcd(0)->shader_engine(0)->compute_unit(0);
  ASSERT_NE(cu, nullptr);
  EXPECT_EQ(cu->wf_size(), 32u);
  EXPECT_EQ(cu->config().num_wf_slots, kGfx1250WaveSlotsPerCu);
  EXPECT_EQ(cu->config().sgprs_per_wf, kGfx1250ScalarSlots);
  EXPECT_EQ(cu->config().vgprs_per_wf, kGfx1250Wave32VgprAllocation);
  EXPECT_EQ(cu->config().lds_size_kb, kGfx1250LdsSizeKb);
  EXPECT_EQ(soc->xcd(0)->command_processor()->vgpr_granularity(), kGfx1250VgprEncodingGranule);
  EXPECT_EQ(soc->xcd(0)->command_processor()->sdma_packet_dialect(),
            amdgpu::SdmaPacketDialect::Gfx1250);
}

TEST(Gfx1250SdmaTest, PollMem64WaitsForFull64BitCondition) {
  Gfx1250Sim sim;
  HostSdmaQueueForTest queue(sim);
  alignas(8) uint64_t value = 1;

  auto *packet = queue.ring();
  packet[0] = kSdmaOpPollRegmem | (kSdmaSubopPollMem64 << 8) | (3u << 28); // 64-bit equal poll.
  write_sdma_qword_address(packet, 1, 2, &value);
  packet[3] = 0;
  packet[4] = 0;
  packet[5] = 0xFFFFFFFFu;
  packet[6] = 0xFFFFFFFFu;
  packet[7] = 0;

  queue.submit(8);
  ASSERT_TRUE(sim.engine->step());
  EXPECT_EQ(queue.read_idx(), 0u);

  std::atomic_ref<uint64_t>(value).store(0, std::memory_order_release);
  sim.engine->schedule_event_now(sim.cp()->doorbell_event());
  ASSERT_TRUE(sim.engine->step());
  EXPECT_EQ(queue.read_idx(), 8u * sizeof(uint32_t));
}

TEST(Gfx1250SdmaTest, Fence64WritesFull64BitValue) {
  Gfx1250Sim sim;
  HostSdmaQueueForTest queue(sim);
  alignas(8) uint64_t value = 0;
  constexpr uint64_t kFenceValue = 0x12345678ABCDEF01ULL;

  auto *packet = queue.ring();
  packet[0] = kSdmaOpFence | (kSdmaSubopFence64 << 8) | (3u << 16);
  write_sdma_qword_address(packet, 1, 2, &value);
  packet[3] = static_cast<uint32_t>(kFenceValue);
  packet[4] = static_cast<uint32_t>(kFenceValue >> 32);

  queue.submit(5);
  ASSERT_TRUE(sim.engine->step());
  EXPECT_EQ(queue.read_idx(), 5u * sizeof(uint32_t));
  EXPECT_EQ(std::atomic_ref<uint64_t>(value).load(std::memory_order_acquire), kFenceValue);
}

// A wrong GCR packet size silently desyncs the SDMA ring read pointer and
// corrupts the following packet. Emit OP_GCR followed by a 32-bit FENCE and
// assert both the read pointer advance and that the FENCE decoded at the right
// boundary (its sentinel lands). gfx11/12 GCR is 5 dwords; gfx1250 is 6.
TEST(Gfx1250SdmaTest, GcrPacketSizeMatchesDialectAndKeepsRingInSync) {
  constexpr uint32_t kGcrLegacySize = 5;
  constexpr uint32_t kGcrGfx1250Size = 6;
  constexpr uint32_t kFenceSize = 4;
  constexpr uint32_t kFenceSentinel = 0xC0FFEE11u;
  // GL2 invalidate control bit position differs by dialect; setting it exercises
  // a realistic invalidate GCR but does not affect the decoded packet size.
  constexpr uint32_t kLegacyGl2InvControlDw = 2;
  constexpr uint32_t kLegacyGl2InvBit = 1u << 30;
  constexpr uint32_t kGfx1250Gl2InvControlDw = 3;
  constexpr uint32_t kGfx1250Gl2InvBit = 1u << 14;

  auto run_dialect = [kFenceSentinel](amdgpu::SdmaPacketDialect dialect, uint32_t gcr_size,
                                      uint32_t control_dw, uint32_t control_bit) {
    Gfx1250Sim sim;
    sim.cp()->set_sdma_packet_dialect(dialect);
    HostSdmaQueueForTest queue(sim);
    alignas(8) uint32_t fence_value = 0;

    auto *packet = queue.ring();
    packet[0] = kSdmaOpGcr;
    packet[control_dw] = control_bit;

    uint32_t *fence = packet + gcr_size;
    fence[0] = kSdmaOpFence; // 32-bit fence (sub_op 0).
    write_sdma_qword_address(fence, 1, 2, &fence_value);
    fence[3] = kFenceSentinel;

    queue.submit(gcr_size + kFenceSize);
    ASSERT_TRUE(sim.engine->step());
    EXPECT_EQ(queue.read_idx(), (gcr_size + kFenceSize) * sizeof(uint32_t));
    EXPECT_EQ(std::atomic_ref<uint32_t>(fence_value).load(std::memory_order_acquire),
              kFenceSentinel);
  };

  run_dialect(amdgpu::SdmaPacketDialect::Gfx11Plus, kGcrLegacySize, kLegacyGl2InvControlDw,
              kLegacyGl2InvBit);
  run_dialect(amdgpu::SdmaPacketDialect::Gfx1250, kGcrGfx1250Size, kGfx1250Gl2InvControlDw,
              kGfx1250Gl2InvBit);
}

// SDMA writes go straight to backing while L2 may still hold a dirty line that
// overlaps the destination. Seed that state explicitly with writeback_line().
// The fix flushes the caches before the direct write, so the dirty line is
// published first and the SDMA data supersedes it. Regression for that ordering.
TEST(Gfx1250SdmaTest, ConstFillSupersedesOverlappingDirtyL2Line) {
  Gfx1250Sim sim;
  // The config-driven topology build wires the XCD's L2 into the CP, so the SDMA
  // cache maintenance operates on the same L2 instance we dirty below.
  auto *l2 = sim.xcd()->l2_cache();
  ASSERT_NE(l2, nullptr);

  TranslatedSdmaQueueForTest queue(sim);
  constexpr uint32_t kProcessId = 1251; // matches TranslatedSdmaQueueForTest.
  constexpr uint32_t kStaleWord = 0x11111111u;
  constexpr uint32_t kFillWord = 0x22222222u;

  // Seed a dirty L2 line overlapping the destination, without touching backing.
  uint8_t stale_line[amdgpu::L2Cache::LINE_SIZE];
  std::memset(stale_line, static_cast<int>(kStaleWord & 0xFF), sizeof(stale_line));
  l2->writeback_line(queue.dst_va(), stale_line, amdgpu::Mtype::RW, kProcessId);

  // CONST_FILL the destination line with a different byte pattern.
  auto *packet = queue.ring();
  packet[0] = kSdmaOpConstFill | (0x2u << 30); // fillsize=2 (dword granularity).
  write_sdma_qword_va(packet, 1, 2, queue.dst_va());
  packet[3] = kFillWord;
  packet[4] = amdgpu::L2Cache::LINE_SIZE - 1; // count-1 bytes.

  queue.submit(5);
  ASSERT_TRUE(sim.engine->step());

  // Backing must reflect the SDMA fill, not the stale cached line.
  EXPECT_EQ(sim.memory->read32(queue.dst_va(), kProcessId), kFillWord);
  EXPECT_NE(sim.memory->read32(queue.dst_va(), kProcessId), kStaleWord);
}

// A scalar L1 (K$) can retain a clean snapshot overlapping an SDMA destination.
// The pre-write maintenance invalidates that snapshot, and later K$ maintenance
// must not publish it over the direct SDMA result.
TEST(Gfx1250SdmaTest, ConstFillSupersedesOverlappingScalarL1Line) {
  Gfx1250Sim sim;
  auto *cu = sim.cu();
  ASSERT_NE(cu, nullptr);

  TranslatedSdmaQueueForTest queue(sim);
  constexpr uint32_t kProcessId = 1251; // matches TranslatedSdmaQueueForTest.
  constexpr uint32_t kStaleWord = 0x11111111u;
  constexpr uint32_t kFillWord = 0x22222222u;

  // Populate a K$ line overlapping the SDMA destination via a write-through
  // scalar store. K$ retains a clean snapshot of the pre-fill value.
  cu->l1_scalar().store(queue.dst_va(), /*num_dwords=*/1, &kStaleWord, kProcessId);

  // CONST_FILL the destination line with a different pattern.
  auto *packet = queue.ring();
  packet[0] = kSdmaOpConstFill | (0x2u << 30); // fillsize=2 (dword granularity).
  write_sdma_qword_va(packet, 1, 2, queue.dst_va());
  packet[3] = kFillWord;
  packet[4] = amdgpu::L2Cache::LINE_SIZE - 1; // count-1 bytes.

  queue.submit(5);
  ASSERT_TRUE(sim.engine->step());

  // The SDMA pre-write maintenance must have invalidated the old K$ snapshot,
  // so the first scalar reload observes the fill rather than kStaleWord.
  uint32_t scalar_value = 0;
  cu->l1_scalar().load(queue.dst_va(), /*num_dwords=*/1, &scalar_value, kProcessId);
  EXPECT_EQ(scalar_value, kFillWord);

  // Mimic later acquire/release maintenance. The clean K$ snapshot must not
  // resurrect stale data over the SDMA fill.
  cu->flush_l1(kProcessId);
  if (auto *l2 = sim.xcd()->l2_cache())
    l2->flush_all();

  // Backing must reflect the SDMA fill, not the stale scalar line.
  EXPECT_EQ(sim.memory->read32(queue.dst_va(), kProcessId), kFillWord);
  EXPECT_NE(sim.memory->read32(queue.dst_va(), kProcessId), kStaleWord);
}

// OP_TIMESTAMP is a direct backing-store write like COPY/FENCE/CONST_FILL, so it
// has the same clobber hazard: a dirty cached line overlapping the timestamp
// address must be published before the store, not written out over it by a later
// flush. Seed a dirty L2 line at the timestamp address, issue OP_TIMESTAMP, then
// force a flush; the stored timestamp must survive (stale word gone, value set).
TEST(Gfx1250SdmaTest, TimestampSupersedesOverlappingDirtyL2Line) {
  Gfx1250Sim sim;
  auto *l2 = sim.xcd()->l2_cache();
  ASSERT_NE(l2, nullptr);

  TranslatedSdmaQueueForTest queue(sim);
  constexpr uint32_t kProcessId = 1251; // matches TranslatedSdmaQueueForTest.
  constexpr uint64_t kStaleQword = 0x1111111111111111ULL;

  // Seed a dirty L2 line overlapping the timestamp destination, without touching
  // backing.
  uint8_t stale_line[amdgpu::L2Cache::LINE_SIZE];
  std::memset(stale_line, 0x11, sizeof(stale_line));
  l2->writeback_line(queue.dst_va(), stale_line, amdgpu::Mtype::RW, kProcessId);

  auto *packet = queue.ring();
  packet[0] = kSdmaOpTimestamp;
  write_sdma_qword_va(packet, 1, 2, queue.dst_va());

  queue.submit(3); // TIMESTAMP is 3 dwords.
  ASSERT_TRUE(sim.engine->step());

  // Force any still-resident dirty line out, mimicking a later flush.
  if (auto *xl2 = sim.xcd()->l2_cache())
    xl2->flush_all();

  // The timestamp value is nondeterministic, but it must not be the stale word
  // and must be a plausible nonzero nanosecond count.
  const uint64_t stored = sim.memory->read64(queue.dst_va(), kProcessId);
  EXPECT_NE(stored, kStaleQword);
  EXPECT_NE(stored, 0u);
}

// The GCR decoder now distinguishes GL2 writeback (publish dirty lines),
// invalidate/discard (drop without writeback), and no-op (no GL2 bits). This is
// the data-loss distinction the PR protects. Dirty an L2 line, then issue each
// GCR flavor and observe whether the dirty data reaches backing.
TEST(Gfx1250SdmaTest, GcrWritebackPublishesInvalidateDropsNoopKeeps) {
  constexpr uint32_t kProcessId = 1251; // matches TranslatedSdmaQueueForTest.
  constexpr uint32_t kDirtyWord = 0x33333333u;
  constexpr uint32_t kBackingWord = 0x44444444u;
  // gfx1250 GCR control dword (DW3) bit positions.
  constexpr uint32_t kControlDw = 3;
  constexpr uint32_t kGl2InvBit = 1u << 14;
  constexpr uint32_t kGl2WbBit = 1u << 15;

  // Outcome of a GCR flavor: the value in backing (read directly through the
  // page table) and the value seen through L2 (which returns the resident dirty
  // line if still present, or re-fetches backing if the line was dropped).
  struct GcrOutcome {
    uint32_t backing = 0;
    uint32_t via_l2 = 0;
  };

  enum class GcrKind { WritebackOnly, InvalidateOnly, Noop };
  // Void return so a missing L2 is a fatal guard (ASSERT_*) before we deref it.
  auto run = [&](GcrKind kind, GcrOutcome &out) {
    Gfx1250Sim sim;
    auto *l2 = sim.xcd()->l2_cache();
    ASSERT_NE(l2, nullptr);
    TranslatedSdmaQueueForTest queue(sim);

    // Put a known value in backing, then a different dirty value in L2 on top.
    for (uint32_t i = 0; i < sizeof(uint32_t); ++i)
      sim.memory->write8(queue.dst_va() + i, static_cast<uint8_t>((kBackingWord >> (i * 8)) & 0xFF),
                         kProcessId);
    uint8_t dirty_line[amdgpu::L2Cache::LINE_SIZE];
    std::memset(dirty_line, static_cast<int>(kDirtyWord & 0xFF), sizeof(dirty_line));
    l2->writeback_line(queue.dst_va(), dirty_line, amdgpu::Mtype::RW, kProcessId);

    auto *packet = queue.ring();
    packet[0] = kSdmaOpGcr;
    if (kind == GcrKind::WritebackOnly)
      packet[kControlDw] = kGl2WbBit;
    else if (kind == GcrKind::InvalidateOnly)
      packet[kControlDw] = kGl2InvBit;
    else
      packet[kControlDw] = 0; // no GL2 bits: no-op.

    queue.submit(6); // gfx1250 GCR is 6 dwords.
    EXPECT_TRUE(sim.engine->step());

    out.backing = sim.memory->read32(queue.dst_va(), kProcessId);
    // Read back through L2: a still-resident dirty line returns kDirtyWord; a
    // dropped line re-fetches from backing on the miss.
    uint32_t l2_word = 0;
    l2->read(queue.dst_va(), reinterpret_cast<uint8_t *>(&l2_word), sizeof(l2_word),
             amdgpu::Mtype::RW, kProcessId);
    out.via_l2 = l2_word;
  };

  // Writeback publishes the dirty line to backing.
  GcrOutcome wb;
  run(GcrKind::WritebackOnly, wb);
  EXPECT_EQ(wb.backing, kDirtyWord);
  // Invalidate/discard drops the dirty line without writeback; backing keeps its
  // original value and the line is no longer resident.
  GcrOutcome inv;
  run(GcrKind::InvalidateOnly, inv);
  EXPECT_EQ(inv.backing, kBackingWord);
  EXPECT_EQ(inv.via_l2, kBackingWord); // line dropped → L2 re-fetches backing.
  // No GL2 bits: no cache maintenance at all. Backing is untouched and the dirty
  // line stays resident in L2 (this is what distinguishes no-op from
  // invalidate-only: an incorrect invalidate would drop the line here too).
  GcrOutcome noop;
  run(GcrKind::Noop, noop);
  EXPECT_EQ(noop.backing, kBackingWord);
  EXPECT_EQ(noop.via_l2, kDirtyWord); // dirty line still resident in L2.
}

TEST(Gfx1250SdmaTest, CopyWaitSignalResolvesTranslatedAddresses) {
  Gfx1250Sim sim;
  TranslatedSdmaQueueForTest queue(sim);
  constexpr uint32_t kCopyBytes = 128;
  queue.signal_value() = 5;
  for (uint32_t i = 0; i < kCopyBytes; ++i) {
    queue.src()[i] = static_cast<uint8_t>(i ^ 0x5a);
    queue.dst()[i] = 0;
  }

  auto *packet = queue.ring();
  packet[0] = kSdmaOpCopy | (kSdmaSubopCopyLinear << 8) | (1u << 31);
  packet[8] = kCopyBytes - 1;
  write_sdma_qword_va(packet, 10, 11, queue.src_va());
  write_sdma_qword_va(packet, 12, 13, queue.dst_va());
  packet[14] = 0x70;
  write_sdma_qword_va(packet, 15, 16, queue.signal_va());
  packet[17] = 1;
  packet[18] = 0;

  queue.submit(19);
  ASSERT_TRUE(sim.engine->step());
  EXPECT_EQ(queue.read_idx(), 19u * sizeof(uint32_t));
  EXPECT_EQ(std::memcmp(queue.dst(), queue.src(), kCopyBytes), 0);
  EXPECT_EQ(queue.signal_value(), 4);
}

TEST(Gfx1250SdmaTest, CopyWaitSignalUnresolvedWaitAddressDoesNotAdvance) {
  Gfx1250Sim sim;
  TranslatedSdmaQueueForTest queue(sim);
  constexpr uint32_t kCopyBytes = 128;
  constexpr uint64_t kUnmappedWaitVa = 0x2000'0000'0000ULL;
  for (uint32_t i = 0; i < kCopyBytes; ++i) {
    queue.src()[i] = static_cast<uint8_t>(i ^ 0xa5);
    queue.dst()[i] = 0;
  }

  auto *packet = queue.ring();
  packet[0] = kSdmaOpCopy | (kSdmaSubopCopyLinear << 8) | (1u << 30);
  packet[1] = 3;
  write_sdma_qword_va(packet, 2, 3, kUnmappedWaitVa);
  packet[4] = 0;
  packet[5] = 0;
  packet[6] = 0xFFFFFFFFu;
  packet[7] = 0xFFFFFFFFu;
  packet[8] = kCopyBytes - 1;
  write_sdma_qword_va(packet, 10, 11, queue.src_va());
  write_sdma_qword_va(packet, 12, 13, queue.dst_va());

  queue.submit(19);
  ASSERT_TRUE(sim.engine->step());
  EXPECT_EQ(queue.read_idx(), 0u);
  EXPECT_NE(std::memcmp(queue.dst(), queue.src(), kCopyBytes), 0);
}

TEST(Gfx1250SdmaTest, CopyWaitSignalUnresolvedDstDoesNotAdvanceOrSignal) {
  Gfx1250Sim sim;
  TranslatedSdmaQueueForTest queue(sim);
  constexpr uint32_t kCopyBytes = 128;
  constexpr uint64_t kUnmappedDstVa = 0x2000'0000'2000ULL;
  queue.signal_value() = 5;
  for (uint32_t i = 0; i < kCopyBytes; ++i) {
    queue.src()[i] = static_cast<uint8_t>(i ^ 0x3c);
    queue.dst()[i] = 0;
  }

  auto *packet = queue.ring();
  packet[0] = kSdmaOpCopy | (kSdmaSubopCopyLinear << 8) | (1u << 31);
  packet[8] = kCopyBytes - 1;
  write_sdma_qword_va(packet, 10, 11, queue.src_va());
  write_sdma_qword_va(packet, 12, 13, kUnmappedDstVa);
  packet[14] = 0x70;
  write_sdma_qword_va(packet, 15, 16, queue.signal_va());
  packet[17] = 1;
  packet[18] = 0;

  queue.submit(19);
  ASSERT_TRUE(sim.engine->step());
  EXPECT_EQ(queue.read_idx(), 0u);
  EXPECT_NE(std::memcmp(queue.dst(), queue.src(), kCopyBytes), 0);
  EXPECT_EQ(queue.signal_value(), 5);
}

TEST(Gfx1250SdmaTest, CopyWaitSignalUnresolvedSignalDoesNotAdvanceOrCopy) {
  Gfx1250Sim sim;
  TranslatedSdmaQueueForTest queue(sim);
  constexpr uint32_t kCopyBytes = 128;
  constexpr uint64_t kUnmappedSignalVa = 0x2000'0000'4000ULL;
  queue.signal_value() = 5;
  for (uint32_t i = 0; i < kCopyBytes; ++i) {
    queue.src()[i] = static_cast<uint8_t>(i ^ 0x69);
    queue.dst()[i] = 0;
  }

  auto *packet = queue.ring();
  packet[0] = kSdmaOpCopy | (kSdmaSubopCopyLinear << 8) | (1u << 31);
  packet[8] = kCopyBytes - 1;
  write_sdma_qword_va(packet, 10, 11, queue.src_va());
  write_sdma_qword_va(packet, 12, 13, queue.dst_va());
  packet[14] = 0x70;
  write_sdma_qword_va(packet, 15, 16, kUnmappedSignalVa);
  packet[17] = 1;
  packet[18] = 0;

  queue.submit(19);
  ASSERT_TRUE(sim.engine->step());
  EXPECT_EQ(queue.read_idx(), 0u);
  EXPECT_NE(std::memcmp(queue.dst(), queue.src(), kCopyBytes), 0);
  EXPECT_EQ(queue.signal_value(), 5);
}

TEST(Gfx1250SdmaTest, CopyLinearUnresolvedDstDoesNotAdvance) {
  Gfx1250Sim sim;
  TranslatedSdmaQueueForTest queue(sim);
  constexpr uint32_t kCopyBytes = 128;
  constexpr uint64_t kUnmappedDstVa = 0x2000'0000'3000ULL;
  for (uint32_t i = 0; i < kCopyBytes; ++i) {
    queue.src()[i] = static_cast<uint8_t>(i ^ 0xc3);
    queue.dst()[i] = 0;
  }

  auto *packet = queue.ring();
  packet[0] = kSdmaOpCopy | (kSdmaSubopCopyLinear << 8);
  packet[1] = kCopyBytes - 1;
  write_sdma_qword_va(packet, 3, 4, queue.src_va());
  write_sdma_qword_va(packet, 5, 6, kUnmappedDstVa);

  queue.submit(7);
  ASSERT_TRUE(sim.engine->step());
  EXPECT_EQ(queue.read_idx(), 0u);
  EXPECT_NE(std::memcmp(queue.dst(), queue.src(), kCopyBytes), 0);
}

TEST(Gfx1250SdmaTest, CopyLinearNpdBitDoesNotDecodeAsBroadcast) {
  Gfx1250Sim sim;
  TranslatedSdmaQueueForTest queue(sim);
  constexpr uint32_t kCopyBytes = 128;
  for (uint32_t i = 0; i < kCopyBytes; ++i) {
    queue.src()[i] = static_cast<uint8_t>(i ^ 0x4d);
    queue.dst()[i] = 0;
  }

  auto *packet = queue.ring();
  packet[0] = kSdmaOpCopy | (kSdmaSubopCopyLinear << 8) | (1u << 28);
  packet[1] = kCopyBytes - 1;
  write_sdma_qword_va(packet, 3, 4, queue.src_va());
  write_sdma_qword_va(packet, 5, 6, queue.dst_va());

  queue.submit(7);
  ASSERT_TRUE(sim.engine->step());
  EXPECT_EQ(queue.read_idx(), 7u * sizeof(uint32_t));
  EXPECT_EQ(std::memcmp(queue.dst(), queue.src(), kCopyBytes), 0);
}

TEST(Gfx1250SdmaTest, PollMem64ResolvesTranslatedAddress) {
  Gfx1250Sim sim;
  TranslatedSdmaQueueForTest queue(sim);
  queue.poll_value() = 1;

  auto *packet = queue.ring();
  packet[0] = kSdmaOpPollRegmem | (kSdmaSubopPollMem64 << 8) | (3u << 28);
  write_sdma_qword_va(packet, 1, 2, queue.poll_va());
  packet[3] = 0;
  packet[4] = 0;
  packet[5] = 0xFFFFFFFFu;
  packet[6] = 0xFFFFFFFFu;
  packet[7] = 0;

  queue.submit(8);
  ASSERT_TRUE(sim.engine->step());
  EXPECT_EQ(queue.read_idx(), 0u);

  std::atomic_ref<uint64_t>(queue.poll_value()).store(0, std::memory_order_release);
  sim.engine->schedule_event_now(sim.cp()->doorbell_event());
  ASSERT_TRUE(sim.engine->step());
  EXPECT_EQ(queue.read_idx(), 8u * sizeof(uint32_t));
}

TEST(Gfx1250SdmaTest, PollMem64UnresolvedAddressDoesNotAdvance) {
  Gfx1250Sim sim;
  TranslatedSdmaQueueForTest queue(sim);
  constexpr uint64_t kUnmappedPollVa = 0x2000'0000'1000ULL;

  auto *packet = queue.ring();
  packet[0] = kSdmaOpPollRegmem | (kSdmaSubopPollMem64 << 8) | (3u << 28);
  write_sdma_qword_va(packet, 1, 2, kUnmappedPollVa);
  packet[3] = 0;
  packet[4] = 0;
  packet[5] = 0xFFFFFFFFu;
  packet[6] = 0xFFFFFFFFu;
  packet[7] = 0;

  queue.submit(8);
  ASSERT_TRUE(sim.engine->step());
  EXPECT_EQ(queue.read_idx(), 0u);
}

TEST(Gfx1250ExecutionTest, TargetProvidesImmutableExecutionBackend) {
  const IsaTargetDescriptor *target = default_isa_target_registry().find("gfx1250");
  ASSERT_NE(target, nullptr);
  EXPECT_TRUE(target->supports_execution);
  EXPECT_TRUE(gfx1250::Operand::full_execution_backend_complete());
}

TEST(Gfx1250ExecutionTest, DivScaleWritesExplicitSdstMask) {
  Gfx1250Sim sim;
  auto *cu = sim.cu();
  auto *wf = cu->dispatch_wf(0, 0, kGfx1250ScalarSlots, 32);
  ASSERT_NE(wf, nullptr);
  wf->set_exec(1u);

  constexpr uint32_t kLane = 0;
  constexpr uint32_t kOne = 0x3f800000u;
  constexpr uint32_t kTwoTo8 = 0x43800000u;
  constexpr uint32_t kTwoTo100 = 0x71800000u;
  const uint32_t vgpr_base = wf->vgpr_alloc().base;
  auto write_vgpr = [&](uint32_t reg, uint32_t value) {
    cu->write_vgpr(vgpr_base + reg, kLane, value);
  };
  auto read_vgpr = [&](uint32_t reg) { return cu->read_vgpr(vgpr_base + reg, kLane); };
  auto write_sgpr = [&](uint32_t reg, uint32_t value) {
    cu->write_sgpr(wf->sgpr_alloc().base + reg, value);
  };

  write_vgpr(1, kOne);
  write_vgpr(2, kTwoTo100);
  wf->set_vcc(0x5a5a5a5au);
  const std::array<uint32_t, 2> null_sdst_words = {
      0xd6fc7c00u, 0x040a0301u}; // v_div_scale_f32 v0, null, v1, v1, v2
  gfx1250::VDivScaleF32Vop3SdstEnc null_sdst(null_sdst_words.data());
  null_sdst.execute_impl(*wf);
  EXPECT_EQ(wf->vcc(), 0x5a5a5a5au);
  EXPECT_EQ(read_vgpr(0), 0x5f800000u); // 2^64

  write_sgpr(7, kTwoTo8);
  write_vgpr(3, kOne);
  wf->set_vcc(0xa5a5a5a5u);
  const std::array<uint32_t, 2> normal_null_sdst_words = {
      0xd6fc7c09u, 0x040c0e07u}; // v_div_scale_f32 v9, null, s7, s7, v3
  gfx1250::VDivScaleF32Vop3SdstEnc normal_null_sdst(normal_null_sdst_words.data());
  normal_null_sdst.execute_impl(*wf);
  EXPECT_EQ(wf->vcc(), 0xa5a5a5a5u);
  EXPECT_EQ(read_vgpr(9), kTwoTo8);

  write_vgpr(4, kOne);
  write_vgpr(5, kTwoTo100);
  wf->set_vcc(0);
  const std::array<uint32_t, 2> vcc_sdst_words = {
      0xd6fc6a03u, 0x04160904u}; // v_div_scale_f32 v3, vcc_lo, v4, v4, v5
  gfx1250::VDivScaleF32Vop3SdstEnc vcc_sdst(vcc_sdst_words.data());
  vcc_sdst.execute_impl(*wf);
  EXPECT_EQ(wf->vcc(), 1u);
  EXPECT_EQ(read_vgpr(3), 0x5f800000u);

  write_vgpr(7, kOne);
  write_vgpr(8, kTwoTo100);
  write_sgpr(3, 0xfefefefeu);
  wf->set_vcc(0x12345678u);
  const std::array<uint32_t, 2> sgpr_sdst_words = {
      0xd6fc0206u, 0x04220f07u}; // v_div_scale_f32 v6, s2, v7, v7, v8
  gfx1250::VDivScaleF32Vop3SdstEnc sgpr_sdst(sgpr_sdst_words.data());
  sgpr_sdst.execute_impl(*wf);
  EXPECT_EQ(wf->vcc(), 0x12345678u);
  EXPECT_EQ(read_wave_sgpr(*cu, *wf, 2), 0x12345679u);
  EXPECT_EQ(read_wave_sgpr(*cu, *wf, 3), 0xfefefefeu);
  EXPECT_EQ(read_vgpr(6), 0x5f800000u);
}

TEST(Gfx1250ExecutionTest, VMovB16HighVdstMergesIntoLowPhysicalVgpr) {
  Gfx1250Sim sim;
  auto *cu = sim.cu();
  auto *wf = cu->dispatch_wf(0, 0, kGfx1250ScalarSlots, kGfx1250Wave32VgprAllocation);
  ASSERT_NE(wf, nullptr);
  wf->set_exec(1u);

  constexpr uint32_t kLane = 0;
  const uint32_t vgpr_base = wf->vgpr_alloc().base;
  cu->write_vgpr(vgpr_base + 1, kLane, 0xAAAA5555u);
  cu->write_vgpr(vgpr_base + 129, kLane, 0xDEADBEEFu);

  const std::array<uint32_t, 1> words = {0x7F023880u}; // v_mov_b16_e32 v1.h, 0
  gfx1250::VMovB16Vop1 high_half_mov(words.data());
  high_half_mov.execute_impl(*wf);

  EXPECT_EQ(cu->read_vgpr(vgpr_base + 1, kLane), 0x00005555u);
  EXPECT_EQ(cu->read_vgpr(vgpr_base + 129, kLane), 0xDEADBEEFu);
}

TEST(Gfx1250ExecutionTest, VNotB16HighVdstMergesIntoLowPhysicalVgpr) {
  Gfx1250Sim sim;
  auto *cu = sim.cu();
  auto *wf = cu->dispatch_wf(0, 0, kGfx1250ScalarSlots, kGfx1250Wave32VgprAllocation);
  ASSERT_NE(wf, nullptr);
  wf->set_exec(1u);

  constexpr uint32_t kLane = 0;
  const uint32_t vgpr_base = wf->vgpr_alloc().base;
  cu->write_vgpr(vgpr_base + 0, kLane, 0x000000FFu);
  cu->write_vgpr(vgpr_base + 1, kLane, 0xAAAA5555u);
  cu->write_vgpr(vgpr_base + 129, kLane, 0xDEADBEEFu);

  const std::array<uint32_t, 1> words = {0x7F02D300u}; // v_not_b16_e32 v1.h, v0.l
  gfx1250::VNotB16Vop1 high_half_not(words.data());
  high_half_not.execute_impl(*wf);

  EXPECT_EQ(cu->read_vgpr(vgpr_base + 1, kLane), 0xFF005555u);
  EXPECT_EQ(cu->read_vgpr(vgpr_base + 129, kLane), 0xDEADBEEFu);
}

TEST(Gfx1250ExecutionTest, VAddF16HighVdstMergesIntoLowPhysicalVgpr) {
  Gfx1250Sim sim;
  auto *cu = sim.cu();
  auto *wf = cu->dispatch_wf(0, 0, kGfx1250ScalarSlots, kGfx1250Wave32VgprAllocation);
  ASSERT_NE(wf, nullptr);
  wf->set_exec(1u);

  constexpr uint32_t kLane = 0;
  const uint32_t vgpr_base = wf->vgpr_alloc().base;
  cu->write_vgpr(vgpr_base + 0, kLane, 0x00003C00u);
  cu->write_vgpr(vgpr_base + 1, kLane, 0xAAAA5555u);
  cu->write_vgpr(vgpr_base + 2, kLane, 0x00003C00u);
  cu->write_vgpr(vgpr_base + 129, kLane, 0xDEADBEEFu);

  const std::array<uint32_t, 1> words = {0x65020500u}; // v_add_f16_e32 v1.h, v0.l, v2.l
  gfx1250::VAddF16Vop2 high_half_add(words.data());
  high_half_add.execute_impl(*wf);

  EXPECT_EQ(cu->read_vgpr(vgpr_base + 1, kLane), 0x40005555u);
  EXPECT_EQ(cu->read_vgpr(vgpr_base + 129, kLane), 0xDEADBEEFu);
}

TEST(Gfx1250ExecutionTest, IreeF16ReductionTailKeepsLane31Sum) {
  Gfx1250Sim sim;
  auto *cu = sim.cu();
  auto *wf = cu->dispatch_wf(0, 0, kGfx1250ScalarSlots, kGfx1250Wave32VgprAllocation);
  ASSERT_NE(wf, nullptr);
  ASSERT_EQ(wf->wf_size(), 32u);
  wf->set_exec(0xffffffffu);

  auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
  ASSERT_NE(decoder, nullptr);

  const uint32_t vgpr_base = wf->vgpr_alloc().base;
  const uint32_t packed_1_2 = 0x40003c00u;
  const uint32_t packed_3_4 = 0x44004200u;
  const uint32_t packed_5_6 = 0x46004500u;
  const uint32_t packed_7_8 = 0x48004700u;
  for (uint32_t lane = 0; lane < wf->wf_size(); ++lane) {
    cu->write_vgpr(vgpr_base + 10, lane, packed_7_8);
    cu->write_vgpr(vgpr_base + 11, lane, packed_1_2);
    cu->write_vgpr(vgpr_base + 16, lane, packed_5_6);
    cu->write_vgpr(vgpr_base + 17, lane, packed_3_4);
  }

  const std::array<std::array<uint32_t, 3>, 20> words = {{
      {0x64021680u, 0, 0},                     // v_add_f16_e32 v1, 0, v11
      {0x32041690u, 0, 0},                     // v_lshrrev_b32_e32 v2, 16, v11
      {0x64020501u, 0, 0},                     // v_add_f16_e32 v1, v1, v2
      {0x32042290u, 0, 0},                     // v_lshrrev_b32_e32 v2, 16, v17
      {0x64022301u, 0, 0},                     // v_add_f16_e32 v1, v1, v17
      {0x64020501u, 0, 0},                     // v_add_f16_e32 v1, v1, v2
      {0x32042090u, 0, 0},                     // v_lshrrev_b32_e32 v2, 16, v16
      {0x64022101u, 0, 0},                     // v_add_f16_e32 v1, v1, v16
      {0x64020501u, 0, 0},                     // v_add_f16_e32 v1, v1, v2
      {0x32041490u, 0, 0},                     // v_lshrrev_b32_e32 v2, 16, v10
      {0x64021501u, 0, 0},                     // v_add_f16_e32 v1, v1, v10
      {0x64020501u, 0, 0},                     // v_add_f16_e32 v1, v1, v2
      {0xd5320001u, 0x000202fau, 0xff08b101u}, // quad_perm:[1,0,3,2]
      {0xd5320001u, 0x000202fau, 0xff084e01u}, // quad_perm:[2,3,0,1]
      {0xd5320001u, 0x000202fau, 0xff094101u}, // row_half_mirror
      {0xd5320001u, 0x000202fau, 0xff094001u}, // row_mirror
      {0xd65c0802u, 0x03058301u, 0},           // v_permlanex16_b32 v2, v1, -1, -1
      {0x64020302u, 0, 0},                     // v_add_f16_e32 v1, v2, v1
      {0xd7600000u, 0x02013f01u, 0},           // v_readlane_b32 s0, v1, 31
      {0xa4808000u, 0, 0},                     // s_add_f16 s0, s0, 0
  }};

  for (const auto &inst_words : words) {
    std::unique_ptr<Instruction> inst(decoder->decode(inst_words.data()));
    ASSERT_NE(inst, nullptr);
    cu->execute_instruction(inst.get(), *wf);
  }

  EXPECT_EQ(read_wave_sgpr(*cu, *wf, 0) & 0xffffu, 0x6480u);
}

TEST(Gfx1250ExecutionTest, VFmacF16Vop3HighVdstUsesHighHalfAddend) {
  Gfx1250Sim sim;
  auto *cu = sim.cu();
  auto *wf = cu->dispatch_wf(0, 0, kGfx1250ScalarSlots, kGfx1250Wave32VgprAllocation);
  ASSERT_NE(wf, nullptr);
  wf->set_exec(1u);

  constexpr uint32_t kLane = 0;
  const uint32_t vgpr_base = wf->vgpr_alloc().base;
  cu->write_vgpr(vgpr_base + 0, kLane, 0x00003C00u);
  cu->write_vgpr(vgpr_base + 1, kLane, 0x40003C00u);
  cu->write_vgpr(vgpr_base + 2, kLane, 0x00003C00u);

  const std::array<uint32_t, 2> words = {
      0xD5364001u, // v_fmac_f16 v1.h, v0.l, v2.l
      0x02020500u,
  };
  gfx1250::VFmacF16Vop3 high_half_fmac(words.data());
  high_half_fmac.execute_impl(*wf);

  EXPECT_EQ(cu->read_vgpr(vgpr_base + 1, kLane), 0x42003C00u);
}

TEST(Gfx1250ExecutionTest, VFmacF16Vop2HighVdstUsesHighHalfAddend) {
  Gfx1250Sim sim;
  auto *cu = sim.cu();
  auto *wf = cu->dispatch_wf(0, 0, kGfx1250ScalarSlots, kGfx1250Wave32VgprAllocation);
  ASSERT_NE(wf, nullptr);
  wf->set_exec(1u);

  constexpr uint32_t kLane = 0;
  const uint32_t vgpr_base = wf->vgpr_alloc().base;
  cu->write_vgpr(vgpr_base + 0, kLane, 0x00003C00u);
  cu->write_vgpr(vgpr_base + 1, kLane, 0x40003C00u);
  cu->write_vgpr(vgpr_base + 2, kLane, 0x00003C00u);
  cu->write_vgpr(vgpr_base + 129, kLane, 0x3C003C00u);

  const std::array<uint32_t, 1> words = {0x6D020500u}; // v_fmac_f16_e32 v1.h, v0.l, v2.l
  gfx1250::VFmacF16Vop2 high_half_fmac(words.data());
  high_half_fmac.execute_impl(*wf);

  EXPECT_EQ(cu->read_vgpr(vgpr_base + 1, kLane), 0x42003C00u);
  EXPECT_EQ(cu->read_vgpr(vgpr_base + 129, kLane), 0x3C003C00u);
}

TEST(Gfx1250ExecutionTest, VMadU32LiteralTimesScalarAddsVector) {
  Gfx1250Sim sim;
  auto *cu = sim.cu();
  auto *wf = cu->dispatch_wf(0, 0, kGfx1250ScalarSlots, 32);
  ASSERT_NE(wf, nullptr);
  wf->set_exec(1u);

  constexpr uint32_t kLane = 0;
  const uint32_t vgpr_base = wf->vgpr_alloc().base;
  write_wave_sgpr(*cu, *wf, 3, 1);
  cu->write_vgpr(vgpr_base + 4, kLane, 0x24u);

  const std::array<uint32_t, 3> words = {
      0xD6350004u, // v_mad_u32 v4, 0x48, s3, v4
      0x041006FFu,
      0x00000048u,
  };
  gfx1250::VMadU32Vop3 mad(words.data());
  mad.execute_impl(*wf);

  EXPECT_EQ(cu->read_vgpr(vgpr_base + 4, kLane), 0x6Cu);
}

TEST(Gfx1250ExecutionTest, VCmpGtU32Wave32ExplicitSdstPreservesHighSgpr) {
  Gfx1250Sim sim;
  auto *cu = sim.cu();
  auto *wf = cu->dispatch_wf(0, 0, kGfx1250ScalarSlots, 32);
  ASSERT_NE(wf, nullptr);
  wf->set_exec(0x3u);

  const uint32_t vgpr_base = wf->vgpr_alloc().base;
  cu->write_vgpr(vgpr_base + 4, 0, 3u);
  cu->write_vgpr(vgpr_base + 4, 1, 5u);
  write_wave_sgpr(*cu, *wf, 2, 0xaaaaaaaau);
  write_wave_sgpr(*cu, *wf, 3, 0xfefefefeu);
  wf->set_vcc(0x12345678u);

  const std::array<uint32_t, 2> words = {
      0xD44C0002u, // v_cmp_gt_u32_e64 s2, 4, v4
      0x02020884u,
  };
  gfx1250::VCmpGtU32Vop3 cmp(words.data());
  cmp.execute_impl(*wf);

  EXPECT_EQ(read_wave_sgpr(*cu, *wf, 2), 0x1u);
  EXPECT_EQ(read_wave_sgpr(*cu, *wf, 3), 0xfefefefeu);
  EXPECT_EQ(wf->vcc(), 0x12345678u);
}

TEST(Gfx1250ExecutionTest, Wave32ScalarVccHiWritePreservesUpperHalf) {
  Gfx1250Sim sim;
  auto *cu = sim.cu();
  auto *wf = cu->dispatch_wf(0, 0, kGfx1250ScalarSlots, 32);
  ASSERT_NE(wf, nullptr);
  ASSERT_EQ(wf->wf_size(), 32u);
  wf->set_exec(0xffff0000u);
  wf->set_vcc(0);

  auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
  ASSERT_NE(decoder, nullptr);

  const uint32_t words[] = {0x8c6b7e6bu, 0}; // s_or_b32 vcc_hi, vcc_hi, exec_lo
  std::unique_ptr<Instruction> inst(decoder->decode(words));
  ASSERT_NE(inst, nullptr);
  ASSERT_EQ(std::string_view(inst->mnemonic()), "s_or_b32");
  cu->execute_instruction(inst.get(), *wf);

  EXPECT_EQ(wf->vcc(), 0xffff0000'00000000ULL);
}

TEST(Gfx1250ExecutionTest, ClusterLoadsDecodeAndPopulateVectorMemState) {
  Gfx1250Sim sim;
  auto *cu = sim.cu();
  auto *wf = cu->dispatch_wf(0, 0, kGfx1250ScalarSlots, 32);
  ASSERT_NE(wf, nullptr);
  wf->set_exec(0x3u);
  wf->set_lds_base(cu->allocate_lds(256));
  wf->set_m0(0xffff0003u);

  constexpr uint64_t kGlobalBase = 0x180000u;
  write_wave_sgpr(*cu, *wf, 0, static_cast<uint32_t>(kGlobalBase));
  write_wave_sgpr(*cu, *wf, 1, static_cast<uint32_t>(kGlobalBase >> 32));

  const uint32_t vgpr_base = wf->vgpr_alloc().base;
  auto write_vgpr_lane_offsets = [&](uint32_t reg, uint32_t stride) {
    cu->write_vgpr(vgpr_base + reg, 0, 0);
    cu->write_vgpr(vgpr_base + reg, 1, stride);
  };
  auto write_vgpr_lane_indices = [&](uint32_t reg) {
    cu->write_vgpr(vgpr_base + reg, 0, 0);
    cu->write_vgpr(vgpr_base + reg, 1, 1);
  };

  struct LoadCase {
    std::array<uint32_t, 3> words;
    std::string_view mnemonic;
    uint32_t num_elems;
    uint32_t dst_reg;
    bool scale_offset;
  };

  const LoadCase load_cases[] = {
      {{0xEE19C000u, 0x00000001u, 0x00000000u}, "cluster_load_b32", 1, 1, false},
      {{0xEE1A0000u, 0x00000000u, 0x00000002u}, "cluster_load_b64", 2, 0, false},
      {{0xEE1A4000u, 0x00010002u, 0x00000000u}, "cluster_load_b128", 4, 2, true},
  };

  for (const LoadCase &tc : load_cases) {
    if (tc.scale_offset)
      write_vgpr_lane_indices(0);
    else
      write_vgpr_lane_offsets(tc.words[2] & 0xffu, 16);

    auto inst = decode_gfx1250(tc.words, tc.mnemonic);
    ASSERT_NE(inst, nullptr);
    inst->execute(*inst, wf);
    auto *state = inst->data_as<amdgpu::VectorMemState>();
    ASSERT_NE(state, nullptr);
    EXPECT_EQ(state->tag(), amdgpu::GLOBAL_MEM);
    EXPECT_EQ(state->dst_reg_base, vgpr_base + tc.dst_reg);
    EXPECT_EQ(state->elem_size, 4u);
    EXPECT_EQ(state->num_elems, tc.num_elems);
    EXPECT_TRUE(state->is_load);
    EXPECT_EQ(state->wait_counter_type, amdgpu::WaitCounterType::LOADCNT);
    EXPECT_FALSE(state->lds_dst);
    EXPECT_FALSE(state->cluster_multicast);
    EXPECT_TRUE(state->request_force_l1_bypass);
    EXPECT_EQ(state->lane_mask, 0x3u);
    EXPECT_EQ(state->per_lane_addr[0], kGlobalBase);
    EXPECT_EQ(state->per_lane_addr[1], kGlobalBase + 16);
  }

  struct AsyncCase {
    std::array<uint32_t, 3> words;
    std::string_view mnemonic;
    uint32_t elem_size;
    uint32_t num_elems;
    uint32_t lds_addr_reg;
    uint32_t global_stride;
    uint32_t lds_stride;
    uint32_t offset;
    bool scale_offset;
    bool cluster_multicast;
  };

  const AsyncCase async_cases[] = {
      {{0xEE1A8000u, 0x00000000u, 0x00000000u},
       "cluster_load_async_to_lds_b8",
       1,
       1,
       0,
       1,
       1,
       0,
       false,
       true},
      {{0xEE180000u, 0x00000000u, 0x00000800u},
       "global_load_async_to_lds_b32",
       4,
       1,
       0,
       16,
       16,
       8,
       false,
       false},
      {{0xEE1AC000u, 0x00000000u, 0x00000800u},
       "cluster_load_async_to_lds_b32",
       4,
       1,
       0,
       16,
       16,
       8,
       false,
       true},
      {{0xEE1B0000u, 0x00000004u, 0x00000004u},
       "cluster_load_async_to_lds_b64",
       4,
       2,
       4,
       16,
       16,
       0,
       false,
       true},
      {{0xEE1B4000u, 0x00010001u, 0x00000000u},
       "cluster_load_async_to_lds_b128",
       4,
       4,
       1,
       16,
       16,
       0,
       true,
       true},
  };

  for (const AsyncCase &tc : async_cases) {
    if (tc.scale_offset) {
      write_vgpr_lane_indices(0);
      write_vgpr_lane_offsets(tc.lds_addr_reg, 16);
    } else {
      write_vgpr_lane_offsets(tc.lds_addr_reg, tc.global_stride);
    }

    auto inst = decode_gfx1250(tc.words, tc.mnemonic);
    ASSERT_NE(inst, nullptr);
    inst->execute(*inst, wf);
    auto *state = inst->data_as<amdgpu::VectorMemState>();
    ASSERT_NE(state, nullptr);
    EXPECT_EQ(state->tag(), amdgpu::GLOBAL_MEM);
    EXPECT_EQ(state->elem_size, tc.elem_size);
    EXPECT_EQ(state->num_elems, tc.num_elems);
    EXPECT_TRUE(state->is_load);
    EXPECT_TRUE(state->lds_dst);
    EXPECT_TRUE(state->lds_per_lane_addr);
    EXPECT_EQ(state->lds_base, wf->lds_base());
    EXPECT_EQ(state->cluster_multicast, tc.cluster_multicast);
    EXPECT_EQ(state->cluster_mcast_mask, tc.cluster_multicast ? 0x3u : 0u);
    EXPECT_EQ(state->request_force_l1_bypass, tc.cluster_multicast);
    EXPECT_EQ(state->wait_counter_type, amdgpu::WaitCounterType::ASYNCCNT);
    EXPECT_EQ(state->lane_mask, 0x3u);
    EXPECT_EQ(state->per_lane_addr[0], kGlobalBase + tc.offset);
    EXPECT_EQ(state->per_lane_addr[1], kGlobalBase + tc.offset + tc.global_stride);
    EXPECT_EQ(state->per_lane_lds_addr[0], wf->lds_base());
    EXPECT_EQ(state->per_lane_lds_addr[1], wf->lds_base() + tc.lds_stride);
  }
}

TEST(Gfx1250ExecutionTest, GlobalStoreAsyncFromLdsKeepsIoffsetOnGlobalDestination) {
  Gfx1250Sim sim;
  auto *cu = sim.cu();
  auto *wf = cu->dispatch_wf(0, 0, kGfx1250ScalarSlots, 32);
  ASSERT_NE(wf, nullptr);
  wf->set_exec(0x3u);
  wf->set_lds_base(cu->allocate_lds(256));

  constexpr uint64_t kGlobalBase = 0x190000u;
  constexpr uint32_t kLane0Value = 0x12345678u;
  constexpr uint32_t kLane1Value = 0xabcdef01u;
  constexpr uint32_t kIoffset = 8;
  write_wave_sgpr(*cu, *wf, 0, static_cast<uint32_t>(kGlobalBase));
  write_wave_sgpr(*cu, *wf, 1, static_cast<uint32_t>(kGlobalBase >> 32));

  const uint32_t vgpr_base = wf->vgpr_alloc().base;
  cu->write_vgpr(vgpr_base, 0, 0);
  cu->write_vgpr(vgpr_base, 1, 16);
  cu->lds().write32(wf->lds_base(), kLane0Value);
  cu->lds().write32(wf->lds_base() + 16, kLane1Value);

  auto inst =
      decode_gfx1250({0xEE190000u, 0x00000000u, 0x00000800u}, "global_store_async_from_lds_b32");
  ASSERT_NE(inst, nullptr);
  inst->execute(*inst, wf);
  auto *state = inst->data_as<amdgpu::VectorMemState>();
  ASSERT_NE(state, nullptr);
  EXPECT_EQ(state->tag(), amdgpu::GLOBAL_MEM);
  EXPECT_FALSE(state->is_load);
  EXPECT_EQ(state->wait_counter_type, amdgpu::WaitCounterType::ASYNCCNT);
  EXPECT_EQ(state->lane_mask, 0x3u);
  EXPECT_EQ(state->per_lane_addr[0], kGlobalBase + kIoffset);
  EXPECT_EQ(state->per_lane_addr[1], kGlobalBase + 16 + kIoffset);
  ASSERT_GE(state->store_data.size(), 8u);

  uint32_t lane0_value = 0;
  uint32_t lane1_value = 0;
  std::memcpy(&lane0_value, &state->store_data[0], sizeof(lane0_value));
  std::memcpy(&lane1_value, &state->store_data[4], sizeof(lane1_value));
  EXPECT_EQ(lane0_value, kLane0Value);
  EXPECT_EQ(lane1_value, kLane1Value);
}

TEST(Gfx1250ExecutionTest, DispatchEntryClusterMathCoversMultiDimensionalShapes) {
  auto expect_rank_period = [](const amdgpu::DispatchEntry &candidate, uint64_t expected_period) {
    ASSERT_TRUE(candidate.cluster_grid_is_complete());
    EXPECT_EQ(candidate.cluster_rank_period(), expected_period);
    const uint32_t workgroup_count =
        candidate.grid_wgs_x * candidate.grid_wgs_y * candidate.grid_wgs_z;
    for (uint32_t flat_wg_id = 0; flat_wg_id < workgroup_count; ++flat_wg_id) {
      EXPECT_EQ(candidate.cluster_rank_for_flat_wg_id(flat_wg_id),
                candidate.cluster_rank_for_flat_wg_id(
                    flat_wg_id + static_cast<uint32_t>(candidate.cluster_rank_period())));
    }
  };

  amdgpu::DispatchEntry entry{};
  entry.grid_wgs_x = 4;
  entry.grid_wgs_y = 4;
  entry.grid_wgs_z = 4;
  entry.cluster_count_x = 2;
  entry.cluster_count_y = 2;
  entry.cluster_count_z = 2;
  entry.cluster_size_x = 2;
  entry.cluster_size_y = 2;
  entry.cluster_size_z = 2;

  EXPECT_TRUE(entry.cluster_grid_is_complete());
  EXPECT_EQ(entry.cluster_size(), 8u);
  expect_rank_period(entry, 32u);
  EXPECT_EQ(entry.cluster_rank_for_flat_wg_id(0), 0u);
  EXPECT_EQ(entry.cluster_rank_for_flat_wg_id(5), 3u);
  EXPECT_EQ(entry.cluster_rank_for_flat_wg_id(21), 7u);
  EXPECT_EQ(entry.cluster_peer_local_wg_id(0, 0), 0u);
  EXPECT_EQ(entry.cluster_peer_local_wg_id(0, 1), 1u);
  EXPECT_EQ(entry.cluster_peer_local_wg_id(0, 2), 4u);
  EXPECT_EQ(entry.cluster_peer_local_wg_id(0, 3), 5u);
  EXPECT_EQ(entry.cluster_peer_local_wg_id(0, 4), 16u);
  EXPECT_EQ(entry.cluster_peer_local_wg_id(0, 7), 21u);
  EXPECT_EQ(entry.cluster_peer_local_wg_id(42, 0), 42u);
  EXPECT_EQ(entry.cluster_peer_local_wg_id(42, 7), 63u);
  EXPECT_EQ(entry.cluster_base_local_wg_id_for_ordinal(0), 0u);
  EXPECT_EQ(entry.cluster_base_local_wg_id_for_ordinal(1), 2u);
  EXPECT_EQ(entry.cluster_base_local_wg_id_for_ordinal(2), 8u);
  EXPECT_EQ(entry.cluster_base_local_wg_id_for_ordinal(4), 32u);

  amdgpu::DispatchEntry one_dimensional{};
  one_dimensional.grid_wgs_x = 8;
  one_dimensional.grid_wgs_y = 1;
  one_dimensional.grid_wgs_z = 1;
  one_dimensional.cluster_count_x = 2;
  one_dimensional.cluster_count_y = 1;
  one_dimensional.cluster_count_z = 1;
  one_dimensional.cluster_size_x = 4;
  one_dimensional.cluster_size_y = 1;
  one_dimensional.cluster_size_z = 1;
  expect_rank_period(one_dimensional, 4u);

  amdgpu::DispatchEntry two_dimensional{};
  two_dimensional.grid_wgs_x = 8;
  two_dimensional.grid_wgs_y = 6;
  two_dimensional.grid_wgs_z = 1;
  two_dimensional.cluster_count_x = 2;
  two_dimensional.cluster_count_y = 2;
  two_dimensional.cluster_count_z = 1;
  two_dimensional.cluster_size_x = 4;
  two_dimensional.cluster_size_y = 3;
  two_dimensional.cluster_size_z = 1;
  expect_rank_period(two_dimensional, 24u);

  entry.grid_wgs_x = 3;
  entry.grid_wgs_y = 2;
  entry.grid_wgs_z = 1;
  entry.cluster_count_x = 2;
  entry.cluster_count_y = 2;
  entry.cluster_count_z = 1;
  entry.cluster_size_x = 2;
  entry.cluster_size_y = 1;
  entry.cluster_size_z = 1;
  EXPECT_FALSE(entry.cluster_grid_is_complete());
}

TEST(Gfx1250ExecutionTest, ClusterLdsMulticastTransactionCapturesRemapState) {
  Gfx1250Sim sim;
  auto *cu = sim.cu();
  auto *wf = cu->dispatch_wf(9, 0, kGfx1250ScalarSlots, 32);
  ASSERT_NE(wf, nullptr);
  wf->set_dispatch_id(7);
  wf->set_cluster_info(/*rank=*/1, /*size=*/4);
  wf->set_lds_base(0x100);

  amdgpu::VectorMemState state(amdgpu::GLOBAL_MEM);
  state.elem_size = 4;
  state.num_elems = 2;
  state.wait_counter_type = amdgpu::WaitCounterType::ASYNCCNT;
  state.lds_base = wf->lds_base();
  state.lds_per_lane_addr = true;
  state.cluster_multicast = true;
  state.cluster_mcast_mask = 0xa;
  state.wf_size = 32;
  state.lane_mask = 0x3;
  state.per_lane_addr[0] = 0x8000;
  state.per_lane_addr[1] = 0x8020;
  state.per_lane_lds_addr[0] = state.lds_base + 0x10;
  state.per_lane_lds_addr[1] = state.lds_base + 0x24;
  state.response_data.resize(state.wf_size * state.num_elems * state.elem_size);

  std::vector<amdgpu::ClusterLdsTarget> targets = {{cu, /*wg_id=*/11, /*lds_base=*/0x400,
                                                    /*cluster_rank=*/3}};
  auto txn = amdgpu::make_cluster_lds_multicast_transaction(state, *wf, std::move(targets));

  EXPECT_EQ(txn.dispatch_id, 7u);
  EXPECT_EQ(txn.source_wg_id, 9u);
  EXPECT_EQ(txn.source_cluster_rank, 1u);
  EXPECT_EQ(txn.source_lds_base, 0x100u);
  EXPECT_EQ(txn.mcast_mask, 0xau);
  EXPECT_EQ(txn.wait_counter_type, amdgpu::WaitCounterType::ASYNCCNT);
  EXPECT_EQ(txn.bytes_per_lane, 8u);
  // Retained for deferred/timing backends that model global request coalescing.
  EXPECT_EQ(txn.per_lane_global_addr[0], 0x8000u);
  EXPECT_EQ(txn.per_lane_global_addr[1], 0x8020u);
  ASSERT_EQ(txn.targets.size(), 1u);
  EXPECT_EQ(txn.targets[0].wg_id, 11u);
  EXPECT_EQ(amdgpu::cluster_lds_lane_addr(txn, 0, txn.targets[0].lds_base), 0x410u);
  EXPECT_EQ(amdgpu::cluster_lds_lane_addr(txn, 1, txn.targets[0].lds_base), 0x424u);
  EXPECT_THROW((void)amdgpu::remap_cluster_lds_addr(0x100, 0x400, 0xfc), std::runtime_error);
}

TEST(Gfx1250ExecutionTest, ClusterLdsSourceRankSelectionCoversDefaultAndMasks) {
  amdgpu::ClusterLdsMulticastTransaction txn{};
  txn.source_cluster_rank = 2;

  txn.mcast_mask = 0;
  EXPECT_TRUE(amdgpu::cluster_lds_source_rank_selected(txn));

  txn.mcast_mask = amdgpu::cluster_multicast_rank_mask(2);
  EXPECT_TRUE(amdgpu::cluster_lds_source_rank_selected(txn));

  txn.mcast_mask = amdgpu::cluster_multicast_rank_mask(1);
  EXPECT_FALSE(amdgpu::cluster_lds_source_rank_selected(txn));
}

TEST(Gfx1250ExecutionTest, ImmediateClusterLdsMulticastEngineWritesOnlyIssuingParticipant) {
  Gfx1250Sim sim;
  auto *cu = sim.cu();
  cu->clear_lds();

  amdgpu::ClusterLdsMulticastTransaction txn{};
  txn.source_wg_id = 1;
  txn.source_cluster_rank = 1;
  txn.source_lds_base = 0x300;
  txn.mcast_mask = 0x3;
  txn.bytes_per_lane = 4;
  txn.wf_size = 4;
  txn.lane_mask = 0x5;
  txn.per_lane_addr = true;
  txn.per_lane_lds_addr[0] = 0x304;
  txn.per_lane_lds_addr[2] = 0x30c;
  txn.payload.resize(txn.wf_size * txn.bytes_per_lane);
  const uint32_t lane0 = 0x11223344;
  const uint32_t lane2 = 0xaabbccdd;
  std::memcpy(&txn.payload[0 * txn.bytes_per_lane], &lane0, sizeof(lane0));
  std::memcpy(&txn.payload[2 * txn.bytes_per_lane], &lane2, sizeof(lane2));
  txn.targets = {{cu, /*wg_id=*/0, /*lds_base=*/0x200, /*cluster_rank=*/0},
                 {cu, /*wg_id=*/1, /*lds_base=*/0x300, /*cluster_rank=*/1}};

  amdgpu::ImmediateClusterLdsMulticastEngine engine;
  bool deferred_callback_called = false;
  EXPECT_EQ(engine.submit(std::move(txn), [&]() { deferred_callback_called = true; }),
            amdgpu::ClusterLdsMulticastResult::Complete);
  EXPECT_FALSE(deferred_callback_called);
  EXPECT_EQ(cu->lds().read32(0x204), 0u);
  EXPECT_EQ(cu->lds().read32(0x20c), 0u);
  EXPECT_EQ(cu->lds().read32(0x304), lane0);
  EXPECT_EQ(cu->lds().read32(0x30c), lane2);
  EXPECT_EQ(cu->lds().read32(0x208), 0u);
}

TEST(Gfx1250ExecutionTest, ImmediateClusterLdsMulticastEngineSkipsUnissuedSelectedPeer) {
  Gfx1250Sim sim;
  auto *cu = sim.cu();
  cu->clear_lds();

  constexpr uint32_t kValue = 0x55667788;
  amdgpu::ClusterLdsMulticastTransaction txn{};
  txn.source_wg_id = 1;
  txn.source_cluster_rank = 1;
  txn.source_lds_base = 0x300;
  txn.mcast_mask = 0x1; // Selects rank 0, not the issuing rank 1.
  txn.bytes_per_lane = sizeof(kValue);
  txn.wf_size = 1;
  txn.lane_mask = 0x1;
  txn.per_lane_addr = true;
  txn.per_lane_lds_addr[0] = 0x310;
  txn.payload.resize(sizeof(kValue));
  std::memcpy(txn.payload.data(), &kValue, sizeof(kValue));
  txn.targets = {{cu, /*wg_id=*/0, /*lds_base=*/0x200, /*cluster_rank=*/0}};

  amdgpu::ImmediateClusterLdsMulticastEngine engine;
  EXPECT_EQ(engine.submit(std::move(txn), []() {}), amdgpu::ClusterLdsMulticastResult::Complete);
  EXPECT_EQ(cu->lds().read32(0x210), 0u);
  EXPECT_EQ(cu->lds().read32(0x310), 0u);
}

TEST(Gfx1250ExecutionTest, ImmediateClusterLdsMulticastEngineUsesRecipientOwnedDestinations) {
  Gfx1250Sim sim;
  auto *cu = sim.cu();
  cu->clear_lds();

  constexpr uint32_t kWg0Value = 0x11112222;
  constexpr uint32_t kWg1Value = 0x33334444;

  auto make_txn = [&](uint32_t wg_id, uint32_t rank, uint32_t lds_base, uint32_t lds_offset,
                      uint32_t value) {
    amdgpu::ClusterLdsMulticastTransaction txn{};
    txn.source_wg_id = wg_id;
    txn.source_cluster_rank = rank;
    txn.source_lds_base = lds_base;
    txn.mcast_mask = 0x3;
    txn.bytes_per_lane = sizeof(value);
    txn.wf_size = 1;
    txn.lane_mask = 0x1;
    txn.per_lane_addr = true;
    txn.per_lane_lds_addr[0] = lds_base + lds_offset;
    txn.payload.resize(sizeof(value));
    std::memcpy(txn.payload.data(), &value, sizeof(value));
    txn.targets = {{cu, /*wg_id=*/0, /*lds_base=*/0x100, /*cluster_rank=*/0},
                   {cu, /*wg_id=*/1, /*lds_base=*/0x200, /*cluster_rank=*/1}};
    return txn;
  };

  amdgpu::ImmediateClusterLdsMulticastEngine engine;
  EXPECT_EQ(engine.submit(make_txn(/*wg_id=*/0, /*rank=*/0, /*lds_base=*/0x100,
                                   /*lds_offset=*/0x10, kWg0Value),
                          []() {}),
            amdgpu::ClusterLdsMulticastResult::Complete);
  EXPECT_EQ(engine.submit(make_txn(/*wg_id=*/1, /*rank=*/1, /*lds_base=*/0x200,
                                   /*lds_offset=*/0x30, kWg1Value),
                          []() {}),
            amdgpu::ClusterLdsMulticastResult::Complete);

  EXPECT_EQ(cu->lds().read32(0x110), kWg0Value);
  EXPECT_EQ(cu->lds().read32(0x230), kWg1Value);
  EXPECT_EQ(cu->lds().read32(0x210), 0u);
  EXPECT_EQ(cu->lds().read32(0x130), 0u);
}

TEST(Gfx1250ExecutionTest, ImmediateClusterLdsMulticastEngineRejectsUndersizedPayload) {
  Gfx1250Sim sim;
  auto *cu = sim.cu();

  amdgpu::ClusterLdsMulticastTransaction txn{};
  txn.bytes_per_lane = 4;
  txn.wf_size = 2;
  txn.lane_mask = 0x3;
  txn.payload.resize(4);
  txn.targets = {{cu, /*wg_id=*/0, /*lds_base=*/0x200, /*cluster_rank=*/0}};

  amdgpu::ImmediateClusterLdsMulticastEngine engine;
  EXPECT_THROW((void)engine.submit(std::move(txn), []() {}), std::runtime_error);
}

TEST(Gfx1250ExecutionTest, ImmediateClusterLdsMulticastEngineRejectsOutOfRangeTarget) {
  Gfx1250Sim sim;
  auto *cu = sim.cu();

  amdgpu::ClusterLdsMulticastTransaction txn{};
  txn.source_lds_base = 0;
  txn.bytes_per_lane = 4;
  txn.wf_size = 1;
  txn.lane_mask = 0x1;
  txn.per_lane_addr = true;
  txn.per_lane_lds_addr[0] = 0;
  txn.payload.resize(4);
  txn.targets = {{cu, /*wg_id=*/0, static_cast<uint32_t>(cu->lds().size_bytes()) - 2,
                  /*cluster_rank=*/0}};

  amdgpu::ImmediateClusterLdsMulticastEngine engine;
  EXPECT_THROW((void)engine.submit(std::move(txn), []() {}), std::runtime_error);
}

TEST(Gfx1250ExecutionTest, ClusterLdsPinPreventsAllocatorReuseUntilClusterCompletes) {
  Gfx1250Sim sim;
  auto *cu = sim.cu();

  EXPECT_EQ(cu->allocate_lds(257), 0u);
  cu->pin_lds_until_cluster_retired(7);
  // A cluster pin blocks LDS reclamation even when the CU has no resident waves.
  cu->maybe_reset_lds_alloc();

  EXPECT_EQ(cu->allocate_lds(257), 512u);
  cu->unpin_lds_for_cluster(7);
  // Once the pin is released and the CU is idle, the LDS allocator resets.
  cu->maybe_reset_lds_alloc();
  EXPECT_EQ(cu->allocate_lds(257), 0u);
}

TEST(Gfx1250ExecutionTest, OrdinaryLdsDstLoadWritesDirectlyAndCompletesAsyncCounter) {
  Gfx1250Sim sim;
  auto *cu = sim.cu();
  auto *wf = cu->dispatch_wf(/*wg_id=*/0, /*pc=*/0, kGfx1250ScalarSlots, 32);
  ASSERT_NE(wf, nullptr);
  wf->set_dispatch_id(3);
  wf->set_lds_base(cu->allocate_lds(256));

  constexpr uint64_t kGlobalAddr = 0x9000;
  constexpr uint32_t kLoadedValue = 0x12345678;
  for (uint32_t byte = 0; byte < sizeof(kLoadedValue); ++byte) {
    sim.memory->write8(kGlobalAddr + byte,
                       static_cast<uint8_t>((kLoadedValue >> (byte * 8)) & 0xffu));
  }

  auto state = std::make_unique<amdgpu::VectorMemState>(amdgpu::GLOBAL_MEM);
  state->elem_size = 4;
  state->num_elems = 1;
  state->is_load = true;
  state->wait_counter_type = amdgpu::WaitCounterType::ASYNCCNT;
  state->lds_dst = true;
  state->lds_per_lane_addr = true;
  state->lds_base = wf->lds_base();
  state->wf_size = 32;
  state->lane_mask = 0x1;
  state->exec_mask = 0x1;
  state->per_lane_addr[0] = kGlobalAddr;
  state->per_lane_lds_addr[0] = wf->lds_base() + 0x20;

  DeferredClusterLdsMulticastEngine deferred_engine;
  cu->set_cluster_lds_multicast_engine(&deferred_engine);
  amdgpu::GlobalMemPipeline pipeline(&cu->l1_vector(), cu->l2());
  pipeline.issue(new TestMemoryInstruction(std::move(state)), *wf);

  EXPECT_EQ(wf->wait_counters().asynccnt, 0u);
  EXPECT_FALSE(static_cast<bool>(deferred_engine.completion));
  EXPECT_EQ(cu->lds().read32(wf->lds_base() + 0x20), kLoadedValue);
  cu->set_cluster_lds_multicast_engine(nullptr);
}

TEST(Gfx1250ExecutionTest, NonClusterClusterLdsLoadDowngradesToOrdinaryAsyncToLds) {
  Gfx1250Sim sim;
  auto *cu = sim.cu();
  auto *wf = cu->dispatch_wf(/*wg_id=*/0, /*pc=*/0, kGfx1250ScalarSlots, 32);
  ASSERT_NE(wf, nullptr);
  wf->set_dispatch_id(5);
  wf->set_lds_base(cu->allocate_lds(256));

  constexpr uint64_t kGlobalAddr = 0x9080;
  constexpr uint32_t kLoadedValue = 0x78563412;
  for (uint32_t byte = 0; byte < sizeof(kLoadedValue); ++byte) {
    sim.memory->write8(kGlobalAddr + byte,
                       static_cast<uint8_t>((kLoadedValue >> (byte * 8)) & 0xffu));
  }

  auto state = std::make_unique<amdgpu::VectorMemState>(amdgpu::GLOBAL_MEM);
  state->elem_size = 4;
  state->num_elems = 1;
  state->is_load = true;
  state->wait_counter_type = amdgpu::WaitCounterType::ASYNCCNT;
  state->lds_dst = true;
  state->lds_per_lane_addr = true;
  state->lds_base = wf->lds_base();
  state->wf_size = 32;
  state->lane_mask = 0x1;
  state->exec_mask = 0x1;
  state->cluster_multicast = true;
  state->cluster_mcast_mask = 0x2; // Excludes rank 0, but non-clustered loads downgrade.
  state->per_lane_addr[0] = kGlobalAddr;
  state->per_lane_lds_addr[0] = wf->lds_base() + 0x24;

  DeferredClusterLdsMulticastEngine deferred_engine;
  cu->set_cluster_lds_multicast_engine(&deferred_engine);
  amdgpu::GlobalMemPipeline pipeline(&cu->l1_vector(), cu->l2());
  pipeline.issue(new TestMemoryInstruction(std::move(state)), *wf);

  EXPECT_EQ(wf->wait_counters().asynccnt, 0u);
  EXPECT_FALSE(static_cast<bool>(deferred_engine.completion));
  EXPECT_EQ(cu->lds().read32(wf->lds_base() + 0x24), kLoadedValue);
  cu->set_cluster_lds_multicast_engine(nullptr);
}

TEST(Gfx1250ExecutionTest, ClusterLoadRequestBypassesStaleL1VectorLine) {
  Gfx1250Sim sim;
  auto *cu = sim.cu();
  auto *wf = cu->dispatch_wf(/*wg_id=*/0, /*pc=*/0, kGfx1250ScalarSlots, 32);
  ASSERT_NE(wf, nullptr);
  wf->set_dispatch_id(6);
  wf->set_lds_base(cu->allocate_lds(256));

  constexpr uint64_t kGlobalAddr = 0xa000;
  constexpr uint32_t kOldValue = 0x11111111;
  constexpr uint32_t kNewValue = 0x22222222;
  for (uint32_t byte = 0; byte < sizeof(kOldValue); ++byte) {
    sim.memory->write8(kGlobalAddr + byte, static_cast<uint8_t>((kOldValue >> (byte * 8)) & 0xffu));
  }

  uint64_t addrs[64] = {};
  addrs[0] = kGlobalAddr;
  uint8_t l1_fill[64 * sizeof(kOldValue)] = {};
  cu->l1_vector().load(addrs, /*lane_mask=*/0x1, /*elem_size=*/4, /*num_elems=*/1, l1_fill,
                       amdgpu::Mtype::RW, /*non_temporal=*/false,
                       /*request_l1_bypass=*/false, /*vmid=*/0);
  uint32_t filled_value = 0;
  std::memcpy(&filled_value, l1_fill, sizeof(filled_value));
  ASSERT_EQ(filled_value, kOldValue);

  cu->l2()->write(kGlobalAddr, reinterpret_cast<const uint8_t *>(&kNewValue), sizeof(kNewValue),
                  amdgpu::Mtype::RW, /*vmid=*/0);

  amdgpu::GlobalMemPipeline pipeline(&cu->l1_vector(), cu->l2());

  auto ordinary = std::make_unique<amdgpu::VectorMemState>(amdgpu::GLOBAL_MEM);
  ordinary->elem_size = 4;
  ordinary->num_elems = 1;
  ordinary->is_load = true;
  ordinary->wait_counter_type = amdgpu::WaitCounterType::LOADCNT;
  ordinary->wf_size = 32;
  ordinary->lane_mask = 0x1;
  ordinary->exec_mask = 0x1;
  ordinary->dst_reg_base = wf->vgpr_alloc().base;
  ordinary->per_lane_addr[0] = kGlobalAddr;
  pipeline.issue(new TestMemoryInstruction(std::move(ordinary)), *wf);
  EXPECT_EQ(cu->read_vgpr(wf->vgpr_alloc().base, 0), kOldValue);

  auto cluster = std::make_unique<amdgpu::VectorMemState>(amdgpu::GLOBAL_MEM);
  cluster->elem_size = 4;
  cluster->num_elems = 1;
  cluster->is_load = true;
  cluster->wait_counter_type = amdgpu::WaitCounterType::ASYNCCNT;
  cluster->request_force_l1_bypass = true;
  cluster->lds_dst = true;
  cluster->lds_per_lane_addr = true;
  cluster->lds_base = wf->lds_base();
  cluster->wf_size = 32;
  cluster->lane_mask = 0x1;
  cluster->exec_mask = 0x1;
  cluster->cluster_multicast = true;
  cluster->cluster_mcast_mask = 0x1;
  cluster->per_lane_addr[0] = kGlobalAddr;
  cluster->per_lane_lds_addr[0] = wf->lds_base() + 0x40;
  pipeline.issue(new TestMemoryInstruction(std::move(cluster)), *wf);

  EXPECT_EQ(cu->lds().read32(wf->lds_base() + 0x40), kNewValue);
}

TEST(Gfx1250ExecutionTest, ClusterLdsFallbackSkipsSelfWhenMaskExcludesSource) {
  Gfx1250Sim sim;
  auto *cu = sim.cu();
  auto *wf = cu->dispatch_wf(/*wg_id=*/3, /*pc=*/0, kGfx1250ScalarSlots, 32);
  ASSERT_NE(wf, nullptr);
  wf->set_dispatch_id(17);
  wf->set_lds_base(cu->allocate_lds(256));
  wf->set_cluster_info(/*rank=*/1, /*size=*/2);

  constexpr uint64_t kGlobalAddr = 0x9100;
  constexpr uint32_t kLoadedValue = 0xabcdef01;
  for (uint32_t byte = 0; byte < sizeof(kLoadedValue); ++byte) {
    sim.memory->write8(kGlobalAddr + byte,
                       static_cast<uint8_t>((kLoadedValue >> (byte * 8)) & 0xffu));
  }

  auto state = std::make_unique<amdgpu::VectorMemState>(amdgpu::GLOBAL_MEM);
  state->elem_size = 4;
  state->num_elems = 1;
  state->is_load = true;
  state->wait_counter_type = amdgpu::WaitCounterType::ASYNCCNT;
  state->lds_dst = true;
  state->lds_per_lane_addr = true;
  state->lds_base = wf->lds_base();
  state->wf_size = 32;
  state->lane_mask = 0x1;
  state->exec_mask = 0x1;
  state->cluster_multicast = true;
  state->cluster_mcast_mask = 0x1; // Selects rank 0, not the source rank 1.
  state->per_lane_addr[0] = kGlobalAddr;
  state->per_lane_lds_addr[0] = wf->lds_base() + 0x20;

  DeferredClusterLdsMulticastEngine deferred_engine;
  cu->set_cluster_lds_multicast_engine(&deferred_engine);
  amdgpu::GlobalMemPipeline pipeline(&cu->l1_vector(), cu->l2());
  pipeline.issue(new TestMemoryInstruction(std::move(state)), *wf);

  EXPECT_EQ(wf->wait_counters().asynccnt, 1u);
  ASSERT_TRUE(static_cast<bool>(deferred_engine.completion));
  EXPECT_TRUE(deferred_engine.txn.targets.empty());

  deferred_engine.completion();
  EXPECT_EQ(wf->wait_counters().asynccnt, 0u);
  cu->set_cluster_lds_multicast_engine(nullptr);
}

TEST(Gfx1250SimulationTest, ClusterLdsTargetsCoversMasksAndLifetime) {
  constexpr uint64_t kernel_addr = 0x10000;
  constexpr uint32_t lds_bytes_per_wg = 256;
  const uint32_t code[] = {S_ENDPGM_GFX12};

  Gfx1250Sim sim;
  uint64_t kernel_object = sim.write_kernel(kernel_addr, code, std::size(code), 128);
  test::AqlQueue queue(sim.memory, sim.cp());
  queue.dispatch_clustered(kernel_object, /*cluster_count_x=*/1, /*cluster_size_x=*/2,
                           /*workgroup_size_x=*/32, /*kernarg_addr=*/0, lds_bytes_per_wg);

  ASSERT_TRUE(sim.engine->step());

  auto all = sim.cp()->cluster_lds_targets(/*dispatch_id=*/1, /*wg_id=*/0, /*mcast_mask=*/0x3);
  ASSERT_EQ(all.size(), 2u);
  EXPECT_EQ(all[0].wg_id, 0u);
  EXPECT_EQ(all[1].wg_id, 1u);

  auto self = sim.cp()->cluster_lds_targets(/*dispatch_id=*/1, /*wg_id=*/0, /*mcast_mask=*/0x1);
  ASSERT_EQ(self.size(), 1u);
  EXPECT_EQ(self[0].wg_id, 0u);

  auto peer = sim.cp()->cluster_lds_targets(/*dispatch_id=*/1, /*wg_id=*/0, /*mcast_mask=*/0x2);
  ASSERT_EQ(peer.size(), 1u);
  EXPECT_EQ(peer[0].wg_id, 1u);

  auto peer_from_rank1 =
      sim.cp()->cluster_lds_targets(/*dispatch_id=*/1, /*wg_id=*/1, /*mcast_mask=*/0x1);
  ASSERT_EQ(peer_from_rank1.size(), 1u);
  EXPECT_EQ(peer_from_rank1[0].wg_id, 0u);

  auto zero_mask = sim.cp()->cluster_lds_targets(/*dispatch_id=*/1, /*wg_id=*/0, /*mcast_mask=*/0);
  ASSERT_EQ(zero_mask.size(), 1u);
  EXPECT_EQ(zero_mask[0].wg_id, 0u);

  auto out_of_range =
      sim.cp()->cluster_lds_targets(/*dispatch_id=*/1, /*wg_id=*/0, /*mcast_mask=*/0x4);
  EXPECT_TRUE(out_of_range.empty());

  auto mixed_range =
      sim.cp()->cluster_lds_targets(/*dispatch_id=*/1, /*wg_id=*/0, /*mcast_mask=*/0x5);
  ASSERT_EQ(mixed_range.size(), 1u);
  EXPECT_EQ(mixed_range[0].wg_id, 0u);

  EXPECT_TRUE(sim.cp()
                  ->cluster_lds_targets(/*dispatch_id=*/999, /*wg_id=*/0,
                                        /*mcast_mask=*/0x3)
                  .empty());

  sim.engine->run();
  EXPECT_TRUE(sim.cp()
                  ->cluster_lds_targets(/*dispatch_id=*/1, /*wg_id=*/0,
                                        /*mcast_mask=*/0x3)
                  .empty());
}

TEST(Gfx1250SimulationTest, ClusterLdsTargetsUseMultiDimensionalClusterPlacement) {
  constexpr uint64_t kernel_addr = 0x10000;
  constexpr uint32_t lds_bytes_per_wg = 256;
  const uint32_t code[] = {S_ENDPGM_GFX12};

  Gfx1250Sim sim;
  uint64_t kernel_object = sim.write_kernel(kernel_addr, code, std::size(code), 128);

  amdgpu::AmdExtKernelDispatchPacket pkt{};
  pkt.header = HSA_PACKET_TYPE_VENDOR_SPECIFIC;
  pkt.amd_format = amdgpu::kHsaAmdPacketTypeExtKernelDispatch;
  pkt.setup = 2;
  pkt.workgroup_size_x = 32;
  pkt.workgroup_size_y = 1;
  pkt.workgroup_size_z = 1;
  pkt.cluster_count_x = 2;
  pkt.cluster_count_y = 2;
  pkt.cluster_count_z = 1;
  pkt.cluster_size_x = 2;
  pkt.cluster_size_y = 2;
  pkt.cluster_size_z = 1;
  pkt.group_segment_size = lds_bytes_per_wg;
  pkt.kernel_object = kernel_object;

  test::AqlQueue queue(sim.memory, sim.cp());
  queue.submit(pkt);
  ASSERT_TRUE(sim.engine->step());

  auto cluster0 = sim.cp()->cluster_lds_targets(/*dispatch_id=*/1, /*wg_id=*/0, /*mcast_mask=*/0xf);
  ASSERT_EQ(cluster0.size(), 4u);
  EXPECT_EQ(cluster0[0].wg_id, 0u);
  EXPECT_EQ(cluster0[1].wg_id, 1u);
  EXPECT_EQ(cluster0[2].wg_id, 4u);
  EXPECT_EQ(cluster0[3].wg_id, 5u);

  auto targets = sim.cp()->cluster_lds_targets(/*dispatch_id=*/1, /*wg_id=*/5, /*mcast_mask=*/0x5);
  ASSERT_EQ(targets.size(), 2u);
  EXPECT_EQ(targets[0].wg_id, 0u);
  EXPECT_EQ(targets[1].wg_id, 4u);

  auto source = sim.cp()->cluster_lds_targets(/*dispatch_id=*/1, /*wg_id=*/5, /*mcast_mask=*/0x8);
  ASSERT_EQ(source.size(), 1u);
  EXPECT_EQ(source[0].wg_id, 5u);

  constexpr uint32_t kValue = 0xfeed1234;
  amdgpu::ClusterLdsMulticastTransaction txn{};
  txn.source_wg_id = source[0].wg_id;
  txn.source_cluster_rank = source[0].cluster_rank;
  txn.source_lds_base = source[0].lds_base;
  txn.mcast_mask = 0x8;
  txn.bytes_per_lane = sizeof(kValue);
  txn.wf_size = 1;
  txn.lane_mask = 0x1;
  txn.per_lane_addr = true;
  txn.per_lane_lds_addr[0] = source[0].lds_base + 0x10;
  txn.payload.resize(sizeof(kValue));
  std::memcpy(txn.payload.data(), &kValue, sizeof(kValue));
  txn.targets = source;

  amdgpu::ImmediateClusterLdsMulticastEngine engine;
  bool deferred_callback_called = false;
  EXPECT_EQ(engine.submit(std::move(txn), [&]() { deferred_callback_called = true; }),
            amdgpu::ClusterLdsMulticastResult::Complete);
  EXPECT_FALSE(deferred_callback_called);
  EXPECT_EQ(targets[0].cu->lds().read32(targets[0].lds_base + 0x10), 0u);
  EXPECT_EQ(targets[1].cu->lds().read32(targets[1].lds_base + 0x10), 0u);
  EXPECT_EQ(source[0].cu->lds().read32(source[0].lds_base + 0x10), kValue);

  sim.engine->run();
}

TEST(Gfx1250SimulationTest, ClusterLdsDoesNotRemapIntoNonParticipatingPeerLdsBase) {
  constexpr uint64_t kernel_addr = 0x10000;
  constexpr uint32_t lds_bytes_per_wg = 256;
  constexpr uint32_t cluster_size = 8;
  const uint32_t code[] = {S_ENDPGM_GFX12};

  Gfx1250Sim sim(make_single_se_gfx1250_config(/*num_cus=*/4));
  uint64_t kernel_object = sim.write_kernel(kernel_addr, code, std::size(code), 128);
  test::AqlQueue queue(sim.memory, sim.cp());
  queue.dispatch_clustered(kernel_object, /*cluster_count_x=*/1, cluster_size,
                           /*workgroup_size_x=*/32, /*kernarg_addr=*/0, lds_bytes_per_wg);
  ASSERT_TRUE(sim.engine->step());

  struct RemapCase {
    amdgpu::ClusterLdsTarget source;
    amdgpu::ClusterLdsTarget target;
  };
  std::optional<RemapCase> remap_case;
  for (uint32_t wg_id = 0; wg_id < cluster_size && !remap_case; ++wg_id) {
    auto source = sim.cp()->cluster_lds_targets(/*dispatch_id=*/1, wg_id, /*mcast_mask=*/0);
    ASSERT_EQ(source.size(), 1u);
    ASSERT_NE(source[0].cu, nullptr);

    auto targets = sim.cp()->cluster_lds_targets(/*dispatch_id=*/1, wg_id,
                                                 /*mcast_mask=*/(1u << cluster_size) - 1);
    for (const auto &target : targets) {
      if (target.wg_id == wg_id)
        continue;
      if (target.cu == source[0].cu && target.lds_base != source[0].lds_base) {
        remap_case = RemapCase{source[0], target};
        break;
      }
    }
  }
  ASSERT_TRUE(remap_case.has_value());
  const auto source = remap_case->source;
  const auto target = remap_case->target;
  ASSERT_NE(target.cu, nullptr);
  EXPECT_EQ(target.cu, source.cu);
  EXPECT_NE(target.lds_base, source.lds_base);

  constexpr uint32_t kValue = 0x13579bdf;
  amdgpu::ClusterLdsMulticastTransaction txn{};
  txn.source_wg_id = source.wg_id;
  txn.source_cluster_rank = source.cluster_rank;
  txn.source_lds_base = source.lds_base;
  txn.mcast_mask = (1u << cluster_size) - 1;
  txn.bytes_per_lane = sizeof(kValue);
  txn.wf_size = 1;
  txn.lane_mask = 0x1;
  txn.per_lane_addr = true;
  txn.per_lane_lds_addr[0] = source.lds_base + 0x20;
  txn.payload.resize(sizeof(kValue));
  std::memcpy(txn.payload.data(), &kValue, sizeof(kValue));
  txn.targets = {target};

  amdgpu::ImmediateClusterLdsMulticastEngine engine;
  EXPECT_EQ(engine.submit(std::move(txn), []() {}), amdgpu::ClusterLdsMulticastResult::Complete);
  EXPECT_EQ(target.cu->lds().read32(target.lds_base + 0x20), 0u);
  EXPECT_EQ(source.cu->lds().read32(source.lds_base + 0x20), 0u);

  sim.engine->run();
}

TEST(Gfx1250SimulationTest, RejectsUnsupportedClusterSize) {
  constexpr uint64_t kernel_addr = 0x10000;
  const uint32_t code[] = {S_ENDPGM_GFX12};

  Gfx1250Sim sim;
  uint64_t kernel_object = sim.write_kernel(kernel_addr, code, std::size(code), 128);
  test::AqlQueue queue(sim.memory, sim.cp());
  queue.dispatch_clustered(kernel_object, /*cluster_count_x=*/1,
                           /*cluster_size_x=*/amdgpu::kClusterMulticastMaskBits + 1,
                           /*workgroup_size_x=*/32);

  EXPECT_THROW((void)sim.engine->step(), std::runtime_error);
}

TEST(Gfx1250SimulationTest, RejectsMisalignedClusteredWorkgroupIdOffset) {
  constexpr uint64_t kernel_addr = 0x10000;
  const uint32_t code[] = {S_ENDPGM_GFX12};

  Gfx1250Sim sim;
  uint64_t kernel_object = sim.write_kernel(kernel_addr, code, std::size(code), 128);
  sim.cp()->set_workgroup_id_offset(1);
  test::AqlQueue queue(sim.memory, sim.cp());
  queue.dispatch_clustered(kernel_object, /*cluster_count_x=*/1, /*cluster_size_x=*/2,
                           /*workgroup_size_x=*/32);

  EXPECT_THROW((void)sim.engine->step(), std::runtime_error);
}

TEST(Gfx1250SimulationTest, RejectsIncompleteClusterGrid) {
  constexpr uint64_t kernel_addr = 0x10000;
  const uint32_t code[] = {S_ENDPGM_GFX12};

  Gfx1250Sim sim;
  uint64_t kernel_object = sim.write_kernel(kernel_addr, code, std::size(code), 128);
  test::AqlQueue queue(sim.memory, sim.cp());
  queue.dispatch_clustered(kernel_object, /*cluster_count_x=*/1, /*cluster_size_x=*/2,
                           /*workgroup_size_x=*/0);

  EXPECT_THROW((void)sim.engine->step(), std::runtime_error);
}

TEST(Gfx1250SimulationTest, ClusterLoadAsyncToLdsDoesNotWriteMaskExcludedParticipant) {
  constexpr uint64_t kernel_addr = 0x10000;
  constexpr uint64_t input_addr = 0x2000;
  constexpr uint32_t lds_bytes_per_wg = 256;

  const uint32_t code[] = {
      0xBE8000FFu,    static_cast<uint32_t>(input_addr), // s_mov_b32 s0, input_addr
      0xBE810080u,                                       // s_mov_b32 s1, 0
      0xBEFD0081u,                                       // s_mov_b32 m0, 0x1
      0x30000082u,                                       // v_lshlrev_b32_e32 v0, 2, v0
      0xEE1AC000u,    0x00000000u,
      0x00000000u, // cluster_load_async_to_lds_b32 v0, v0, s[0:1]
      0xBFCA0000u, // s_wait_asynccnt 0
      S_ENDPGM_GFX12,
  };

  Gfx1250Sim sim;
  uint64_t kernel_object = sim.write_kernel(kernel_addr, code, std::size(code), 128);
  for (uint32_t byte = 0; byte < 256; ++byte)
    sim.memory->write8(input_addr + byte, static_cast<uint8_t>(0x40u + byte));

  test::AqlQueue queue(sim.memory, sim.cp());
  queue.dispatch_clustered(kernel_object, /*cluster_count_x=*/1, /*cluster_size_x=*/2,
                           /*workgroup_size_x=*/32, /*kernarg_addr=*/0, lds_bytes_per_wg);
  ASSERT_TRUE(sim.engine->step());

  auto targets = sim.cp()->cluster_lds_targets(/*dispatch_id=*/1, /*wg_id=*/0, /*mcast_mask=*/0x3);
  ASSERT_EQ(targets.size(), 2u);
  ASSERT_NE(targets[0].cu, nullptr);
  ASSERT_NE(targets[1].cu, nullptr);
  EXPECT_NE(targets[0].cu, targets[1].cu);
  sim.engine->run();
  EXPECT_TRUE(sim.cp()
                  ->cluster_lds_targets(/*dispatch_id=*/1, /*wg_id=*/0,
                                        /*mcast_mask=*/0x3)
                  .empty());

  auto read_unaligned_input = [&](uint32_t byte_offset) {
    uint32_t value = 0;
    for (uint32_t byte = 0; byte < 4; ++byte)
      value |= static_cast<uint32_t>(sim.memory->read8(input_addr + byte_offset + byte))
               << (byte * 8);
    return value;
  };

  for (uint32_t lane = 0; lane < 32; ++lane) {
    uint32_t value0 = targets[0].cu->lds().read32(targets[0].lds_base + lane * 4);
    uint32_t value1 = targets[1].cu->lds().read32(targets[1].lds_base + lane * 4);
    uint32_t wg0_value = read_unaligned_input(lane * 4);
    EXPECT_EQ(value0, wg0_value) << "lane " << lane;
    EXPECT_EQ(value1, 0u) << "lane " << lane;
  }
}

TEST(Gfx1250SimulationTest, ClusterLoadAsyncToLdsWritesEachIssuingParticipant) {
  constexpr uint64_t kernel_addr = 0x10000;
  constexpr uint64_t input_addr = 0x2400;
  constexpr uint32_t lds_bytes_per_wg = 256;

  const uint32_t code[] = {
      0xBE8000FFu,    static_cast<uint32_t>(input_addr), // s_mov_b32 s0, input_addr
      0xBE810080u,                                       // s_mov_b32 s1, 0
      0xBEFD0083u,                                       // s_mov_b32 m0, 0x3
      0x30000082u,                                       // v_lshlrev_b32_e32 v0, 2, v0
      0xEE1AC000u,    0x00000000u,
      0x00000000u, // cluster_load_async_to_lds_b32 v0, v0, s[0:1]
      0xBFCA0000u, // s_wait_asynccnt 0
      S_ENDPGM_GFX12,
  };

  Gfx1250Sim sim;
  uint64_t kernel_object = sim.write_kernel(kernel_addr, code, std::size(code), 128);
  for (uint32_t byte = 0; byte < 256; ++byte)
    sim.memory->write8(input_addr + byte, static_cast<uint8_t>(0x80u + byte));

  test::AqlQueue queue(sim.memory, sim.cp());
  queue.dispatch_clustered(kernel_object, /*cluster_count_x=*/1, /*cluster_size_x=*/2,
                           /*workgroup_size_x=*/32, /*kernarg_addr=*/0, lds_bytes_per_wg);
  ASSERT_TRUE(sim.engine->step());

  auto targets = sim.cp()->cluster_lds_targets(/*dispatch_id=*/1, /*wg_id=*/0, /*mcast_mask=*/0x3);
  ASSERT_EQ(targets.size(), 2u);
  ASSERT_NE(targets[0].cu, nullptr);
  ASSERT_NE(targets[1].cu, nullptr);
  EXPECT_NE(targets[0].cu, targets[1].cu);
  sim.engine->run();

  auto read_unaligned_input = [&](uint32_t byte_offset) {
    uint32_t value = 0;
    for (uint32_t byte = 0; byte < 4; ++byte)
      value |= static_cast<uint32_t>(sim.memory->read8(input_addr + byte_offset + byte))
               << (byte * 8);
    return value;
  };

  for (uint32_t lane = 0; lane < 32; ++lane) {
    uint32_t value0 = targets[0].cu->lds().read32(targets[0].lds_base + lane * 4);
    uint32_t value1 = targets[1].cu->lds().read32(targets[1].lds_base + lane * 4);
    uint32_t expected_value = read_unaligned_input(lane * 4);
    EXPECT_EQ(value0, expected_value) << "lane " << lane;
    EXPECT_EQ(value1, expected_value) << "lane " << lane;
  }
}

TEST(Gfx1250ExecutionTest, TensorDmaD2CopiesGlobalAndLds) {
  Gfx1250Sim sim;
  auto *cu = sim.cu();
  auto *wf = cu->dispatch_wf(0, 0, kGfx1250ScalarSlots, 32);
  ASSERT_NE(wf, nullptr);
  wf->set_lds_base(cu->allocate_lds(256));

  constexpr uint32_t kElements = 16;
  constexpr uint64_t kLoadGlobal = 0x100000;
  constexpr uint64_t kStoreGlobal = 0x110000;
  auto write_sgpr = [&](uint32_t reg, uint32_t value) {
    cu->write_sgpr(wf->sgpr_alloc().base + reg, value);
  };
  auto write_d0 = [&](uint32_t reg, uint64_t global_addr) {
    write_sgpr(reg + 0, 1);
    write_sgpr(reg + 1, 0);
    write_sgpr(reg + 2, static_cast<uint32_t>(global_addr));
    write_sgpr(reg + 3, static_cast<uint32_t>((global_addr >> 32) & 0x01ffffffu) | 0x80000000u);
  };

  write_d0(0, kLoadGlobal);
  write_d0(8, kStoreGlobal);
  write_sgpr(12, 2u << 16);        // i32 elements.
  write_sgpr(13, kElements << 16); // tensor dim0.
  write_sgpr(14, 0);
  write_sgpr(15, kElements << 16); // tile dim0.
  write_sgpr(16, 0);
  write_sgpr(17, 0);
  write_sgpr(18, 0);
  write_sgpr(19, 0);

  for (uint32_t i = 0; i < kElements; ++i) {
    const uint32_t value = 0x11000000u + i * 0x101u;
    for (uint32_t byte = 0; byte < 4; ++byte)
      sim.memory->write8(kLoadGlobal + i * 4 + byte, static_cast<uint8_t>(value >> (byte * 8)));
  }

  const std::array<uint32_t, 3> load_words = {0xd0710001u, 0x7c000000u, 0x7c7c0c00u};
  gfx1250::TensorLoadToLdsVimage load_inst(load_words.data());
  load_inst.execute_impl(*wf);
  for (uint32_t i = 0; i < kElements; ++i)
    EXPECT_EQ(cu->lds().read32(wf->lds_base() + i * 4), 0x11000000u + i * 0x101u);

  for (uint32_t i = 0; i < kElements; ++i)
    cu->lds().write32(wf->lds_base() + i * 4, 0x22000000u + i * 0x303u);

  const std::array<uint32_t, 3> store_words = {0xd0714001u, 0x7c000000u, 0x7c7c0c08u};
  gfx1250::TensorStoreFromLdsVimage store_inst(store_words.data());
  store_inst.execute_impl(*wf);
  for (uint32_t i = 0; i < kElements; ++i) {
    const uint32_t actual = read_global_u32(*sim.memory, kStoreGlobal + i * 4);
    EXPECT_EQ(actual, 0x22000000u + i * 0x303u);
  }
}

TEST(Gfx1250ExecutionTest, TensorDmaDecodeExecuteCoversLoadStoreD2D4Forms) {
  constexpr uint32_t kNullSgpr = 124;
  constexpr std::array<uint32_t, 3> kLoadD2 = {0xd0710001u, 0x7c000000u, 0x7c7c0c00u};
  constexpr std::array<uint32_t, 3> kStoreD2 = {0xd0714001u, 0x7c000000u, 0x7c7c0c08u};
  constexpr std::array<uint32_t, 3> kLoadD4 = {0xd0710001u, 0x7c000000u, 0x18140c00u};
  constexpr std::array<uint32_t, 3> kStoreD4 = {0xd0714001u, 0x7c000000u, 0x18140c08u};

  {
    Gfx1250Sim sim;
    auto *cu = sim.cu();
    auto *wf = cu->dispatch_wf(0, 0, kGfx1250ScalarSlots, 32);
    ASSERT_NE(wf, nullptr);
    wf->set_lds_base(cu->allocate_lds(256));

    constexpr uint32_t kElements = 4;
    constexpr uint64_t kLoadGlobal = 0x1a0000;
    constexpr uint64_t kStoreGlobal = 0x1b0000;

    write_tensor_dma_d0(*cu, *wf, 0, kLoadGlobal);
    write_tensor_dma_d0(*cu, *wf, 8, kStoreGlobal);
    write_wave_sgpr(*cu, *wf, 12, 2u << 16);        // i32 elements.
    write_wave_sgpr(*cu, *wf, 13, kElements << 16); // Tensor dim0.
    write_wave_sgpr(*cu, *wf, 14, 0);
    write_wave_sgpr(*cu, *wf, 15, kElements << 16); // Tile dim0.
    write_wave_sgpr(*cu, *wf, 16, 0);
    write_wave_sgpr(*cu, *wf, 17, 0);
    write_wave_sgpr(*cu, *wf, 18, 0);
    write_wave_sgpr(*cu, *wf, 19, 0);

    for (uint32_t i = 0; i < kElements; ++i)
      write_global_u32(*sim.memory, kLoadGlobal + i * 4, 0x71000000u + i);

    auto load = decode_gfx1250(kLoadD2, "tensor_load_to_lds");
    ASSERT_NE(load, nullptr);
    ASSERT_EQ(load->num_src_operands(), 4);
    EXPECT_EQ(load->src_operand(2)->encoding_value(), static_cast<int>(kNullSgpr));
    EXPECT_EQ(load->src_operand(3)->encoding_value(), static_cast<int>(kNullSgpr));
    load->execute(*load, wf);

    for (uint32_t i = 0; i < kElements; ++i)
      EXPECT_EQ(cu->lds().read32(wf->lds_base() + i * 4), 0x71000000u + i);

    for (uint32_t i = 0; i < kElements; ++i)
      cu->lds().write32(wf->lds_base() + i * 4, 0x72000000u + i);

    auto store = decode_gfx1250(kStoreD2, "tensor_store_from_lds");
    ASSERT_NE(store, nullptr);
    ASSERT_EQ(store->num_src_operands(), 4);
    EXPECT_EQ(store->src_operand(2)->encoding_value(), static_cast<int>(kNullSgpr));
    EXPECT_EQ(store->src_operand(3)->encoding_value(), static_cast<int>(kNullSgpr));
    store->execute(*store, wf);

    for (uint32_t i = 0; i < kElements; ++i)
      EXPECT_EQ(read_global_u32(*sim.memory, kStoreGlobal + i * 4), 0x72000000u + i);
  }

  {
    Gfx1250Sim sim;
    auto *cu = sim.cu();
    auto *wf = cu->dispatch_wf(0, 0, kGfx1250ScalarSlots, 32);
    ASSERT_NE(wf, nullptr);
    wf->set_lds_base(cu->allocate_lds(256));

    constexpr uint32_t kCols = 2;
    constexpr uint32_t kRows = 2;
    constexpr uint32_t kTileRows = 3;
    constexpr uint32_t kDepth = 2;
    constexpr uint32_t kPlaneStride = kTileRows * kCols;
    constexpr uint64_t kLoadGlobal = 0x1c0000;
    constexpr uint64_t kStoreGlobal = 0x1d0000;
    constexpr uint32_t kSentinel = 0x7e7e7e7eu;

    write_tensor_dma_d0(*cu, *wf, 0, kLoadGlobal);
    write_tensor_dma_d0(*cu, *wf, 8, kStoreGlobal);
    write_wave_sgpr(*cu, *wf, 12, 2u << 16);    // i32 elements.
    write_wave_sgpr(*cu, *wf, 13, kCols << 16); // Tensor dim0.
    write_wave_sgpr(*cu, *wf, 14, kRows << 16); // Tensor dim1.
    write_wave_sgpr(*cu, *wf, 15, kCols << 16); // Tile dim0.
    write_wave_sgpr(*cu, *wf, 16, kTileRows | (kDepth << 16));
    write_wave_sgpr(*cu, *wf, 17, kCols); // Tensor dim0 stride.
    write_wave_sgpr(*cu, *wf, 18, kPlaneStride << 16);
    write_wave_sgpr(*cu, *wf, 19, 0);
    write_wave_sgpr(*cu, *wf, 20, kDepth); // Tensor dim2, from D2.
    write_wave_sgpr(*cu, *wf, 21, 0);
    write_wave_sgpr(*cu, *wf, 22, 0);
    write_wave_sgpr(*cu, *wf, 23, 0);
    write_wave_sgpr(*cu, *wf, 24, 0);
    write_wave_sgpr(*cu, *wf, 25, 0);
    write_wave_sgpr(*cu, *wf, 26, 0);
    write_wave_sgpr(*cu, *wf, 27, 0);

    for (uint32_t z = 0; z < kDepth; ++z) {
      for (uint32_t y = 0; y < kTileRows; ++y) {
        for (uint32_t x = 0; x < kCols; ++x) {
          const uint64_t offset = static_cast<uint64_t>(z * kPlaneStride + y * kCols + x) * 4;
          const uint32_t value = 0x73000000u + z * 0x100u + y * 0x10u + x;
          write_global_u32(*sim.memory, kLoadGlobal + offset, value);
          write_global_u32(*sim.memory, kStoreGlobal + offset, kSentinel);
        }
      }
    }

    auto load = decode_gfx1250(kLoadD4, "tensor_load_to_lds");
    ASSERT_NE(load, nullptr);
    ASSERT_EQ(load->num_src_operands(), 4);
    EXPECT_EQ(load->src_operand(2)->encoding_value(), 20);
    EXPECT_EQ(load->src_operand(3)->encoding_value(), 24);
    load->execute(*load, wf);

    for (uint32_t z = 0; z < kDepth; ++z) {
      for (uint32_t y = 0; y < kTileRows; ++y) {
        for (uint32_t x = 0; x < kCols; ++x) {
          const uint32_t lds_index = z * kTileRows * kCols + y * kCols + x;
          const uint32_t expected = (y < kRows) ? (0x73000000u + z * 0x100u + y * 0x10u + x) : 0u;
          EXPECT_EQ(cu->lds().read32(wf->lds_base() + lds_index * 4), expected);
        }
      }
    }

    for (uint32_t i = 0; i < kDepth * kTileRows * kCols; ++i)
      cu->lds().write32(wf->lds_base() + i * 4, 0x74000000u + i);

    auto store = decode_gfx1250(kStoreD4, "tensor_store_from_lds");
    ASSERT_NE(store, nullptr);
    ASSERT_EQ(store->num_src_operands(), 4);
    EXPECT_EQ(store->src_operand(2)->encoding_value(), 20);
    EXPECT_EQ(store->src_operand(3)->encoding_value(), 24);
    store->execute(*store, wf);

    for (uint32_t z = 0; z < kDepth; ++z) {
      for (uint32_t y = 0; y < kTileRows; ++y) {
        for (uint32_t x = 0; x < kCols; ++x) {
          const uint64_t offset = static_cast<uint64_t>(z * kPlaneStride + y * kCols + x) * 4;
          const uint32_t lds_index = z * kTileRows * kCols + y * kCols + x;
          const uint32_t expected = (y < kRows) ? (0x74000000u + lds_index) : kSentinel;
          EXPECT_EQ(read_global_u32(*sim.memory, kStoreGlobal + offset), expected);
        }
      }
    }
  }
}

TEST(Gfx1250ExecutionTest, TensorDmaLoadAppliesLdsPadding) {
  Gfx1250Sim sim;
  auto *cu = sim.cu();
  auto *wf = cu->dispatch_wf(0, 0, kGfx1250ScalarSlots, 32);
  ASSERT_NE(wf, nullptr);
  wf->set_lds_base(cu->allocate_lds(256));

  constexpr uint64_t kGlobal = 0x120000;
  constexpr uint32_t kSentinel = 0xCAFECAFEu;
  write_tensor_dma_d0(*cu, *wf, 0, kGlobal);
  write_wave_sgpr(*cu, *wf, 12, (2u << 16) | (1u << 20)); // i32, pad enabled.
  write_wave_sgpr(*cu, *wf, 13, 4u << 16);                // tensor dim0.
  write_wave_sgpr(*cu, *wf, 14, 0);
  write_wave_sgpr(*cu, *wf, 15, 4u << 16); // tile dim0.
  write_wave_sgpr(*cu, *wf, 16, 0);
  write_wave_sgpr(*cu, *wf, 17, 0);
  write_wave_sgpr(*cu, *wf, 18, 0);
  write_wave_sgpr(*cu, *wf, 19, 0);

  for (uint32_t i = 0; i < 4; ++i) {
    write_global_u32(*sim.memory, kGlobal + i * 4, 0x33000000u + i);
    cu->lds().write32(wf->lds_base() + i * 4, kSentinel);
  }
  cu->lds().write32(wf->lds_base() + 4 * 4, kSentinel);

  const std::array<uint32_t, 3> load_words = {0xd0710001u, 0x7c000000u, 0x7c7c0c00u};
  gfx1250::TensorLoadToLdsVimage load_inst(load_words.data());
  load_inst.execute_impl(*wf);

  EXPECT_EQ(cu->lds().read32(wf->lds_base() + 0 * 4), 0x33000000u);
  EXPECT_EQ(cu->lds().read32(wf->lds_base() + 1 * 4), 0x33000001u);
  EXPECT_EQ(cu->lds().read32(wf->lds_base() + 2 * 4), kSentinel);
  EXPECT_EQ(cu->lds().read32(wf->lds_base() + 3 * 4), 0x33000002u);
  EXPECT_EQ(cu->lds().read32(wf->lds_base() + 4 * 4), 0x33000003u);
}

TEST(Gfx1250ExecutionTest, TensorDmaByteLoadUsesDwordPaddingUnits) {
  Gfx1250Sim sim;
  auto *cu = sim.cu();
  auto *wf = cu->dispatch_wf(0, 0, kGfx1250ScalarSlots, 32);
  ASSERT_NE(wf, nullptr);
  wf->set_lds_base(cu->allocate_lds(512));

  constexpr uint64_t kGlobal = 0x121000;
  constexpr uint8_t kSentinel = 0xA5u;
  write_tensor_dma_d0(*cu, *wf, 0, kGlobal);
  write_wave_sgpr(*cu, *wf, 12,
                  (1u << 20) | (5u << 22) | (3u << 25)); // u8, pad 4 dwords per 64 dwords.
  write_wave_sgpr(*cu, *wf, 13, 260u << 16);             // tensor dim0.
  write_wave_sgpr(*cu, *wf, 14, 0);
  write_wave_sgpr(*cu, *wf, 15, 260u << 16); // tile dim0.
  write_wave_sgpr(*cu, *wf, 16, 0);
  write_wave_sgpr(*cu, *wf, 17, 0);
  write_wave_sgpr(*cu, *wf, 18, 0);
  write_wave_sgpr(*cu, *wf, 19, 0);

  for (uint32_t i = 0; i < 260; ++i)
    sim.memory->write8(kGlobal + i, static_cast<uint8_t>(i));
  for (uint32_t i = 0; i < 300; ++i)
    cu->lds().write8(wf->lds_base() + i, kSentinel);

  const std::array<uint32_t, 3> load_words = {0xd0710001u, 0x7c000000u, 0x7c7c0c00u};
  gfx1250::TensorLoadToLdsVimage load_inst(load_words.data());
  load_inst.execute_impl(*wf);

  EXPECT_EQ(cu->lds().read8(wf->lds_base() + 63), 63u);
  EXPECT_EQ(cu->lds().read8(wf->lds_base() + 64), 64u);
  EXPECT_EQ(cu->lds().read8(wf->lds_base() + 255), 255u);
  for (uint32_t i = 0; i < 16; ++i)
    EXPECT_EQ(cu->lds().read8(wf->lds_base() + 256 + i), kSentinel);
  EXPECT_EQ(cu->lds().read8(wf->lds_base() + 272), 0u);
  EXPECT_EQ(cu->lds().read8(wf->lds_base() + 275), 3u);
}

TEST(Gfx1250ExecutionTest, TensorDmaPaddedStoreIsRejected) {
  Gfx1250Sim sim;
  auto *cu = sim.cu();
  auto *wf = cu->dispatch_wf(0, 0, kGfx1250ScalarSlots, 32);
  ASSERT_NE(wf, nullptr);
  wf->set_lds_base(cu->allocate_lds(256));

  write_tensor_dma_d0(*cu, *wf, 0, 0x121000);
  write_wave_sgpr(*cu, *wf, 12, (2u << 16) | (1u << 20)); // i32, pad enabled.
  write_wave_sgpr(*cu, *wf, 13, 4u << 16);
  write_wave_sgpr(*cu, *wf, 14, 0);
  write_wave_sgpr(*cu, *wf, 15, 4u << 16);
  write_wave_sgpr(*cu, *wf, 16, 0);
  write_wave_sgpr(*cu, *wf, 17, 0);
  write_wave_sgpr(*cu, *wf, 18, 0);
  write_wave_sgpr(*cu, *wf, 19, 0);

  const std::array<uint32_t, 3> store_words = {0xd0714001u, 0x7c000000u, 0x7c7c0c00u};
  gfx1250::TensorStoreFromLdsVimage store_inst(store_words.data());
  EXPECT_THROW(store_inst.execute_impl(*wf), util::UnimplementedInst);
}

TEST(Gfx1250ExecutionTest, TensorDmaIterateCopiesMultipleTiles) {
  Gfx1250Sim sim;
  auto *cu = sim.cu();
  auto *wf = cu->dispatch_wf(0, 0, kGfx1250ScalarSlots, 32);
  ASSERT_NE(wf, nullptr);
  wf->set_lds_base(cu->allocate_lds(256));

  constexpr uint64_t kGlobal = 0x130000;
  constexpr uint32_t kSentinel = 0xFEEDFEEDu;
  write_tensor_dma_d0(*cu, *wf, 0, kGlobal);
  write_wave_sgpr(*cu, *wf, 12, (2u << 16) | (1u << 19)); // i32, iterate enabled.
  write_wave_sgpr(*cu, *wf, 13, 2u << 16);                // tensor dim0.
  write_wave_sgpr(*cu, *wf, 14, 2u << 16);                // tensor dim1.
  write_wave_sgpr(*cu, *wf, 15, 2u << 16);                // tile dim0.
  write_wave_sgpr(*cu, *wf, 16, 1u);                      // tile dim1.
  write_wave_sgpr(*cu, *wf, 17, 0);
  write_wave_sgpr(*cu, *wf, 18, 0);
  write_wave_sgpr(*cu, *wf, 19, 0);
  write_wave_sgpr(*cu, *wf, 20, 0);
  write_wave_sgpr(*cu, *wf, 21, 4);        // LDS increment in elements.
  write_wave_sgpr(*cu, *wf, 22, 2);        // Global increment in elements.
  write_wave_sgpr(*cu, *wf, 23, 1u << 16); // iteration_count - 1.

  for (uint32_t i = 0; i < 4; ++i)
    write_global_u32(*sim.memory, kGlobal + i * 4, 0x44000000u + i);
  for (uint32_t i = 0; i < 8; ++i)
    cu->lds().write32(wf->lds_base() + i * 4, kSentinel);

  const std::array<uint32_t, 3> load_words = {0xd0710001u, 0x7c000000u, 0x7c140c00u};
  gfx1250::TensorLoadToLdsVimage load_inst(load_words.data());
  load_inst.execute_impl(*wf);

  EXPECT_EQ(cu->lds().read32(wf->lds_base() + 0 * 4), 0x44000000u);
  EXPECT_EQ(cu->lds().read32(wf->lds_base() + 1 * 4), 0x44000001u);
  EXPECT_EQ(cu->lds().read32(wf->lds_base() + 2 * 4), kSentinel);
  EXPECT_EQ(cu->lds().read32(wf->lds_base() + 3 * 4), kSentinel);
  EXPECT_EQ(cu->lds().read32(wf->lds_base() + 4 * 4), 0x44000002u);
  EXPECT_EQ(cu->lds().read32(wf->lds_base() + 5 * 4), 0x44000003u);
  EXPECT_EQ(cu->lds().read32(wf->lds_base() + 6 * 4), kSentinel);
  EXPECT_EQ(cu->lds().read32(wf->lds_base() + 7 * 4), kSentinel);
}

TEST(Gfx1250ExecutionTest, TensorDmaAtomicBarrierArrivesAfterCopy) {
  Gfx1250Sim sim;
  auto *cu = sim.cu();
  auto *wf = cu->dispatch_wf(0, 0, kGfx1250ScalarSlots, 32);
  ASSERT_NE(wf, nullptr);
  wf->set_lds_base(cu->allocate_lds(256));

  constexpr uint64_t kGlobal = 0x140000;
  constexpr uint32_t kBarrierLdsAddr = 64;
  write_tensor_dma_d0(*cu, *wf, 0, kGlobal);
  write_wave_sgpr(*cu, *wf, 12, (2u << 16) | (1u << 18)); // i32, atomic barrier enabled.
  write_wave_sgpr(*cu, *wf, 13, (2u << 16) | (kBarrierLdsAddr >> 3));
  write_wave_sgpr(*cu, *wf, 14, 0);
  write_wave_sgpr(*cu, *wf, 15, 2u << 16);
  write_wave_sgpr(*cu, *wf, 16, 0);
  write_wave_sgpr(*cu, *wf, 17, 0);
  write_wave_sgpr(*cu, *wf, 18, 0);
  write_wave_sgpr(*cu, *wf, 19, 0);

  write_global_u32(*sim.memory, kGlobal + 0 * 4, 0x55000000u);
  write_global_u32(*sim.memory, kGlobal + 1 * 4, 0x55000001u);
  cu->lds().write64(wf->lds_base() + kBarrierLdsAddr, 0);

  const std::array<uint32_t, 3> load_words = {0xd0710001u, 0x7c000000u, 0x7c7c0c00u};
  gfx1250::TensorLoadToLdsVimage load_inst(load_words.data());
  load_inst.execute_impl(*wf);

  EXPECT_TRUE(wf->wait_counters().empty());
  EXPECT_EQ(wf->state(), amdgpu::WfState::RUNNING);
  EXPECT_EQ(cu->lds().read32(wf->lds_base() + 0 * 4), 0x55000000u);
  EXPECT_EQ(cu->lds().read32(wf->lds_base() + 1 * 4), 0x55000001u);
  const uint64_t state = cu->lds().read64(wf->lds_base() + kBarrierLdsAddr);
  EXPECT_EQ(state, 0xffffull << 16);
  EXPECT_TRUE(amdgpu::lds_barrier_cell_phase_parity(state));
}

TEST(Gfx1250ExecutionTest, SBarrierWaitEntersBarrierState) {
  Gfx1250Sim sim;
  auto *cu = sim.cu();
  auto *wf = cu->dispatch_wf(0, 0, kGfx1250ScalarSlots, 32);
  ASSERT_NE(wf, nullptr);
  ASSERT_EQ(wf->state(), amdgpu::WfState::RUNNING);

  auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
  ASSERT_NE(decoder, nullptr);

  const std::array<uint32_t, 2> wait_words = {0xBF940000u, 0u};
  std::unique_ptr<Instruction> wait_inst(decoder->decode(wait_words.data()));
  ASSERT_NE(wait_inst, nullptr);
  ASSERT_EQ(std::string_view(wait_inst->mnemonic()), "s_barrier_wait");

  cu->execute_instruction(wait_inst.get(), *wf);

  EXPECT_EQ(wf->state(), amdgpu::WfState::BARRIER);
}

TEST(Gfx1250ExecutionTest, SBarrierWaitReleasesOnlyAfterAllSiblingsWait) {
  Gfx1250Sim sim;
  auto *cu = sim.cu();
  constexpr uint64_t kEndPgmPc = 0x150000;
  const std::array<uint32_t, 1> endpgm_words = {S_ENDPGM_GFX12};
  sim.memory->load_image(reinterpret_cast<const uint8_t *>(endpgm_words.data()), sizeof(uint32_t),
                         kEndPgmPc);

  auto *wf0 = cu->dispatch_wf(0, kEndPgmPc, kGfx1250ScalarSlots, 32);
  auto *wf1 = cu->dispatch_wf(0, kEndPgmPc, kGfx1250ScalarSlots, 32);
  ASSERT_NE(wf0, nullptr);
  ASSERT_NE(wf1, nullptr);
  cu->begin_workgroup(0, 0, 2);

  auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
  ASSERT_NE(decoder, nullptr);

  const std::array<uint32_t, 2> wait_words = {0xBF940000u, 0u};
  std::unique_ptr<Instruction> wait_inst(decoder->decode(wait_words.data()));
  ASSERT_NE(wait_inst, nullptr);
  ASSERT_EQ(std::string_view(wait_inst->mnemonic()), "s_barrier_wait");

  cu->execute_instruction(wait_inst.get(), *wf0);
  wf1->wait_counters().increment(amdgpu::WaitCounterType::TENSORCNT);
  wf1->set_wait_target_tensorcnt(0);
  wf1->set_state(amdgpu::WfState::WAITCNT);

  ASSERT_TRUE(cu->step());
  EXPECT_EQ(wf0->state(), amdgpu::WfState::BARRIER);
  EXPECT_EQ(wf1->state(), amdgpu::WfState::WAITCNT);

  wf1->release_wait_counter(amdgpu::WaitCounterType::TENSORCNT);
  ASSERT_EQ(wf1->state(), amdgpu::WfState::RUNNING);
  cu->execute_instruction(wait_inst.get(), *wf1);
  ASSERT_EQ(wf1->state(), amdgpu::WfState::BARRIER);

  EXPECT_FALSE(cu->step());
  EXPECT_TRUE(wf0->is_halted());
  EXPECT_TRUE(wf1->is_halted());
}

TEST(Gfx1250ExecutionTest, ReleaseWaitCounterWakesWaitcntWhenTargetIsSatisfied) {
  Gfx1250Sim sim;
  auto *cu = sim.cu();
  auto *wf = cu->dispatch_wf(0, 0, kGfx1250ScalarSlots, 32);
  ASSERT_NE(wf, nullptr);

  wf->wait_counters().increment(amdgpu::WaitCounterType::TENSORCNT);
  wf->set_wait_target_tensorcnt(0);
  ASSERT_EQ(wf->state(), amdgpu::WfState::WAITCNT);

  wf->release_wait_counter(amdgpu::WaitCounterType::TENSORCNT);

  EXPECT_TRUE(wf->wait_counters().empty());
  EXPECT_EQ(wf->state(), amdgpu::WfState::RUNNING);
}

TEST(Gfx1250ExecutionTest, ReleaseWaitCounterHaltsEndingWaveWhenLastCounterRetires) {
  Gfx1250Sim sim;
  auto *cu = sim.cu();
  auto *wf = cu->dispatch_wf(0, 0, kGfx1250ScalarSlots, 32);
  ASSERT_NE(wf, nullptr);

  wf->wait_counters().increment(amdgpu::WaitCounterType::TENSORCNT);
  wf->end();
  ASSERT_EQ(wf->state(), amdgpu::WfState::ENDING);

  wf->release_wait_counter(amdgpu::WaitCounterType::TENSORCNT);

  EXPECT_TRUE(wf->wait_counters().empty());
  EXPECT_TRUE(wf->is_halted());
}

TEST(Gfx1250ExecutionTest, DsAtomicAsyncBarrierArriveFlipsRawBarrierPhase) {
  Gfx1250Sim sim;
  auto *cu = sim.cu();
  auto *wf = cu->dispatch_wf(0, 0, kGfx1250ScalarSlots, 32);
  ASSERT_NE(wf, nullptr);
  wf->set_exec(1);
  wf->set_lds_base(cu->allocate_lds(256));

  constexpr uint32_t kBarrierLdsAddr = 64;
  cu->write_vgpr(wf->vgpr_alloc().base + 0, 0, kBarrierLdsAddr);
  cu->lds().write64(wf->lds_base() + kBarrierLdsAddr, amdgpu::lds_barrier_cell_init_state(1));

  const std::array<uint32_t, 2> words = {0xd9580000u, 0x00000000u};
  auto *arrive_inst = new gfx1250::DsAtomicAsyncBarrierArriveB64Vds(words.data());
  arrive_inst->execute_impl(*wf);
  amdgpu::LocalMemPipeline local_pipeline;
  local_pipeline.issue(arrive_inst, *wf);

  const uint64_t state = cu->lds().read64(wf->lds_base() + kBarrierLdsAddr);
  EXPECT_EQ(state, 0xffffull << 16);
  EXPECT_TRUE(amdgpu::lds_barrier_cell_phase_parity(state));
  EXPECT_TRUE(wf->wait_counters().empty());
}

TEST(Gfx1250ExecutionTest, LocalMemPipelineUsesInjectedBarrierDecrementPayload) {
  Gfx1250Sim sim;
  auto *cu = sim.cu();
  auto *wf = cu->dispatch_wf(0, 0, kGfx1250ScalarSlots, 32);
  ASSERT_NE(wf, nullptr);
  wf->set_exec(1);
  wf->set_lds_base(cu->allocate_lds(256));

  constexpr uint32_t kBarrierLdsAddr = 64;
  cu->write_vgpr(wf->vgpr_alloc().base + 0, 0, kBarrierLdsAddr);
  cu->lds().write64(wf->lds_base() + kBarrierLdsAddr, amdgpu::lds_barrier_cell_init_state(2));

  const std::array<uint32_t, 2> words = {0xd9580000u, 0x00000000u};
  auto *arrive_inst = new gfx1250::DsAtomicAsyncBarrierArriveB64Vds(words.data());
  arrive_inst->execute_impl(*wf);

  auto *state = arrive_inst->data_as<amdgpu::VectorMemState>();
  const uint64_t decrement = 2;
  state->store_data.resize(static_cast<size_t>(wf->wf_size()) * sizeof(decrement));
  std::memcpy(state->store_data.data(), &decrement, sizeof(decrement));

  amdgpu::LocalMemPipeline local_pipeline;
  local_pipeline.issue(arrive_inst, *wf);

  const uint64_t expected =
      amdgpu::lds_barrier_cell_update_arrive(amdgpu::lds_barrier_cell_init_state(2), decrement);
  EXPECT_EQ(cu->lds().read64(wf->lds_base() + kBarrierLdsAddr), expected);
  EXPECT_EQ(expected, (1ull << 32) | (0xffffull << 16) | 1ull);
  EXPECT_TRUE(wf->wait_counters().empty());
}

TEST(Gfx1250ExecutionTest, LdsBarrierCellHandlesSingleAndBatchedArrivals) {
  uint64_t state = amdgpu::lds_barrier_cell_init_state(2);
  EXPECT_EQ(state, (1ull << 32) | 1ull);

  state = amdgpu::lds_barrier_cell_update_arrive(state);
  EXPECT_EQ(state, 1ull << 32);
  EXPECT_FALSE(amdgpu::lds_barrier_cell_phase_parity(state));

  state = amdgpu::lds_barrier_cell_update_arrive(state);
  EXPECT_EQ(state, (1ull << 32) | (0xffffull << 16) | 1ull);
  EXPECT_TRUE(amdgpu::lds_barrier_cell_phase_parity(state));

  const uint64_t drained =
      amdgpu::lds_barrier_cell_update_arrive(amdgpu::lds_barrier_cell_init_state(2));
  ASSERT_EQ(amdgpu::lds_barrier_cell_pending_count(drained), 0ull);
  EXPECT_EQ(amdgpu::lds_barrier_cell_update_arrive(drained, 0), drained);
  EXPECT_EQ(amdgpu::lds_barrier_cell_update_arrive(amdgpu::lds_barrier_cell_init_state(3), 0),
            amdgpu::lds_barrier_cell_init_state(3));

  uint64_t iterated = amdgpu::lds_barrier_cell_init_state(2);
  for (int i = 0; i < 5; ++i)
    iterated = amdgpu::lds_barrier_cell_update_arrive(iterated);
  const uint64_t batched =
      amdgpu::lds_barrier_cell_update_arrive(amdgpu::lds_barrier_cell_init_state(2), 5);
  EXPECT_EQ(batched, iterated);
  EXPECT_EQ(batched, (1ull << 32) | (0xfffeull << 16));

  for (uint32_t arrivals_per_phase : {0u, 1u}) {
    state = amdgpu::lds_barrier_cell_init_state(arrivals_per_phase);
    EXPECT_EQ(amdgpu::lds_barrier_cell_pending_count(state), 0ull);
    state = amdgpu::lds_barrier_cell_update_arrive(state);
    EXPECT_EQ(amdgpu::lds_barrier_cell_pending_count(state), 0ull);
    EXPECT_EQ(amdgpu::lds_barrier_cell_phase(state), amdgpu::kLdsBarrierCellPhaseMask);
    EXPECT_TRUE(amdgpu::lds_barrier_cell_phase_parity(state));
  }

  const uint64_t reserved = 0xabcdull << 48;
  const uint64_t reserved_state =
      amdgpu::lds_barrier_cell_update_arrive(reserved | amdgpu::lds_barrier_cell_init_state(2));
  EXPECT_EQ(reserved_state & amdgpu::kLdsBarrierCellReservedMask, reserved);
  EXPECT_EQ(amdgpu::lds_barrier_cell_init_count(reserved_state), 1ull);
}

TEST(Gfx1250ExecutionTest, TensorDmaDenseDescriptorCopiesDenseRows) {
  Gfx1250Sim sim;
  auto *cu = sim.cu();
  auto *wf = cu->dispatch_wf(0, 0, kGfx1250ScalarSlots, 32);
  ASSERT_NE(wf, nullptr);
  wf->set_lds_base(cu->allocate_lds(512));

  constexpr uint64_t kLoadGlobal = 0x150000;
  constexpr uint64_t kStoreGlobal = 0x160000;
  constexpr uint32_t kRows = 8;
  constexpr uint32_t kCols = 4;
  constexpr uint32_t kTileDim1RowCount = 3;
  constexpr uint32_t kSentinel = 0xABCDABCDu;
  constexpr uint32_t kIgnoredD2 = 0xFFFFu;

  write_tensor_dma_d0(*cu, *wf, 0, kLoadGlobal);
  write_wave_sgpr(*cu, *wf, 0, 1u);
  write_tensor_dma_d0(*cu, *wf, 8, kStoreGlobal);
  write_wave_sgpr(*cu, *wf, 8, 1u);
  write_wave_sgpr(*cu, *wf, 12, 2u << 16);    // i32 elements.
  write_wave_sgpr(*cu, *wf, 13, kCols << 16); // Tensor dim0.
  write_wave_sgpr(*cu, *wf, 14, kRows << 16); // Tensor dim1.
  write_wave_sgpr(*cu, *wf, 15, kCols << 16); // Tile dim0.
  write_wave_sgpr(*cu, *wf, 16, kTileDim1RowCount);
  write_wave_sgpr(*cu, *wf, 17, kCols); // Tensor dim0 stride.
  write_wave_sgpr(*cu, *wf, 18, 0);
  write_wave_sgpr(*cu, *wf, 19, 0);
  write_wave_sgpr(*cu, *wf, 20, kIgnoredD2);
  write_wave_sgpr(*cu, *wf, 21, kIgnoredD2);
  write_wave_sgpr(*cu, *wf, 22, kIgnoredD2);
  write_wave_sgpr(*cu, *wf, 23, 0);
  write_wave_sgpr(*cu, *wf, 24, 0);
  write_wave_sgpr(*cu, *wf, 25, 0);
  write_wave_sgpr(*cu, *wf, 26, 0);
  write_wave_sgpr(*cu, *wf, 27, 0);

  for (uint32_t row = 0; row < kRows; ++row) {
    for (uint32_t col = 0; col < kCols; ++col) {
      write_global_u32(*sim.memory, kLoadGlobal + (row * kCols + col) * 4,
                       0x66000000u + row * 0x100u + col);
      write_global_u32(*sim.memory, kStoreGlobal + (row * kCols + col) * 4, kSentinel);
    }
  }
  for (uint32_t i = 0; i < (kTileDim1RowCount + 1) * kCols; ++i)
    cu->lds().write32(wf->lds_base() + i * 4, kSentinel);

  const std::array<uint32_t, 3> load_words = {0xd0710001u, 0x7c000000u, 0x18140c00u};
  gfx1250::TensorLoadToLdsVimage load_inst(load_words.data());
  load_inst.execute_impl(*wf);

  for (uint32_t row_idx = 0; row_idx < kTileDim1RowCount; ++row_idx) {
    const uint32_t src_row = row_idx;
    for (uint32_t col = 0; col < kCols; ++col) {
      EXPECT_EQ(cu->lds().read32(wf->lds_base() + (row_idx * kCols + col) * 4),
                0x66000000u + src_row * 0x100u + col);
    }
  }
  for (uint32_t col = 0; col < kCols; ++col)
    EXPECT_EQ(cu->lds().read32(wf->lds_base() + (kTileDim1RowCount * kCols + col) * 4), kSentinel);

  for (uint32_t i = 0; i < kTileDim1RowCount * kCols; ++i)
    cu->lds().write32(wf->lds_base() + i * 4, 0x77000000u + i);

  const std::array<uint32_t, 3> store_words = {0xd0714001u, 0x7c000000u, 0x18140c08u};
  gfx1250::TensorStoreFromLdsVimage store_inst(store_words.data());
  store_inst.execute_impl(*wf);

  for (uint32_t row = 0; row < kRows; ++row) {
    for (uint32_t col = 0; col < kCols; ++col) {
      const uint32_t actual = read_global_u32(*sim.memory, kStoreGlobal + (row * kCols + col) * 4);
      if (row < kTileDim1RowCount)
        EXPECT_EQ(actual, 0x77000000u + row * kCols + col);
      else
        EXPECT_EQ(actual, kSentinel);
    }
  }
}

TEST(Gfx1250ExecutionTest, TensorDmaGatherSupportsI32Indices) {
  Gfx1250Sim sim;
  auto *cu = sim.cu();
  auto *wf = cu->dispatch_wf(0, 0, kGfx1250ScalarSlots, 32);
  ASSERT_NE(wf, nullptr);
  wf->set_lds_base(cu->allocate_lds(512));

  constexpr uint64_t kGlobal = 0x170000;
  constexpr uint32_t kRows = 8;
  constexpr uint32_t kCols = 2;
  constexpr std::array<uint32_t, 5> kIndices = {7, 0, 3, 6, 1};
  constexpr uint32_t kSentinel = 0xBCADBCADu;

  write_tensor_dma_d0(*cu, *wf, 0, kGlobal);
  write_wave_sgpr(*cu, *wf, 0, 1u | (1u << 30) | (1u << 31));
  write_wave_sgpr(*cu, *wf, 12, 2u << 16);    // i32 elements.
  write_wave_sgpr(*cu, *wf, 13, kCols << 16); // Tensor dim0.
  write_wave_sgpr(*cu, *wf, 14, kRows << 16); // Tensor dim1.
  write_wave_sgpr(*cu, *wf, 15, kCols << 16); // Tile dim0.
  write_wave_sgpr(*cu, *wf, 16, static_cast<uint32_t>(kIndices.size()));
  write_wave_sgpr(*cu, *wf, 17, kCols); // Tensor dim0 stride.
  write_wave_sgpr(*cu, *wf, 18, 0);
  write_wave_sgpr(*cu, *wf, 19, 0);
  write_wave_sgpr(*cu, *wf, 20, kIndices[0]);
  write_wave_sgpr(*cu, *wf, 21, kIndices[1]);
  write_wave_sgpr(*cu, *wf, 22, kIndices[2]);
  write_wave_sgpr(*cu, *wf, 23, kIndices[3]);
  write_wave_sgpr(*cu, *wf, 24, kIndices[4]);
  write_wave_sgpr(*cu, *wf, 25, 0);
  write_wave_sgpr(*cu, *wf, 26, 0);
  write_wave_sgpr(*cu, *wf, 27, 0);

  for (uint32_t row = 0; row < kRows; ++row) {
    for (uint32_t col = 0; col < kCols; ++col)
      write_global_u32(*sim.memory, kGlobal + (row * kCols + col) * 4,
                       0x68000000u + row * 0x100u + col);
  }
  for (uint32_t i = 0; i < kIndices.size() * kCols; ++i)
    cu->lds().write32(wf->lds_base() + i * 4, kSentinel);

  const std::array<uint32_t, 3> load_words = {0xd0710001u, 0x7c000000u, 0x18140c00u};
  gfx1250::TensorLoadToLdsVimage load_inst(load_words.data());
  load_inst.execute_impl(*wf);

  for (uint32_t row_idx = 0; row_idx < kIndices.size(); ++row_idx) {
    const uint32_t src_row = kIndices[row_idx];
    for (uint32_t col = 0; col < kCols; ++col) {
      EXPECT_EQ(cu->lds().read32(wf->lds_base() + (row_idx * kCols + col) * 4),
                0x68000000u + src_row * 0x100u + col);
    }
  }
}

TEST(Gfx1250ExecutionTest, TensorDmaGatherSupportsI16Indices) {
  Gfx1250Sim sim;
  auto *cu = sim.cu();
  auto *wf = cu->dispatch_wf(0, 0, kGfx1250ScalarSlots, 32);
  ASSERT_NE(wf, nullptr);
  wf->set_lds_base(cu->allocate_lds(512));

  constexpr uint64_t kGlobal = 0x180000;
  constexpr uint32_t kRows = 8;
  constexpr uint32_t kCols = 2;
  constexpr std::array<uint32_t, 10> kIndices = {7, 0, 3, 6, 1, 4, 2, 5, 7, 3};
  constexpr uint32_t kSentinel = 0xCDAECDAEu;

  write_tensor_dma_d0(*cu, *wf, 0, kGlobal);
  write_wave_sgpr(*cu, *wf, 0, 1u | (1u << 31));
  write_wave_sgpr(*cu, *wf, 12, 2u << 16);    // i32 elements.
  write_wave_sgpr(*cu, *wf, 13, kCols << 16); // Tensor dim0.
  write_wave_sgpr(*cu, *wf, 14, kRows << 16); // Tensor dim1.
  write_wave_sgpr(*cu, *wf, 15, kCols << 16); // Tile dim0.
  write_wave_sgpr(*cu, *wf, 16, static_cast<uint32_t>(kIndices.size()));
  write_wave_sgpr(*cu, *wf, 17, kCols); // Tensor dim0 stride.
  write_wave_sgpr(*cu, *wf, 18, 0);
  write_wave_sgpr(*cu, *wf, 19, 0);
  write_wave_sgpr(*cu, *wf, 20, kIndices[0] | (kIndices[1] << 16));
  write_wave_sgpr(*cu, *wf, 21, kIndices[2] | (kIndices[3] << 16));
  write_wave_sgpr(*cu, *wf, 22, kIndices[4] | (kIndices[5] << 16));
  write_wave_sgpr(*cu, *wf, 23, kIndices[6] | (kIndices[7] << 16));
  write_wave_sgpr(*cu, *wf, 24, kIndices[8] | (kIndices[9] << 16));
  write_wave_sgpr(*cu, *wf, 25, 0);
  write_wave_sgpr(*cu, *wf, 26, 0);
  write_wave_sgpr(*cu, *wf, 27, 0);

  for (uint32_t row = 0; row < kRows; ++row) {
    for (uint32_t col = 0; col < kCols; ++col)
      write_global_u32(*sim.memory, kGlobal + (row * kCols + col) * 4,
                       0x69000000u + row * 0x100u + col);
  }
  for (uint32_t i = 0; i < kIndices.size() * kCols; ++i)
    cu->lds().write32(wf->lds_base() + i * 4, kSentinel);

  const std::array<uint32_t, 3> load_words = {0xd0710001u, 0x7c000000u, 0x18140c00u};
  gfx1250::TensorLoadToLdsVimage load_inst(load_words.data());
  load_inst.execute_impl(*wf);

  for (uint32_t row_idx = 0; row_idx < kIndices.size(); ++row_idx) {
    const uint32_t src_row = kIndices[row_idx];
    for (uint32_t col = 0; col < kCols; ++col) {
      EXPECT_EQ(cu->lds().read32(wf->lds_base() + (row_idx * kCols + col) * 4),
                0x69000000u + src_row * 0x100u + col);
    }
  }
}

TEST(Gfx1250CodeObjectTest, MachineFlagMapsToTarget) {
  auto image = make_minimal_gfx1250_elf();
  AmdGpuCodeObject co(image.data(), image.size());
  ASSERT_TRUE(co.is_valid());
  EXPECT_EQ(co.target_id(), ROCJITSU_CODE_TARGET_GFX1250);
  ASSERT_EQ(co.text_sections().size(), 1u);

  auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
  ASSERT_NE(decoder, nullptr);
  const auto *text = co.text_sections()[0];
  auto *words = reinterpret_cast<const uint32_t *>(text->data());
  std::unique_ptr<Instruction> inst(decoder->decode(words));
  ASSERT_NE(inst, nullptr);
  EXPECT_EQ(inst->mnemonic(), "s_endpgm");
}

TEST(Gfx1250DecodeTest, SMovB64Literal64ConsumesThreeDwords) {
  const uint32_t words[] = {
      0xBEB801FEu, // s_mov_b64 s[56:57], literal64
      0xFFFFFF80u,
      0xFFFFFFFFu,
  };

  auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
  ASSERT_NE(decoder, nullptr);
  std::unique_ptr<Instruction> inst(decoder->decode(words));
  ASSERT_NE(inst, nullptr);
  EXPECT_EQ(inst->mnemonic(), "s_mov_b64");
  EXPECT_EQ(inst->size(), sizeof(words));
}

TEST(Gfx1250DecodeTest, Vop3LiteralConsumesThreeDwords) {
  const uint32_t words[] = {
      0xD6570001u, // v_and_or_b32 v1, 0xf8, v1, v2
      0x040A02FFu,
      0x000000F8u,
  };

  auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
  ASSERT_NE(decoder, nullptr);
  std::unique_ptr<Instruction> inst(decoder->decode(words));
  ASSERT_NE(inst, nullptr);
  EXPECT_EQ(inst->mnemonic(), "v_and_or_b32");
  EXPECT_EQ(inst->size(), sizeof(words));
}

TEST(Gfx1250DecodeTest, Vop3SdstLiteralConsumesThreeDwords) {
  const uint32_t words[] = {
      0xD7020001u, // v_subrev_co_u32 v1, s0, 0x60, s12
      0x020018FFu,
      0x00000060u,
  };

  auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
  ASSERT_NE(decoder, nullptr);
  std::unique_ptr<Instruction> inst(decoder->decode(words));
  ASSERT_NE(inst, nullptr);
  EXPECT_EQ(inst->mnemonic(), "v_subrev_co_u32");
  EXPECT_EQ(inst->size(), sizeof(words));
}

TEST(Gfx1250DecodeTest, VFmamkF64ImpliedLiteralConsumesThreeDwords) {
  const uint32_t words[] = {
      0x46040504u, // v_fmamk_f64 v[2:3], v[4:5], -30.0, v[2:3]
      0x00000000u, 0xC1F00000u,
      0x7E042B02u, // v_cvt_u32_f64_e32 v2, v[2:3]
  };

  auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
  ASSERT_NE(decoder, nullptr);
  std::unique_ptr<Instruction> fmamk(decoder->decode(words));
  ASSERT_NE(fmamk, nullptr);
  EXPECT_EQ(fmamk->mnemonic(), "v_fmamk_f64_e32");
  EXPECT_EQ(fmamk->size(), 3 * sizeof(uint32_t));

  std::unique_ptr<Instruction> next(decoder->decode(words + 3));
  ASSERT_NE(next, nullptr);
  EXPECT_EQ(next->mnemonic(), "v_cvt_u32_f64_e32");
}

TEST(Gfx1250DecodeTest, SWaitXcntHasWaitcntMetadata) {
  const uint32_t words[] = {
      0xBFC50000u, // s_wait_xcnt 0
  };

  auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
  ASSERT_NE(decoder, nullptr);
  std::unique_ptr<Instruction> inst(decoder->decode(words));
  ASSERT_NE(inst, nullptr);
  EXPECT_EQ(inst->mnemonic(), "s_wait_xcnt");
  EXPECT_TRUE(inst->is_waitcnt());
  EXPECT_EQ(inst->disassemble(), "s_wait_xcnt 0");
}

TEST(Gfx1250DecodeTest, BufferOffenUsesSingleVaddrRegister) {
  const uint32_t words[] = {
      0xC405C07Cu, // buffer_load_b128 v[32:35], v7, s[4:7], s0 offen
      0x40800820u,
      0x00000007u,
  };

  auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
  ASSERT_NE(decoder, nullptr);
  std::unique_ptr<Instruction> inst(decoder->decode(words));
  ASSERT_NE(inst, nullptr);
  ASSERT_EQ(inst->mnemonic(), "buffer_load_b128");
  ASSERT_EQ(inst->num_dst_operands(), 1);
  ASSERT_EQ(inst->num_src_operands(), 4);

  const Operand *vdst = inst->dst_operand(0);
  ASSERT_NE(vdst, nullptr);
  EXPECT_FALSE(vdst->is_fieldless());
  EXPECT_EQ(vdst->name(), "v[32:35]");

  const Operand *vaddr = inst->src_operand(0);
  ASSERT_NE(vaddr, nullptr);
  EXPECT_FALSE(vaddr->is_fieldless());
  EXPECT_EQ(vaddr->size_bits(), 32);
  ASSERT_TRUE(vaddr->to_register_ref().has_value());
  EXPECT_EQ(*vaddr->to_register_ref(), (RegisterRef{RegClass::VGPR, 7, 1}));

  const Operand *gpumem = inst->src_operand(3);
  ASSERT_NE(gpumem, nullptr);
  EXPECT_TRUE(gpumem->is_fieldless());
  EXPECT_EQ(gpumem->size_bits(), 128);
  EXPECT_FALSE(gpumem->to_register_ref().has_value());
  // End-to-end: the decoded memory pseudo-operand is inert through the normal
  // accessors, driven by the capability flags the generated ctor applies.
  EXPECT_FALSE(gpumem->reads_value());
  EXPECT_FALSE(gpumem->is_writable());
  EXPECT_FALSE(gpumem->is_vgpr());

  EXPECT_EQ(inst->disassemble(), "buffer_load_b128 v[32:35], v7, s[4:7], NULL offen");
}

TEST(Gfx1250DecodeTest, BufferWithoutIdxenOffenDoesNotExposeVaddrRegister) {
  const uint32_t words[] = {
      0xC405C07Cu, // buffer_load_b128 v[32:35], s[4:7], s0
      0x00800820u,
      0x00000007u,
  };

  auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
  ASSERT_NE(decoder, nullptr);
  std::unique_ptr<Instruction> inst(decoder->decode(words));
  ASSERT_NE(inst, nullptr);
  ASSERT_EQ(inst->mnemonic(), "buffer_load_b128");
  ASSERT_EQ(inst->num_dst_operands(), 1);
  ASSERT_EQ(inst->num_src_operands(), 4);

  const Operand *vaddr = inst->src_operand(0);
  ASSERT_NE(vaddr, nullptr);
  EXPECT_FALSE(vaddr->is_fieldless());
  EXPECT_EQ(vaddr->size_bits(), 0);
  EXPECT_FALSE(vaddr->to_register_ref().has_value());

  const Operand *gpumem = inst->src_operand(3);
  ASSERT_NE(gpumem, nullptr);
  EXPECT_TRUE(gpumem->is_fieldless());
  EXPECT_EQ(gpumem->size_bits(), 128);
  EXPECT_FALSE(gpumem->to_register_ref().has_value());

  EXPECT_EQ(inst->disassemble(), "buffer_load_b128 v[32:35], s[4:7], NULL");
}

TEST(Gfx1250DecodeTest, WmmaF8f6f4UsesMatrixFormatOperandWidths) {
  const uint32_t words[] = {
      0xCC336010u,
      0x04421100u,
  };

  auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
  ASSERT_NE(decoder, nullptr);
  std::unique_ptr<Instruction> inst(decoder->decode(words));
  ASSERT_NE(inst, nullptr);
  EXPECT_EQ(inst->disassemble(), "v_wmma_f32_16x16x128_f8f6f4 v[16:23], v[0:7], v[8:15], v[16:23] "
                                 "matrix_a_fmt:MATRIX_FMT_FP4 matrix_b_fmt:MATRIX_FMT_FP4");
}

TEST(Gfx1250DecodeTest, WmmaScaleF8f6f4ConsumesVop3px2Pair) {
  const uint32_t words[] = {
      0xCC350000u,
      0x02020900u,
      0xCC330006u,
      0x02026912u,
  };

  auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
  ASSERT_NE(decoder, nullptr);
  std::unique_ptr<Instruction> inst(decoder->decode(words));
  ASSERT_NE(inst, nullptr);
  EXPECT_EQ(inst->mnemonic(), "v_wmma_scale_f32_16x16x128_f8f6f4");
  EXPECT_EQ(inst->size(), sizeof(words));
  EXPECT_EQ(inst->disassemble(),
            "v_wmma_scale_f32_16x16x128_f8f6f4 v[6:13], v[18:33], v[52:67], 0, v0, v4");
}

TEST(Gfx1250DecodeTest, WmmaScale16F8f6f4ConsumesVop3px2Pair) {
  const uint32_t words[] = {
      0xCC3A0000u,
      0x0202391Au,
      0xCC336012u,
      0x02021502u,
  };

  auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
  ASSERT_NE(decoder, nullptr);
  std::unique_ptr<Instruction> inst(decoder->decode(words));
  ASSERT_NE(inst, nullptr);
  EXPECT_EQ(inst->mnemonic(), "v_wmma_scale16_f32_16x16x128_f8f6f4");
  EXPECT_EQ(inst->size(), sizeof(words));
  EXPECT_EQ(inst->disassemble(),
            "v_wmma_scale16_f32_16x16x128_f8f6f4 v[18:25], v[2:9], v[10:17], 0, "
            "v[26:27], v[28:29] matrix_a_fmt:MATRIX_FMT_FP4 matrix_b_fmt:MATRIX_FMT_FP4");
}

TEST(Gfx1250DecodeTest, WmmaScaleF4_32x16x128ConsumesVop3px2Pair) {
  const uint32_t words[] = {
      0xCC350000u,
      0x02025328u,
      0xCC884000u,
      0x1A024110u,
  };

  auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
  ASSERT_NE(decoder, nullptr);
  std::unique_ptr<Instruction> inst(decoder->decode(words));
  ASSERT_NE(inst, nullptr);
  EXPECT_EQ(inst->mnemonic(), "v_wmma_scale_f32_32x16x128_f4");
  EXPECT_EQ(inst->size(), sizeof(words));
  EXPECT_EQ(inst->disassemble(),
            "v_wmma_scale_f32_32x16x128_f4 v[0:15], v[16:31], v[32:39], 0, v40, v41");
}

TEST(Gfx1250ExecutionTest, WmmaRegularScaleInlineZeroMatchesNeutralScalarSources) {
  Gfx1250Sim sim;
  auto *cu = sim.cu();
  auto *wf = sim.dispatch_scratch_wf();
  ASSERT_NE(wf, nullptr);
  wf->set_exec(0xffffffffu);

  constexpr uint16_t kVgprEncoding = 256;
  constexpr auto scalar_prefix =
      gfx1250::build_vop3p(0x35, {.src0 = 0, .src1 = 1, .src2 = kVgprEncoding});
  constexpr auto inline_prefix =
      gfx1250::build_vop3p(0x35, {.src0 = 128, .src1 = 128, .src2 = kVgprEncoding});
  constexpr auto scalar_matrix = gfx1250::build_vop3p(
      gfx1250::kVWmmaF3216x16x128F8f6f4Vop3p,
      {.vdst = 32, .src0 = kVgprEncoding, .src1 = kVgprEncoding + 16, .src2 = kVgprEncoding + 32});
  constexpr auto inline_matrix = gfx1250::build_vop3p(
      gfx1250::kVWmmaF3216x16x128F8f6f4Vop3p,
      {.vdst = 40, .src0 = kVgprEncoding, .src1 = kVgprEncoding + 16, .src2 = kVgprEncoding + 40});
  const std::array<uint32_t, 4> scalar_words = {scalar_prefix[0], scalar_prefix[1],
                                                scalar_matrix[0], scalar_matrix[1]};
  const std::array<uint32_t, 4> inline_words = {inline_prefix[0], inline_prefix[1],
                                                inline_matrix[0], inline_matrix[1]};

  write_wave_sgpr(*cu, *wf, 0, 0x7f7f7f7fu);
  write_wave_sgpr(*cu, *wf, 1, 0x7f7f7f7fu);
  const uint32_t vgpr_base = wf->vgpr_alloc().base;
  for (uint32_t reg = 0; reg < 32; ++reg)
    for (uint32_t lane = 0; lane < wf->wf_size(); ++lane)
      cu->write_vgpr(vgpr_base + reg, lane, 0x38383838u);

  auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
  ASSERT_NE(decoder, nullptr);
  std::unique_ptr<Instruction> scalar_inst(decoder->decode(scalar_words.data()));
  std::unique_ptr<Instruction> inline_inst(decoder->decode(inline_words.data()));
  ASSERT_NE(scalar_inst, nullptr);
  ASSERT_NE(inline_inst, nullptr);
  cu->execute_instruction(scalar_inst.get(), *wf);
  cu->execute_instruction(inline_inst.get(), *wf);

  for (uint32_t reg = 0; reg < 8; ++reg)
    for (uint32_t lane = 0; lane < wf->wf_size(); ++lane) {
      const uint32_t scalar_result = cu->read_vgpr(vgpr_base + 32 + reg, lane);
      EXPECT_EQ(scalar_result, std::bit_cast<uint32_t>(128.0f))
          << "reg " << reg << ", lane " << lane;
      EXPECT_EQ(cu->read_vgpr(vgpr_base + 40 + reg, lane), scalar_result)
          << "reg " << reg << ", lane " << lane;
    }
}

TEST(Gfx1250DecodeTest, SwmmacPrintsIndexKeyAndReuseModifiers) {
  auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
  ASSERT_NE(decoder, nullptr);

  const uint32_t index_key_words[] = {
      0xCC65081Au,
      0x1C8E0112u,
  };
  std::unique_ptr<Instruction> index_key_inst(decoder->decode(index_key_words));
  ASSERT_NE(index_key_inst, nullptr);
  EXPECT_EQ(index_key_inst->disassemble(),
            "v_swmmac_f32_16x16x64_f16 v[26:33], v[18:25], v[0:15], v35 index_key:1");

  const uint32_t matrix_a_reuse_words[] = {
      0xCC65201Au,
      0x1C8E0112u,
  };
  std::unique_ptr<Instruction> matrix_a_reuse_inst(decoder->decode(matrix_a_reuse_words));
  ASSERT_NE(matrix_a_reuse_inst, nullptr);
  EXPECT_EQ(matrix_a_reuse_inst->disassemble(),
            "v_swmmac_f32_16x16x64_f16 v[26:33], v[18:25], v[0:15], v35 matrix_a_reuse");

  const uint32_t matrix_b_reuse_words[] = {
      0xCC65401Au,
      0x1C8E0112u,
  };
  std::unique_ptr<Instruction> matrix_b_reuse_inst(decoder->decode(matrix_b_reuse_words));
  ASSERT_NE(matrix_b_reuse_inst, nullptr);
  EXPECT_EQ(matrix_b_reuse_inst->disassemble(),
            "v_swmmac_f32_16x16x64_f16 v[26:33], v[18:25], v[0:15], v35 matrix_b_reuse");
}

TEST(Gfx1250DecodeTest, VopdXyConsumesTwoDwords) {
  const uint32_t words[] = {
      0xCA500501u, // v_dual_cndmask_b32 v2, v1, v2 :: v_dual_mov_b32 v1, 0
      0x02000080u,
  };

  auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
  ASSERT_NE(decoder, nullptr);
  std::unique_ptr<Instruction> inst(decoder->decode(words));
  ASSERT_NE(inst, nullptr);
  EXPECT_EQ(inst->mnemonic(), "v_dual_cndmask_b32 :: v_dual_mov_b32");
  EXPECT_EQ(inst->size(), sizeof(words));
  EXPECT_NE(inst->disassemble().find("v_dual_cndmask_b32"), std::string::npos);
  EXPECT_NE(inst->disassemble().find("v_dual_mov_b32"), std::string::npos);
}

TEST(Gfx1250DecodeTest, Vopd3ConsumesThreeDwords) {
  const uint32_t words[] = {
      0xCF455083u, // v_dual_lshlrev_b32 v1, 3, v0 :: v_dual_lshrrev_b32 v10, 6, v0
      0x00000086u,
      0x0A000001u,
  };

  auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
  ASSERT_NE(decoder, nullptr);
  std::unique_ptr<Instruction> inst(decoder->decode(words));
  ASSERT_NE(inst, nullptr);
  EXPECT_EQ(inst->mnemonic(), "v_dual_lshlrev_b32 :: v_dual_lshrrev_b32");
  EXPECT_EQ(inst->size(), sizeof(words));
  EXPECT_NE(inst->disassemble().find("v_dual_lshlrev_b32"), std::string::npos);
  EXPECT_NE(inst->disassemble().find("v_dual_lshrrev_b32"), std::string::npos);
}

TEST(Gfx1250DecodeTest, VopdLiteralConsumesThreeDwords) {
  const uint32_t words[] = {
      0xC8D006FFu, // v_dual_mul_f32 v4, 0x4f7ffffe, v3 :: v_dual_mov_b32 v3, 0
      0x04020080u,
      0x4F7FFFFEu,
  };

  auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
  ASSERT_NE(decoder, nullptr);
  std::unique_ptr<Instruction> inst(decoder->decode(words));
  ASSERT_NE(inst, nullptr);
  EXPECT_EQ(inst->mnemonic(), "v_dual_mul_f32 :: v_dual_mov_b32");
  EXPECT_EQ(inst->size(), sizeof(words));
  EXPECT_NE(inst->disassemble().find("0x4f7ffffe"), std::string::npos);
}

TEST(Gfx1250DecodeTest, VopdSourceOperandsFollowPrintedSlots) {
  const uint32_t words[] = {
      0xCF448082u, // v_dual_lshlrev_b32 v17, 2, v9 :: v_dual_mov_b32 v9, s11
      0x0009000Bu,
      0x09000011u,
  };

  auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
  ASSERT_NE(decoder, nullptr);
  std::unique_ptr<Instruction> inst(decoder->decode(words));
  ASSERT_NE(inst, nullptr);
  EXPECT_EQ(inst->mnemonic(), "v_dual_lshlrev_b32 :: v_dual_mov_b32");
  EXPECT_EQ(inst->num_src_operands(), 3);
  ASSERT_NE(inst->src_operand(2), nullptr);
  EXPECT_EQ(inst->src_operand(2)->name(), "s11");
  ASSERT_TRUE(inst->src_operand(2)->to_register_ref().has_value());
  EXPECT_EQ(*inst->src_operand(2)->to_register_ref(), (RegisterRef{RegClass::SGPR, 11, 1}));
}

TEST(Gfx1250SimulationTest, DispatchesEndpgmThroughConfig) {
  Gfx1250Sim sim;
  const uint32_t code[] = {S_ENDPGM_GFX12};
  uint64_t kernel_object = sim.write_kernel(0x10000, code, std::size(code));

  test::AqlQueue queue(sim.memory, sim.cp());
  queue.dispatch(kernel_object, 32, 32);
  step_until_halted(*sim.engine, *sim.cu());

  EXPECT_EQ(sim.cp()->dispatched_count(), 1u);
  ASSERT_EQ(sim.snapshot->snapshots().size(), 1u);
  EXPECT_EQ(sim.snapshot->snapshots().front().wf_size, 32u);
  EXPECT_EQ(sim.cu()->num_wfs(), 0u);
}

TEST(Gfx1250SimulationTest, MultiWaveDispatchHonorsPackedTidComponentCount) {
  const uint32_t code[] = {S_ENDPGM_GFX12};

  for (uint32_t component_count = 0; component_count <= 1; ++component_count) {
    SCOPED_TRACE("component_count=" + std::to_string(component_count));
    Gfx1250Sim sim;
    uint64_t kernel_object = sim.write_kernel(0x10000, code, std::size(code), 104, 32, 2, false,
                                              false, false, 0, 0, 0, 0, component_count);

    test::AqlQueue queue(sim.memory, sim.cp());
    hsa_kernel_dispatch_packet_t pkt{};
    pkt.header = HSA_PACKET_TYPE_KERNEL_DISPATCH;
    pkt.setup = 2;
    pkt.workgroup_size_x = 32;
    pkt.workgroup_size_y = 4;
    pkt.workgroup_size_z = 1;
    pkt.grid_size_x = 32;
    pkt.grid_size_y = 4;
    pkt.grid_size_z = 1;
    pkt.kernel_object = kernel_object;
    queue.submit(pkt);
    step_until_xcd_halted(sim);

    std::vector<uint32_t> lane0_values;
    std::vector<uint32_t> lane31_values;
    for (const auto &wf : sim.snapshot->snapshots()) {
      lane0_values.push_back(wf.vgpr(0, 0));
      lane31_values.push_back(wf.vgpr(0, 31));
    }

    std::sort(lane0_values.begin(), lane0_values.end());
    std::sort(lane31_values.begin(), lane31_values.end());
    const uint32_t y_scale = component_count >= 1 ? 1u << 10 : 0;
    const std::vector<uint32_t> expected_lane0{0u, y_scale, 2 * y_scale, 3 * y_scale};
    const std::vector<uint32_t> expected_lane31{31u, 31u | y_scale, 31u | (2 * y_scale),
                                                31u | (3 * y_scale)};
    EXPECT_EQ(lane0_values, expected_lane0);
    EXPECT_EQ(lane31_values, expected_lane31);
  }
}

// Collect the EXEC mask each wavefront had at halt (captured before it freed).
std::vector<uint64_t> collect_active_exec_masks(Gfx1250Sim &sim) {
  std::vector<uint64_t> exec_masks;
  for (const auto &wf : sim.snapshot->snapshots())
    exec_masks.push_back(wf.exec);
  std::sort(exec_masks.begin(), exec_masks.end());
  return exec_masks;
}

TEST(Gfx1250SimulationTest, PartialWorkgroupMasksTailWaveExec) {
  Gfx1250Sim sim;
  const uint32_t code[] = {S_ENDPGM_GFX12};
  uint64_t kernel_object = sim.write_kernel(0x10000, code, std::size(code));

  test::AqlQueue queue(sim.memory, sim.cp());
  queue.dispatch(kernel_object, 33, 33);
  step_until_xcd_halted(sim);

  // One 33-thread workgroup exercises the intra-workgroup tail wave.
  const std::vector<uint64_t> expected{1ULL, 0xFFFFFFFFULL};
  EXPECT_EQ(collect_active_exec_masks(sim), expected);
}

// Count the distinct workgroup ids across all wavefronts activated for the
// (single) dispatch. dispatched_count() counts dispatch packets, not
// workgroups, so it cannot observe the rounding; wavefront wg_ids can.
uint32_t count_dispatched_workgroups(Gfx1250Sim &sim) {
  std::set<uint32_t> wg_ids;
  for (const auto &wf : sim.snapshot->snapshots())
    wg_ids.insert(wf.wg_id);
  return static_cast<uint32_t>(wg_ids.size());
}

// The grid-to-workgroup count must round up: a partial final workgroup still
// counts. With grid_size_x=65 and workgroup_size_x=32, HW dispatches 3 WGs
// (ceil(65/32)); the old floor division dropped the tail WG and dispatched 2.
TEST(Gfx1250SimulationTest, PartialFinalWorkgroupRoundsUpDispatchCount) {
  Gfx1250Sim sim;
  const uint32_t code[] = {S_ENDPGM_GFX12};
  uint64_t kernel_object = sim.write_kernel(0x10000, code, std::size(code));

  test::AqlQueue queue(sim.memory, sim.cp());
  queue.dispatch(kernel_object, /*grid_size_x=*/65, /*workgroup_size_x=*/32);
  step_until_xcd_halted(sim);

  EXPECT_EQ(count_dispatched_workgroups(sim), 3u);
}

// Same rounding rule in 2D: grid 65x33 with workgroups 32x16 dispatches
// ceil(65/32) * ceil(33/16) = 3 * 3 = 9 workgroups, not floor's 2 * 2 = 4.
TEST(Gfx1250SimulationTest, PartialFinalWorkgroupRoundsUpDispatchCount2D) {
  Gfx1250Sim sim;
  const uint32_t code[] = {S_ENDPGM_GFX12};
  uint64_t kernel_object = sim.write_kernel(0x10000, code, std::size(code));

  test::AqlQueue queue(sim.memory, sim.cp());
  hsa_kernel_dispatch_packet_t pkt{};
  pkt.header = HSA_PACKET_TYPE_KERNEL_DISPATCH;
  pkt.setup = 2; // 2D grid.
  pkt.workgroup_size_x = 32;
  pkt.workgroup_size_y = 16;
  pkt.workgroup_size_z = 1;
  pkt.grid_size_x = 65;
  pkt.grid_size_y = 33;
  pkt.grid_size_z = 1;
  pkt.kernel_object = kernel_object;
  queue.submit(pkt);
  step_until_xcd_halted(sim);

  EXPECT_EQ(count_dispatched_workgroups(sim), 9u);
}

TEST(Gfx1250SimulationTest, PartialGridTailMasksFinalWorkgroupExec) {
  Gfx1250Sim sim;
  const uint32_t code[] = {S_ENDPGM_GFX12};
  uint64_t kernel_object = sim.write_kernel(0x10000, code, std::size(code));

  test::AqlQueue queue(sim.memory, sim.cp());
  queue.dispatch(kernel_object, 33, 32);
  step_until_xcd_halted(sim);

  // Unlike PartialWorkgroupMasksTailWaveExec, this uses two full-size
  // workgroups and exercises the grid-bounds mask on the final workgroup.
  const std::vector<uint64_t> expected{1ULL, 0xFFFFFFFFULL};
  EXPECT_EQ(collect_active_exec_masks(sim), expected);
}

TEST(Gfx1250SimulationTest, Partial2DGridTailMasksNonContiguousExecLanes) {
  Gfx1250Sim sim;
  const uint32_t code[] = {S_ENDPGM_GFX12};
  uint64_t kernel_object = sim.write_kernel(0x10000, code, std::size(code));

  test::AqlQueue queue(sim.memory, sim.cp());
  hsa_kernel_dispatch_packet_t pkt{};
  pkt.header = HSA_PACKET_TYPE_KERNEL_DISPATCH;
  pkt.setup = 2;
  pkt.workgroup_size_x = 8;
  pkt.workgroup_size_y = 2;
  pkt.workgroup_size_z = 1;
  pkt.grid_size_x = 13;
  pkt.grid_size_y = 2;
  pkt.grid_size_z = 1;
  pkt.kernel_object = kernel_object;
  queue.submit(pkt);
  step_until_xcd_halted(sim);

  const std::vector<uint64_t> expected{0x1F1FULL, 0xFFFFULL};
  EXPECT_EQ(collect_active_exec_masks(sim), expected);
}

TEST(Gfx1250SimulationTest, DispatchPreloadsKernargDwordsIntoUserSgprs) {
  using namespace rocr::llvm::amdhsa;

  constexpr uint64_t kKernelAddr = 0x10000;
  constexpr uint64_t kKernargAddr = 0x400000;
  const uint32_t code[] = {S_ENDPGM_GFX12};
  struct Args {
    uint32_t skip;
    uint32_t first;
    uint32_t second;
  } args{0x11111111u, 0x22222222u, 0x33333333u};

  uint32_t kernel_code_properties = 0;
  AMDHSA_BITS_SET(kernel_code_properties, KERNEL_CODE_PROPERTY_ENABLE_SGPR_KERNARG_SEGMENT_PTR, 1);

  Gfx1250Sim sim;
  sim.memory->load_image(reinterpret_cast<const uint8_t *>(&args), sizeof(args), kKernargAddr);
  uint64_t kernel_object =
      sim.write_kernel(kKernelAddr, code, std::size(code), 104, 32, 4, false, false, false,
                       kernel_code_properties, sizeof(args), 2, 1);

  test::AqlQueue queue(sim.memory, sim.cp());
  queue.dispatch(kernel_object, 32, 32, kKernargAddr);
  step_until_halted(*sim.engine, *sim.cu());

  ASSERT_EQ(sim.snapshot->snapshots().size(), 1u);
  const auto &wf = sim.snapshot->snapshots().front();
  EXPECT_EQ(wf.sgpr64(0), kKernargAddr);
  EXPECT_EQ(wf.sgpr(2), args.first);
  EXPECT_EQ(wf.sgpr(3), args.second);
}

TEST(Gfx1250SimulationTest, DispatchPreloadsKernargWhenDescriptorSizeIsUnknown) {
  using namespace rocr::llvm::amdhsa;

  constexpr uint64_t kKernelAddr = 0x10000;
  constexpr uint64_t kKernargAddr = 0x400000;
  const uint32_t code[] = {S_ENDPGM_GFX12};
  const std::array<uint32_t, 3> args{0x11111111u, 0x22222222u, 0x33333333u};

  uint32_t kernel_code_properties = 0;
  AMDHSA_BITS_SET(kernel_code_properties, KERNEL_CODE_PROPERTY_ENABLE_SGPR_KERNARG_SEGMENT_PTR, 1);

  Gfx1250Sim sim;
  sim.memory->load_image(reinterpret_cast<const uint8_t *>(args.data()),
                         args.size() * sizeof(args[0]), kKernargAddr);
  uint64_t kernel_object = sim.write_kernel(kKernelAddr, code, std::size(code), 104, 32, 4, false,
                                            false, false, kernel_code_properties, 0, 2, 1);

  test::AqlQueue queue(sim.memory, sim.cp());
  queue.dispatch(kernel_object, 32, 32, kKernargAddr);
  step_until_halted(*sim.engine, *sim.cu());

  ASSERT_EQ(sim.snapshot->snapshots().size(), 1u);
  const auto &wf = sim.snapshot->snapshots().front();
  EXPECT_EQ(wf.sgpr64(0), kKernargAddr);
  EXPECT_EQ(wf.sgpr(2), args[1]);
  EXPECT_EQ(wf.sgpr(3), args[2]);
}

TEST(Gfx1250SimulationTest, SLoadB32DoesNotScaleImmediateOffset) {
  using namespace rocr::llvm::amdhsa;

  constexpr uint64_t kKernelAddr = 0x10000;
  constexpr uint64_t kKernargAddr = 0x400000;
  constexpr uint32_t kExpected = 0x12345678u;

  std::vector<uint32_t> code;
  // s_load_b32 s4, s[0:1], 0x4 scale_offset
  append_instruction(code, make_s_load_b32_scaled_imm(4, 0, 4));
  append_instruction(code, S_WAIT_KMCNT_0_GFX12);
  append_instruction(code, S_ENDPGM_GFX12);

  uint32_t kernel_code_properties = 0;
  AMDHSA_BITS_SET(kernel_code_properties, KERNEL_CODE_PROPERTY_ENABLE_SGPR_KERNARG_SEGMENT_PTR, 1);

  Gfx1250Sim sim;
  write_global_u32(*sim.memory, kKernargAddr + 4, kExpected);
  uint64_t kernel_object = sim.write_kernel(kKernelAddr, code.data(), code.size(), 104, 32, 2,
                                            false, false, false, kernel_code_properties, 16);

  test::AqlQueue queue(sim.memory, sim.cp());
  queue.dispatch(kernel_object, 32, 32, kKernargAddr);
  step_until_halted(*sim.engine, *sim.cu());

  ASSERT_EQ(sim.snapshot->snapshots().size(), 1u);
  EXPECT_EQ(sim.snapshot->snapshots().front().sgpr(4), kExpected);
}

TEST(Gfx1250SimulationTest, TtmpWorkgroupIdsUseGridCoordinatesFor2DDispatch) {
  Gfx1250Sim sim;
  const uint32_t code[] = {S_ENDPGM_GFX12};
  uint64_t kernel_object =
      sim.write_kernel(0x10000, code, std::size(code), 104, 32, 2, true, false, false);

  test::AqlQueue queue(sim.memory, sim.cp());
  hsa_kernel_dispatch_packet_t pkt{};
  pkt.header = HSA_PACKET_TYPE_KERNEL_DISPATCH;
  pkt.setup = 2;
  pkt.workgroup_size_x = 32;
  pkt.workgroup_size_y = 1;
  pkt.workgroup_size_z = 1;
  pkt.grid_size_x = 96;
  pkt.grid_size_y = 2;
  pkt.grid_size_z = 1;
  pkt.kernel_object = kernel_object;
  queue.submit(pkt);
  step_until_xcd_halted(sim);

  const auto *target = sim.snapshot->by_wg_id(4);
  ASSERT_NE(target, nullptr);
  // TTMP9 (s117) holds grid_wg_id_x; TTMP7 (s115) packs wg_id_y/z.
  EXPECT_EQ(target->sgpr(117), 1u);
  EXPECT_EQ(target->sgpr(115), 1u);
}

TEST(Gfx1250SimulationTest, Ttmp7ClusterGridYDoesNotBleedIntoZAt16BitBoundary) {
  const uint32_t code[] = {S_ENDPGM_GFX12};

  Gfx1250Sim sim;
  uint64_t kernel_object = sim.write_kernel(0x10000, code, std::size(code), 128);
  constexpr uint32_t kWorkgroupIdOffset = 0x10000;
  sim.cp()->set_workgroup_id_offset(kWorkgroupIdOffset);

  hsa_kernel_dispatch_packet_t pkt{};
  pkt.header = HSA_PACKET_TYPE_KERNEL_DISPATCH;
  pkt.setup = 2;
  pkt.workgroup_size_x = 32;
  pkt.workgroup_size_y = 1;
  pkt.workgroup_size_z = 1;
  pkt.grid_size_x = 32;
  pkt.grid_size_y = 0x10001;
  pkt.grid_size_z = 1;
  pkt.kernel_object = kernel_object;

  test::AqlQueue queue(sim.memory, sim.cp());
  queue.submit(pkt);
  ASSERT_TRUE(sim.engine->step());

  amdgpu::Wavefront *target = nullptr;
  amdgpu::ComputeUnitCore *target_cu = nullptr;
  for (uint32_t se_idx = 0; se_idx < sim.xcd()->num_shader_engines(); ++se_idx) {
    auto *se = sim.xcd()->shader_engine(se_idx);
    for (uint32_t cu_idx = 0; cu_idx < se->num_compute_units(); ++cu_idx) {
      auto *cu = se->compute_unit(cu_idx);
      for (uint32_t wf_idx = 0; wf_idx < cu->num_wf_slots(); ++wf_idx) {
        auto *wf = cu->wf(wf_idx);
        if (wf && wf->sgpr_alloc().count > 0 && wf->wg_id() == kWorkgroupIdOffset) {
          target = wf;
          target_cu = cu;
          break;
        }
      }
      if (target)
        break;
    }
    if (target)
      break;
  }

  ASSERT_NE(target, nullptr);
  ASSERT_NE(target_cu, nullptr);
  EXPECT_EQ(target_cu->read_sgpr(target->sgpr_alloc().base + 115), 0u);
}

TEST(Gfx1250SimulationTest, IbSts2ClusterFieldsAreZeroForOrdinaryDispatch) {
  const uint32_t code[] = {
      0xB882199Cu, // s_getreg_b32 s2, hwreg(IB_STS2, 6, 4)
      0xB8831D5Cu, // s_getreg_b32 s3, hwreg(IB_STS2, 21, 4)
      S_ENDPGM_GFX12,
  };

  Gfx1250Sim sim;
  uint64_t kernel_object = sim.write_kernel(0x10000, code, std::size(code), 128);
  test::AqlQueue queue(sim.memory, sim.cp());
  queue.dispatch(kernel_object, /*grid_size_x=*/32, /*workgroup_size_x=*/32);
  step_until_halted(*sim.engine, *sim.cu());

  auto *wf = sim.cu()->wf(0);
  ASSERT_NE(wf, nullptr);
  const uint32_t sbase = wf->sgpr_alloc().base;
  EXPECT_EQ(sim.cu()->read_sgpr(sbase + 2), 0u);
  EXPECT_EQ(sim.cu()->read_sgpr(sbase + 3), 0u);
}

TEST(Gfx1250SimulationTest, DynamicClusterLaunchStateMatchesCompilerAbiWithAlignedOffset) {
  // Scalar workgroup-ID reconstruction emitted by clang 23 when cluster
  // dimensions are not fixed by an amdgpu-cluster-dims attribute.
  const uint32_t code[] = {
      0x9302FF72u,    0x0004000Cu, // s_bfe_u32 s2, ttmp6, 0x4000c
      0x9305FF72u,    0x00040010u, // s_bfe_u32 s5, ttmp6, 0x40010
      0x9309FF72u,    0x00040014u, // s_bfe_u32 s9, ttmp6, 0x40014
      0x81038102u,                 // s_add_co_i32 s3, s2, 1
      0x8B06FF73u,    0x0000FFFFu, // s_and_b32 s6, ttmp7, 0xffff
      0x81078105u,                 // s_add_co_i32 s7, s5, 1
      0x850A9073u,                 // s_lshr_b32 s10, ttmp7, 16
      0x810B8109u,                 // s_add_co_i32 s11, s9, 1
      0x8B048F72u,                 // s_and_b32 s4, ttmp6, 15
      0x96030375u,                 // s_mul_i32 s3, ttmp9, s3
      0x96070706u,                 // s_mul_i32 s7, s6, s7
      0x9308FF72u,    0x00040004u, // s_bfe_u32 s8, ttmp6, 0x40004
      0x960B0B0Au,                 // s_mul_i32 s11, s10, s11
      0x930CFF72u,    0x00040008u, // s_bfe_u32 s12, ttmp6, 0x40008
      0xB88D199Cu,                 // s_getreg_b32 s13, hwreg(IB_STS2, 6, 4)
      0x81030304u,                 // s_add_co_i32 s3, s4, s3
      0x81070708u,                 // s_add_co_i32 s7, s8, s7
      0x810B0B0Cu,                 // s_add_co_i32 s11, s12, s11
      0xBF06800Du,                 // s_cmp_eq_u32 s13, 0
      0x980A0B0Au,                 // s_cselect_b32 s10, s10, s11
      0x98030375u,                 // s_cselect_b32 s3, ttmp9, s3
      0x98060706u,                 // s_cselect_b32 s6, s6, s7
      0xB8821D5Cu,                 // s_getreg_b32 s2, hwreg(IB_STS2, 21, 4)
      S_ENDPGM_GFX12,
  };

  Gfx1250Sim sim;
  uint64_t kernel_object = sim.write_kernel(0x10000, code, std::size(code), 128);
  constexpr uint32_t kWorkgroupIdOffset = 16;
  sim.cp()->set_workgroup_id_offset(kWorkgroupIdOffset);

  amdgpu::AmdExtKernelDispatchPacket pkt{};
  pkt.header = HSA_PACKET_TYPE_VENDOR_SPECIFIC;
  pkt.amd_format = amdgpu::kHsaAmdPacketTypeExtKernelDispatch;
  pkt.setup = 3;
  pkt.workgroup_size_x = 32;
  pkt.workgroup_size_y = 1;
  pkt.workgroup_size_z = 1;
  pkt.cluster_count_x = 2;
  pkt.cluster_count_y = 2;
  pkt.cluster_count_z = 2;
  pkt.cluster_size_x = 4;
  pkt.cluster_size_y = 2;
  pkt.cluster_size_z = 1;
  pkt.kernel_object = kernel_object;

  test::AqlQueue queue(sim.memory, sim.cp());
  queue.submit(pkt);
  step_until_xcd_halted(sim);

  struct LaunchState {
    uint32_t workgroup_id;
    uint32_t ttmp6;
    uint32_t ttmp7;
    uint32_t ttmp9;
    uint32_t cluster_rank;
    uint32_t global_x;
    uint32_t global_y;
    uint32_t global_z;
    uint32_t cluster_id;
  };
  std::vector<LaunchState> states;
  for (const auto &wf : sim.snapshot->snapshots())
    states.push_back({wf.wg_id, wf.sgpr(114), wf.sgpr(115), wf.sgpr(117), wf.sgpr(2), wf.sgpr(3),
                      wf.sgpr(6), wf.sgpr(10), wf.sgpr(13)});
  std::sort(states.begin(), states.end(), [](const LaunchState &lhs, const LaunchState &rhs) {
    return lhs.workgroup_id < rhs.workgroup_id;
  });

  ASSERT_EQ(states.size(), 64u);
  for (uint32_t local_workgroup_id = 0; local_workgroup_id < states.size(); ++local_workgroup_id) {
    const uint32_t workgroup_id = local_workgroup_id + kWorkgroupIdOffset;
    const uint32_t workgroup_x = workgroup_id % 8;
    const uint32_t workgroup_y = (workgroup_id / 8) % 4;
    const uint32_t workgroup_z = workgroup_id / 32;
    const uint32_t local_x = workgroup_x % 4;
    const uint32_t local_y = workgroup_y % 2;
    const uint32_t local_z = workgroup_z % 1;
    const auto &state = states[local_workgroup_id];
    EXPECT_EQ(state.workgroup_id, workgroup_id);
    EXPECT_EQ(state.ttmp6, 0x07013000u | local_x | (local_y << 4) | (local_z << 8));
    EXPECT_EQ(state.ttmp7, (workgroup_z << 16) | (workgroup_y / 2));
    EXPECT_EQ(state.ttmp9, workgroup_x / 4);
    EXPECT_EQ(state.cluster_rank, local_x + 4 * local_y);
    EXPECT_EQ(state.global_x, workgroup_x);
    EXPECT_EQ(state.global_y, workgroup_y);
    EXPECT_EQ(state.global_z, workgroup_z);
    EXPECT_EQ(state.cluster_id, 1u);
  }
}

TEST(Gfx1250SimulationTest, SGetPcI64ReturnsNextInstructionAddress) {
  constexpr uint64_t kKernelAddr = 0x10000;
  const uint32_t code[] = {
      0xBE844700u, // s_get_pc_i64 s[4:5]
      S_ENDPGM_GFX12,
  };

  Gfx1250Sim sim;
  uint64_t kernel_object = sim.write_kernel(kKernelAddr, code, std::size(code));
  test::AqlQueue queue(sim.memory, sim.cp());
  queue.dispatch(kernel_object, 32, 32);
  step_until_halted(*sim.engine, *sim.cu());

  ASSERT_EQ(sim.snapshot->snapshots().size(), 1u);
  const auto &wf = sim.snapshot->snapshots().front();
  uint64_t entry_pc = kKernelAddr + sizeof(rocr::llvm::amdhsa::kernel_descriptor_t);
  EXPECT_EQ(wf.sgpr64(4), entry_pc + sizeof(uint32_t));
}

TEST(Gfx1250SimulationTest, SAddPcI64SkipsRelativeToNextPc) {
  const uint32_t code[] = {
      0xBE804B84u, // s_add_pc_i64 4
      0xBE840081u, // s_mov_b32 s4, 1
      0xBE840082u, // s_mov_b32 s4, 2
      S_ENDPGM_GFX12,
  };

  Gfx1250Sim sim;
  const auto *wf = dispatch_one_wave(sim, code, std::size(code));
  ASSERT_NE(wf, nullptr);
  EXPECT_EQ(wf->sgpr(4), 2u);
}

TEST(Gfx1250SimulationTest, SSetPcI64JumpsToScalarAddress) {
  constexpr uint64_t kKernelAddr = 0x10000;
  constexpr uint32_t kTargetWord = 5;
  uint64_t entry_pc = kKernelAddr + sizeof(rocr::llvm::amdhsa::kernel_descriptor_t);
  uint64_t target_pc = entry_pc + kTargetWord * sizeof(uint32_t);
  std::vector<uint32_t> code = {
      0xBE8400FFu,    static_cast<uint32_t>(target_pc), // s_mov_b32 s4, target_pc[31:0]
      0xBE850080u,                                      // s_mov_b32 s5, 0
      0xBE804804u,                                      // s_set_pc_i64 s[4:5]
      0xBE860081u,                                      // s_mov_b32 s6, 1
      0xBE860082u,                                      // s_mov_b32 s6, 2
      S_ENDPGM_GFX12,
  };

  Gfx1250Sim sim;
  uint64_t kernel_object = sim.write_kernel(kKernelAddr, code.data(), code.size());
  test::AqlQueue queue(sim.memory, sim.cp());
  queue.dispatch(kernel_object, 32, 32);
  step_until_halted(*sim.engine, *sim.cu());

  ASSERT_EQ(sim.snapshot->snapshots().size(), 1u);
  EXPECT_EQ(sim.snapshot->snapshots().front().sgpr(6), 2u);
}

TEST(Gfx1250SimulationTest, SSwapPcI64StoresReturnAddressAndJumps) {
  constexpr uint64_t kKernelAddr = 0x10000;
  constexpr uint32_t kReturnWord = 4;
  constexpr uint32_t kTargetWord = 5;
  uint64_t entry_pc = kKernelAddr + sizeof(rocr::llvm::amdhsa::kernel_descriptor_t);
  uint64_t return_pc = entry_pc + kReturnWord * sizeof(uint32_t);
  uint64_t target_pc = entry_pc + kTargetWord * sizeof(uint32_t);
  std::vector<uint32_t> code = {
      0xBE8400FFu,    static_cast<uint32_t>(target_pc), // s_mov_b32 s4, target_pc[31:0]
      0xBE850080u,                                      // s_mov_b32 s5, 0
      0xBE864904u,                                      // s_swap_pc_i64 s[6:7], s[4:5]
      0xBE880081u,                                      // s_mov_b32 s8, 1
      0xBE880082u,                                      // s_mov_b32 s8, 2
      S_ENDPGM_GFX12,
  };

  Gfx1250Sim sim;
  uint64_t kernel_object = sim.write_kernel(kKernelAddr, code.data(), code.size());
  test::AqlQueue queue(sim.memory, sim.cp());
  queue.dispatch(kernel_object, 32, 32);
  step_until_halted(*sim.engine, *sim.cu());

  ASSERT_EQ(sim.snapshot->snapshots().size(), 1u);
  const auto &wf = sim.snapshot->snapshots().front();
  EXPECT_EQ(wf.sgpr64(6), return_pc);
  EXPECT_EQ(wf.sgpr(8), 2u);
}

TEST(Gfx1250SimulationTest, SBitreplicateB64B32DuplicatesEachSourceBit) {
  const uint32_t code[] = {
      0xBE8200FFu,
      0x80000001u, // s_mov_b32 s2, 0x80000001
      0xBE841402u, // s_bitreplicate_b64_b32 s[4:5], s2
      S_ENDPGM_GFX12,
  };

  Gfx1250Sim sim;
  const auto *wf = dispatch_one_wave(sim, code, std::size(code));
  ASSERT_NE(wf, nullptr);
  EXPECT_EQ(wf->sgpr64(4), 0xC000000000000003ULL);
}

TEST(Gfx1250SimulationTest, SGetShaderCyclesU64ReadsSimulationTime) {
  const uint32_t code[] = {
      0xBE840600u, // s_get_shader_cycles_u64 s[4:5]
      S_ENDPGM_GFX12,
  };

  Gfx1250Sim sim;
  sim.engine->schedule_event_async(sim.cp()->doorbell_event(), 17);
  ASSERT_TRUE(sim.engine->step());
  ASSERT_EQ(sim.engine->global_time(), 17u);

  const auto *wf = dispatch_one_wave(sim, code, std::size(code));
  ASSERT_NE(wf, nullptr);
  const auto observed_time = wf->sgpr64(4);
  EXPECT_GE(observed_time, 17u);
  EXPECT_LE(observed_time, sim.engine->global_time());
}

TEST(Gfx1250SimulationTest, SSendmsgRtnB64ReadsRealtimeAndB32UsesPlaceholder) {
  const uint32_t code[] = {
      0xBE844D83u, // s_sendmsg_rtn_b64 s[4:5], sendmsg(MSG_RTN_GET_REALTIME)
      0xBE864C80u, // s_sendmsg_rtn_b32 s6, sendmsg(MSG_RTN_GET_DOORBELL)
      S_ENDPGM_GFX12,
  };

  Gfx1250Sim sim;
  sim.engine->schedule_event_async(sim.cp()->doorbell_event(), 23);
  ASSERT_TRUE(sim.engine->step());
  ASSERT_EQ(sim.engine->global_time(), 23u);

  const auto *wf = dispatch_one_wave(sim, code, std::size(code));
  ASSERT_NE(wf, nullptr);
  const auto observed_time = wf->sgpr64(4);
  EXPECT_GE(observed_time, 23u);
  EXPECT_LE(observed_time, sim.engine->global_time());
  EXPECT_EQ(wf->sgpr(6), 0u);
}

TEST(Gfx1250SimulationTest, SMovrelsReadsM0IndexedScalarSources) {
  const uint32_t code[] = {
      0xBEFD0081u,                 // s_mov_b32 m0, 1
      0xBE8200FFu,    0x11111111u, // s_mov_b32 s2, 0x11111111
      0xBE8300FFu,    0x22222222u, // s_mov_b32 s3, 0x22222222
      0xBE8400FFu,    0x33333333u, // s_mov_b32 s4, 0x33333333
      0xBE8500FFu,    0x44444444u, // s_mov_b32 s5, 0x44444444
      0xBE884002u,                 // s_movrels_b32 s8, s2
      0xBE8A4102u,                 // s_movrels_b64 s[10:11], s[2:3]
      S_ENDPGM_GFX12,
  };

  Gfx1250Sim sim;
  const auto *wf = dispatch_one_wave(sim, code, std::size(code));
  ASSERT_NE(wf, nullptr);
  EXPECT_EQ(wf->sgpr(8), 0x22222222u);
  EXPECT_EQ(wf->sgpr64(10), 0x4444444433333333ULL);
}

TEST(Gfx1250SimulationTest, SMovreldWritesM0IndexedScalarDestinations) {
  const uint32_t code[] = {
      0xBEFD0081u,                 // s_mov_b32 m0, 1
      0xBE8200FFu,    0x55555555u, // s_mov_b32 s2, 0x55555555
      0xBE8300FFu,    0x66666666u, // s_mov_b32 s3, 0x66666666
      0xBE884202u,                 // s_movreld_b32 s8, s2
      0xBE8A4302u,                 // s_movreld_b64 s[10:11], s[2:3]
      S_ENDPGM_GFX12,
  };

  Gfx1250Sim sim;
  const auto *wf = dispatch_one_wave(sim, code, std::size(code));
  ASSERT_NE(wf, nullptr);
  EXPECT_EQ(wf->sgpr(9), 0x55555555u);
  EXPECT_EQ(wf->sgpr64(12), 0x6666666655555555ULL);
}

TEST(Gfx1250SimulationTest, SMovrelsd2B32UsesSeparatePackedM0Offsets) {
  const uint32_t code[] = {
      0xBEFD00FFu,    0x00000201u, // s_mov_b32 m0, 0x201
      0xBE8200FFu,    0x77777777u, // s_mov_b32 s2, 0x77777777
      0xBE8300FFu,    0x88888888u, // s_mov_b32 s3, 0x88888888
      0xBE884402u,                 // s_movrelsd_2_b32 s8, s2
      S_ENDPGM_GFX12,
  };

  Gfx1250Sim sim;
  const auto *wf = dispatch_one_wave(sim, code, std::size(code));
  ASSERT_NE(wf, nullptr);
  EXPECT_EQ(wf->sgpr(10), 0x88888888u);
}

TEST(Gfx1250SimulationTest, SplitNamedBarrierOpsReportIdleState) {
  const uint32_t code[] = {
      0xBEFD0081u,                 // s_mov_b32 m0, 1
      0xBE8400FFu,    0xFFFFFFFFu, // s_mov_b32 s4, -1
      0xBE80517Du,                 // s_barrier_init m0
      0xBE80527Du,                 // s_barrier_join m0
      0xBE80577Du,                 // s_wakeup_barrier m0
      0xBE84507Du,                 // s_get_barrier_state s4, m0
      S_ENDPGM_GFX12,
  };

  Gfx1250Sim sim;
  const auto *wf = dispatch_one_wave(sim, code, std::size(code));
  ASSERT_NE(wf, nullptr);
  EXPECT_EQ(wf->sgpr(4), 0u);
}

TEST(Gfx1250SimulationTest, VgprMsbModeTracksModeRegisterLayout) {
  Gfx1250Sim sim;
  // Resident wave: this test mutates and reads live wavefront MODE state.
  amdgpu::Wavefront *wf = sim.dispatch_scratch_wf();
  ASSERT_NE(wf, nullptr);

  constexpr uint8_t kSetLayout = 0xB9;  // src0=1, src1=2, src2=3, dst=2.
  constexpr uint8_t kModeLayout = 0xE6; // dst,src0,src1,src2 packed in MODE.
  static_assert(amdgpu::set_vgpr_msb_to_mode_layout(kSetLayout) == kModeLayout);
  static_assert(amdgpu::mode_layout_to_set_vgpr_msb(kModeLayout) == kSetLayout);
  for (uint32_t layout = 0; layout <= 0xFF; ++layout) {
    EXPECT_EQ(amdgpu::mode_layout_to_set_vgpr_msb(
                  amdgpu::set_vgpr_msb_to_mode_layout(static_cast<uint8_t>(layout))),
              layout);
    EXPECT_EQ(amdgpu::set_vgpr_msb_to_mode_layout(
                  amdgpu::mode_layout_to_set_vgpr_msb(static_cast<uint8_t>(layout))),
              layout);
  }

  wf->set_vgpr_msb_mode(kSetLayout);
  EXPECT_EQ(wf->vgpr_msb_mode(), kSetLayout);
  EXPECT_EQ((wf->mode_raw() & amdgpu::VGPR_MSB_MODE_MASK) >> amdgpu::VGPR_MSB_MODE_SHIFT,
            kModeLayout);
  EXPECT_EQ(wf->vgpr_msb_for_role(amdgpu::VgprMsbRole::Src0), 1u);
  EXPECT_EQ(wf->vgpr_msb_for_role(amdgpu::VgprMsbRole::Src1), 2u);
  EXPECT_EQ(wf->vgpr_msb_for_role(amdgpu::VgprMsbRole::Src2), 3u);
  EXPECT_EQ(wf->vgpr_msb_for_role(amdgpu::VgprMsbRole::Dst), 2u);

  wf->set_mode_raw(kModeLayout << amdgpu::VGPR_MSB_MODE_SHIFT);
  EXPECT_EQ(wf->vgpr_msb_mode(), kSetLayout);
}

TEST(Gfx1250SimulationTest, SSetVgprMsbUpdatesWavefrontMode) {
  constexpr uint8_t kSetLayout = 0xB9;
  constexpr uint8_t kModeLayout = amdgpu::set_vgpr_msb_to_mode_layout(kSetLayout);
  Gfx1250Sim sim;
  const uint32_t code[] = {S_SET_VGPR_MSB | kSetLayout, S_ENDPGM_GFX12};

  // The s_set_vgpr_msb result lives in the wavefront MODE register; capture it at
  // halt so it survives the wave freeing itself.
  auto *snapshot = sim.snapshot;
  uint64_t kernel_object =
      sim.write_kernel(0x10000, code, std::size(code), 104, kGfx1250Wave32VgprAllocation);
  {
    test::AqlQueue queue(sim.memory, sim.cp());
    queue.dispatch(kernel_object, 32, 32);
  }
  step_until_halted(*sim.engine, *sim.cu());
  ASSERT_EQ(snapshot->snapshots().size(), 1u);
  const auto &wf = snapshot->snapshots().front();

  EXPECT_EQ(wf.vgpr_msb_mode, kSetLayout);
  EXPECT_EQ((wf.mode_raw & amdgpu::VGPR_MSB_MODE_MASK) >> amdgpu::VGPR_MSB_MODE_SHIFT, kModeLayout);
}

TEST(Gfx1250SimulationTest, VgprMsbRolesSelectHighVgprBanks) {
  Gfx1250Sim sim;
  // Resident wave: this test drives Operand read/write against live VGPR banks.
  amdgpu::Wavefront *wf = sim.dispatch_scratch_wf();
  ASSERT_NE(wf, nullptr);
  ASSERT_GE(wf->vgpr_alloc().count, kGfx1250Wave32VgprAllocation);

  constexpr uint32_t kLane = 0;
  constexpr uint8_t kSetLayout = 0xB9; // src0=1, src1=2, src2=3, dst=2.
  const uint32_t vb = wf->vgpr_alloc().base;
  auto &cu = *sim.cu();

  wf->set_vgpr_msb_mode(kSetLayout);
  cu.write_vgpr(vb + 5, kLane, 0xDEADBEEFu);
  cu.write_vgpr(vb + 1 * 256 + 2, kLane, 0x11111111u);
  cu.write_vgpr(vb + 2 * 256 + 3, kLane, 0x22222222u);
  cu.write_vgpr(vb + 3 * 256 + 4, kLane, 0x33333333u);

  gfx1250::Operand src0(32, gfx1250::OperandType::OPR_SRC, 256 + 2);
  gfx1250::Operand src1(32, gfx1250::OperandType::OPR_VGPR, 3);
  gfx1250::Operand src2(32, gfx1250::OperandType::OPR_SRC_VGPR, 256 + 4);
  gfx1250::Operand dst(32, gfx1250::OperandType::OPR_VGPR, 5);
  src0.set_vgpr_msb_role(amdgpu::VgprMsbRole::Src0);
  src1.set_vgpr_msb_role(amdgpu::VgprMsbRole::Src1);
  src2.set_vgpr_msb_role(amdgpu::VgprMsbRole::Src2);
  dst.set_vgpr_msb_role(amdgpu::VgprMsbRole::Dst);

  EXPECT_EQ(amdgpu::RegisterAccess(*wf).read_lane(src0, kLane), 0x11111111u);
  EXPECT_EQ(amdgpu::RegisterAccess(*wf).read_lane(src1, kLane), 0x22222222u);
  EXPECT_EQ(amdgpu::RegisterAccess(*wf).read_lane(src2, kLane), 0x33333333u);

  amdgpu::RegisterAccess(*wf).write_lane(dst, kLane, 0x44444444u);
  EXPECT_EQ(cu.read_vgpr(vb + 2 * 256 + 5, kLane), 0x44444444u);
  EXPECT_EQ(cu.read_vgpr(vb + 5, kLane), 0xDEADBEEFu);
}

TEST(Gfx1250SimulationTest, VMovDppReadsSelectedHighVgprBank) {
  Gfx1250Sim sim;
  // Resident wave: these tests inject instructions directly (execute_impl) and read
  // live register state, so they need a live wavefront, not a run-to-halt snapshot.
  amdgpu::Wavefront *wf = sim.dispatch_scratch_wf(kGfx1250Wave32VgprAllocation);
  ASSERT_NE(wf, nullptr);
  ASSERT_GE(wf->vgpr_alloc().count, kGfx1250Wave32VgprAllocation);

  constexpr uint32_t kSrc = 7;
  constexpr uint32_t kDst = 8;
  constexpr uint32_t kHighBank = 256;
  constexpr uint32_t kHighValueBase = 0x42000000u;
  constexpr uint32_t kLowAliasBase = 0xDEAD0000u;
  const uint32_t vb = wf->vgpr_alloc().base;
  auto &cu = *sim.cu();

  gfx1250::Vop1VopDpp16MachineInst raw{};
  raw.src0 = amdgpu::SRC_DPP;
  raw.vsrc0 = kSrc;
  raw.vdst = kDst;
  raw.dpp_ctrl = 0xB1; // quad_perm:[1,0,3,2]
  raw.bound_ctrl = 1;
  raw.bank_mask = 0xF;
  raw.row_mask = 0xF;

  for (const uint8_t mode : {uint8_t{0x01}, uint8_t{0x41}}) {
    SCOPED_TRACE(testing::Message() << "vgpr_msb_mode=" << static_cast<uint32_t>(mode));
    wf->set_vgpr_msb_mode(mode); // SRC0=bank 1; DST=bank 0 or bank 1.
    for (uint32_t lane = 0; lane < wf->wf_size(); ++lane) {
      cu.write_vgpr(vb + kSrc, lane, kLowAliasBase + lane);
      cu.write_vgpr(vb + kHighBank + kSrc, lane, kHighValueBase + lane);
      cu.write_vgpr(vb + kDst, lane, 0u);
      cu.write_vgpr(vb + kHighBank + kDst, lane, 0u);
    }

    gfx1250::VMovB32Vop1 inst(reinterpret_cast<const gfx1250::MachineInst *>(&raw));
    inst.execute_impl(*wf);

    const uint32_t dst_bank = (mode >> 6) & 0x3u;
    for (uint32_t lane = 0; lane < wf->wf_size(); ++lane) {
      EXPECT_EQ(cu.read_vgpr(vb + dst_bank * kHighBank + kDst, lane), kHighValueBase + (lane ^ 1u))
          << "lane " << lane;
    }
  }
}

TEST(Gfx1250SimulationTest, VMovDpp8ReadsSelectedHighVgprBank) {
  Gfx1250Sim sim;
  // Resident wave: these tests inject instructions directly (execute_impl) and read
  // live register state, so they need a live wavefront, not a run-to-halt snapshot.
  amdgpu::Wavefront *wf = sim.dispatch_scratch_wf(kGfx1250Wave32VgprAllocation);
  ASSERT_NE(wf, nullptr);
  ASSERT_GE(wf->vgpr_alloc().count, kGfx1250Wave32VgprAllocation);

  constexpr uint32_t kSrc = 7;
  constexpr uint32_t kDst = 8;
  constexpr uint32_t kHighBank = 256;
  constexpr uint32_t kHighValueBase = 0x43000000u;
  const uint32_t vb = wf->vgpr_alloc().base;
  auto &cu = *sim.cu();

  wf->set_vgpr_msb_mode(0x01); // SRC0=bank 1; DST=bank 0.
  for (uint32_t lane = 0; lane < wf->wf_size(); ++lane) {
    cu.write_vgpr(vb + kSrc, lane, 0u);
    cu.write_vgpr(vb + kHighBank + kSrc, lane, kHighValueBase + lane);
  }

  gfx1250::Vop1VopDpp8MachineInst raw{};
  raw.src0 = amdgpu::SRC_DPP8_FI_1;
  raw.vsrc0 = kSrc;
  raw.vdst = kDst;
  raw.lane_sel_0 = 1;
  raw.lane_sel_1 = 0;
  raw.lane_sel_2 = 3;
  raw.lane_sel_3 = 2;
  raw.lane_sel_4 = 5;
  raw.lane_sel_5 = 4;
  raw.lane_sel_6 = 7;
  raw.lane_sel_7 = 6;

  gfx1250::VMovB32Vop1 inst(reinterpret_cast<const gfx1250::MachineInst *>(&raw));
  inst.execute_impl(*wf);

  for (uint32_t lane = 0; lane < wf->wf_size(); ++lane)
    EXPECT_EQ(cu.read_vgpr(vb + kDst, lane), kHighValueBase + (lane ^ 1u)) << "lane " << lane;
}

TEST(Gfx1250SimulationTest, VMovDppPreservesMaskedHighDestinationLanes) {
  Gfx1250Sim sim;
  // Resident wave: these tests inject instructions directly (execute_impl) and read
  // live register state, so they need a live wavefront, not a run-to-halt snapshot.
  amdgpu::Wavefront *wf = sim.dispatch_scratch_wf(kGfx1250Wave32VgprAllocation);
  ASSERT_NE(wf, nullptr);

  constexpr uint32_t kSrc = 7;
  constexpr uint32_t kDst = 8;
  constexpr uint32_t kHighBank = 256;
  constexpr uint32_t kSourceBase = 0x44000000u;
  constexpr uint32_t kOldHighDstBase = 0x55000000u;
  constexpr uint32_t kLowDstBase = 0x66000000u;
  const uint32_t vb = wf->vgpr_alloc().base;
  auto &cu = *sim.cu();

  wf->set_vgpr_msb_mode(0x41); // SRC0=bank 1; DST=bank 1.
  for (uint32_t lane = 0; lane < wf->wf_size(); ++lane) {
    cu.write_vgpr(vb + kHighBank + kSrc, lane, kSourceBase + lane);
    cu.write_vgpr(vb + kHighBank + kDst, lane, kOldHighDstBase + lane);
    cu.write_vgpr(vb + kDst, lane, kLowDstBase + lane);
  }

  gfx1250::Vop1VopDpp16MachineInst raw{};
  raw.src0 = amdgpu::SRC_DPP;
  raw.vsrc0 = kSrc;
  raw.vdst = kDst;
  raw.dpp_ctrl = 0xB1; // quad_perm:[1,0,3,2]
  raw.bound_ctrl = 1;
  raw.bank_mask = 0xF;
  raw.row_mask = 0x1; // Only lanes 0-15 may be updated.

  gfx1250::VMovB32Vop1 inst(reinterpret_cast<const gfx1250::MachineInst *>(&raw));
  inst.execute_impl(*wf);

  for (uint32_t lane = 0; lane < wf->wf_size(); ++lane) {
    const uint32_t expected = lane < 16 ? kSourceBase + (lane ^ 1u) : kOldHighDstBase + lane;
    EXPECT_EQ(cu.read_vgpr(vb + kHighBank + kDst, lane), expected) << "lane " << lane;
    EXPECT_EQ(cu.read_vgpr(vb + kDst, lane), kLowDstBase + lane) << "lane " << lane;
  }
}

TEST(Gfx1250SimulationTest, VAddDppReadsHighBanksAndPreservesMaskedHighDst) {
  Gfx1250Sim sim;
  // Resident wave: these tests inject instructions directly (execute_impl) and read
  // live register state, so they need a live wavefront, not a run-to-halt snapshot.
  amdgpu::Wavefront *wf = sim.dispatch_scratch_wf(kGfx1250Wave32VgprAllocation);
  ASSERT_NE(wf, nullptr);

  constexpr uint32_t kSrc0 = 2;
  constexpr uint32_t kSrc1 = 3;
  constexpr uint32_t kDst = 4;
  constexpr uint32_t kBankStride = 256;
  constexpr uint32_t kSrc0Base = 1000;
  constexpr uint32_t kSrc1Base = 2000;
  constexpr uint32_t kOldHighDstBase = 3000;
  constexpr uint32_t kLowDstBase = 4000;
  const uint32_t vb = wf->vgpr_alloc().base;
  auto &cu = *sim.cu();

  wf->set_vgpr_msb_mode(0x49); // SRC0=bank 1; SRC1=bank 2; DST=bank 1.
  for (uint32_t lane = 0; lane < wf->wf_size(); ++lane) {
    cu.write_vgpr(vb + kSrc0, lane, 10u);
    cu.write_vgpr(vb + kSrc1, lane, 20u);
    cu.write_vgpr(vb + kBankStride + kSrc0, lane, kSrc0Base + lane);
    cu.write_vgpr(vb + 2 * kBankStride + kSrc1, lane, kSrc1Base + lane);
    cu.write_vgpr(vb + kBankStride + kDst, lane, kOldHighDstBase + lane);
    cu.write_vgpr(vb + kDst, lane, kLowDstBase + lane);
  }

  gfx1250::Vop2VopDpp16MachineInst raw{};
  raw.src0 = amdgpu::SRC_DPP;
  raw.vsrc0 = kSrc0;
  raw.vsrc1 = kSrc1;
  raw.vdst = kDst;
  raw.dpp_ctrl = 0xB1; // quad_perm:[1,0,3,2]
  raw.bound_ctrl = 1;
  raw.bank_mask = 0xF;
  raw.row_mask = 0x1;

  gfx1250::VAddNcU32Vop2 inst(reinterpret_cast<const gfx1250::MachineInst *>(&raw));
  inst.execute_impl(*wf);

  for (uint32_t lane = 0; lane < wf->wf_size(); ++lane) {
    const uint32_t expected =
        lane < 16 ? kSrc0Base + (lane ^ 1u) + kSrc1Base + lane : kOldHighDstBase + lane;
    EXPECT_EQ(cu.read_vgpr(vb + kBankStride + kDst, lane), expected) << "lane " << lane;
    EXPECT_EQ(cu.read_vgpr(vb + kDst, lane), kLowDstBase + lane) << "lane " << lane;
  }
}

TEST(Gfx1250SimulationTest, VCvtF64DppPreservesBothMaskedHighDstDwords) {
  Gfx1250Sim sim;
  // Resident wave: these tests inject instructions directly (execute_impl) and read
  // live register state, so they need a live wavefront, not a run-to-halt snapshot.
  amdgpu::Wavefront *wf = sim.dispatch_scratch_wf(kGfx1250Wave32VgprAllocation);
  ASSERT_NE(wf, nullptr);

  constexpr uint32_t kSrc = 7;
  constexpr uint32_t kDst = 20;
  constexpr uint32_t kHighBank = 256;
  constexpr uint32_t kSourceBase = 100;
  constexpr uint64_t kOldDstBase = 0x5500000066000000ULL;
  constexpr uint32_t kLowDstLoBase = 0x77000000u;
  constexpr uint32_t kLowDstHiBase = 0x88000000u;
  const uint32_t vb = wf->vgpr_alloc().base;
  auto &cu = *sim.cu();

  wf->set_vgpr_msb_mode(0x41); // SRC0=bank 1; DST=bank 1.
  for (uint32_t lane = 0; lane < wf->wf_size(); ++lane) {
    const uint64_t old_dst = kOldDstBase + lane;
    cu.write_vgpr(vb + kHighBank + kSrc, lane, kSourceBase + lane);
    cu.write_vgpr(vb + kHighBank + kDst, lane, static_cast<uint32_t>(old_dst));
    cu.write_vgpr(vb + kHighBank + kDst + 1, lane, static_cast<uint32_t>(old_dst >> 32));
    cu.write_vgpr(vb + kDst, lane, kLowDstLoBase + lane);
    cu.write_vgpr(vb + kDst + 1, lane, kLowDstHiBase + lane);
  }

  gfx1250::Vop1VopDpp16MachineInst raw{};
  raw.src0 = amdgpu::SRC_DPP;
  raw.vsrc0 = kSrc;
  raw.vdst = kDst;
  raw.dpp_ctrl = 0xB1; // quad_perm:[1,0,3,2]
  raw.bound_ctrl = 1;
  raw.bank_mask = 0xF;
  raw.row_mask = 0x1;

  gfx1250::VCvtF64I32Vop1 inst(reinterpret_cast<const gfx1250::MachineInst *>(&raw));
  inst.execute_impl(*wf);

  for (uint32_t lane = 0; lane < wf->wf_size(); ++lane) {
    const uint64_t actual =
        static_cast<uint64_t>(cu.read_vgpr(vb + kHighBank + kDst, lane)) |
        (static_cast<uint64_t>(cu.read_vgpr(vb + kHighBank + kDst + 1, lane)) << 32);
    const uint64_t expected =
        lane < 16 ? std::bit_cast<uint64_t>(static_cast<double>(kSourceBase + (lane ^ 1u)))
                  : kOldDstBase + lane;
    EXPECT_EQ(actual, expected) << "lane " << lane;
    EXPECT_EQ(cu.read_vgpr(vb + kDst, lane), kLowDstLoBase + lane) << "lane " << lane;
    EXPECT_EQ(cu.read_vgpr(vb + kDst + 1, lane), kLowDstHiBase + lane) << "lane " << lane;
  }
}

TEST(Gfx1250SimulationTest, VAddF32Vop3DppPreservesMaskedHighDestinationLanes) {
  Gfx1250Sim sim;
  // Resident wave: these tests inject instructions directly (execute_impl) and read
  // live register state, so they need a live wavefront, not a run-to-halt snapshot.
  amdgpu::Wavefront *wf = sim.dispatch_scratch_wf(kGfx1250Wave32VgprAllocation);
  ASSERT_NE(wf, nullptr);

  constexpr uint32_t kSrc0 = 2;
  constexpr uint32_t kSrc1 = 3;
  constexpr uint32_t kDst = 4;
  constexpr uint32_t kBankStride = 256;
  constexpr uint32_t kOldHighDstBase = 0x55000000u;
  constexpr uint32_t kLowDstBase = 0x66000000u;
  const uint32_t vb = wf->vgpr_alloc().base;
  auto &cu = *sim.cu();

  wf->set_vgpr_msb_mode(0x49); // SRC0=bank 1; SRC1=bank 2; DST=bank 1.
  for (uint32_t lane = 0; lane < wf->wf_size(); ++lane) {
    cu.write_vgpr(vb + kBankStride + kSrc0, lane,
                  std::bit_cast<uint32_t>(static_cast<float>(lane + 1)));
    cu.write_vgpr(vb + 2 * kBankStride + kSrc1, lane, std::bit_cast<uint32_t>(100.0f));
    cu.write_vgpr(vb + kBankStride + kDst, lane, kOldHighDstBase + lane);
    cu.write_vgpr(vb + kDst, lane, kLowDstBase + lane);
  }

  gfx1250::Vop3VopDpp16MachineInst raw{};
  raw.vdst = kDst;
  raw.src0 = amdgpu::SRC_DPP;
  raw.src1 = 256 + kSrc1;
  raw.vsrc0 = kSrc0;
  raw.dpp_ctrl = 0xB1; // quad_perm:[1,0,3,2]
  raw.bound_ctrl = 1;
  raw.bank_mask = 0xF;
  raw.row_mask = 0x1;

  gfx1250::VAddF32Vop3 inst(reinterpret_cast<const gfx1250::MachineInst *>(&raw));
  inst.execute_impl(*wf);

  for (uint32_t lane = 0; lane < wf->wf_size(); ++lane) {
    const uint32_t expected = lane < 16
                                  ? std::bit_cast<uint32_t>(static_cast<float>((lane ^ 1u) + 101))
                                  : kOldHighDstBase + lane;
    EXPECT_EQ(cu.read_vgpr(vb + kBankStride + kDst, lane), expected) << "lane " << lane;
    EXPECT_EQ(cu.read_vgpr(vb + kDst, lane), kLowDstBase + lane) << "lane " << lane;
  }
}

TEST(Gfx1250SimulationTest, VMovDppComposesVgprMsbWithGprIdx) {
  Gfx1250Sim sim;
  // Resident wave: these tests inject instructions directly (execute_impl) and read
  // live register state, so they need a live wavefront, not a run-to-halt snapshot.
  amdgpu::Wavefront *wf = sim.dispatch_scratch_wf(kGfx1250Wave32VgprAllocation);
  ASSERT_NE(wf, nullptr);

  constexpr uint32_t kSrc = 7;
  constexpr uint32_t kDst = 8;
  constexpr uint32_t kHighBank = 256;
  constexpr uint32_t kGprIdxOffset = 16;
  constexpr uint32_t kExpectedBase = 0x77000000u;
  const uint32_t vb = wf->vgpr_alloc().base;
  auto &cu = *sim.cu();

  wf->set_vgpr_msb_mode(0x01); // SRC0=bank 1; DST=bank 0.
  wf->set_mode_raw(wf->mode_raw() | amdgpu::Wavefront::GPR_IDX_EN_BIT);
  wf->set_m0((1u << 8u) | kGprIdxOffset); // Index SRC0 only.
  for (uint32_t lane = 0; lane < wf->wf_size(); ++lane) {
    cu.write_vgpr(vb + kHighBank + kSrc, lane, 0u);
    cu.write_vgpr(vb + kHighBank + kGprIdxOffset + kSrc, lane, kExpectedBase + lane);
  }

  gfx1250::Vop1VopDpp16MachineInst raw{};
  raw.src0 = amdgpu::SRC_DPP;
  raw.vsrc0 = kSrc;
  raw.vdst = kDst;
  raw.dpp_ctrl = 0xB1; // quad_perm:[1,0,3,2]
  raw.bound_ctrl = 1;
  raw.bank_mask = 0xF;
  raw.row_mask = 0xF;

  gfx1250::VMovB32Vop1 inst(reinterpret_cast<const gfx1250::MachineInst *>(&raw));
  inst.execute_impl(*wf);

  for (uint32_t lane = 0; lane < wf->wf_size(); ++lane)
    EXPECT_EQ(cu.read_vgpr(vb + kDst, lane), kExpectedBase + (lane ^ 1u)) << "lane " << lane;
}

TEST(Gfx1250SimulationTest, PackedTrue16SourcesHonorGprIdx) {
  Gfx1250Sim sim;
  // Resident wave: this test drives Operand reads against live VGPR banks.
  amdgpu::Wavefront *wf = sim.dispatch_scratch_wf();
  ASSERT_NE(wf, nullptr);
  ASSERT_GE(wf->vgpr_alloc().count, kGfx1250Wave32VgprAllocation);

  constexpr uint32_t kLane = 0;
  const uint32_t vb = wf->vgpr_alloc().base;
  auto &cu = *sim.cu();

  wf->set_mode_raw(amdgpu::Wavefront::GPR_IDX_EN_BIT);
  wf->set_m0((1u << 8u) | 16u);
  cu.write_vgpr(vb + 2, kLane, 0xAAAA1111u);
  cu.write_vgpr(vb + 18, kLane, 0xBBBB2222u);

  gfx1250::Operand lo(16, gfx1250::OperandType::OPR_VGPR, 2, true);
  gfx1250::Operand hi(16, gfx1250::OperandType::OPR_VGPR, 128 + 2, true);
  EXPECT_EQ(amdgpu::RegisterAccess(*wf).read_lane(lo, kLane), 0x2222u);
  EXPECT_EQ(amdgpu::RegisterAccess(*wf).read_lane(hi, kLane), 0xBBBBu);
}

TEST(Gfx1250SimulationTest, PackedTrue16InstructionComposesHighBanksWithGprIdx) {
  Gfx1250Sim sim;
  // Resident wave: these tests inject instructions directly (execute_impl) and read
  // live register state, so they need a live wavefront, not a run-to-halt snapshot.
  amdgpu::Wavefront *wf = sim.dispatch_scratch_wf(kGfx1250Wave32VgprAllocation);
  ASSERT_NE(wf, nullptr);
  wf->set_exec(1u);

  constexpr uint32_t kSrc = 2;
  constexpr uint32_t kDst = 3;
  constexpr uint32_t kBankStride = 256;
  constexpr uint32_t kGprIdxOffset = 16;
  constexpr uint32_t kLowAlias = 0x55556666u;
  const uint32_t vb = wf->vgpr_alloc().base;
  auto &cu = *sim.cu();

  wf->set_vgpr_msb_mode(0x41); // SRC0=bank 1; DST=bank 1.
  wf->set_mode_raw(wf->mode_raw() | amdgpu::Wavefront::GPR_IDX_EN_BIT);
  wf->set_m0((0x9u << 8u) | kGprIdxOffset); // Index SRC0 and DST.
  cu.write_vgpr(vb + kSrc, 0, kLowAlias);
  cu.write_vgpr(vb + kDst, 0, kLowAlias);
  cu.write_vgpr(vb + kBankStride + kSrc, 0, kLowAlias);
  cu.write_vgpr(vb + kBankStride + kDst, 0, kLowAlias);
  cu.write_vgpr(vb + kBankStride + kGprIdxOffset + kSrc, 0, 0xABCD1234u);
  cu.write_vgpr(vb + kBankStride + kGprIdxOffset + kDst, 0, 0xEEEE1111u);

  gfx1250::Vop1MachineInst raw{};
  raw.src0 = 256 + 128 + kSrc; // High half of the encoded VGPR source.
  raw.vdst = kDst;
  gfx1250::VMovB16Vop1 inst(reinterpret_cast<const gfx1250::MachineInst *>(&raw));
  inst.execute_impl(*wf);

  EXPECT_EQ(cu.read_vgpr(vb + kBankStride + kGprIdxOffset + kDst, 0), 0xEEEEABCDu);
  EXPECT_EQ(cu.read_vgpr(vb + kDst, 0), kLowAlias);
  EXPECT_EQ(cu.read_vgpr(vb + kBankStride + kDst, 0), kLowAlias);
}

TEST(Gfx1250SimulationTest, DsPermuteUsesIndependentHighOperandBanks) {
  Gfx1250Sim sim;
  // Resident wave: these tests inject instructions directly (execute_impl) and read
  // live register state, so they need a live wavefront, not a run-to-halt snapshot.
  amdgpu::Wavefront *wf = sim.dispatch_scratch_wf(kGfx1250Wave32VgprAllocation);
  ASSERT_NE(wf, nullptr);

  constexpr uint32_t kAddr = 1;
  constexpr uint32_t kData = 2;
  constexpr uint32_t kDst = 3;
  constexpr uint32_t kBankStride = 256;
  constexpr uint32_t kValueBase = 0x12340000u;
  constexpr uint32_t kLowAlias = 0xDEADBEEFu;
  const uint32_t vb = wf->vgpr_alloc().base;
  auto &cu = *sim.cu();

  wf->set_vgpr_msb_mode(0xC9); // SRC0=bank 1; SRC1=bank 2; DST=bank 3.
  for (uint32_t lane = 0; lane < wf->wf_size(); ++lane) {
    cu.write_vgpr(vb + kBankStride + kAddr, lane, lane * 4);
    cu.write_vgpr(vb + 2 * kBankStride + kData, lane, kValueBase + lane);
    cu.write_vgpr(vb + 3 * kBankStride + kDst, lane, 0u);
    cu.write_vgpr(vb + kAddr, lane, kLowAlias);
    cu.write_vgpr(vb + kData, lane, kLowAlias);
    cu.write_vgpr(vb + kDst, lane, kLowAlias);
  }

  gfx1250::VdsMachineInst raw{};
  raw.addr = kAddr;
  raw.data0 = kData;
  raw.vdst = kDst;
  gfx1250::DsPermuteB32Vds inst(reinterpret_cast<const gfx1250::MachineInst *>(&raw));
  inst.execute_impl(*wf);

  for (uint32_t lane = 0; lane < wf->wf_size(); ++lane) {
    EXPECT_EQ(cu.read_vgpr(vb + 3 * kBankStride + kDst, lane), kValueBase + lane)
        << "lane " << lane;
    EXPECT_EQ(cu.read_vgpr(vb + kDst, lane), kLowAlias) << "lane " << lane;
  }
}

TEST(Gfx1250SimulationTest, DsStore2addrUsesSrc2HighBank) {
  Gfx1250Sim sim;
  // Resident wave: these tests inject instructions directly (execute_impl) and read
  // live register state, so they need a live wavefront, not a run-to-halt snapshot.
  amdgpu::Wavefront *wf = sim.dispatch_scratch_wf(kGfx1250Wave32VgprAllocation);
  ASSERT_NE(wf, nullptr);
  wf->set_exec(1u);

  constexpr uint32_t kAddr = 1;
  constexpr uint32_t kData0 = 2;
  constexpr uint32_t kData1 = 3;
  constexpr uint32_t kSrc0Bank = 1;
  constexpr uint32_t kSrc1Bank = 2;
  constexpr uint32_t kSrc2Bank = 3;
  constexpr uint32_t kBankStride = 256;
  constexpr uint32_t kAddress = 0x20;
  constexpr uint32_t kExpected0 = 0x12345678u;
  constexpr uint32_t kExpected1 = 0x9ABCDEF0u;
  const uint32_t vb = wf->vgpr_alloc().base;
  auto &cu = *sim.cu();

  wf->set_vgpr_msb_mode(kSrc0Bank | (kSrc1Bank << 2) | (kSrc2Bank << 4));
  cu.write_vgpr(vb + kSrc0Bank * kBankStride + kAddr, 0, kAddress);
  cu.write_vgpr(vb + kSrc1Bank * kBankStride + kData0, 0, kExpected0);
  cu.write_vgpr(vb + kSrc2Bank * kBankStride + kData1, 0, kExpected1);
  cu.write_vgpr(vb + kAddr, 0, 0x100u);
  cu.write_vgpr(vb + kData0, 0, 0xDEADBEEFu);
  cu.write_vgpr(vb + kData1, 0, 0xDEADBEEFu);

  gfx1250::VdsMachineInst raw{};
  raw.addr = kAddr;
  raw.data0 = kData0;
  raw.data1 = kData1;
  raw.offset0 = 1;
  raw.offset1 = 2;
  gfx1250::DsStore2addrB32Vds inst(reinterpret_cast<const gfx1250::MachineInst *>(&raw));
  inst.execute_impl(*wf);

  auto *state = inst.data_as<amdgpu::VectorMemState>();
  ASSERT_NE(state, nullptr);
  EXPECT_EQ(state->per_lane_addr[0], wf->lds_base() + kAddress + 4);
  EXPECT_EQ(state->ds2_per_lane_addr[0], wf->lds_base() + kAddress + 8);
  uint32_t actual0 = 0;
  uint32_t actual1 = 0;
  std::memcpy(&actual0, state->store_data.data(), sizeof(actual0));
  std::memcpy(&actual1, state->ds2_store_data.data(), sizeof(actual1));
  EXPECT_EQ(actual0, kExpected0);
  EXPECT_EQ(actual1, kExpected1);
}

TEST(Gfx1250SimulationTest, DsStorexchg2addrB64ExchangesBothAddresses) {
  Gfx1250Sim sim;
  auto *wf = sim.dispatch_scratch_wf(kGfx1250Wave32VgprAllocation);
  ASSERT_NE(wf, nullptr);
  wf->set_exec(1u);

  auto &cu = *sim.cu();
  wf->set_lds_base(cu.allocate_lds(256));
  constexpr uint32_t kAddr = 1;
  constexpr uint32_t kData0 = 2;
  constexpr uint32_t kData1 = 4;
  constexpr uint32_t kDst = 6;
  constexpr uint32_t kAddress = 0x20;
  constexpr uint64_t kOld0 = 0x1122334455667788ull;
  constexpr uint64_t kOld1 = 0x99AABBCCDDEEFF00ull;
  constexpr uint64_t kNew0 = 0x0123456789ABCDEFull;
  constexpr uint64_t kNew1 = 0xFEDCBA9876543210ull;
  const uint32_t vb = wf->vgpr_alloc().base;

  cu.write_vgpr(vb + kAddr, 0, kAddress);
  cu.write_vgpr(vb + kData0, 0, static_cast<uint32_t>(kNew0));
  cu.write_vgpr(vb + kData0 + 1, 0, static_cast<uint32_t>(kNew0 >> 32));
  cu.write_vgpr(vb + kData1, 0, static_cast<uint32_t>(kNew1));
  cu.write_vgpr(vb + kData1 + 1, 0, static_cast<uint32_t>(kNew1 >> 32));
  cu.lds().write64(wf->lds_base() + kAddress + 8, kOld0);
  cu.lds().write64(wf->lds_base() + kAddress + 24, kOld1);

  gfx1250::VdsMachineInst raw{};
  raw.addr = kAddr;
  raw.data0 = kData0;
  raw.data1 = kData1;
  raw.vdst = kDst;
  raw.offset0 = 1;
  raw.offset1 = 3;
  auto *inst =
      new gfx1250::DsStorexchg2addrRtnB64Vds(reinterpret_cast<const gfx1250::MachineInst *>(&raw));
  inst->execute_impl(*wf);

  auto *state = inst->data_as<amdgpu::VectorMemState>();
  ASSERT_NE(state, nullptr);
  EXPECT_EQ(state->wf_size, wf->wf_size());
  EXPECT_EQ(state->per_lane_addr[0], wf->lds_base() + kAddress + 8);
  EXPECT_EQ(state->ds2_per_lane_addr[0], wf->lds_base() + kAddress + 24);

  amdgpu::LocalMemPipeline local_pipeline;
  local_pipeline.issue(inst, *wf);

  EXPECT_EQ(cu.lds().read64(wf->lds_base() + kAddress + 8), kNew0);
  EXPECT_EQ(cu.lds().read64(wf->lds_base() + kAddress + 24), kNew1);
  const uint64_t returned0 =
      cu.read_vgpr(vb + kDst, 0) | (static_cast<uint64_t>(cu.read_vgpr(vb + kDst + 1, 0)) << 32);
  const uint64_t returned1 = cu.read_vgpr(vb + kDst + 2, 0) |
                             (static_cast<uint64_t>(cu.read_vgpr(vb + kDst + 3, 0)) << 32);
  EXPECT_EQ(returned0, kOld0);
  EXPECT_EQ(returned1, kOld1);
}

TEST(Gfx1250SimulationTest, DsStorexchg2addrStride64B32UsesScaledOffsetsForActiveLanes) {
  Gfx1250Sim sim;
  auto *wf = sim.dispatch_scratch_wf(kGfx1250Wave32VgprAllocation);
  ASSERT_NE(wf, nullptr);
  wf->set_exec(0x5u);

  auto &cu = *sim.cu();
  wf->set_lds_base(cu.allocate_lds(1024));
  constexpr uint32_t kAddr = 1;
  constexpr uint32_t kData0 = 2;
  constexpr uint32_t kData1 = 3;
  constexpr uint32_t kDst = 4;
  constexpr uint32_t kLane0Address = 0x20;
  constexpr uint32_t kLane2Address = 0x40;
  constexpr uint32_t kOffset0 = 256;
  constexpr uint32_t kOffset1 = 768;
  constexpr uint32_t kLane0New0 = 0x01020304;
  constexpr uint32_t kLane0New1 = 0x11121314;
  constexpr uint32_t kLane2New0 = 0x21222324;
  constexpr uint32_t kLane2New1 = 0x31323334;
  constexpr uint32_t kLane0Old0 = 0x41424344;
  constexpr uint32_t kLane0Old1 = 0x51525354;
  constexpr uint32_t kLane2Old0 = 0x61626364;
  constexpr uint32_t kLane2Old1 = 0x71727374;
  constexpr uint32_t kInactiveSentinel = 0xDEADBEEF;
  const uint32_t vb = wf->vgpr_alloc().base;

  cu.write_vgpr(vb + kAddr, 0, kLane0Address);
  cu.write_vgpr(vb + kAddr, 2, kLane2Address);
  cu.write_vgpr(vb + kData0, 0, kLane0New0);
  cu.write_vgpr(vb + kData1, 0, kLane0New1);
  cu.write_vgpr(vb + kData0, 2, kLane2New0);
  cu.write_vgpr(vb + kData1, 2, kLane2New1);
  cu.write_vgpr(vb + kDst, 1, kInactiveSentinel);
  cu.write_vgpr(vb + kDst + 1, 1, kInactiveSentinel);

  const uint32_t lane0_addr0 = wf->lds_base() + kLane0Address + kOffset0;
  const uint32_t lane0_addr1 = wf->lds_base() + kLane0Address + kOffset1;
  const uint32_t lane2_addr0 = wf->lds_base() + kLane2Address + kOffset0;
  const uint32_t lane2_addr1 = wf->lds_base() + kLane2Address + kOffset1;
  cu.lds().write32(lane0_addr0, kLane0Old0);
  cu.lds().write32(lane0_addr1, kLane0Old1);
  cu.lds().write32(lane2_addr0, kLane2Old0);
  cu.lds().write32(lane2_addr1, kLane2Old1);

  gfx1250::VdsMachineInst raw{};
  raw.addr = kAddr;
  raw.data0 = kData0;
  raw.data1 = kData1;
  raw.vdst = kDst;
  raw.offset0 = 1;
  raw.offset1 = 3;
  auto *inst = new gfx1250::DsStorexchg2addrStride64RtnB32Vds(
      reinterpret_cast<const gfx1250::MachineInst *>(&raw));
  inst->execute_impl(*wf);

  auto *state = inst->data_as<amdgpu::VectorMemState>();
  ASSERT_NE(state, nullptr);
  EXPECT_EQ(state->per_lane_addr[0], lane0_addr0);
  EXPECT_EQ(state->ds2_per_lane_addr[0], lane0_addr1);
  EXPECT_EQ(state->per_lane_addr[2], lane2_addr0);
  EXPECT_EQ(state->ds2_per_lane_addr[2], lane2_addr1);

  amdgpu::LocalMemPipeline local_pipeline;
  local_pipeline.issue(inst, *wf);

  EXPECT_EQ(cu.lds().read32(lane0_addr0), kLane0New0);
  EXPECT_EQ(cu.lds().read32(lane0_addr1), kLane0New1);
  EXPECT_EQ(cu.lds().read32(lane2_addr0), kLane2New0);
  EXPECT_EQ(cu.lds().read32(lane2_addr1), kLane2New1);
  EXPECT_EQ(cu.read_vgpr(vb + kDst, 0), kLane0Old0);
  EXPECT_EQ(cu.read_vgpr(vb + kDst + 1, 0), kLane0Old1);
  EXPECT_EQ(cu.read_vgpr(vb + kDst, 2), kLane2Old0);
  EXPECT_EQ(cu.read_vgpr(vb + kDst + 1, 2), kLane2Old1);
  EXPECT_EQ(cu.read_vgpr(vb + kDst, 1), kInactiveSentinel);
  EXPECT_EQ(cu.read_vgpr(vb + kDst + 1, 1), kInactiveSentinel);
}

TEST(Gfx1250SimulationTest, DsStorexchg2addrSameAddressAppliesBothExchangesInOrder) {
  Gfx1250Sim sim;
  auto *wf = sim.dispatch_scratch_wf(kGfx1250Wave32VgprAllocation);
  ASSERT_NE(wf, nullptr);
  wf->set_exec(1u);

  auto &cu = *sim.cu();
  wf->set_lds_base(cu.allocate_lds(64));
  constexpr uint32_t kAddr = 1;
  constexpr uint32_t kData0 = 2;
  constexpr uint32_t kData1 = 3;
  constexpr uint32_t kDst = 4;
  constexpr uint32_t kAddress = 0x20;
  constexpr uint32_t kOld = 0x11223344;
  constexpr uint32_t kNew0 = 0x55667788;
  constexpr uint32_t kNew1 = 0x99AABBCC;
  const uint32_t vb = wf->vgpr_alloc().base;

  cu.write_vgpr(vb + kAddr, 0, kAddress);
  cu.write_vgpr(vb + kData0, 0, kNew0);
  cu.write_vgpr(vb + kData1, 0, kNew1);
  cu.lds().write32(wf->lds_base() + kAddress, kOld);

  gfx1250::VdsMachineInst raw{};
  raw.addr = kAddr;
  raw.data0 = kData0;
  raw.data1 = kData1;
  raw.vdst = kDst;
  auto *inst =
      new gfx1250::DsStorexchg2addrRtnB32Vds(reinterpret_cast<const gfx1250::MachineInst *>(&raw));
  inst->execute_impl(*wf);

  amdgpu::LocalMemPipeline local_pipeline;
  local_pipeline.issue(inst, *wf);

  EXPECT_EQ(cu.read_vgpr(vb + kDst, 0), kOld);
  EXPECT_EQ(cu.read_vgpr(vb + kDst + 1, 0), kNew0);
  EXPECT_EQ(cu.lds().read32(wf->lds_base() + kAddress), kNew1);
}

TEST(Gfx1250SimulationTest, DsTransposeLoadUsesHighDestinationBank) {
  Gfx1250Sim sim;
  // Resident wave: these tests inject instructions directly (execute_impl) and read
  // live register state, so they need a live wavefront, not a run-to-halt snapshot.
  amdgpu::Wavefront *wf = sim.dispatch_scratch_wf(kGfx1250Wave32VgprAllocation);
  ASSERT_NE(wf, nullptr);
  wf->set_exec(1u);

  constexpr uint32_t kAddr = 1;
  constexpr uint32_t kDst = 20;
  constexpr uint32_t kDstBank = 2;
  constexpr uint32_t kBankStride = 256;
  const uint32_t vb = wf->vgpr_alloc().base;
  sim.cu()->write_vgpr(vb + kAddr, 0, 0u);
  wf->set_vgpr_msb_mode(kDstBank << 6);

  gfx1250::VdsMachineInst raw{};
  raw.addr = kAddr;
  raw.vdst = kDst;
  gfx1250::DsLoadTr4B64Vds inst(reinterpret_cast<const gfx1250::MachineInst *>(&raw));
  inst.execute_impl(*wf);

  auto *state = inst.data_as<amdgpu::VectorMemState>();
  ASSERT_NE(state, nullptr);
  EXPECT_EQ(state->dst_reg_base, vb + kDstBank * kBankStride + kDst);
}

TEST(Gfx1250SimulationTest, Vop2FmamkUsesSrc2HighBank) {
  Gfx1250Sim sim;
  // Resident wave: these tests inject instructions directly (execute_impl) and read
  // live register state, so they need a live wavefront, not a run-to-halt snapshot.
  amdgpu::Wavefront *wf = sim.dispatch_scratch_wf(kGfx1250Wave32VgprAllocation);
  ASSERT_NE(wf, nullptr);
  wf->set_exec(1u);

  constexpr uint32_t kAddend = 3;
  constexpr uint32_t kDst = 4;
  constexpr uint32_t kSrc2Bank = 2;
  constexpr uint32_t kBankStride = 256;
  const uint32_t vb = wf->vgpr_alloc().base;
  auto &cu = *sim.cu();

  wf->set_vgpr_msb_mode(kSrc2Bank << 4);
  cu.write_vgpr(vb + kAddend, 0, std::bit_cast<uint32_t>(100.0f));
  cu.write_vgpr(vb + kSrc2Bank * kBankStride + kAddend, 0, std::bit_cast<uint32_t>(4.0f));

  gfx1250::Vop2InstLiteralMachineInst raw{};
  raw.src0 = 242; // Inline 1.0f.
  raw.vsrc1 = kAddend;
  raw.vdst = kDst;
  raw.simm32 = std::bit_cast<uint32_t>(2.0f);
  gfx1250::VFmamkF32Vop2 inst(reinterpret_cast<const gfx1250::MachineInst *>(&raw));
  inst.execute_impl(*wf);

  EXPECT_EQ(cu.read_vgpr(vb + kDst, 0), std::bit_cast<uint32_t>(6.0f));
}

TEST(Gfx1250SimulationTest, VopdFmamkUsesSrc2HighBank) {
  constexpr uint32_t kSrc0InlineOne = 242;
  constexpr uint32_t kAddend = 3;
  constexpr uint32_t kDst = 4;
  constexpr uint32_t kSrc2Bank = 2;
  constexpr uint32_t kBankStride = 256;
  const std::array<uint32_t, 3> words = {
      (0x32u << 26) | (static_cast<uint32_t>(VopdOp::FmamkF32) << 22) |
          (static_cast<uint32_t>(VopdOp::MovB32) << 17) | (kAddend << 9) | kSrc0InlineOne,
      (kDst << 24) | 128u,
      std::bit_cast<uint32_t>(2.0f),
  };

  Gfx1250Sim sim;
  auto *cu = sim.cu();
  auto *wf = cu->dispatch_wf(0, 0, kGfx1250ScalarSlots, kGfx1250Wave32VgprAllocation);
  ASSERT_NE(wf, nullptr);
  wf->set_exec(1u);
  wf->set_vgpr_msb_mode(kSrc2Bank << 4);
  const uint32_t vb = wf->vgpr_alloc().base;
  cu->write_vgpr(vb + kAddend, 0, std::bit_cast<uint32_t>(100.0f));
  cu->write_vgpr(vb + kSrc2Bank * kBankStride + kAddend, 0, std::bit_cast<uint32_t>(4.0f));

  auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
  ASSERT_NE(decoder, nullptr);
  std::unique_ptr<Instruction> inst(decoder->decode(words.data()));
  ASSERT_NE(inst, nullptr);
  ASSERT_EQ(std::string_view(inst->mnemonic()), "v_dual_fmamk_f32 :: v_dual_mov_b32");
  cu->execute_instruction(inst.get(), *wf);

  EXPECT_EQ(cu->read_vgpr(vb + kDst, 0), std::bit_cast<uint32_t>(6.0f));
}

TEST(Gfx1250SimulationTest, VSwapUsesIndependentSourceAndDestinationBanks) {
  Gfx1250Sim sim;
  // Resident wave: these tests inject instructions directly (execute_impl) and read
  // live register state, so they need a live wavefront, not a run-to-halt snapshot.
  amdgpu::Wavefront *wf = sim.dispatch_scratch_wf(kGfx1250Wave32VgprAllocation);
  ASSERT_NE(wf, nullptr);
  wf->set_exec(1u);

  constexpr uint32_t kSrc = 1;
  constexpr uint32_t kDst = 2;
  constexpr uint32_t kBankStride = 256;
  constexpr uint32_t kOldDst = 0x11112222u;
  constexpr uint32_t kHighSrc = 0x33334444u;
  constexpr uint32_t kLowSrc = 0x55556666u;
  const uint32_t vb = wf->vgpr_alloc().base;
  auto &cu = *sim.cu();

  wf->set_vgpr_msb_mode(0x01); // SRC0=bank 1; DST=bank 0.
  cu.write_vgpr(vb + kSrc, 0, kLowSrc);
  cu.write_vgpr(vb + kBankStride + kSrc, 0, kHighSrc);
  cu.write_vgpr(vb + kDst, 0, kOldDst);

  gfx1250::Vop1MachineInst raw{};
  raw.src0 = 256 + kSrc;
  raw.vdst = kDst;
  gfx1250::VSwapB32Vop1 inst(reinterpret_cast<const gfx1250::MachineInst *>(&raw));
  inst.execute_impl(*wf);

  EXPECT_EQ(cu.read_vgpr(vb + kDst, 0), kHighSrc);
  EXPECT_EQ(cu.read_vgpr(vb + kBankStride + kSrc, 0), kOldDst);
  EXPECT_EQ(cu.read_vgpr(vb + kSrc, 0), kLowSrc);
}

TEST(Gfx1250SimulationTest, AsyncToLdsUsesDstAndSrc0HighBanks) {
  Gfx1250Sim sim;
  // Resident wave: these tests inject instructions directly (execute_impl) and read
  // live register state, so they need a live wavefront, not a run-to-halt snapshot.
  amdgpu::Wavefront *wf = sim.dispatch_scratch_wf(kGfx1250Wave32VgprAllocation);
  ASSERT_NE(wf, nullptr);
  wf->set_exec(1u);

  constexpr uint32_t kVaddr = 10;
  constexpr uint32_t kLdsAddr = 20;
  constexpr uint32_t kSrc0Bank = 1;
  constexpr uint32_t kDstBank = 2;
  constexpr uint32_t kBankStride = 256;
  constexpr uint64_t kGlobalAddr = 0x1000;
  constexpr uint32_t kLdsOffset = 0x40;
  const uint32_t vb = wf->vgpr_alloc().base;
  auto &cu = *sim.cu();
  wf->set_lds_base(cu.allocate_lds(256));
  wf->set_vgpr_msb_mode((kDstBank << 6) | kSrc0Bank);
  cu.write_vgpr(vb + kSrc0Bank * kBankStride + kVaddr, 0, static_cast<uint32_t>(kGlobalAddr));
  cu.write_vgpr(vb + kSrc0Bank * kBankStride + kVaddr + 1, 0,
                static_cast<uint32_t>(kGlobalAddr >> 32));
  cu.write_vgpr(vb + kDstBank * kBankStride + kLdsAddr, 0, kLdsOffset);
  cu.write_vgpr(vb + kLdsAddr, 0, 0x80u);

  gfx1250::VglobalMachineInst raw{};
  raw.saddr = 124; // null
  raw.vaddr = kVaddr;
  raw.vdst = kLdsAddr;
  gfx1250::GlobalLoadAsyncToLdsB8Vglobal inst(reinterpret_cast<const gfx1250::MachineInst *>(&raw));
  inst.execute_impl(*wf);

  auto *state = inst.data_as<amdgpu::VectorMemState>();
  ASSERT_NE(state, nullptr);
  EXPECT_EQ(state->per_lane_addr[0], kGlobalAddr);
  EXPECT_EQ(state->per_lane_lds_addr[0], wf->lds_base() + kLdsOffset);
}

TEST(Gfx1250SimulationTest, AddtidStoresUseSrc1HighBank) {
  Gfx1250Sim sim;
  // Resident wave: these tests inject instructions directly (execute_impl) and read
  // live register state, so they need a live wavefront, not a run-to-halt snapshot.
  amdgpu::Wavefront *wf = sim.dispatch_scratch_wf(kGfx1250Wave32VgprAllocation);
  ASSERT_NE(wf, nullptr);
  wf->set_exec(1u);

  constexpr uint32_t kData = 5;
  constexpr uint32_t kSrc1Bank = 2;
  constexpr uint32_t kBankStride = 256;
  constexpr uint32_t kExpected = 0x12345678u;
  const uint32_t vb = wf->vgpr_alloc().base;
  auto &cu = *sim.cu();
  wf->set_vgpr_msb_mode(kSrc1Bank << 2);
  cu.write_vgpr(vb + kData, 0, 0xDEADBEEFu);
  cu.write_vgpr(vb + kSrc1Bank * kBankStride + kData, 0, kExpected);

  gfx1250::VdsMachineInst ds_raw{};
  ds_raw.data0 = kData;
  gfx1250::DsStoreAddtidB32Vds ds_inst(reinterpret_cast<const gfx1250::MachineInst *>(&ds_raw));
  ds_inst.execute_impl(*wf);
  auto *ds_state = ds_inst.data_as<amdgpu::VectorMemState>();
  ASSERT_NE(ds_state, nullptr);
  uint32_t ds_value = 0;
  std::memcpy(&ds_value, ds_state->store_data.data(), sizeof(ds_value));
  EXPECT_EQ(ds_value, kExpected);

  write_wave_sgpr(cu, *wf, 0, 0u);
  write_wave_sgpr(cu, *wf, 1, 0u);
  gfx1250::VglobalMachineInst global_raw{};
  global_raw.saddr = 0;
  global_raw.vsrc = kData;
  gfx1250::GlobalStoreAddtidB32Vglobal global_inst(
      reinterpret_cast<const gfx1250::MachineInst *>(&global_raw));
  global_inst.execute_impl(*wf);
  auto *global_state = global_inst.data_as<amdgpu::VectorMemState>();
  ASSERT_NE(global_state, nullptr);
  uint32_t global_value = 0;
  std::memcpy(&global_value, global_state->store_data.data(), sizeof(global_value));
  EXPECT_EQ(global_value, kExpected);
}

TEST(Gfx1250SimulationTest, DsAddtidLoadAndStoreUseM0ByteBaseAddresses) {
  Gfx1250Sim sim;
  amdgpu::Wavefront *wf = sim.dispatch_scratch_wf(kGfx1250Wave32VgprAllocation);
  ASSERT_NE(wf, nullptr);
  wf->set_exec(0x3u);

  auto &cu = *sim.cu();
  wf->set_lds_base(cu.allocate_lds(0x4000));
  constexpr uint32_t kM0ByteBase = 0x1000;
  wf->set_m0(kM0ByteBase);

  auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
  ASSERT_NE(decoder, nullptr);
  const uint32_t vb = wf->vgpr_alloc().base;
  constexpr uint32_t kLane0Data = 0x12345678u;
  constexpr uint32_t kLane1Data = 0x9ABCDEF0u;
  cu.write_vgpr(vb + 5, 0, kLane0Data);
  cu.write_vgpr(vb + 5, 1, kLane1Data);
  const uint32_t expected0 = wf->lds_base() + kM0ByteBase + 0x1234;
  const uint32_t expected1 = expected0 + sizeof(uint32_t);

  const uint32_t store_words[] = {0xDAC01234u, 0x00000500u};
  std::unique_ptr<Instruction> store(decoder->decode(store_words));
  ASSERT_NE(store, nullptr);
  ASSERT_EQ(std::string_view(store->mnemonic()), "ds_store_addtid_b32");
  cu.execute_instruction(store.get(), *wf);
  auto *store_state = store->data_as<amdgpu::VectorMemState>();
  ASSERT_NE(store_state, nullptr);
  EXPECT_EQ(store_state->per_lane_addr[0], expected0);
  EXPECT_EQ(store_state->per_lane_addr[1], expected1);
  uint32_t lane0_data = 0;
  uint32_t lane1_data = 0;
  std::memcpy(&lane0_data, &store_state->store_data[0], sizeof(lane0_data));
  std::memcpy(&lane1_data, &store_state->store_data[4], sizeof(lane1_data));
  EXPECT_EQ(lane0_data, kLane0Data);
  EXPECT_EQ(lane1_data, kLane1Data);
  amdgpu::LocalMemPipeline local_pipeline;
  local_pipeline.issue(store.release(), *wf);
  EXPECT_EQ(cu.lds().read32(expected0), kLane0Data);
  EXPECT_EQ(cu.lds().read32(expected1), kLane1Data);

  const uint32_t load_words[] = {0xDAC41234u, 0x08000000u};
  std::unique_ptr<Instruction> load(decoder->decode(load_words));
  ASSERT_NE(load, nullptr);
  ASSERT_EQ(std::string_view(load->mnemonic()), "ds_load_addtid_b32");
  cu.execute_instruction(load.get(), *wf);
  auto *load_state = load->data_as<amdgpu::VectorMemState>();
  ASSERT_NE(load_state, nullptr);
  EXPECT_EQ(load_state->per_lane_addr[0], expected0);
  EXPECT_EQ(load_state->per_lane_addr[1], expected1);
  const uint32_t load_dst_base = load_state->dst_reg_base;
  local_pipeline.issue(load.release(), *wf);
  EXPECT_EQ(cu.read_vgpr(load_dst_base, 0), kLane0Data);
  EXPECT_EQ(cu.read_vgpr(load_dst_base, 1), kLane1Data);

  wf->set_m0(0);
  std::unique_ptr<Instruction> default_m0_store(decoder->decode(store_words));
  ASSERT_NE(default_m0_store, nullptr);
  cu.execute_instruction(default_m0_store.get(), *wf);
  auto *default_m0_state = default_m0_store->data_as<amdgpu::VectorMemState>();
  ASSERT_NE(default_m0_state, nullptr);
  EXPECT_EQ(default_m0_state->per_lane_addr[0], wf->lds_base() + 0x1234);
  EXPECT_EQ(default_m0_state->per_lane_addr[1], wf->lds_base() + 0x1238);
}

TEST(Gfx1250SimulationTest, VMovrelsReadsM0RelativeVgpr) {
  const uint32_t code[] = {
      0xBEFD0082u, // s_mov_b32 m0, 2
      0x7E1402FFu,
      0x00000063u, // v_mov_b32_e32 v10, 99
      0x7E028708u, // v_movrels_b32_e32 v1, v8
      S_ENDPGM_GFX12,
  };

  Gfx1250Sim sim;
  const auto *wf = dispatch_one_wave(sim, code, std::size(code), 16);
  ASSERT_NE(wf, nullptr);

  for (uint32_t lane = 0; lane < wf->wf_size; ++lane)
    EXPECT_EQ(wf->vgpr(1, lane), 99u) << "lane " << lane;
}

TEST(Gfx1250SimulationTest, VopdMulDx9ZeroOverridesNanProducts) {
  constexpr auto dx9_mul = make_vopd3_pair(
      {.op = VopdOp::MulDx9ZeroF32, .src0 = vopd_src0_vgpr(0), .src1 = 1, .src2 = 0, .dst = 4},
      {.op = VopdOp::MulDx9ZeroF32, .src0 = vopd_src0_vgpr(0), .src1 = 2, .src2 = 0, .dst = 5});
  constexpr auto ieee_mul = make_vopd3_pair(
      {.op = VopdOp::MulF32, .src0 = vopd_src0_vgpr(0), .src1 = 1, .src2 = 0, .dst = 6},
      {.op = VopdOp::MulF32, .src0 = vopd_src0_vgpr(0), .src1 = 2, .src2 = 0, .dst = 7});

  std::vector<uint32_t> code;
  append_instruction(code, make_vmov_b32_literal(0, 0x7FC00000u)); // quiet NaN
  append_instruction(code, make_vmov_b32_literal(1, 0x00000000u)); // +0.0f
  append_instruction(code, make_vmov_b32_literal(2, 0x80000000u)); // -0.0f
  append_instruction(code, dx9_mul);
  append_instruction(code, ieee_mul);
  append_instruction(code, S_ENDPGM_GFX12);

  Gfx1250Sim sim;
  const auto *wf = dispatch_one_wave(sim, code.data(), code.size(), 16);
  ASSERT_NE(wf, nullptr);

  for (uint32_t lane = 0; lane < wf->wf_size; ++lane) {
    EXPECT_EQ(wf->vgpr(4, lane), 0x00000000u) << "lane " << lane;
    EXPECT_EQ(wf->vgpr(5, lane), 0x00000000u) << "lane " << lane;
    EXPECT_TRUE(std::isnan(std::bit_cast<float>(wf->vgpr(6, lane)))) << "lane " << lane;
    EXPECT_TRUE(std::isnan(std::bit_cast<float>(wf->vgpr(7, lane)))) << "lane " << lane;
  }
}

TEST(Gfx1250SimulationTest, VopdFmaUsesSingleRounding) {
  constexpr uint32_t kSrc0 = 0x3F800001u;
  constexpr uint32_t kSrc1 = 0x3F7FFFFFu;
  constexpr uint32_t kSrc2 = 0xBF800000u;
  constexpr auto fma = make_vopd3_pair(
      {.op = VopdOp::FmaF32, .src0 = vopd_src0_vgpr(0), .src1 = 1, .src2 = 2, .dst = 4},
      {.op = VopdOp::FmaF32, .src0 = vopd_src0_vgpr(0), .src1 = 1, .src2 = 2, .dst = 5});
  const uint32_t expected = std::bit_cast<uint32_t>(std::fma(
      std::bit_cast<float>(kSrc0), std::bit_cast<float>(kSrc1), std::bit_cast<float>(kSrc2)));
  ASSERT_EQ(expected, 0x337FFFFEu);

  std::vector<uint32_t> code;
  append_instruction(code, make_vmov_b32_literal(0, kSrc0));
  append_instruction(code, make_vmov_b32_literal(1, kSrc1));
  append_instruction(code, make_vmov_b32_literal(2, kSrc2));
  append_instruction(code, fma);
  append_instruction(code, S_ENDPGM_GFX12);

  Gfx1250Sim sim;
  const auto *wf = dispatch_one_wave(sim, code.data(), code.size(), 16);
  ASSERT_NE(wf, nullptr);

  for (uint32_t lane = 0; lane < wf->wf_size; ++lane) {
    EXPECT_EQ(wf->vgpr(4, lane), expected) << "lane " << lane;
    EXPECT_EQ(wf->vgpr(5, lane), expected) << "lane " << lane;
  }
}

TEST(Gfx1250SimulationTest, VopdFmacUsesDestinationAccumulator) {
  const uint32_t code[] = {
      0x7E0002FFu,
      0x40000000u, // v_mov_b32_e32 v0, 2.0f
      0x7E1202FFu,
      0x3F800000u, // v_mov_b32_e32 v9, 1.0f
      0x7E1402FFu,
      0x40400000u, // v_mov_b32_e32 v10, 3.0f
      0x7E1C02FFu,
      0x40800000u, // v_mov_b32_e32 v14, 4.0f
      0xC900150Eu,    0x0A0800FFu,
      0x3F000000u, // v_dual_add_f32 v10, v14, v10 :: v_dual_fmac_f32 v9, 0.5f, v0
      S_ENDPGM_GFX12,
  };

  Gfx1250Sim sim;
  const auto *wf = dispatch_one_wave(sim, code, std::size(code), 16);
  ASSERT_NE(wf, nullptr);

  for (uint32_t lane = 0; lane < wf->wf_size; ++lane) {
    EXPECT_EQ(wf->vgpr(10, lane), 0x40E00000u) << "lane " << lane;
    EXPECT_EQ(wf->vgpr(9, lane), 0x40000000u) << "lane " << lane;
  }
}

TEST(Gfx1250ExecutionTest, Vopd3CndmaskAppliesB32NegModifiers) {
  constexpr auto cndmask = make_vopd3_pair(
      {.op = VopdOp::CndmaskB32, .src0 = vopd_src0_vgpr(0), .src1 = 1, .src2 = 106, .dst = 2},
      {.op = VopdOp::CndmaskB32, .src0 = vopd_src0_vgpr(3), .src1 = 4, .src2 = 106, .dst = 5},
      /*negx=*/0x1, /*negy=*/0x2);

  Gfx1250Sim sim;
  auto *cu = sim.cu();
  auto *wf = cu->dispatch_wf(0, 0, kGfx1250ScalarSlots, kGfx1250Wave32VgprAllocation);
  ASSERT_NE(wf, nullptr);
  ASSERT_EQ(wf->wf_size(), 32u);
  wf->set_exec(0x3u);
  wf->set_vcc(0x1u);

  auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
  ASSERT_NE(decoder, nullptr);
  std::unique_ptr<Instruction> inst(decoder->decode(cndmask.data()));
  ASSERT_NE(inst, nullptr);
  ASSERT_EQ(std::string_view(inst->mnemonic()), "v_dual_cndmask_b32 :: v_dual_cndmask_b32");

  const uint32_t vb = wf->vgpr_alloc().base;
  cu->write_vgpr(vb + 0, 0, 0x3F800000u);
  cu->write_vgpr(vb + 0, 1, 0x3F000000u);
  cu->write_vgpr(vb + 1, 0, 0x11223344u);
  cu->write_vgpr(vb + 1, 1, 0x55667788u);
  cu->write_vgpr(vb + 3, 0, 0x40000000u);
  cu->write_vgpr(vb + 3, 1, 0x40400000u);
  cu->write_vgpr(vb + 4, 0, 0x3F800000u);
  cu->write_vgpr(vb + 4, 1, 0x3F000000u);

  cu->execute_instruction(inst.get(), *wf);

  EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0x11223344u);
  EXPECT_EQ(cu->read_vgpr(vb + 2, 1), 0xBF000000u);
  EXPECT_EQ(cu->read_vgpr(vb + 5, 0), 0xBF800000u);
  EXPECT_EQ(cu->read_vgpr(vb + 5, 1), 0x40400000u);
}

TEST(Gfx1250SimulationTest, GlobalStoreWritesVisibleMemory) {
  constexpr uint64_t kernel_addr = 0x10000;
  constexpr uint64_t output_addr = 0x2000;

  const uint32_t code[] = {
      0xBE8400FFu,    static_cast<uint32_t>(output_addr), // s_mov_b32 s4, output_addr
      0xBE850080u,                                        // s_mov_b32 s5, 0
      0x30000082u,                                        // v_lshlrev_b32_e32 v0, 2, v0
      0x7E020281u,                                        // v_mov_b32_e32 v1, 1
      0xEE068004u,    0x00800000u,
      0x00000000u, // global_store_b32 v0, v1, s[4:5]
      0xBFC10000u, // s_wait_storecnt 0
      S_ENDPGM_GFX12,
  };

  Gfx1250Sim sim;
  uint64_t kernel_object = sim.write_kernel(kernel_addr, code, std::size(code));
  for (uint32_t lane = 0; lane < 32; ++lane)
    sim.memory->write32(output_addr + lane * sizeof(uint32_t), 0);

  test::AqlQueue queue(sim.memory, sim.cp());
  queue.dispatch(kernel_object, 32, 32);
  sim.engine->run();
  sim.soc->flush_all();

  for (uint32_t lane = 0; lane < 32; ++lane)
    EXPECT_EQ(sim.memory->read32(output_addr + lane * sizeof(uint32_t)), 1u) << "lane " << lane;
}

TEST(Gfx1250SimulationTest, BufferStoreUsesM0Soffset) {
  constexpr uint64_t kernel_addr = 0x10000;
  constexpr uint64_t output_addr = 0x2000;

  const uint32_t code[] = {
      0xBE8400FFu,    static_cast<uint32_t>(output_addr), // s_mov_b32 s4, output_addr
      0xBE850080u,                                        // s_mov_b32 s5, 0
      0xBE8600FFu,    0x1000u,                            // s_mov_b32 s6, num_records
      0xBEFD0090u,                                        // s_mov_b32 m0, 16
      0x300A0085u,                                        // v_lshlrev_b32_e32 v5, 5, v0
      0x7E000287u,                                        // v_mov_b32_e32 v0, 7
      0xC406807Du,    0x40800800u,
      0x00000005u, // buffer_store_b32 v0, v5, s[4:7], m0 offen
      0xBFC10000u, // s_wait_storecnt 0
      S_ENDPGM_GFX12,
  };

  Gfx1250Sim sim;
  uint64_t kernel_object = sim.write_kernel(kernel_addr, code, std::size(code), 128);
  for (uint32_t lane = 0; lane < 32; ++lane) {
    sim.memory->write32(output_addr + lane * 32, 0);
    sim.memory->write32(output_addr + 16 + lane * 32, 0);
  }

  test::AqlQueue queue(sim.memory, sim.cp());
  queue.dispatch(kernel_object, 32, 32);
  sim.engine->run();
  sim.soc->flush_all();

  for (uint32_t lane = 0; lane < 32; ++lane) {
    EXPECT_EQ(sim.memory->read32(output_addr + lane * 32), 0u) << "lane " << lane;
    EXPECT_EQ(sim.memory->read32(output_addr + 16 + lane * 32), 7u) << "lane " << lane;
  }
}

TEST(Gfx1250SimulationTest, MemorySideCacheFlushSerializesConcurrentAccess) {
  constexpr uint64_t output_addr = 0x2000;
  constexpr uint32_t iterations = 32;

  Gfx1250Sim sim;
  auto *msc = sim.soc->iod(0)->msc();
  std::barrier start(3);

  std::thread writer([&] {
    start.arrive_and_wait();
    for (uint32_t value = 1; value <= iterations; ++value) {
      msc->write(output_addr, reinterpret_cast<const uint8_t *>(&value), sizeof(value));
      std::this_thread::yield();
    }
  });
  std::thread reader([&] {
    start.arrive_and_wait();
    for (uint32_t i = 0; i < iterations; ++i) {
      uint32_t value = 0;
      msc->read(output_addr, reinterpret_cast<uint8_t *>(&value), sizeof(value));
      EXPECT_LE(value, iterations);
      std::this_thread::yield();
    }
  });
  std::thread flusher([&] {
    start.arrive_and_wait();
    for (uint32_t i = 0; i < iterations; ++i) {
      msc->flush_all();
      std::this_thread::yield();
    }
  });

  writer.join();
  reader.join();
  flusher.join();
  msc->flush_all();

  EXPECT_EQ(sim.memory->read32(output_addr), iterations);
}

TEST(WriterPreferredAccessGateTest, WaitingWriterBlocksLateSharedEntrant) {
  amdgpu::WriterPreferredAccessGate gate;
  std::shared_lock active_reader(gate);
  bool writer_acquired = false;
  std::thread writer([&] {
    std::unique_lock waiting_writer(gate);
    writer_acquired = true;
  });

  const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(60);
  while (!amdgpu::WriterPreferredAccessGateTestAccess::has_waiting_writer(gate) &&
         std::chrono::steady_clock::now() < deadline)
    std::this_thread::yield();

  const bool writer_waiting = amdgpu::WriterPreferredAccessGateTestAccess::has_waiting_writer(gate);
  EXPECT_TRUE(writer_waiting);
  if (writer_waiting) {
    const bool late_reader_acquired = gate.try_lock_shared();
    EXPECT_FALSE(late_reader_acquired);
    if (late_reader_acquired)
      gate.unlock_shared();
  }

  active_reader.unlock();
  writer.join();
  EXPECT_TRUE(writer_acquired);
}

TEST(Gfx1250SimulationTest, MemorySideCacheFlushMakesProgressUnderContinuousReads) {
  constexpr uint64_t output_addr = 0x2000;
  constexpr uint32_t reader_count = 4;

  Gfx1250Sim sim;
  auto *msc = sim.soc->iod(0)->msc();
  std::barrier start(reader_count + 1);
  std::atomic<bool> stop_readers = false;
  std::atomic<uint32_t> readers_started = 0;
  std::vector<std::thread> readers;
  readers.reserve(reader_count);
  for (uint32_t i = 0; i < reader_count; ++i) {
    readers.emplace_back([&] {
      start.arrive_and_wait();
      bool first_read = true;
      while (!stop_readers.load(std::memory_order_acquire)) {
        uint32_t value = 0;
        msc->read(output_addr, reinterpret_cast<uint8_t *>(&value), sizeof(value));
        if (first_read) {
          readers_started.fetch_add(1, std::memory_order_release);
          first_read = false;
        }
      }
    });
  }

  start.arrive_and_wait();
  while (readers_started.load(std::memory_order_acquire) < reader_count)
    std::this_thread::yield();

  std::atomic<bool> flush_completed = false;
  std::thread flusher([&] {
    msc->flush_all();
    flush_completed.store(true, std::memory_order_release);
  });

  const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(60);
  while (!flush_completed.load(std::memory_order_acquire) &&
         std::chrono::steady_clock::now() < deadline)
    std::this_thread::yield();
  const bool completed_while_readers_active = flush_completed.load(std::memory_order_acquire);

  stop_readers.store(true, std::memory_order_release);
  for (auto &reader : readers)
    reader.join();
  flusher.join();

  EXPECT_TRUE(completed_while_readers_active);
}

} // namespace