// Copyright (c) 2026 Advanced Micro Devices, Inc.
// SPDX-License-Identifier: MIT

/// @file instruction_execution_harness_test.cpp
/// @brief Phase E parameterized test: execute safe scalar encodings on every ISA.
///
/// For each ISA, iterates auto-generated encodings that are safe to execute on
/// a zeroed wavefront, decodes them, and calls execute(). Decode failures are
/// rejected, and UnimplementedInst results must exactly match the explicit
/// expectation for that ISA.

#include "rocjitsu/base/rj_compiler.h"
#include "rocjitsu/code/rj_code.h"
#include "rocjitsu/isa/arch/amdgpu/cdna1/opcodes.h"
#include "rocjitsu/isa/arch/amdgpu/cdna2/opcodes.h"
#include "rocjitsu/isa/arch/amdgpu/cdna3/opcodes.h"
#include "rocjitsu/isa/arch/amdgpu/cdna4/opcodes.h"
#include "rocjitsu/isa/arch/amdgpu/gfx1250/isa.h"
#include "rocjitsu/isa/arch/amdgpu/gfx1250/opcodes.h"
#include "rocjitsu/isa/arch/amdgpu/rdna1/opcodes.h"
#include "rocjitsu/isa/arch/amdgpu/rdna2/opcodes.h"
#include "rocjitsu/isa/arch/amdgpu/rdna3/opcodes.h"
#include "rocjitsu/isa/arch/amdgpu/rdna3_5/opcodes.h"
#include "rocjitsu/isa/arch/amdgpu/rdna4/opcodes.h"
#include "rocjitsu/isa/arch/amdgpu/rdna4/vop3.h"
#include "rocjitsu/isa/arch/amdgpu/shared/dpp_sdwa_ops.h"
#include "rocjitsu/isa/arch/amdgpu/shared/mma_exec.h"
#include "rocjitsu/isa/decoder.h"
#include "rocjitsu/isa/instruction.h"
#include "rocjitsu/vm/amdgpu/compute_unit.h"
#include "rocjitsu/vm/amdgpu/gpu_memory.h"
#include "rocjitsu/vm/amdgpu/hwreg.h"
#include "rocjitsu/vm/amdgpu/l2_cache.h"
#include "rocjitsu/vm/amdgpu/wavefront.h"
#include "util/data_types.h"
#include "util/except.h"
#include "util/simd.h"
#include "util/simd_test_hooks.h"

// Per-ISA test encoding tables (auto-generated by amdisa codegen).
#include "rocjitsu/isa/arch/amdgpu/cdna4/builders.h"
#include "rocjitsu/isa/arch/amdgpu/cdna4/opcodes.h"

#include "rocjitsu/isa/arch/amdgpu/cdna1/test_encodings.h"
#include "rocjitsu/isa/arch/amdgpu/cdna2/test_encodings.h"
#include "rocjitsu/isa/arch/amdgpu/cdna3/test_encodings.h"
#include "rocjitsu/isa/arch/amdgpu/cdna4/test_encodings.h"
#include "rocjitsu/isa/arch/amdgpu/gfx1250/test_encodings.h"
#include "rocjitsu/isa/arch/amdgpu/rdna1/test_encodings.h"
#include "rocjitsu/isa/arch/amdgpu/rdna2/test_encodings.h"
#include "rocjitsu/isa/arch/amdgpu/rdna3/test_encodings.h"
#include "rocjitsu/isa/arch/amdgpu/rdna3_5/test_encodings.h"
#include "rocjitsu/isa/arch/amdgpu/rdna4/test_encodings.h"

#include <gtest/gtest.h>

#include <algorithm>
#include <array>
#include <bit>
#include <cmath>
#include <cstdint>
#include <exception>
#include <limits>
#include <memory>
#include <span>
#include <string>
#include <string_view>
#include <utility>
#include <vector>

namespace {

using namespace rocjitsu;

/// @brief Common test encoding entry (matches all ISA test_encodings.h).
struct TestEncEntry {
  std::string_view mnemonic;
  std::array<uint32_t, 2> words;
};

constexpr uint32_t pack16(uint16_t lo, uint16_t hi) {
  return static_cast<uint32_t>(lo) | (static_cast<uint32_t>(hi) << 16);
}

constexpr uint32_t vgpr_src(uint32_t reg) { return 256u + reg; }

constexpr std::array<uint32_t, 2> encode_vop3(uint32_t op, uint32_t vdst, uint32_t src0,
                                              uint32_t src1, uint32_t src2, uint32_t abs = 0,
                                              uint32_t opsel = 0, uint32_t clamp = 0,
                                              uint32_t omod = 0, uint32_t neg = 0) {
  return {vdst | ((abs & 0x7u) << 8) | ((opsel & 0xFu) << 11) | ((clamp & 0x1u) << 15) |
              ((op & 0x3FFu) << 16) | (0x35u << 26),
          (src0 & 0x1FFu) | ((src1 & 0x1FFu) << 9) | ((src2 & 0x1FFu) << 18) |
              ((omod & 0x3u) << 27) | ((neg & 0x7u) << 29)};
}

constexpr std::array<uint32_t, 2> encode_cdna_vop3(uint32_t op, uint32_t vdst, uint32_t src0,
                                                   uint32_t src1, uint32_t src2,
                                                   uint32_t opsel = 0) {
  return {vdst | ((opsel & 0xFu) << 11) | ((op & 0x3FFu) << 16) | (0x34u << 26),
          (src0 & 0x1FFu) | ((src1 & 0x1FFu) << 9) | ((src2 & 0x1FFu) << 18)};
}

constexpr std::array<uint32_t, 2> encode_vop2(uint32_t op, uint32_t vdst, uint32_t src0,
                                              uint32_t vsrc1) {
  return {(src0 & 0x1FFu) | ((vsrc1 & 0xFFu) << 9) | ((vdst & 0xFFu) << 17) | ((op & 0x3Fu) << 25),
          0u};
}

constexpr std::array<uint32_t, 2> encode_sop1(uint32_t op, uint32_t sdst, uint32_t ssrc0) {
  return {(ssrc0 & 0xFFu) | ((op & 0xFFu) << 8) | ((sdst & 0x7Fu) << 16) | 0xBE800000u, 0u};
}

constexpr std::array<uint32_t, 2> encode_sop2(uint32_t op, uint32_t sdst, uint32_t ssrc0,
                                              uint32_t ssrc1) {
  return {(ssrc0 & 0xFFu) | ((ssrc1 & 0xFFu) << 8) | ((sdst & 0x7Fu) << 16) | ((op & 0x7Fu) << 23) |
              (0x2u << 30),
          0u};
}

constexpr uint32_t encode_hwreg(uint32_t id, uint32_t offset = 0, uint32_t size = 32) {
  return (id & 0x3Fu) | ((offset & 0x1Fu) << 6) | (((size - 1u) & 0x1Fu) << 11);
}

constexpr std::array<uint32_t, 2> encode_sopk(uint32_t op, uint32_t sdst, uint32_t simm16,
                                              uint32_t literal = 0) {
  return {(0xBu << 28) | ((op & 0x1Fu) << 23) | ((sdst & 0x7Fu) << 16) | (simm16 & 0xFFFFu),
          literal};
}

struct HwregArchCase {
  rj_code_arch_t arch;
  const char *name;
  uint32_t getreg_op;
  uint32_t setreg_op;
  uint32_t setreg_imm_op;
};

constexpr HwregArchCase kHwregArchCases[] = {
    {ROCJITSU_CODE_ARCH_CDNA1, "cdna1", cdna1::kSGetregB32Sopk, cdna1::kSSetregB32Sopk,
     cdna1::kSSetregImm32B32Sopk},
    {ROCJITSU_CODE_ARCH_CDNA2, "cdna2", cdna2::kSGetregB32Sopk, cdna2::kSSetregB32Sopk,
     cdna2::kSSetregImm32B32Sopk},
    {ROCJITSU_CODE_ARCH_CDNA3, "cdna3", cdna3::kSGetregB32Sopk, cdna3::kSSetregB32Sopk,
     cdna3::kSSetregImm32B32Sopk},
    {ROCJITSU_CODE_ARCH_CDNA4, "cdna4", cdna4::kSGetregB32Sopk, cdna4::kSSetregB32Sopk,
     cdna4::kSSetregImm32B32Sopk},
    {ROCJITSU_CODE_ARCH_RDNA1, "rdna1", rdna1::kSGetregB32Sopk, rdna1::kSSetregB32Sopk,
     rdna1::kSSetregImm32B32Sopk},
    {ROCJITSU_CODE_ARCH_RDNA2, "rdna2", rdna2::kSGetregB32Sopk, rdna2::kSSetregB32Sopk,
     rdna2::kSSetregImm32B32Sopk},
    {ROCJITSU_CODE_ARCH_RDNA3, "rdna3", rdna3::kSGetregB32Sopk, rdna3::kSSetregB32Sopk,
     rdna3::kSSetregImm32B32Sopk},
    {ROCJITSU_CODE_ARCH_RDNA3_5, "rdna3_5", rdna3_5::kSGetregB32Sopk, rdna3_5::kSSetregB32Sopk,
     rdna3_5::kSSetregImm32B32Sopk},
    {ROCJITSU_CODE_ARCH_RDNA4, "rdna4", rdna4::kSGetregB32Sopk, rdna4::kSSetregB32Sopk,
     rdna4::kSSetregImm32B32Sopk},
    {ROCJITSU_CODE_ARCH_GFX1250, "gfx1250", gfx1250::kSGetregB32Sopk, gfx1250::kSSetregB32Sopk,
     gfx1250::kSSetregImm32B32Sopk},
};

struct ForceScalarGuard {
  explicit ForceScalarGuard(bool force_scalar) : old_force_scalar(util::force_scalar()) {
    util::set_force_scalar_for_testing(force_scalar);
  }
  ~ForceScalarGuard() { util::set_force_scalar_for_testing(old_force_scalar); }

  bool old_force_scalar;
};

class Gfx1250MemoryTestCu
    : public amdgpu::IsaExecComputeUnit<simdojo::ExecMode::FUNCTIONAL, gfx1250::Isa> {
public:
  using Base = amdgpu::IsaExecComputeUnit<simdojo::ExecMode::FUNCTIONAL, gfx1250::Isa>;

  Gfx1250MemoryTestCu(std::string name, const amdgpu::ComputeUnitCore::Config &config,
                      amdgpu::GpuMemory *memory, amdgpu::L2Cache *l2)
      : Base(std::move(name), config, memory, l2) {
    if (l2)
      l2->set_backing_memory(memory);
    set_memory(memory);
    set_l2(l2);
  }

  void execute_and_route(Instruction *inst, amdgpu::Wavefront &wf) {
    execute_instruction(inst, wf);
    if (inst->is_memory_op())
      route_memory_inst(inst, wf);
    else
      delete inst;
  }
};

/// @brief Check if a mnemonic should be skipped in the execution harness.
///
/// Memory instructions require valid addresses and will crash on zeroed state.
/// Control flow instructions (branch, endpgm, barrier, waitcnt, trap) may
/// halt, loop, or enter wait states that hang the test.
inline bool should_skip_inst(std::string_view mn) {
  // Only execute scalar ALU instructions (s_* that don't touch memory or
  // control flow). Vector instructions may crash because the synthesized
  // encoding has src0=0 which maps to SGPR[0] for scalar source, but in
  // VOP encodings src0=0 is VGPR[0] and VGPR reading with zeroed register
  // allocation can segfault.
  //
  // Allowlist approach: only s_* instructions that are pure ALU.
  if (mn.empty() || mn[0] != 's' || mn[1] != '_')
    return true; // Skip all non-scalar instructions.
  static constexpr std::string_view SKIP_PREFIXES[] = {
      // Memory instructions
      "s_load_",
      "s_store_",
      "s_buffer_",
      "s_dcache_",
      "s_atomic_",
      // Control flow that halts or waits
      "s_endpgm",
      "s_branch",
      "s_cbranch",
      "s_setpc",
      "s_swappc",
      "s_call",
      "s_waitcnt",
      "s_wait_",
      "s_barrier",
      "s_trap",
      "s_sleep",
      "s_sethalt",
      "s_sendmsg",
      "s_sendmsghalt",
      "s_icache_inv",
      "s_nop",
      "s_getpc",
      "s_getreg",
      "s_setreg",
      "s_rfe",
  };
  for (auto p : SKIP_PREFIXES)
    if (mn.substr(0, p.size()) == p)
      return true;
  return false;
}

constexpr std::array<std::string_view, 1> EXPECTED_CDNA_UNIMPLEMENTED = {"s_setvskip"};
constexpr std::array<std::string_view, 3> EXPECTED_RDNA1_UNIMPLEMENTED = {
    "s_subvector_loop_begin", "s_subvector_loop_end", "s_get_waveid_in_workgroup"};
constexpr std::array<std::string_view, 2> EXPECTED_RDNA2_UNIMPLEMENTED = {"s_subvector_loop_begin",
                                                                          "s_subvector_loop_end"};
constexpr std::array<std::string_view, 0> EXPECTED_NO_UNIMPLEMENTED = {};

struct HarnessExpectation {
  std::string_view arch_name;
  std::span<const std::string_view> unimplemented;
};

constexpr HarnessExpectation HARNESS_EXPECTATIONS[] = {
    {"cdna1", EXPECTED_CDNA_UNIMPLEMENTED},  {"cdna2", EXPECTED_CDNA_UNIMPLEMENTED},
    {"cdna3", EXPECTED_CDNA_UNIMPLEMENTED},  {"cdna4", EXPECTED_CDNA_UNIMPLEMENTED},
    {"rdna1", EXPECTED_RDNA1_UNIMPLEMENTED}, {"rdna2", EXPECTED_RDNA2_UNIMPLEMENTED},
    {"rdna3", EXPECTED_NO_UNIMPLEMENTED},    {"rdna3_5", EXPECTED_NO_UNIMPLEMENTED},
    {"rdna4", EXPECTED_NO_UNIMPLEMENTED},    {"gfx1250", EXPECTED_NO_UNIMPLEMENTED},
};

const HarnessExpectation *harness_expectation(std::string_view arch_name) {
  for (const auto &expectation : HARNESS_EXPECTATIONS)
    if (expectation.arch_name == arch_name)
      return &expectation;
  return nullptr;
}

/// @brief Helper to run all test encodings for a given ISA.
void run_execution_harness(rj_code_arch_t arch, std::string_view arch_name,
                           const TestEncEntry *encodings, size_t num_encodings) {
  // Set up minimal CU with zeroed memory.
  amdgpu::GpuMemory gpu_mem(std::string(arch_name) + "_mem");
  amdgpu::L2Cache l2(std::string(arch_name) + "_l2");

  amdgpu::ComputeUnitCore::Config cfg{};
  cfg.arch = arch;
  cfg.num_wf_slots = 1;
  cfg.sgprs_per_wf = 106; // max across all ISAs
  cfg.vgprs_per_wf = 256;
  cfg.lds_size_kb = 64;

  auto cu = amdgpu::ComputeUnitCore::create(std::string(arch_name), cfg, &gpu_mem, &l2);
  ASSERT_NE(cu, nullptr) << "Failed to create CU for " << arch_name;

  auto decoder = Decoder::create(arch);
  ASSERT_NE(decoder, nullptr) << "Failed to create decoder for " << arch_name;

  size_t total = num_encodings;
  size_t decoded = 0;
  size_t executed = 0;
  size_t skipped = 0;
  std::vector<std::string_view> unimpl_list;
  std::vector<std::string> decode_fail_list;
  std::vector<std::string> mnemonic_mismatch_list;
  std::vector<std::string> execution_fail_list;

  for (size_t i = 0; i < total; ++i) {
    const auto &te = encodings[i];

    // Skip memory instructions — they require valid addresses and will
    // crash or hang when executed on zeroed wavefront/memory state.
    if (should_skip_inst(te.mnemonic)) {
      ++skipped;
      continue;
    }

    // Decode the instruction.
    std::unique_ptr<Instruction> inst;
    try {
      inst.reset(decoder->decode(te.words.data()));
    } catch (const util::InvalidInst &) {
      decode_fail_list.emplace_back(te.mnemonic);
      continue;
    } catch (...) {
      decode_fail_list.emplace_back(te.mnemonic);
      continue;
    }
    if (!inst) {
      decode_fail_list.emplace_back(te.mnemonic);
      continue;
    }

    const std::string_view decoded_mnemonic = inst->mnemonic();
    if (decoded_mnemonic != te.mnemonic) {
      mnemonic_mismatch_list.emplace_back(
          std::string(te.mnemonic).append(" decoded as ").append(decoded_mnemonic));
      continue;
    }
    ++decoded;

    // Dispatch a fresh wavefront for each instruction to avoid state leaks. The
    // previous iteration halted its wave (freeing the slot), so this always
    // succeeds on the single-slot scratch CU.
    amdgpu::Wavefront *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
    ASSERT_NE(wf, nullptr) << "Failed to dispatch WF for " << te.mnemonic;

    // Execute and catch UnimplementedInst.
    try {
      cu->execute_instruction(inst.get(), *wf);
      ++executed;
    } catch (const util::UnimplementedInst &) {
      unimpl_list.emplace_back(te.mnemonic);
    } catch (const std::exception &error) {
      execution_fail_list.emplace_back(std::string(te.mnemonic).append(": ").append(error.what()));
    } catch (...) {
      execution_fail_list.emplace_back(te.mnemonic);
    }

    // Reset the wavefront for the next instruction.
    if (!wf->is_halted())
      wf->halt();
  }

  size_t testable = total - skipped;
  std::printf("\n  %.*s: %zu/%zu testable encodings executed, %zu unimplemented, "
              "%zu decode fail, %zu execution fail, %zu skipped\n",
              static_cast<int>(arch_name.size()), arch_name.data(), executed, testable,
              unimpl_list.size(), decode_fail_list.size(), execution_fail_list.size(), skipped);

  if (!unimpl_list.empty()) {
    std::printf("  Unimplemented (%zu):", unimpl_list.size());
    for (size_t i = 0; i < std::min(unimpl_list.size(), size_t{20}); ++i) {
      const auto mnemonic = unimpl_list[i];
      std::printf(" %.*s", static_cast<int>(mnemonic.size()), mnemonic.data());
    }
    if (unimpl_list.size() > 20)
      std::printf(" ... +%zu more", unimpl_list.size() - 20);
    std::printf("\n");
  }

  // Every encoding admitted by should_skip_inst must decode. Known
  // UnimplementedInst results are named explicitly so failures identify the
  // exact instruction that regressed instead of only crossing a numeric
  // coverage threshold.
  EXPECT_GT(decoded, 0u) << "No instructions decoded for " << arch_name;
  EXPECT_EQ(decode_fail_list.size(), 0u) << arch_name << " gained decode failures";
  EXPECT_TRUE(mnemonic_mismatch_list.empty())
      << arch_name << " encodings decoded as a different instruction: "
      << ::testing::PrintToString(mnemonic_mismatch_list);
  EXPECT_EQ(execution_fail_list.size(), 0u)
      << arch_name << " failed to execute implemented instructions";

  const auto *expectation = harness_expectation(arch_name);
  ASSERT_NE(expectation, nullptr) << "Missing harness expectation for " << arch_name;
  std::sort(unimpl_list.begin(), unimpl_list.end());
  std::vector<std::string_view> expected(expectation->unimplemented.begin(),
                                         expectation->unimplemented.end());
  std::sort(expected.begin(), expected.end());
  EXPECT_EQ(unimpl_list, expected) << arch_name << " unimplemented instruction set changed";
}

// --- Parameterized tests per ISA ---

// Helper macro: cast the ISA-specific TestEncoding array to the common layout.
// The struct layout (string_view + array<uint32_t,2>) is identical across all ISAs.
#define RUN_HARNESS(ISA_NS, ARCH_ENUM, ARCH_STR)                                                   \
  run_execution_harness(ARCH_ENUM, ARCH_STR,                                                       \
                        reinterpret_cast<const TestEncEntry *>(ISA_NS::test_data::ENCODINGS),      \
                        ISA_NS::test_data::NUM_ENCODINGS)

TEST(InstructionExecutionHarness, Cdna1) { RUN_HARNESS(cdna1, ROCJITSU_CODE_ARCH_CDNA1, "cdna1"); }
TEST(InstructionExecutionHarness, Cdna2) { RUN_HARNESS(cdna2, ROCJITSU_CODE_ARCH_CDNA2, "cdna2"); }
TEST(InstructionExecutionHarness, Cdna3) { RUN_HARNESS(cdna3, ROCJITSU_CODE_ARCH_CDNA3, "cdna3"); }
TEST(InstructionExecutionHarness, Cdna4) { RUN_HARNESS(cdna4, ROCJITSU_CODE_ARCH_CDNA4, "cdna4"); }
TEST(InstructionExecutionHarness, Rdna1) { RUN_HARNESS(rdna1, ROCJITSU_CODE_ARCH_RDNA1, "rdna1"); }
TEST(InstructionExecutionHarness, Rdna2) { RUN_HARNESS(rdna2, ROCJITSU_CODE_ARCH_RDNA2, "rdna2"); }
TEST(InstructionExecutionHarness, Rdna3) { RUN_HARNESS(rdna3, ROCJITSU_CODE_ARCH_RDNA3, "rdna3"); }
TEST(InstructionExecutionHarness, Rdna3_5) {
  RUN_HARNESS(rdna3_5, ROCJITSU_CODE_ARCH_RDNA3_5, "rdna3_5");
}
TEST(InstructionExecutionHarness, Rdna4) { RUN_HARNESS(rdna4, ROCJITSU_CODE_ARCH_RDNA4, "rdna4"); }

TEST(Rdna4ExecMaskTest, Wave32ExecHiRemainsAvailableAsScalarScratch) {
  amdgpu::GpuMemory gpu_mem("rdna4_exec_hi_mem");
  amdgpu::L2Cache l2("rdna4_exec_hi_l2");

  amdgpu::ComputeUnitCore::Config cfg{};
  cfg.arch = ROCJITSU_CODE_ARCH_RDNA4;
  cfg.num_wf_slots = 1;
  cfg.sgprs_per_wf = 128;
  cfg.vgprs_per_wf = 256;
  cfg.lds_size_kb = 64;

  auto cu = amdgpu::ComputeUnitCore::create("rdna4_exec_hi", cfg, &gpu_mem, &l2);
  ASSERT_NE(cu, nullptr);
  auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
  ASSERT_NE(wf, nullptr);
  ASSERT_EQ(wf->wf_size(), 32u);

  auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_RDNA4);
  ASSERT_NE(decoder, nullptr);
  cu->write_sgpr(wf->sgpr_alloc().base, 0xffffffffu);
  wf->set_exec(0xffffffffu);

  constexpr uint32_t kInline16 = 128u + 16u;
  const std::array<std::pair<uint32_t, std::string_view>, 3> sequence{{
      {encode_sop2(/*s_lshr_b32=*/10, /*exec_lo=*/126, /*s0=*/0, kInline16)[0], "s_lshr_b32"},
      {encode_sop2(/*s_lshl_b32=*/8, /*exec_hi=*/127, /*s0=*/0, kInline16)[0], "s_lshl_b32"},
      {encode_sop2(/*s_or_b32=*/24, /*exec_lo=*/126, /*exec_lo=*/126, /*exec_hi=*/127)[0],
       "s_or_b32"},
  }};

  for (size_t i = 0; i < sequence.size(); ++i) {
    const uint32_t words[] = {sequence[i].first, 0};
    std::unique_ptr<Instruction> inst(decoder->decode(words));
    ASSERT_NE(inst, nullptr);
    ASSERT_EQ(std::string_view(inst->mnemonic()), sequence[i].second);
    cu->execute_instruction(inst.get(), *wf);
    if (i == 1) {
      EXPECT_EQ(wf->exec(), 0x0000ffffu);
      EXPECT_EQ(wf->exec_raw(), 0xffff0000'0000ffffULL);
    }
  }

  EXPECT_EQ(wf->exec(), 0xffffffffu);
  EXPECT_EQ(wf->exec_raw(), 0xffff0000'ffffffffULL);

  constexpr uint64_t kRawExec = 0x2468ace0'13579bdfULL;
  cu->write_sgpr(wf->sgpr_alloc().base, static_cast<uint32_t>(kRawExec));
  cu->write_sgpr(wf->sgpr_alloc().base + 1, static_cast<uint32_t>(kRawExec >> 32));
  wf->set_exec_raw(0xdeadbeef'ffffffffULL);

  const auto mov_b64 = encode_sop1(/*s_mov_b64=*/1, /*exec=*/126, /*s0=*/0);
  std::unique_ptr<Instruction> inst(decoder->decode(mov_b64.data()));
  ASSERT_NE(inst, nullptr);
  ASSERT_EQ(std::string_view(inst->mnemonic()), "s_mov_b64");
  cu->execute_instruction(inst.get(), *wf);

  EXPECT_EQ(wf->exec_raw(), kRawExec);
  EXPECT_EQ(wf->exec(), static_cast<uint32_t>(kRawExec));
}
TEST(InstructionExecutionHarness, Gfx1250) {
  RUN_HARNESS(gfx1250, ROCJITSU_CODE_ARCH_GFX1250, "gfx1250");
}

#undef RUN_HARNESS

TEST(Gfx1250MemoryExecutionHarness, ExecutesRepresentativeValidAddressStores) {
  amdgpu::GpuMemory gpu_mem("gfx1250_memory_harness_mem");
  amdgpu::L2Cache l2("gfx1250_memory_harness_l2");

  amdgpu::ComputeUnitCore::Config cfg{};
  cfg.arch = ROCJITSU_CODE_ARCH_GFX1250;
  cfg.num_wf_slots = 1;
  cfg.sgprs_per_wf = 106;
  cfg.vgprs_per_wf = 256;
  cfg.lds_size_kb = 64;

  Gfx1250MemoryTestCu cu("gfx1250", cfg, &gpu_mem, &l2);

  auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
  ASSERT_NE(decoder, nullptr);

  auto *wf = cu.dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
  ASSERT_NE(wf, nullptr);
  wf->set_exec(0x3u);

  constexpr uint64_t kGlobalAddr = 0x2000;
  const uint32_t sb = wf->sgpr_alloc().base;
  const uint32_t vb = wf->vgpr_alloc().base;

  cu.write_sgpr(sb + 4, static_cast<uint32_t>(kGlobalAddr));
  cu.write_sgpr(sb + 5, static_cast<uint32_t>(kGlobalAddr >> 32));
  for (uint32_t lane = 0; lane < 2; ++lane) {
    gpu_mem.write32(kGlobalAddr + lane * sizeof(uint32_t), 0u);
    cu.write_vgpr(vb + 0, lane, lane * sizeof(uint32_t));
    cu.write_vgpr(vb + 1, lane, 0x11000000u + lane);
  }

  // global_store_b32 v0, v1, s[4:5]
  const uint32_t global_store_words[] = {0xEE068004u, 0x00800000u, 0x00000000u};
  std::unique_ptr<Instruction> global_store(decoder->decode(global_store_words));
  ASSERT_NE(global_store, nullptr);
  ASSERT_EQ(std::string_view(global_store->mnemonic()), "global_store_b32");
  cu.execute_and_route(global_store.release(), *wf);
  cu.flush_all();

  EXPECT_EQ(gpu_mem.read32(kGlobalAddr), 0x11000000u);
  EXPECT_EQ(gpu_mem.read32(kGlobalAddr + sizeof(uint32_t)), 0x11000001u);

  constexpr uint64_t kBufferAddr = 0x3000;
  cu.write_sgpr(sb + 4, static_cast<uint32_t>(kBufferAddr));
  cu.write_sgpr(sb + 5, static_cast<uint32_t>(kBufferAddr >> 32));
  cu.write_sgpr(sb + 6, 0x1000u);
  cu.write_sgpr(sb + 7, 0u);
  wf->set_m0(16u);
  for (uint32_t lane = 0; lane < 2; ++lane) {
    gpu_mem.write32(kBufferAddr + lane * 32, 0u);
    gpu_mem.write32(kBufferAddr + 16 + lane * 32, 0u);
    cu.write_vgpr(vb + 0, lane, 0x22000000u + lane);
    cu.write_vgpr(vb + 5, lane, lane * 32);
  }

  // buffer_store_b32 v0, v5, s[4:7], m0 offen
  const uint32_t buffer_store_words[] = {0xC406807Du, 0x40800800u, 0x00000005u};
  std::unique_ptr<Instruction> buffer_store(decoder->decode(buffer_store_words));
  ASSERT_NE(buffer_store, nullptr);
  ASSERT_EQ(std::string_view(buffer_store->mnemonic()), "buffer_store_b32");
  cu.execute_and_route(buffer_store.release(), *wf);
  cu.flush_all();

  EXPECT_EQ(gpu_mem.read32(kBufferAddr), 0u);
  EXPECT_EQ(gpu_mem.read32(kBufferAddr + 16), 0x22000000u);
  EXPECT_EQ(gpu_mem.read32(kBufferAddr + 32), 0u);
  EXPECT_EQ(gpu_mem.read32(kBufferAddr + 48), 0x22000001u);

  if (!wf->is_halted())
    wf->halt();
}

TEST(Rdna4ScalarSccTest, AddSubCoI32UseSignedOverflow) {
  amdgpu::GpuMemory gpu_mem("rdna4_scc_i32_overflow_mem");
  amdgpu::L2Cache l2("rdna4_scc_i32_overflow_l2");

  amdgpu::ComputeUnitCore::Config cfg{};
  cfg.arch = ROCJITSU_CODE_ARCH_RDNA4;
  cfg.num_wf_slots = 1;
  cfg.sgprs_per_wf = 106;
  cfg.vgprs_per_wf = 256;
  cfg.lds_size_kb = 64;

  auto cu = amdgpu::ComputeUnitCore::create("rdna4", cfg, &gpu_mem, &l2);
  ASSERT_NE(cu, nullptr);

  auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_RDNA4);
  ASSERT_NE(decoder, nullptr);

  auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
  ASSERT_NE(wf, nullptr);

  const uint32_t sb = wf->sgpr_alloc().base;
  auto run = [&](const std::array<uint32_t, 2> &words, std::string_view mnemonic, uint32_t lhs,
                 uint32_t rhs, uint32_t expected_result, bool expected_scc) {
    std::unique_ptr<Instruction> inst(decoder->decode(words.data()));
    ASSERT_NE(inst, nullptr);
    ASSERT_EQ(std::string_view(inst->mnemonic()), mnemonic);

    cu->write_sgpr(sb + 0, lhs);
    cu->write_sgpr(sb + 1, rhs);
    wf->write_scc(!expected_scc);
    cu->execute_instruction(inst.get(), *wf);

    EXPECT_EQ(cu->read_sgpr(sb + 2), expected_result) << mnemonic;
    EXPECT_EQ(wf->read_scc(), expected_scc) << mnemonic;
  };

  const auto add = encode_sop2(/*op=*/2, /*sdst=*/2, /*ssrc0=*/0, /*ssrc1=*/1);
  const auto sub = encode_sop2(/*op=*/3, /*sdst=*/2, /*ssrc0=*/0, /*ssrc1=*/1);

  run(add, "s_add_co_i32", 0x7FFFFFFFu, 1u, 0x80000000u, true);
  run(add, "s_add_co_i32", 0x7FFFFFFFu, 0u, 0x7FFFFFFFu, false);
  run(sub, "s_sub_co_i32", 0x80000000u, 1u, 0x7FFFFFFFu, true);
  run(sub, "s_sub_co_i32", 5u, 3u, 2u, false);

  if (!wf->is_halted())
    wf->halt();
}

struct ScalarCvtSccCase {
  uint32_t op;
  const char *mnemonic;
  uint32_t src;
  uint32_t expected;
};

void run_scalar_cvt_preserves_scc(rj_code_arch_t arch, std::string_view arch_name,
                                  const ScalarCvtSccCase *cases, size_t num_cases) {
  amdgpu::GpuMemory gpu_mem(std::string(arch_name) + "_s_cvt_scc_mem");
  amdgpu::L2Cache l2(std::string(arch_name) + "_s_cvt_scc_l2");

  amdgpu::ComputeUnitCore::Config cfg{};
  cfg.arch = arch;
  cfg.num_wf_slots = 1;
  cfg.sgprs_per_wf = 106;
  cfg.vgprs_per_wf = 256;
  cfg.lds_size_kb = 64;

  auto cu = amdgpu::ComputeUnitCore::create(std::string(arch_name), cfg, &gpu_mem, &l2);
  ASSERT_NE(cu, nullptr) << arch_name;

  auto decoder = Decoder::create(arch);
  ASSERT_NE(decoder, nullptr) << arch_name;

  auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
  ASSERT_NE(wf, nullptr) << arch_name;

  const uint32_t sb = wf->sgpr_alloc().base;

  for (size_t i = 0; i < num_cases; ++i) {
    const auto &tc = cases[i];
    const auto words = encode_sop1(tc.op, /*sdst=*/1, /*ssrc0=*/0);
    for (bool initial_scc : std::array<bool, 2>{false, true}) {
      std::unique_ptr<Instruction> inst(decoder->decode(words.data()));
      ASSERT_NE(inst, nullptr);
      ASSERT_EQ(std::string_view(inst->mnemonic()), tc.mnemonic) << arch_name;

      cu->write_sgpr(sb + 0, tc.src);
      cu->write_sgpr(sb + 1, 0u);
      wf->write_scc(initial_scc);
      cu->execute_instruction(inst.get(), *wf);

      EXPECT_EQ(cu->read_sgpr(sb + 1), tc.expected) << arch_name << " " << tc.mnemonic;
      EXPECT_EQ(wf->read_scc(), initial_scc)
          << arch_name << " " << tc.mnemonic << " initial_scc=" << initial_scc;
    }
  }

  if (!wf->is_halted())
    wf->halt();
}

TEST(ScalarSccTest, ScalarCvtPreservesScc) {
  const uint32_t one_f32 = std::bit_cast<uint32_t>(1.0f);
  const uint32_t one_f16 = util::f32_to_f16(1.0f);
  const uint32_t qnan_f32 = 0x7FC00000u;
  const uint32_t i32_overflow_f32 = std::bit_cast<uint32_t>(2147483648.0f);
  const uint32_t below_i32_min_f32 = std::bit_cast<uint32_t>(-2147483904.0f);
  const uint32_t u32_overflow_f32 = std::bit_cast<uint32_t>(4294967296.0f);

  const std::array<ScalarCvtSccCase, 13> cases{{
      {0x64u, "s_cvt_f32_i32", 1u, one_f32},
      {0x65u, "s_cvt_f32_u32", 1u, one_f32},
      {0x66u, "s_cvt_i32_f32", one_f32, 1u},
      {0x66u, "s_cvt_i32_f32", qnan_f32, 0u},
      {0x66u, "s_cvt_i32_f32", i32_overflow_f32, 0x7FFFFFFFu},
      {0x66u, "s_cvt_i32_f32", below_i32_min_f32, 0x80000000u},
      {0x67u, "s_cvt_u32_f32", one_f32, 1u},
      {0x67u, "s_cvt_u32_f32", qnan_f32, 0u},
      {0x67u, "s_cvt_u32_f32", std::bit_cast<uint32_t>(-1.0f), 0u},
      {0x67u, "s_cvt_u32_f32", u32_overflow_f32, 0xFFFFFFFFu},
      {0x68u, "s_cvt_f16_f32", one_f32, one_f16},
      {0x69u, "s_cvt_f32_f16", one_f16, one_f32},
      {0x6Au, "s_cvt_hi_f32_f16", one_f16 << 16, one_f32},
  }};

  run_scalar_cvt_preserves_scc(ROCJITSU_CODE_ARCH_RDNA3_5, "rdna3_5", cases.data(), cases.size());
  run_scalar_cvt_preserves_scc(ROCJITSU_CODE_ARCH_RDNA4, "rdna4", cases.data(), cases.size());
  run_scalar_cvt_preserves_scc(ROCJITSU_CODE_ARCH_GFX1250, "gfx1250", cases.data(), cases.size());
}

TEST(Cdna4Vop3Test, CmpClassF16WritesWave64UpperMaskDword) {
  struct Case {
    uint32_t lane;
    uint16_t value;
    uint16_t mask;
    bool expected;
  };

  constexpr Case cases[] = {
      {0, 0x0000u, 0x0040u, true},   // +0
      {5, 0x7C01u, 0x0002u, false},  // sNaN is not qNaN
      {32, 0x0000u, 0x0040u, true},  // upper SGPR dword low bit
      {45, 0x7E00u, 0x0001u, false}, // qNaN is not sNaN
      {63, 0x7C00u, 0x0200u, true},  // upper SGPR dword high bit
  };

  for (bool force_scalar : {false, true}) {
    ForceScalarGuard guard(force_scalar);
    amdgpu::GpuMemory gpu_mem(force_scalar ? "cdna4_true16_class64_scalar_mem"
                                           : "cdna4_true16_class64_simd_mem");
    amdgpu::L2Cache l2(force_scalar ? "cdna4_true16_class64_scalar_l2"
                                    : "cdna4_true16_class64_simd_l2");

    amdgpu::ComputeUnitCore::Config cfg{};
    cfg.arch = ROCJITSU_CODE_ARCH_CDNA4;
    cfg.num_wf_slots = 1;
    cfg.sgprs_per_wf = 106;
    cfg.vgprs_per_wf = 256;
    cfg.lds_size_kb = 64;

    auto cu = amdgpu::ComputeUnitCore::create("cdna4", cfg, &gpu_mem, &l2);
    ASSERT_NE(cu, nullptr);

    auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_CDNA4);
    ASSERT_NE(decoder, nullptr);

    auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
    ASSERT_NE(wf, nullptr);
    ASSERT_EQ(wf->wf_size(), 64u);

    const uint32_t vb = wf->vgpr_alloc().base;
    const uint32_t sb = wf->sgpr_alloc().base;
    auto &src0 = cu->raw_vgpr_reg<64>(vb + 0);
    auto &src1 = cu->raw_vgpr_reg<64>(vb + 1);

    uint64_t active_mask = 0;
    uint64_t expected_mask = 0;
    for (const auto &test : cases) {
      active_mask |= (1ULL << test.lane);
      if (test.expected)
        expected_mask |= (1ULL << test.lane);
      src0[test.lane] = pack16(test.value, 0xCAFEu);
      src1[test.lane] = pack16(test.mask, 0x1234u);
    }
    wf->set_exec(active_mask);
    cu->write_sgpr(sb + 0, 0xFFFFFFFFu);
    cu->write_sgpr(sb + 1, 0xDEADBEEFu);

    // CDNA4 v_cmp_class_f16 s0, v0, v1.
    const uint32_t words[] = {0xD0140000U, 0x00020300U};
    std::unique_ptr<Instruction> inst(decoder->decode(words));
    ASSERT_NE(inst, nullptr);
    ASSERT_EQ(std::string_view(inst->mnemonic()), "v_cmp_class_f16");
    cu->execute_instruction(inst.get(), *wf);

    EXPECT_EQ(cu->read_sgpr(sb + 0), static_cast<uint32_t>(expected_mask));
    EXPECT_EQ(cu->read_sgpr(sb + 1), static_cast<uint32_t>(expected_mask >> 32));

    if (!wf->is_halted())
      wf->halt();
  }
}

TEST(CdnaVop3True16Test, B16I16U16OpsUseOpSelAndCdnaDestinationPolicy) {
  struct ArchCase {
    rj_code_arch_t arch;
    std::string_view name;
  };

  constexpr ArchCase arch_cases[] = {
      {ROCJITSU_CODE_ARCH_CDNA1, "cdna1"},
      {ROCJITSU_CODE_ARCH_CDNA2, "cdna2"},
      {ROCJITSU_CODE_ARCH_CDNA3, "cdna3"},
      {ROCJITSU_CODE_ARCH_CDNA4, "cdna4"},
  };

  for (const auto &arch : arch_cases) {
    for (bool force_scalar : {false, true}) {
      ForceScalarGuard guard(force_scalar);
      const std::string suffix = force_scalar ? "_scalar" : "_simd";
      amdgpu::GpuMemory gpu_mem(std::string(arch.name) + "_true16_vop3_mem" + suffix);
      amdgpu::L2Cache l2(std::string(arch.name) + "_true16_vop3_l2" + suffix);

      amdgpu::ComputeUnitCore::Config cfg{};
      cfg.arch = arch.arch;
      cfg.num_wf_slots = 1;
      cfg.sgprs_per_wf = 106;
      cfg.vgprs_per_wf = 256;
      cfg.lds_size_kb = 64;

      auto cu = amdgpu::ComputeUnitCore::create(std::string(arch.name), cfg, &gpu_mem, &l2);
      ASSERT_NE(cu, nullptr);

      auto decoder = Decoder::create(arch.arch);
      ASSERT_NE(decoder, nullptr);

      auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
      ASSERT_NE(wf, nullptr);
      ASSERT_EQ(wf->wf_size(), 64u);
      wf->set_exec(~uint64_t{0});

      const uint32_t vb = wf->vgpr_alloc().base;

      auto execute = [&](const std::array<uint32_t, 2> &words, std::string_view mnemonic) {
        std::unique_ptr<Instruction> inst(decoder->decode(words.data()));
        ASSERT_NE(inst, nullptr) << arch.name << " " << mnemonic;
        ASSERT_EQ(std::string_view(inst->mnemonic()), mnemonic) << arch.name;
        cu->execute_instruction(inst.get(), *wf);
      };

      for (uint32_t lane = 0; lane < wf->wf_size(); ++lane) {
        cu->write_vgpr(vb + 0, lane, pack16(1u, 4u));
        cu->write_vgpr(vb + 1, lane, pack16(0x8888u, 0x1234u));
        cu->write_vgpr(vb + 16, lane, 0xBEEFCAFEu);
      }
      // v_lshrrev_b16 v16.h, v0.h, v1.h op_sel:[1,1,0,1]
      execute(encode_cdna_vop3(/*op=*/0x12B, /*vdst=*/16, /*src0=*/256, /*src1=*/257,
                               /*src2=*/0, /*opsel=*/0xBu),
              "v_lshrrev_b16");
      for (uint32_t lane = 0; lane < wf->wf_size(); ++lane) {
        EXPECT_EQ(cu->read_vgpr(vb + 16, lane), pack16(0xCAFEu, 0x0123u))
            << arch.name << " force_scalar=" << force_scalar << " lane " << lane;
      }

      for (uint32_t lane = 0; lane < wf->wf_size(); ++lane) {
        cu->write_vgpr(vb + 0, lane, pack16(0x0011u, 2u));
        cu->write_vgpr(vb + 1, lane, pack16(0x0022u, 3u));
        cu->write_vgpr(vb + 2, lane, pack16(0x0033u, 4u));
        cu->write_vgpr(vb + 4, lane, 0xBEEFCAFEu);
      }
      // v_mad_u16 v4.l, v0.h, v1.h, v2.h op_sel:[1,1,1,0]
      execute(encode_cdna_vop3(/*op=*/0x204, /*vdst=*/4, /*src0=*/256, /*src1=*/257,
                               /*src2=*/258, /*opsel=*/0x7u),
              "v_mad_u16");
      for (uint32_t lane = 0; lane < wf->wf_size(); ++lane) {
        EXPECT_EQ(cu->read_vgpr(vb + 4, lane), 10u)
            << arch.name << " force_scalar=" << force_scalar << " lane " << lane;
      }

      for (uint32_t lane = 0; lane < wf->wf_size(); ++lane) {
        cu->write_vgpr(vb + 0, lane, pack16(0x0002u, 0xFFFEu));
        cu->write_vgpr(vb + 1, lane, pack16(0x0004u, 3u));
        cu->write_vgpr(vb + 2, lane, pack16(0x0006u, 5u));
        cu->write_vgpr(vb + 5, lane, 0xCAFE1234u);
      }
      // v_mad_i16 v5.h, v0.h, v1.h, v2.h op_sel:[1,1,1,1]
      execute(encode_cdna_vop3(/*op=*/0x205, /*vdst=*/5, /*src0=*/256, /*src1=*/257,
                               /*src2=*/258, /*opsel=*/0xFu),
              "v_mad_i16");
      for (uint32_t lane = 0; lane < wf->wf_size(); ++lane) {
        EXPECT_EQ(cu->read_vgpr(vb + 5, lane), 0xFFFF1234u)
            << arch.name << " force_scalar=" << force_scalar << " lane " << lane;
      }

      if (!wf->is_halted())
        wf->halt();
    }
  }
}

TEST(Rdna3Dot2ExecutionTest, Vop2Dot2accF32F16AccumulatesDst) {
  amdgpu::GpuMemory gpu_mem("rdna3_dot2acc_vop2_mem");
  amdgpu::L2Cache l2("rdna3_dot2acc_vop2_l2");

  amdgpu::ComputeUnitCore::Config cfg{};
  cfg.arch = ROCJITSU_CODE_ARCH_RDNA3;
  cfg.num_wf_slots = 1;
  cfg.sgprs_per_wf = 106;
  cfg.vgprs_per_wf = 256;
  cfg.lds_size_kb = 64;

  auto cu = amdgpu::ComputeUnitCore::create("rdna3", cfg, &gpu_mem, &l2);
  ASSERT_NE(cu, nullptr);

  auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_RDNA3);
  ASSERT_NE(decoder, nullptr);

  auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
  ASSERT_NE(wf, nullptr);
  wf->set_exec(0x1u);

  const uint32_t vb = wf->vgpr_alloc().base;
  cu->write_vgpr(vb + 0, 0, pack16(util::f32_to_f16(1.0f), util::f32_to_f16(2.0f)));
  cu->write_vgpr(vb + 1, 0, pack16(util::f32_to_f16(3.0f), util::f32_to_f16(4.0f)));
  cu->write_vgpr(vb + 2, 0, std::bit_cast<uint32_t>(5.0f));

  const auto words = encode_vop2(/*op=*/2, /*vdst=*/2, /*src0=*/256, /*vsrc1=*/1);
  std::unique_ptr<Instruction> inst(decoder->decode(words.data()));
  ASSERT_NE(inst, nullptr);
  ASSERT_EQ(std::string_view(inst->mnemonic()), "v_dot2acc_f32_f16_e32");
  cu->execute_instruction(inst.get(), *wf);

  EXPECT_EQ(cu->read_vgpr(vb + 2, 0), std::bit_cast<uint32_t>(16.0f));

  if (!wf->is_halted())
    wf->halt();
}

TEST(Rdna4Dot2True16ExecutionTest, F16AppliesVop3ModifiersAndSelectedHalves) {
  amdgpu::GpuMemory gpu_mem("rdna4_dot2_true16_f16_mem");
  amdgpu::L2Cache l2("rdna4_dot2_true16_f16_l2");

  amdgpu::ComputeUnitCore::Config cfg{};
  cfg.arch = ROCJITSU_CODE_ARCH_RDNA4;
  cfg.num_wf_slots = 1;
  cfg.sgprs_per_wf = 106;
  cfg.vgprs_per_wf = 256;
  cfg.lds_size_kb = 64;

  auto cu = amdgpu::ComputeUnitCore::create("rdna4", cfg, &gpu_mem, &l2);
  ASSERT_NE(cu, nullptr);

  auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_RDNA4);
  ASSERT_NE(decoder, nullptr);

  auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
  ASSERT_NE(wf, nullptr);
  wf->set_exec(0x1u);

  const uint32_t vb = wf->vgpr_alloc().base;
  cu->write_vgpr(vb + 0, 0, pack16(util::f32_to_f16(-2.0f), util::f32_to_f16(3.0f)));
  cu->write_vgpr(vb + 1, 0, pack16(util::f32_to_f16(-4.0f), util::f32_to_f16(-5.0f)));
  cu->write_vgpr(vb + 2, 0, pack16(util::f32_to_f16(0.25f), util::f32_to_f16(0.5f)));
  cu->write_vgpr(vb + 3, 0, 0xCAFEBEEFu);

  const auto words = encode_vop3(/*op=*/0x266, /*vdst=*/3, /*src0=*/256, /*src1=*/257,
                                 /*src2=*/258, /*abs=*/0x1, /*opsel=*/0xCu, /*clamp=*/1,
                                 /*omod=*/3, /*neg=*/0x2);
  std::unique_ptr<Instruction> inst(decoder->decode(words.data()));
  ASSERT_NE(inst, nullptr);
  ASSERT_EQ(std::string_view(inst->mnemonic()), "v_dot2_f16_f16");
  cu->execute_instruction(inst.get(), *wf);

  EXPECT_EQ(cu->read_vgpr(vb + 3, 0), pack16(0xBEEFu, util::f32_to_f16(1.0f)));

  if (!wf->is_halted())
    wf->halt();
}

TEST(Rdna4Dot2True16ExecutionTest, Bf16UsesSelectedAccumulatorAndDestinationHalf) {
  amdgpu::GpuMemory gpu_mem("rdna4_dot2_true16_bf16_mem");
  amdgpu::L2Cache l2("rdna4_dot2_true16_bf16_l2");

  amdgpu::ComputeUnitCore::Config cfg{};
  cfg.arch = ROCJITSU_CODE_ARCH_RDNA4;
  cfg.num_wf_slots = 1;
  cfg.sgprs_per_wf = 106;
  cfg.vgprs_per_wf = 256;
  cfg.lds_size_kb = 64;

  auto cu = amdgpu::ComputeUnitCore::create("rdna4", cfg, &gpu_mem, &l2);
  ASSERT_NE(cu, nullptr);

  auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_RDNA4);
  ASSERT_NE(decoder, nullptr);

  auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
  ASSERT_NE(wf, nullptr);
  wf->set_exec(0x1u);

  const uint32_t vb = wf->vgpr_alloc().base;
  cu->write_vgpr(vb + 0, 0, pack16(util::f32_to_bf16(1.0f), util::f32_to_bf16(2.0f)));
  cu->write_vgpr(vb + 1, 0, pack16(util::f32_to_bf16(3.0f), util::f32_to_bf16(4.0f)));
  cu->write_vgpr(vb + 2, 0, pack16(util::f32_to_bf16(0.5f), util::f32_to_bf16(5.0f)));
  cu->write_vgpr(vb + 3, 0, 0xCAFEBEEFu);

  const auto words = encode_vop3(/*op=*/0x267, /*vdst=*/3, /*src0=*/256, /*src1=*/257,
                                 /*src2=*/258, /*abs=*/0, /*opsel=*/0x4u);
  std::unique_ptr<Instruction> inst(decoder->decode(words.data()));
  ASSERT_NE(inst, nullptr);
  ASSERT_EQ(std::string_view(inst->mnemonic()), "v_dot2_bf16_bf16");
  cu->execute_instruction(inst.get(), *wf);

  EXPECT_EQ(cu->read_vgpr(vb + 3, 0), pack16(util::f32_to_bf16(16.0f), 0xCAFEu));

  if (!wf->is_halted())
    wf->halt();
}

// VOP3P v_dot2_f32_bf16 must widen each 16-bit half as BF16 (8-bit exponent),
// not F16 (5-bit exponent). Values like 3.0/4.0 have BF16 and F16 bit patterns
// that decode to completely different floats, so a wrong widening yields a wildly
// wrong dot product rather than a small rounding difference. This asserts the
// exact result against hand-computed BF16 arithmetic. Runs both the SIMD fast
// path and the forced-scalar path so a divergence in either is caught.
void run_cdna4_dot2_f32_bf16(bool force_scalar) {
  ForceScalarGuard guard(force_scalar);
  amdgpu::GpuMemory gpu_mem("cdna4_dot2_f32_bf16_mem");
  amdgpu::L2Cache l2("cdna4_dot2_f32_bf16_l2");

  amdgpu::ComputeUnitCore::Config cfg{};
  cfg.arch = ROCJITSU_CODE_ARCH_CDNA4;
  cfg.num_wf_slots = 1;
  cfg.sgprs_per_wf = 106;
  cfg.vgprs_per_wf = 256;
  cfg.lds_size_kb = 64;

  auto cu = amdgpu::ComputeUnitCore::create("cdna4", cfg, &gpu_mem, &l2);
  ASSERT_NE(cu, nullptr);
  auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_CDNA4);
  ASSERT_NE(decoder, nullptr);

  auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
  ASSERT_NE(wf, nullptr);
  wf->set_exec(0x1u);

  const uint32_t vb = wf->vgpr_alloc().base;
  // src0 = {1.0, 2.0}, src1 = {3.0, 4.0}, acc = 0.5.
  cu->write_vgpr(vb + 0, 0, pack16(util::f32_to_bf16(1.0f), util::f32_to_bf16(2.0f)));
  cu->write_vgpr(vb + 1, 0, pack16(util::f32_to_bf16(3.0f), util::f32_to_bf16(4.0f)));
  cu->write_vgpr(vb + 2, 0, std::bit_cast<uint32_t>(0.5f));

  // Default packing (op_sel = 0, op_sel_hi = 3).
  const auto words =
      cdna4::build_vop3p(cdna4::kVDot2F32Bf16Vop3p, {.vdst = static_cast<uint8_t>(3),
                                                     .op_sel_hi_2 = 1,
                                                     .src0 = static_cast<uint16_t>(256),
                                                     .src1 = static_cast<uint16_t>(257),
                                                     .src2 = static_cast<uint16_t>(258),
                                                     .op_sel_hi = 3});
  std::unique_ptr<Instruction> inst(decoder->decode(words.data()));
  ASSERT_NE(inst, nullptr);
  ASSERT_EQ(std::string_view(inst->mnemonic()), "v_dot2_f32_bf16");
  cu->execute_instruction(inst.get(), *wf);

  // 1.0*3.0 + 2.0*4.0 + 0.5 = 11.5, computed in exact f32 (all inputs represent
  // exactly in BF16). A wrong f16 widening would not produce this.
  EXPECT_EQ(cu->read_vgpr(vb + 3, 0), std::bit_cast<uint32_t>(11.5f));

  if (!wf->is_halted())
    wf->halt();
}

TEST(Cdna4Dot2Vop3pExecutionTest, Bf16WidensAsBf16Simd) {
  run_cdna4_dot2_f32_bf16(/*force_scalar=*/false);
}

TEST(Cdna4Dot2Vop3pExecutionTest, Bf16WidensAsBf16Scalar) {
  run_cdna4_dot2_f32_bf16(/*force_scalar=*/true);
}

// v_permlane16_swap_b32 on a wave64 swaps ALL FOUR 16-lane groups: it exchanges
// vdst[16..31] with src0[0..15] AND vdst[48..63] with src0[32..47]. An earlier
// simulator only swapped the first block (lanes 0-31), silently leaving lanes
// 32-63 untouched. This test writes a distinct value per lane and asserts the
// swap happened in BOTH 32-lane halves, so a regression that drops the upper
// half is caught.
TEST(Cdna4Permlane16SwapExecutionTest, Wave64SwapsAllFourGroups) {
  amdgpu::GpuMemory gpu_mem("cdna4_permlane16_swap_mem");
  amdgpu::L2Cache l2("cdna4_permlane16_swap_l2");

  amdgpu::ComputeUnitCore::Config cfg{};
  cfg.arch = ROCJITSU_CODE_ARCH_CDNA4;
  cfg.num_wf_slots = 1;
  cfg.sgprs_per_wf = 106;
  cfg.vgprs_per_wf = 256;
  cfg.lds_size_kb = 64;

  auto cu = amdgpu::ComputeUnitCore::create("cdna4", cfg, &gpu_mem, &l2);
  ASSERT_NE(cu, nullptr);
  auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_CDNA4);
  ASSERT_NE(decoder, nullptr);

  auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
  ASSERT_NE(wf, nullptr);
  ASSERT_EQ(wf->wf_size(), 64u);
  wf->set_exec(~0ULL);

  const uint32_t vb = wf->vgpr_alloc().base;
  // Distinct, decodable per-lane values: vdst[lane] = 0xD0000000 | lane,
  // src0[lane] = 0x50000000 | lane.
  const uint32_t vdst_reg = 0;
  const uint32_t vsrc_reg = 1;
  for (uint32_t lane = 0; lane < 64; ++lane) {
    cu->write_vgpr(vb + vdst_reg, lane, 0xD0000000u | lane);
    cu->write_vgpr(vb + vsrc_reg, lane, 0x50000000u | lane);
  }

  const auto words = cdna4::build_vop1(
      cdna4::kVPermlane16SwapB32Vop1,
      {.src0 = static_cast<uint16_t>(256 + vsrc_reg), .vdst = static_cast<uint8_t>(vdst_reg)});
  std::unique_ptr<Instruction> inst(decoder->decode(words.data()));
  ASSERT_NE(inst, nullptr);
  ASSERT_EQ(std::string_view(inst->mnemonic()), "v_permlane16_swap_b32_e32");
  cu->execute_instruction(inst.get(), *wf);

  for (uint32_t base = 0; base < 64; base += 32) {
    for (uint32_t i = 0; i < 16; ++i) {
      const uint32_t lo = base + i;       // group L of this 32-lane block
      const uint32_t hi = base + 16u + i; // group H of this 32-lane block
      // src0[lo] gets the old vdst[hi]; vdst[hi] gets the old src0[lo].
      EXPECT_EQ(cu->read_vgpr(vb + vsrc_reg, lo), 0xD0000000u | hi)
          << "src0 lane " << lo << " should hold old vdst lane " << hi;
      EXPECT_EQ(cu->read_vgpr(vb + vdst_reg, hi), 0x50000000u | lo)
          << "vdst lane " << hi << " should hold old src0 lane " << lo;
      // The untouched halves of each block keep their originals.
      EXPECT_EQ(cu->read_vgpr(vb + vdst_reg, lo), 0xD0000000u | lo);
      EXPECT_EQ(cu->read_vgpr(vb + vsrc_reg, hi), 0x50000000u | hi);
    }
  }

  if (!wf->is_halted())
    wf->halt();
}

uint32_t wmma64_ab_lane(uint32_t row_or_col, uint32_t k) {
  return row_or_col + 16u * ((k >> 2) & 1u);
}

uint32_t wmma64_ab_index(uint32_t k) {
  const uint32_t reg =
      ((k >> 1) & 1u) + 2u * ((k >> 3) & 1u) + 4u * ((k >> 4) & 1u) + 8u * ((k >> 5) & 1u);
  return 2u * reg + (k & 1u);
}

void write_packed_byte(amdgpu::ComputeUnitCore &cu, uint32_t reg, uint32_t lane, uint32_t byte,
                       uint8_t value) {
  const uint32_t shift = 8u * byte;
  const uint32_t old = cu.read_vgpr(reg, lane);
  cu.write_vgpr(reg, lane, (old & ~(0xFFu << shift)) | (static_cast<uint32_t>(value) << shift));
}

void write_packed_half(amdgpu::ComputeUnitCore &cu, uint32_t reg, uint32_t lane, uint32_t half,
                       uint16_t value) {
  const uint32_t shift = 16u * half;
  const uint32_t old = cu.read_vgpr(reg, lane);
  cu.write_vgpr(reg, lane, (old & ~(0xFFFFu << shift)) | (static_cast<uint32_t>(value) << shift));
}

float wmma_test_a(uint32_t row, uint32_t k) {
  return static_cast<float>(static_cast<int>((row + k) % 5u) - 2);
}

float wmma_test_b(uint32_t k, uint32_t col) {
  return static_cast<float>(static_cast<int>((col + 3u * k) % 5u) - 2);
}

float wmma_test_c(uint32_t row, uint32_t col) {
  return static_cast<float>(static_cast<int>((2u * row + col) % 7u) - 3);
}

TEST(Gfx1250WmmaTest, F16Fp8K64MatchesReferenceLayout) {
  amdgpu::GpuMemory gpu_mem("gfx1250_wmma_f16_fp8_k64_mem");
  amdgpu::L2Cache l2("gfx1250_wmma_f16_fp8_k64_l2");

  amdgpu::ComputeUnitCore::Config cfg{};
  cfg.arch = ROCJITSU_CODE_ARCH_GFX1250;
  cfg.num_wf_slots = 1;
  cfg.sgprs_per_wf = 106;
  cfg.vgprs_per_wf = 256;
  cfg.lds_size_kb = 64;

  auto cu = amdgpu::ComputeUnitCore::create("gfx1250", cfg, &gpu_mem, &l2);
  ASSERT_NE(cu, nullptr);

  auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
  ASSERT_NE(wf, nullptr);
  ASSERT_EQ(wf->wf_size(), 32u);
  wf->set_exec((1ULL << wf->wf_size()) - 1ULL);

  auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
  ASSERT_NE(decoder, nullptr);

  const uint32_t vb = wf->vgpr_alloc().base;
  const uint32_t a_base = vb + 10;
  const uint32_t b_base = vb + 20;
  const uint32_t c_base = vb + 30;
  const uint32_t d_base = vb + 40;

  for (uint32_t reg = 0; reg < 8; ++reg) {
    for (uint32_t lane = 0; lane < wf->wf_size(); ++lane) {
      cu->write_vgpr(a_base + reg, lane, 0);
      cu->write_vgpr(b_base + reg, lane, 0);
    }
  }
  for (uint32_t reg = 0; reg < 4; ++reg) {
    for (uint32_t lane = 0; lane < wf->wf_size(); ++lane) {
      cu->write_vgpr(c_base + reg, lane, 0);
      cu->write_vgpr(d_base + reg, lane, 0xDEADBEEFu);
    }
  }

  for (uint32_t row = 0; row < 16; ++row) {
    for (uint32_t k = 0; k < 64; ++k) {
      const uint32_t idx = wmma64_ab_index(k);
      write_packed_byte(*cu, a_base + idx / 4u, wmma64_ab_lane(row, k), idx % 4u,
                        util::f32_to_fp8_e4m3_rne(wmma_test_a(row, k)));
    }
  }
  for (uint32_t k = 0; k < 64; ++k) {
    for (uint32_t col = 0; col < 16; ++col) {
      const uint32_t idx = wmma64_ab_index(k);
      write_packed_byte(*cu, b_base + idx / 4u, wmma64_ab_lane(col, k), idx % 4u,
                        util::f32_to_fp8_e4m3_rne(wmma_test_b(k, col)));
    }
  }
  for (uint32_t row = 0; row < 16; ++row) {
    for (uint32_t col = 0; col < 16; ++col) {
      const auto out = amdgpu::wmma_output_loc_16(16, 16, row, col);
      write_packed_half(*cu, c_base + out.reg, out.lane, out.sub_element,
                        util::f32_to_f16(wmma_test_c(row, col)));
    }
  }

  amdgpu::exec_wmma_f16(*cu, 16, 16, 64, 8, d_base, a_base, b_base, c_base, amdgpu::extract_fp8,
                        amdgpu::extract_fp8, amdgpu::ACC_FROM_VGPR);

  auto expect_reference = [&] {
    for (uint32_t row = 0; row < 16; ++row) {
      for (uint32_t col = 0; col < 16; ++col) {
        float ref = wmma_test_c(row, col);
        for (uint32_t k = 0; k < 64; ++k)
          ref += wmma_test_a(row, k) * wmma_test_b(k, col);
        const auto out = amdgpu::wmma_output_loc_16(16, 16, row, col);
        const uint32_t raw = cu->read_vgpr(d_base + out.reg, out.lane);
        const auto actual = static_cast<uint16_t>((raw >> (16u * out.sub_element)) & 0xFFFFu);
        EXPECT_EQ(actual, util::f32_to_f16(ref)) << "row=" << row << " col=" << col;
      }
    }
  };
  expect_reference();

  for (uint32_t reg = 0; reg < 4; ++reg)
    for (uint32_t lane = 0; lane < wf->wf_size(); ++lane)
      cu->write_vgpr(d_base + reg, lane, 0xDEADBEEFu);

  const uint32_t wmma_words[] = {
      0xCC6E0000U | 40u,
      (256u + 10u) | ((256u + 20u) << 9u) | ((256u + 30u) << 18u),
  };
  std::unique_ptr<Instruction> inst(decoder->decode(wmma_words));
  ASSERT_NE(inst, nullptr);
  ASSERT_EQ(std::string_view(inst->mnemonic()), "v_wmma_f16_16x16x64_fp8_fp8");
  cu->execute_instruction(inst.get(), *wf);
  expect_reference();

  if (!wf->is_halted())
    wf->halt();
}

TEST(Gfx1250WmmaTest, CModifierAppliesAbsBeforeNegate) {
  EXPECT_EQ(amdgpu::wmma_c_modifier(0u, 0u), 0u);
  EXPECT_EQ(amdgpu::wmma_c_modifier(1u << 2u, 0u), 1u);
  EXPECT_EQ(amdgpu::wmma_c_modifier(0u, 1u << 2u), 2u);
  EXPECT_EQ(amdgpu::wmma_c_modifier(1u << 2u, 1u << 2u), 3u);

  EXPECT_EQ(amdgpu::apply_wmma_c_modifier(-3.0f, 0u), -3.0f);
  EXPECT_EQ(amdgpu::apply_wmma_c_modifier(-3.0f, 1u), 3.0f);
  EXPECT_EQ(amdgpu::apply_wmma_c_modifier(-3.0f, 2u), 3.0f);
  EXPECT_EQ(amdgpu::apply_wmma_c_modifier(-3.0f, 3u), -3.0f);
}

TEST(Gfx1250WmmaTest, E8M0ScaleZeroMatchesUtilContract) {
  EXPECT_EQ(std::bit_cast<uint32_t>(amdgpu::decode_wmma_scale_byte(0x00, 0)),
            std::bit_cast<uint32_t>(util::e8m0_to_f32(0x00)));
  EXPECT_TRUE(std::isnan(amdgpu::decode_wmma_scale_byte(0xff, 0)));
}

TEST(Gfx1250WmmaTest, Bf16F32K32Bf16PreservesAccumulatorLayout) {
  amdgpu::GpuMemory gpu_mem("gfx1250_wmma_bf16f32_k32_bf16_mem");
  amdgpu::L2Cache l2("gfx1250_wmma_bf16f32_k32_bf16_l2");

  amdgpu::ComputeUnitCore::Config cfg{};
  cfg.arch = ROCJITSU_CODE_ARCH_GFX1250;
  cfg.num_wf_slots = 1;
  cfg.sgprs_per_wf = 106;
  cfg.vgprs_per_wf = 256;
  cfg.lds_size_kb = 64;

  auto cu = amdgpu::ComputeUnitCore::create("gfx1250", cfg, &gpu_mem, &l2);
  ASSERT_NE(cu, nullptr);

  auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
  ASSERT_NE(wf, nullptr);
  ASSERT_EQ(wf->wf_size(), 32u);
  wf->set_exec((1ULL << wf->wf_size()) - 1ULL);

  const uint32_t vb = wf->vgpr_alloc().base;
  const uint32_t a_base = vb + 10;
  const uint32_t b_base = vb + 20;
  const uint32_t c_base = vb + 30;
  const uint32_t d_base = vb + 40;

  for (uint32_t reg = 0; reg < 8; ++reg) {
    for (uint32_t lane = 0; lane < wf->wf_size(); ++lane) {
      cu->write_vgpr(a_base + reg, lane, 0);
      cu->write_vgpr(b_base + reg, lane, 0);
      cu->write_vgpr(c_base + reg, lane, 0);
    }
  }
  for (uint32_t reg = 0; reg < 4; ++reg)
    for (uint32_t lane = 0; lane < wf->wf_size(); ++lane)
      cu->write_vgpr(d_base + reg, lane, 0xDEADBEEFu);

  for (uint32_t row = 0; row < 16; ++row) {
    for (uint32_t k = 0; k < 32; ++k) {
      const auto in = amdgpu::wmma_input_loc(16, 32, row, k, 16);
      write_packed_half(*cu, a_base + in.vgpr_offset, in.lane, in.sub_element,
                        util::f32_to_bf16(wmma_test_a(row, k)));
    }
  }
  for (uint32_t k = 0; k < 32; ++k) {
    for (uint32_t col = 0; col < 16; ++col) {
      const auto in = amdgpu::wmma_input_loc(16, 32, col, k, 16);
      write_packed_half(*cu, b_base + in.vgpr_offset, in.lane, in.sub_element,
                        util::f32_to_bf16(wmma_test_b(k, col)));
    }
  }
  for (uint32_t row = 0; row < 16; ++row) {
    for (uint32_t col = 0; col < 16; ++col) {
      const auto out = amdgpu::wmma_output_loc_32(16, 16, row, col);
      cu->write_vgpr(c_base + out.reg, out.lane, std::bit_cast<uint32_t>(wmma_test_c(row, col)));
    }
  }

  amdgpu::exec_wmma_bf16f32_16x16x32_bf16(*cu, d_base, a_base, b_base, c_base);

  for (uint32_t row = 0; row < 16; ++row) {
    for (uint32_t col = 0; col < 16; ++col) {
      float ref = wmma_test_c(row, col);
      for (uint32_t k = 0; k < 32; ++k)
        ref += wmma_test_a(row, k) * wmma_test_b(k, col);
      const auto out = amdgpu::wmma_output_loc_16(16, 16, row, col);
      const uint32_t raw = cu->read_vgpr(d_base + out.reg, out.lane);
      const auto actual = static_cast<uint16_t>((raw >> (16u * out.sub_element)) & 0xFFFFu);
      EXPECT_EQ(actual, util::f32_to_bf16(ref)) << "row=" << row << " col=" << col;
    }
  }

  if (!wf->is_halted())
    wf->halt();
}

TEST(Gfx1250Dpp8Test, Vop2AddF16UsesPermutedSourceLanes) {
  amdgpu::GpuMemory gpu_mem("gfx1250_dpp8_add_f16_mem");
  amdgpu::L2Cache l2("gfx1250_dpp8_add_f16_l2");

  amdgpu::ComputeUnitCore::Config cfg{};
  cfg.arch = ROCJITSU_CODE_ARCH_GFX1250;
  cfg.num_wf_slots = 1;
  cfg.sgprs_per_wf = 106;
  cfg.vgprs_per_wf = 256;
  cfg.lds_size_kb = 64;

  auto cu = amdgpu::ComputeUnitCore::create("gfx1250", cfg, &gpu_mem, &l2);
  ASSERT_NE(cu, nullptr);

  auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
  ASSERT_NE(decoder, nullptr);

  auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
  ASSERT_NE(wf, nullptr);
  ASSERT_EQ(wf->wf_size(), 32u);
  wf->set_exec((1ULL << wf->wf_size()) - 1ULL);

  const uint32_t vb = wf->vgpr_alloc().base;
  constexpr uint32_t kSrc0 = 2;
  constexpr uint32_t kSrc1 = 3;
  constexpr uint32_t kDst = 4;
  const uint32_t lane_sel = (7u << 0u) | (0u << 3u) | (3u << 6u) | (2u << 9u) | (5u << 12u) |
                            (4u << 15u) | (1u << 18u) | (6u << 21u);

  for (uint32_t lane = 0; lane < wf->wf_size(); ++lane) {
    cu->write_vgpr(vb + kSrc0, lane, util::f32_to_f16(static_cast<float>(lane + 1)));
    cu->write_vgpr(vb + kSrc1, lane, util::f32_to_f16(0.0f));
    cu->write_vgpr(vb + kDst, lane, 0xDEAD0000u);
  }

  const uint32_t add_f16_dpp8_words[] = {
      0x64000000u | amdgpu::SRC_DPP8_FI_0 | (kSrc1 << 9u) | (kDst << 17u),
      kSrc0 | (lane_sel << 8u),
  };
  std::unique_ptr<Instruction> inst(decoder->decode(add_f16_dpp8_words));
  ASSERT_NE(inst, nullptr);
  ASSERT_EQ(std::string_view(inst->mnemonic()), "v_add_f16_e32");
  cu->execute_instruction(inst.get(), *wf);

  for (uint32_t lane = 0; lane < wf->wf_size(); ++lane) {
    const uint32_t src_lane = amdgpu::dpp::dpp8_src_lane(lane, lane_sel);
    const uint32_t raw = cu->read_vgpr(vb + kDst, lane);
    EXPECT_EQ(raw >> 16, 0xDEADu) << "lane=" << lane;
    EXPECT_EQ(raw & 0xFFFFu, util::f32_to_f16(static_cast<float>(src_lane + 1))) << "lane=" << lane;
  }

  if (!wf->is_halted())
    wf->halt();
}

TEST(Gfx1250ExecMaskTest, CmpxNeqAndScalarOrRestoreExec) {
  amdgpu::GpuMemory gpu_mem("gfx1250_exec_mask_mem");
  amdgpu::L2Cache l2("gfx1250_exec_mask_l2");

  amdgpu::ComputeUnitCore::Config cfg{};
  cfg.arch = ROCJITSU_CODE_ARCH_GFX1250;
  cfg.num_wf_slots = 1;
  cfg.sgprs_per_wf = 106;
  cfg.vgprs_per_wf = 256;
  cfg.lds_size_kb = 64;

  auto cu = amdgpu::ComputeUnitCore::create("gfx1250", cfg, &gpu_mem, &l2);
  ASSERT_NE(cu, nullptr);

  auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
  ASSERT_NE(decoder, nullptr);

  auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
  ASSERT_NE(wf, nullptr);
  ASSERT_LE(wf->wf_size(), 64u);
  const uint64_t all_lanes = wf->wf_size() == 64 ? ~0ULL : ((1ULL << wf->wf_size()) - 1ULL);
  wf->set_exec(all_lanes);

  // s_mov_b32 s1, exec_lo
  const uint32_t save_exec_words[] = {0xBE81007EU, 0};
  std::unique_ptr<Instruction> save_exec(decoder->decode(save_exec_words));
  ASSERT_NE(save_exec, nullptr);
  ASSERT_EQ(std::string_view(save_exec->mnemonic()), "s_mov_b32");
  cu->execute_instruction(save_exec.get(), *wf);
  EXPECT_EQ(cu->read_sgpr(wf->sgpr_alloc().base + 1), static_cast<uint32_t>(all_lanes));

  const uint32_t v4 = wf->vgpr_alloc().base + 4;
  for (uint32_t lane = 0; lane < wf->wf_size(); ++lane) {
    const float value = lane == 4 ? 0.0f : 1.0f;
    cu->write_vgpr(v4, lane, std::bit_cast<uint32_t>(value));
  }

  // v_cmpx_neq_f32_e32 0, v4
  const uint32_t mask_zero_lane_words[] = {0x7D3A0880U, 0};
  std::unique_ptr<Instruction> mask_zero_lane(decoder->decode(mask_zero_lane_words));
  ASSERT_NE(mask_zero_lane, nullptr);
  ASSERT_EQ(std::string_view(mask_zero_lane->mnemonic()), "v_cmpx_neq_f32_e32");
  cu->execute_instruction(mask_zero_lane.get(), *wf);
  EXPECT_EQ(wf->exec(), all_lanes & ~(1ULL << 4));

  // s_or_b32 exec_lo, exec_lo, s1
  const uint32_t restore_exec_words[] = {0x8C7E017EU, 0};
  std::unique_ptr<Instruction> restore_exec(decoder->decode(restore_exec_words));
  ASSERT_NE(restore_exec, nullptr);
  ASSERT_EQ(std::string_view(restore_exec->mnemonic()), "s_or_b32");
  cu->execute_instruction(restore_exec.get(), *wf);
  EXPECT_EQ(wf->exec(), all_lanes);

  if (!wf->is_halted())
    wf->halt();
}

TEST(Gfx1250ExecMaskTest, Vop3CmpxGtI32KeepsInRangeLanesActive) {
  amdgpu::GpuMemory gpu_mem("gfx1250_cmpx_gt_i32_mem");
  amdgpu::L2Cache l2("gfx1250_cmpx_gt_i32_l2");

  amdgpu::ComputeUnitCore::Config cfg{};
  cfg.arch = ROCJITSU_CODE_ARCH_GFX1250;
  cfg.num_wf_slots = 1;
  cfg.sgprs_per_wf = 106;
  cfg.vgprs_per_wf = 256;
  cfg.lds_size_kb = 64;

  auto cu = amdgpu::ComputeUnitCore::create("gfx1250", cfg, &gpu_mem, &l2);
  ASSERT_NE(cu, nullptr);

  auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
  ASSERT_NE(decoder, nullptr);

  auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
  ASSERT_NE(wf, nullptr);
  ASSERT_LE(wf->wf_size(), 64u);
  const uint64_t all_lanes = wf->wf_size() == 64 ? ~0ULL : ((1ULL << wf->wf_size()) - 1ULL);
  wf->set_exec(all_lanes);

  cu->write_sgpr(wf->sgpr_alloc().base + 3, 64);
  const uint32_t v2 = wf->vgpr_alloc().base + 2;
  for (uint32_t lane = 0; lane < wf->wf_size(); ++lane)
    cu->write_vgpr(v2, lane, lane);

  // v_cmpx_gt_i32_e64 s3, v2
  const uint32_t in_range_words[] = {0xD4C4007EU, 0x02020403U};
  std::unique_ptr<Instruction> in_range(decoder->decode(in_range_words));
  ASSERT_NE(in_range, nullptr);
  ASSERT_EQ(std::string_view(in_range->mnemonic()), "v_cmpx_gt_i32");
  cu->execute_instruction(in_range.get(), *wf);
  EXPECT_EQ(wf->exec(), all_lanes);

  if (!wf->is_halted())
    wf->halt();
}

TEST(Gfx1250ExecMaskTest, Vop3CmpF32PreservesHighMaskSgprOnWave32) {
  amdgpu::GpuMemory gpu_mem("gfx1250_cmp_f32_mask_mem");
  amdgpu::L2Cache l2("gfx1250_cmp_f32_mask_l2");

  amdgpu::ComputeUnitCore::Config cfg{};
  cfg.arch = ROCJITSU_CODE_ARCH_GFX1250;
  cfg.num_wf_slots = 1;
  cfg.sgprs_per_wf = 106;
  cfg.vgprs_per_wf = 256;
  cfg.lds_size_kb = 64;

  auto cu = amdgpu::ComputeUnitCore::create("gfx1250", cfg, &gpu_mem, &l2);
  ASSERT_NE(cu, nullptr);

  auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
  ASSERT_NE(decoder, nullptr);

  auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
  ASSERT_NE(wf, nullptr);
  ASSERT_EQ(wf->wf_size(), 32u);
  const uint64_t all_lanes = (1ULL << wf->wf_size()) - 1ULL;
  wf->set_exec(all_lanes);

  constexpr uint32_t saved_exec = 0xDEADBEEFu;
  cu->write_sgpr(wf->sgpr_alloc().base + 1, saved_exec);

  const uint32_t v4 = wf->vgpr_alloc().base + 4;
  for (uint32_t lane = 0; lane < wf->wf_size(); ++lane) {
    const float value = lane == 4 ? 0.0f : 1.0f;
    cu->write_vgpr(v4, lane, std::bit_cast<uint32_t>(value));
  }

  // v_cmp_eq_f32_e64 s0, 0, v4
  const uint32_t compare_words[] = {0xD4120000U, 0x02020880U};
  std::unique_ptr<Instruction> compare(decoder->decode(compare_words));
  ASSERT_NE(compare, nullptr);
  ASSERT_EQ(std::string_view(compare->mnemonic()), "v_cmp_eq_f32");
  cu->execute_instruction(compare.get(), *wf);

  EXPECT_EQ(cu->read_sgpr(wf->sgpr_alloc().base + 0), 1u << 4);
  EXPECT_EQ(cu->read_sgpr(wf->sgpr_alloc().base + 1), saved_exec);

  if (!wf->is_halted())
    wf->halt();
}

TEST(Gfx1250True16Vop3Test, SelectedHalfArithmeticPreservesDestinationHalf) {
  amdgpu::GpuMemory gpu_mem("gfx1250_true16_arith_mem");
  amdgpu::L2Cache l2("gfx1250_true16_arith_l2");

  amdgpu::ComputeUnitCore::Config cfg{};
  cfg.arch = ROCJITSU_CODE_ARCH_GFX1250;
  cfg.num_wf_slots = 1;
  cfg.sgprs_per_wf = 106;
  cfg.vgprs_per_wf = 256;
  cfg.lds_size_kb = 64;

  auto cu = amdgpu::ComputeUnitCore::create("gfx1250", cfg, &gpu_mem, &l2);
  ASSERT_NE(cu, nullptr);

  auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
  ASSERT_NE(decoder, nullptr);

  auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
  ASSERT_NE(wf, nullptr);
  ASSERT_EQ(wf->wf_size(), 32u);
  wf->set_exec((1ULL << wf->wf_size()) - 1ULL);

  const uint32_t v2 = wf->vgpr_alloc().base + 2;
  const uint32_t v3 = wf->vgpr_alloc().base + 3;
  for (uint32_t lane = 0; lane < wf->wf_size(); ++lane) {
    cu->write_vgpr(v2, lane, 0xCAFE0005u);
    cu->write_vgpr(v3, lane, 0xBEEF0007u);
  }

  auto execute = [&](const uint32_t(&words)[2], std::string_view mnemonic) {
    std::unique_ptr<Instruction> inst(decoder->decode(words));
    ASSERT_NE(inst, nullptr);
    ASSERT_EQ(std::string_view(inst->mnemonic()), mnemonic);
    cu->execute_instruction(inst.get(), *wf);
  };

  // v_mul_lo_u16 v3.h, v3.l, v2.l op_sel:[0,0,1]
  const uint32_t mul_hi_words[] = {0xD7054003U, 0x02020503U};
  execute(mul_hi_words, "v_mul_lo_u16");
  for (uint32_t lane = 0; lane < wf->wf_size(); ++lane)
    EXPECT_EQ(cu->read_vgpr(v3, lane), 0x00230007u);

  // v_sub_nc_u16 v3.h, 2, v3.h op_sel:[0,1,1]
  const uint32_t sub_hi_words[] = {0xD7045003U, 0x02020682U};
  execute(sub_hi_words, "v_sub_nc_u16");
  for (uint32_t lane = 0; lane < wf->wf_size(); ++lane)
    EXPECT_EQ(cu->read_vgpr(v3, lane), 0xFFDF0007u);

  for (uint32_t lane = 0; lane < wf->wf_size(); ++lane) {
    cu->write_vgpr(v2, lane, 0xCAFE0005u);
    cu->write_vgpr(v3, lane, 0x12340007u);
  }

  // v_mad_u16 v3.h, v3.l, v2.l, v3.h op_sel:[0,0,1,1]
  const uint32_t mad_hi_words[] = {0xD6416003U, 0x040E0503U};
  execute(mad_hi_words, "v_mad_u16");
  for (uint32_t lane = 0; lane < wf->wf_size(); ++lane)
    EXPECT_EQ(cu->read_vgpr(v3, lane), 0x12570007u);

  for (uint32_t lane = 0; lane < wf->wf_size(); ++lane) {
    cu->write_vgpr(v2, lane, 0xCAFE0005u);
    cu->write_vgpr(v3, lane, 0xBEEF0007u);
  }

  // v_mul_lo_u16 v3.l, v3.l, v2.l op_sel:[0,0,0]
  const uint32_t mul_lo_words[] = {0xD7050003U, 0x02020503U};
  execute(mul_lo_words, "v_mul_lo_u16");
  for (uint32_t lane = 0; lane < wf->wf_size(); ++lane)
    EXPECT_EQ(cu->read_vgpr(v3, lane), 0xBEEF0023u);

  if (!wf->is_halted())
    wf->halt();
}

TEST(Gfx1250True16Vop2Test, AddF16UsesSelectedHighVsrc1AndPreservesLowDestination) {
  amdgpu::GpuMemory gpu_mem("gfx1250_true16_vop2_add_mem");
  amdgpu::L2Cache l2("gfx1250_true16_vop2_add_l2");

  amdgpu::ComputeUnitCore::Config cfg{};
  cfg.arch = ROCJITSU_CODE_ARCH_GFX1250;
  cfg.num_wf_slots = 1;
  cfg.sgprs_per_wf = 106;
  cfg.vgprs_per_wf = 256;
  cfg.lds_size_kb = 64;

  auto cu = amdgpu::ComputeUnitCore::create("gfx1250", cfg, &gpu_mem, &l2);
  ASSERT_NE(cu, nullptr);

  auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
  ASSERT_NE(decoder, nullptr);

  auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
  ASSERT_NE(wf, nullptr);
  wf->set_exec(1);

  const uint32_t vb = wf->vgpr_alloc().base;
  cu->write_vgpr(vb + 0, 0, 0x40003C00u); // low=1.0h, high=2.0h
  cu->write_vgpr(vb + 1, 0, 0x44004200u); // low=3.0h, high=4.0h
  cu->write_vgpr(vb + 2, 0, 0xDEADCAFEu);

  // v_add_f16_e32 v2.h, v0.h, v1.h
  const uint32_t words[] = {0x65050380U, 0x00000000U};
  std::unique_ptr<Instruction> inst(decoder->decode(words));
  ASSERT_NE(inst, nullptr);
  ASSERT_EQ(std::string_view(inst->mnemonic()), "v_add_f16_e32");
  EXPECT_NE(inst->disassemble().find("v2.h"), std::string::npos);
  EXPECT_NE(inst->disassemble().find("v0.h"), std::string::npos);
  EXPECT_NE(inst->disassemble().find("v1.h"), std::string::npos);
  cu->execute_instruction(inst.get(), *wf);

  EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0x4600CAFEu); // high=6.0h, low preserved

  if (!wf->is_halted())
    wf->halt();
}

TEST(Gfx1250True16Vop2Test, FmacF16HighDestinationUsesSelectedAccumulatorHalf) {
  amdgpu::GpuMemory gpu_mem("gfx1250_true16_vop2_fmac_mem");
  amdgpu::L2Cache l2("gfx1250_true16_vop2_fmac_l2");

  amdgpu::ComputeUnitCore::Config cfg{};
  cfg.arch = ROCJITSU_CODE_ARCH_GFX1250;
  cfg.num_wf_slots = 1;
  cfg.sgprs_per_wf = 106;
  cfg.vgprs_per_wf = 256;
  cfg.lds_size_kb = 64;

  auto cu = amdgpu::ComputeUnitCore::create("gfx1250", cfg, &gpu_mem, &l2);
  ASSERT_NE(cu, nullptr);

  auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
  ASSERT_NE(decoder, nullptr);

  auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
  ASSERT_NE(wf, nullptr);
  wf->set_exec(1);

  const uint32_t vb = wf->vgpr_alloc().base;
  cu->write_vgpr(vb + 0, 0, 0x40003C00u); // low=1.0h, high=2.0h accumulator
  cu->write_vgpr(vb + 1, 0, 0x42000000u); // high=3.0h
  cu->write_vgpr(vb + 2, 0, 0x44000000u); // high=4.0h

  // v_fmac_f16_e32 v0.h, v1.h, v2.h
  const uint32_t words[] = {0x6D010581U, 0x00000000U};
  std::unique_ptr<Instruction> inst(decoder->decode(words));
  ASSERT_NE(inst, nullptr);
  ASSERT_EQ(std::string_view(inst->mnemonic()), "v_fmac_f16_e32");
  EXPECT_NE(inst->disassemble().find("v0.h"), std::string::npos);
  EXPECT_NE(inst->disassemble().find("v1.h"), std::string::npos);
  EXPECT_NE(inst->disassemble().find("v2.h"), std::string::npos);
  cu->execute_instruction(inst.get(), *wf);

  EXPECT_EQ(cu->read_vgpr(vb + 0, 0), 0x4B003C00u); // high=14.0h, low preserved

  if (!wf->is_halted())
    wf->halt();
}

TEST(Gfx1250True16Vop3Test, MulLoU16UsesSelectedHighSourceHalf) {
  amdgpu::GpuMemory gpu_mem("gfx1250_true16_mul_src_mem");
  amdgpu::L2Cache l2("gfx1250_true16_mul_src_l2");

  amdgpu::ComputeUnitCore::Config cfg{};
  cfg.arch = ROCJITSU_CODE_ARCH_GFX1250;
  cfg.num_wf_slots = 1;
  cfg.sgprs_per_wf = 106;
  cfg.vgprs_per_wf = 256;
  cfg.lds_size_kb = 64;

  auto cu = amdgpu::ComputeUnitCore::create("gfx1250", cfg, &gpu_mem, &l2);
  ASSERT_NE(cu, nullptr);

  auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
  ASSERT_NE(decoder, nullptr);

  auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
  ASSERT_NE(wf, nullptr);
  wf->set_exec(1);

  const uint32_t vb = wf->vgpr_alloc().base;
  cu->write_vgpr(vb + 0, 0, 0x12345678u);
  cu->write_vgpr(vb + 2, 0, 0xAAAA5555u);

  // v_mul_lo_u16 v2.l, 0x12b3, v0.h op_sel:[0,1,0]
  const uint32_t words[] = {0xD7051002U, 0x020200FFU, 0x000012B3U};
  std::unique_ptr<Instruction> inst(decoder->decode(words));
  ASSERT_NE(inst, nullptr);
  ASSERT_EQ(std::string_view(inst->mnemonic()), "v_mul_lo_u16");
  cu->execute_instruction(inst.get(), *wf);

  EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0xAAAA625Cu);

  if (!wf->is_halted())
    wf->halt();
}

TEST(Gfx1250True16Vop3Test, CmpLtI16UsesSelectedHighSourceHalf) {
  amdgpu::GpuMemory gpu_mem("gfx1250_true16_cmp_src_mem");
  amdgpu::L2Cache l2("gfx1250_true16_cmp_src_l2");

  amdgpu::ComputeUnitCore::Config cfg{};
  cfg.arch = ROCJITSU_CODE_ARCH_GFX1250;
  cfg.num_wf_slots = 1;
  cfg.sgprs_per_wf = 106;
  cfg.vgprs_per_wf = 256;
  cfg.lds_size_kb = 64;

  auto cu = amdgpu::ComputeUnitCore::create("gfx1250", cfg, &gpu_mem, &l2);
  ASSERT_NE(cu, nullptr);

  auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
  ASSERT_NE(decoder, nullptr);

  auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
  ASSERT_NE(wf, nullptr);
  ASSERT_EQ(wf->wf_size(), 32u);
  wf->set_exec(1);

  const uint32_t vb = wf->vgpr_alloc().base;
  const uint32_t sb = wf->sgpr_alloc().base;
  cu->write_vgpr(vb + 0, 0, 0xC0003C00u);
  cu->write_sgpr(sb + 0, 0xFFFFFFFFu);
  cu->write_sgpr(sb + 1, 0xDEADBEEFu);

  // v_cmp_lt_i16 s0, -1, v0.h op_sel:[0,1,0]
  const uint32_t words[] = {0xD4311000U, 0x020200C1U};
  std::unique_ptr<Instruction> inst(decoder->decode(words));
  ASSERT_NE(inst, nullptr);
  ASSERT_EQ(std::string_view(inst->mnemonic()), "v_cmp_lt_i16");
  cu->execute_instruction(inst.get(), *wf);

  EXPECT_EQ(cu->read_sgpr(sb + 0), 0u);
  EXPECT_EQ(cu->read_sgpr(sb + 1), 0xDEADBEEFu);

  if (!wf->is_halted())
    wf->halt();
}

TEST(Gfx1250True16Vop3Test, CmpClassF16UsesSelectedMaskHalf) {
  for (bool force_scalar : {false, true}) {
    ForceScalarGuard guard(force_scalar);
    amdgpu::GpuMemory gpu_mem(force_scalar ? "gfx1250_true16_class_scalar_mem"
                                           : "gfx1250_true16_class_simd_mem");
    amdgpu::L2Cache l2(force_scalar ? "gfx1250_true16_class_scalar_l2"
                                    : "gfx1250_true16_class_simd_l2");

    amdgpu::ComputeUnitCore::Config cfg{};
    cfg.arch = ROCJITSU_CODE_ARCH_GFX1250;
    cfg.num_wf_slots = 1;
    cfg.sgprs_per_wf = 106;
    cfg.vgprs_per_wf = 256;
    cfg.lds_size_kb = 64;

    auto cu = amdgpu::ComputeUnitCore::create("gfx1250", cfg, &gpu_mem, &l2);
    ASSERT_NE(cu, nullptr);

    auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
    ASSERT_NE(decoder, nullptr);

    auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
    ASSERT_NE(wf, nullptr);
    ASSERT_EQ(wf->wf_size(), 32u);
    wf->set_exec(0x3);

    const uint32_t vb = wf->vgpr_alloc().base;
    const uint32_t sb = wf->sgpr_alloc().base;
    cu->write_vgpr(vb + 0, 0, pack16(0x0000u, 0x0040u)); // +0h, mask=+0
    cu->write_vgpr(vb + 0, 1, pack16(0x7C01u, 0x0002u)); // sNaN, mask=qNaN
    cu->write_sgpr(sb + 0, 0xFFFFFFFFu);
    cu->write_sgpr(sb + 1, 0xDEADBEEFu);

    constexpr auto words = encode_vop3(/*op=*/125, /*vdst=*/0, /*src0=*/256, /*src1=*/256,
                                       /*src2=*/0, /*abs=*/0, /*opsel=*/0x2);
    std::unique_ptr<Instruction> inst(decoder->decode(words.data()));
    ASSERT_NE(inst, nullptr);
    ASSERT_EQ(std::string_view(inst->mnemonic()), "v_cmp_class_f16");
    cu->execute_instruction(inst.get(), *wf);

    EXPECT_EQ(cu->read_sgpr(sb + 0), 0x1u);
    EXPECT_EQ(cu->read_sgpr(sb + 1), 0xDEADBEEFu);

    if (!wf->is_halted())
      wf->halt();
  }
}

TEST(Gfx1250True16Vop3Test, CmpxClassF16UsesSelectedMaskHalf) {
  for (bool force_scalar : {false, true}) {
    ForceScalarGuard guard(force_scalar);
    amdgpu::GpuMemory gpu_mem(force_scalar ? "gfx1250_true16_cmpx_class_scalar_mem"
                                           : "gfx1250_true16_cmpx_class_simd_mem");
    amdgpu::L2Cache l2(force_scalar ? "gfx1250_true16_cmpx_class_scalar_l2"
                                    : "gfx1250_true16_cmpx_class_simd_l2");

    amdgpu::ComputeUnitCore::Config cfg{};
    cfg.arch = ROCJITSU_CODE_ARCH_GFX1250;
    cfg.num_wf_slots = 1;
    cfg.sgprs_per_wf = 106;
    cfg.vgprs_per_wf = 256;
    cfg.lds_size_kb = 64;

    auto cu = amdgpu::ComputeUnitCore::create("gfx1250", cfg, &gpu_mem, &l2);
    ASSERT_NE(cu, nullptr);

    auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
    ASSERT_NE(decoder, nullptr);

    auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
    ASSERT_NE(wf, nullptr);
    ASSERT_EQ(wf->wf_size(), 32u);
    wf->set_exec(0x3);
    wf->set_vcc(0xA5A5u);

    const uint32_t vb = wf->vgpr_alloc().base;
    cu->write_vgpr(vb + 0, 0, pack16(0x0000u, 0x0040u)); // +0h, mask=+0
    cu->write_vgpr(vb + 0, 1, pack16(0x7C01u, 0x0002u)); // sNaN, mask=qNaN

    constexpr auto words = encode_vop3(/*op=*/253, /*vdst=*/0, /*src0=*/256, /*src1=*/256,
                                       /*src2=*/0, /*abs=*/0, /*opsel=*/0x2);
    std::unique_ptr<Instruction> inst(decoder->decode(words.data()));
    ASSERT_NE(inst, nullptr);
    ASSERT_EQ(std::string_view(inst->mnemonic()), "v_cmpx_class_f16");
    cu->execute_instruction(inst.get(), *wf);

    EXPECT_EQ(wf->exec(), 0x1u);
    EXPECT_EQ(wf->vcc(), 0xA5A5u);

    if (!wf->is_halted())
      wf->halt();
  }
}

TEST(Gfx1250True16Vop3Test, SpecialVop3OpsUseSelectedHalves) {
  for (bool force_scalar : {false, true}) {
    ForceScalarGuard guard(force_scalar);
    amdgpu::GpuMemory gpu_mem(force_scalar ? "gfx1250_true16_special_scalar_mem"
                                           : "gfx1250_true16_special_simd_mem");
    amdgpu::L2Cache l2(force_scalar ? "gfx1250_true16_special_scalar_l2"
                                    : "gfx1250_true16_special_simd_l2");

    amdgpu::ComputeUnitCore::Config cfg{};
    cfg.arch = ROCJITSU_CODE_ARCH_GFX1250;
    cfg.num_wf_slots = 1;
    cfg.sgprs_per_wf = 106;
    cfg.vgprs_per_wf = 256;
    cfg.lds_size_kb = 64;

    auto cu = amdgpu::ComputeUnitCore::create("gfx1250", cfg, &gpu_mem, &l2);
    ASSERT_NE(cu, nullptr);

    auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
    ASSERT_NE(decoder, nullptr);

    auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
    ASSERT_NE(wf, nullptr);
    ASSERT_EQ(wf->wf_size(), 32u);
    wf->set_exec(0x1);

    const uint32_t vb = wf->vgpr_alloc().base;
    auto execute = [&](const std::array<uint32_t, 2> &words, std::string_view mnemonic) {
      std::unique_ptr<Instruction> inst(decoder->decode(words.data()));
      ASSERT_NE(inst, nullptr);
      ASSERT_EQ(std::string_view(inst->mnemonic()), mnemonic);
      cu->execute_instruction(inst.get(), *wf);
    };

    cu->write_vgpr(vb + 0, 0, pack16(2u, 7u));
    cu->write_vgpr(vb + 1, 0, pack16(3u, 11u));
    cu->write_vgpr(vb + 2, 0, 0x12340064u);
    execute(encode_vop3(/*op=*/601, /*vdst=*/3, /*src0=*/256, /*src1=*/257,
                        /*src2=*/258, /*abs=*/0, /*opsel=*/0x3),
            "v_mad_u32_u16");
    EXPECT_EQ(cu->read_vgpr(vb + 3, 0), 0x123400B1u);

    cu->write_vgpr(vb + 4, 0, pack16(0xFFFEu, 0xFFF9u)); // low=-2, high=-7
    cu->write_vgpr(vb + 5, 0, pack16(5u, 0xFFFDu));      // low=5, high=-3
    cu->write_vgpr(vb + 6, 0, 0x12340032u);
    execute(encode_vop3(/*op=*/602, /*vdst=*/7, /*src0=*/260, /*src1=*/261,
                        /*src2=*/262, /*abs=*/0, /*opsel=*/0x3),
            "v_mad_i32_i16");
    EXPECT_EQ(cu->read_vgpr(vb + 7, 0), 0x12340047u);

    cu->write_vgpr(vb + 0, 0, pack16(0x1111u, 0x2222u));
    cu->write_vgpr(vb + 1, 0, pack16(0x3333u, 0x4444u));
    execute(encode_vop3(/*op=*/785, /*vdst=*/8, /*src0=*/256, /*src1=*/257,
                        /*src2=*/0, /*abs=*/0, /*opsel=*/0x3),
            "v_pack_b32_f16");
    EXPECT_EQ(cu->read_vgpr(vb + 8, 0), 0x44442222u);

    cu->write_vgpr(vb + 0, 0, pack16(util::f32_to_f16(0.0f), util::f32_to_f16(0.5f)));
    cu->write_vgpr(vb + 1, 0, pack16(util::f32_to_f16(0.0f), util::f32_to_f16(1.0f)));
    execute(encode_vop3(/*op=*/786, /*vdst=*/9, /*src0=*/256, /*src1=*/257,
                        /*src2=*/0, /*abs=*/0, /*opsel=*/0x3),
            "v_cvt_pk_norm_i16_f16");
    EXPECT_EQ(cu->read_vgpr(vb + 9, 0), 0x7FFF3FFFu);

    execute(encode_vop3(/*op=*/787, /*vdst=*/10, /*src0=*/256, /*src1=*/257,
                        /*src2=*/0, /*abs=*/0, /*opsel=*/0x3),
            "v_cvt_pk_norm_u16_f16");
    EXPECT_EQ(cu->read_vgpr(vb + 10, 0), 0xFFFF7FFFu);

    cu->write_vgpr(vb + 0, 0, pack16(util::f32_to_f16(0.25f), util::f32_to_f16(2.0f)));
    cu->write_vgpr(vb + 1, 0, pack16(util::f32_to_f16(1.0f), util::f32_to_f16(3.0f)));
    cu->write_vgpr(vb + 2, 0, pack16(util::f32_to_f16(1.0f), util::f32_to_f16(4.0f)));
    cu->write_vgpr(vb + 12, 0, pack16(0xBEEFu, 0xCAFEu));
    execute(encode_vop3(/*op=*/596, /*vdst=*/12, /*src0=*/256, /*src1=*/257,
                        /*src2=*/258, /*abs=*/0, /*opsel=*/0xF),
            "v_div_fixup_f16");
    EXPECT_EQ(cu->read_vgpr(vb + 12, 0), pack16(0xBEEFu, util::f32_to_f16(2.0f)));

    if (!wf->is_halted())
      wf->halt();
  }
}

TEST(Gfx1250True16VopcTest, CmpLtI16ReadsPackedHighVsrc1) {
  amdgpu::GpuMemory gpu_mem("gfx1250_true16_vopc_cmp_src_mem");
  amdgpu::L2Cache l2("gfx1250_true16_vopc_cmp_src_l2");

  amdgpu::ComputeUnitCore::Config cfg{};
  cfg.arch = ROCJITSU_CODE_ARCH_GFX1250;
  cfg.num_wf_slots = 1;
  cfg.sgprs_per_wf = 106;
  cfg.vgprs_per_wf = 256;
  cfg.lds_size_kb = 64;

  auto cu = amdgpu::ComputeUnitCore::create("gfx1250", cfg, &gpu_mem, &l2);
  ASSERT_NE(cu, nullptr);

  auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
  ASSERT_NE(decoder, nullptr);

  auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
  ASSERT_NE(wf, nullptr);
  ASSERT_EQ(wf->wf_size(), 32u);
  wf->set_exec(0x3);
  wf->set_vcc(0);

  const uint32_t vb = wf->vgpr_alloc().base;
  cu->write_vgpr(vb + 0, 0, 0xC0003C00u);
  cu->write_vgpr(vb + 0, 1, 0x3C00C000u);

  // v_cmp_lt_i16_e32 vcc_lo, -1, v0.h
  const uint32_t words[] = {0x7C6300C1U};
  std::unique_ptr<Instruction> inst(decoder->decode(words));
  ASSERT_NE(inst, nullptr);
  ASSERT_EQ(std::string_view(inst->mnemonic()), "v_cmp_lt_i16_e32");
  cu->execute_instruction(inst.get(), *wf);

  EXPECT_EQ(wf->vcc(), 0x2u);

  if (!wf->is_halted())
    wf->halt();
}

TEST(Rdna4True16Vop3Test, CmpGeF16UsesSelectedHighSourceHalf) {
  for (bool force_scalar : {false, true}) {
    SCOPED_TRACE(force_scalar ? "scalar" : "simd");
    ForceScalarGuard guard(force_scalar);
    amdgpu::GpuMemory gpu_mem(force_scalar ? "rdna4_true16_cmp_scalar_mem"
                                           : "rdna4_true16_cmp_simd_mem");
    amdgpu::L2Cache l2(force_scalar ? "rdna4_true16_cmp_scalar_l2" : "rdna4_true16_cmp_simd_l2");

    amdgpu::ComputeUnitCore::Config cfg{};
    cfg.arch = ROCJITSU_CODE_ARCH_RDNA4;
    cfg.num_wf_slots = 1;
    cfg.sgprs_per_wf = 106;
    cfg.vgprs_per_wf = 256;
    cfg.lds_size_kb = 64;

    auto cu = amdgpu::ComputeUnitCore::create("rdna4", cfg, &gpu_mem, &l2);
    ASSERT_NE(cu, nullptr);

    auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_RDNA4);
    ASSERT_NE(decoder, nullptr);

    auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
    ASSERT_NE(wf, nullptr);
    ASSERT_EQ(wf->wf_size(), 32u);
    wf->set_exec(0x3);

    const uint32_t vb = wf->vgpr_alloc().base;
    const uint32_t sb = wf->sgpr_alloc().base;
    cu->write_vgpr(vb + 0, 0, 0x34003C00u); // low=1.0h, high=0.25h
    cu->write_vgpr(vb + 0, 1, 0x38000000u); // low=0.0h, high=0.5h
    cu->write_sgpr(sb + 0, 0xFFFFFFFFu);
    cu->write_sgpr(sb + 1, 0xDEADBEEFu);

    // v_cmp_ge_f16_e64 s0, |v0.h|, 0.5 op_sel:[1,0,0]
    const uint32_t words[] = {0xD4060900U, 0x0201E100U};
    std::unique_ptr<Instruction> inst(decoder->decode(words));
    ASSERT_NE(inst, nullptr);
    ASSERT_EQ(std::string_view(inst->mnemonic()), "v_cmp_ge_f16");
    cu->execute_instruction(inst.get(), *wf);

    EXPECT_EQ(cu->read_sgpr(sb + 0), 0x2u);
    EXPECT_EQ(cu->read_sgpr(sb + 1), 0xDEADBEEFu);

    if (!wf->is_halted())
      wf->halt();
  }
}

TEST(Rdna4True16Vop3Test, CmpClassF16UsesSelectedMaskHalf) {
  for (bool force_scalar : {false, true}) {
    ForceScalarGuard guard(force_scalar);
    amdgpu::GpuMemory gpu_mem(force_scalar ? "rdna4_true16_class_scalar_mem"
                                           : "rdna4_true16_class_simd_mem");
    amdgpu::L2Cache l2(force_scalar ? "rdna4_true16_class_scalar_l2"
                                    : "rdna4_true16_class_simd_l2");

    amdgpu::ComputeUnitCore::Config cfg{};
    cfg.arch = ROCJITSU_CODE_ARCH_RDNA4;
    cfg.num_wf_slots = 1;
    cfg.sgprs_per_wf = 106;
    cfg.vgprs_per_wf = 256;
    cfg.lds_size_kb = 64;

    auto cu = amdgpu::ComputeUnitCore::create("rdna4", cfg, &gpu_mem, &l2);
    ASSERT_NE(cu, nullptr);

    auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_RDNA4);
    ASSERT_NE(decoder, nullptr);

    auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
    ASSERT_NE(wf, nullptr);
    ASSERT_EQ(wf->wf_size(), 32u);
    wf->set_exec(0x3);

    const uint32_t vb = wf->vgpr_alloc().base;
    const uint32_t sb = wf->sgpr_alloc().base;
    cu->write_vgpr(vb + 0, 0, pack16(0x0000u, 0x0040u)); // +0h, mask=+0
    cu->write_vgpr(vb + 0, 1, pack16(0x7C01u, 0x0002u)); // sNaN, mask=qNaN
    cu->write_sgpr(sb + 0, 0xFFFFFFFFu);
    cu->write_sgpr(sb + 1, 0xDEADBEEFu);

    constexpr auto words = encode_vop3(/*op=*/125, /*vdst=*/0, /*src0=*/256, /*src1=*/256,
                                       /*src2=*/0, /*abs=*/0, /*opsel=*/0x2);
    std::unique_ptr<Instruction> inst(decoder->decode(words.data()));
    ASSERT_NE(inst, nullptr);
    ASSERT_EQ(std::string_view(inst->mnemonic()), "v_cmp_class_f16");
    cu->execute_instruction(inst.get(), *wf);

    EXPECT_EQ(cu->read_sgpr(sb + 0), 0x1u);
    EXPECT_EQ(cu->read_sgpr(sb + 1), 0xDEADBEEFu);

    if (!wf->is_halted())
      wf->halt();
  }
}

TEST(Rdna4True16Vop3Test, CmpClassF16InlineConstantUsesF16Broadcast) {
  for (bool force_scalar : {false, true}) {
    SCOPED_TRACE(force_scalar ? "scalar" : "simd");
    ForceScalarGuard guard(force_scalar);
    amdgpu::GpuMemory gpu_mem(force_scalar ? "rdna4_true16_class_inline_scalar_mem"
                                           : "rdna4_true16_class_inline_simd_mem");
    amdgpu::L2Cache l2(force_scalar ? "rdna4_true16_class_inline_scalar_l2"
                                    : "rdna4_true16_class_inline_simd_l2");

    amdgpu::ComputeUnitCore::Config cfg{};
    cfg.arch = ROCJITSU_CODE_ARCH_RDNA4;
    cfg.num_wf_slots = 1;
    cfg.sgprs_per_wf = 106;
    cfg.vgprs_per_wf = 256;
    cfg.lds_size_kb = 64;

    auto cu = amdgpu::ComputeUnitCore::create("rdna4", cfg, &gpu_mem, &l2);
    ASSERT_NE(cu, nullptr);

    auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_RDNA4);
    ASSERT_NE(decoder, nullptr);

    auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
    ASSERT_NE(wf, nullptr);
    ASSERT_EQ(wf->wf_size(), 32u);
    wf->set_exec(0x3);

    const uint32_t sb = wf->sgpr_alloc().base;
    cu->write_sgpr(sb + 0, 0xFFFFFFFFu);
    cu->write_sgpr(sb + 1, 0xDEADBEEFu);
    cu->write_sgpr(sb + 4, 0x0100u);

    constexpr auto words = encode_vop3(/*op=*/125, /*vdst=*/0, /*src0=*/242, /*src1=*/4,
                                       /*src2=*/0, /*abs=*/0, /*opsel=*/0);
    std::unique_ptr<Instruction> inst(decoder->decode(words.data()));
    ASSERT_NE(inst, nullptr);
    ASSERT_EQ(std::string_view(inst->mnemonic()), "v_cmp_class_f16");
    cu->execute_instruction(inst.get(), *wf);

    EXPECT_EQ(cu->read_sgpr(sb + 0), 0x3u);
    EXPECT_EQ(cu->read_sgpr(sb + 1), 0xDEADBEEFu);

    if (!wf->is_halted())
      wf->halt();
  }
}

TEST(Rdna4True16Vop3Test, CmpxClassF16UsesSelectedMaskHalf) {
  for (bool force_scalar : {false, true}) {
    ForceScalarGuard guard(force_scalar);
    amdgpu::GpuMemory gpu_mem(force_scalar ? "rdna4_true16_cmpx_class_scalar_mem"
                                           : "rdna4_true16_cmpx_class_simd_mem");
    amdgpu::L2Cache l2(force_scalar ? "rdna4_true16_cmpx_class_scalar_l2"
                                    : "rdna4_true16_cmpx_class_simd_l2");

    amdgpu::ComputeUnitCore::Config cfg{};
    cfg.arch = ROCJITSU_CODE_ARCH_RDNA4;
    cfg.num_wf_slots = 1;
    cfg.sgprs_per_wf = 106;
    cfg.vgprs_per_wf = 256;
    cfg.lds_size_kb = 64;

    auto cu = amdgpu::ComputeUnitCore::create("rdna4", cfg, &gpu_mem, &l2);
    ASSERT_NE(cu, nullptr);

    auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_RDNA4);
    ASSERT_NE(decoder, nullptr);

    auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
    ASSERT_NE(wf, nullptr);
    ASSERT_EQ(wf->wf_size(), 32u);
    wf->set_exec(0x3);
    wf->set_vcc(0xA5A5u);

    const uint32_t vb = wf->vgpr_alloc().base;
    cu->write_vgpr(vb + 0, 0, pack16(0x0000u, 0x0040u)); // +0h, mask=+0
    cu->write_vgpr(vb + 0, 1, pack16(0x7C01u, 0x0002u)); // sNaN, mask=qNaN

    constexpr auto words = encode_vop3(/*op=*/253, /*vdst=*/0, /*src0=*/256, /*src1=*/256,
                                       /*src2=*/0, /*abs=*/0, /*opsel=*/0x2);
    std::unique_ptr<Instruction> inst(decoder->decode(words.data()));
    ASSERT_NE(inst, nullptr);
    ASSERT_EQ(std::string_view(inst->mnemonic()), "v_cmpx_class_f16");
    cu->execute_instruction(inst.get(), *wf);

    EXPECT_EQ(wf->exec(), 0x1u);
    EXPECT_EQ(wf->vcc(), 0xA5A5u);

    if (!wf->is_halted())
      wf->halt();
  }
}

TEST(Rdna4True16Vop3Test, ClassF16HelperSequenceMovesMaskIntoSelectedHighHalf) {
  struct Case {
    uint32_t lane;
    uint16_t value;
    uint16_t mask;
    bool expected;
  };

  constexpr Case cases[] = {
      {0, 0x0000u, 0x0040u, true},   // +0
      {1, 0x8000u, 0x0020u, true},   // -0
      {2, 0x3C00u, 0x0100u, true},   // +normal
      {3, 0xBC00u, 0x0008u, true},   // -normal
      {4, 0x0001u, 0x0080u, true},   // +denormal
      {5, 0x8001u, 0x0010u, true},   // -denormal
      {6, 0x7C00u, 0x0200u, true},   // +inf
      {7, 0xFC00u, 0x0004u, true},   // -inf
      {8, 0x7C01u, 0x0001u, true},   // sNaN
      {9, 0x7E00u, 0x0002u, true},   // qNaN
      {10, 0x7C01u, 0x0002u, false}, // sNaN is not qNaN
  };

  for (bool force_scalar : {false, true}) {
    ForceScalarGuard guard(force_scalar);
    amdgpu::GpuMemory gpu_mem(force_scalar ? "rdna4_true16_class_seq_scalar_mem"
                                           : "rdna4_true16_class_seq_simd_mem");
    amdgpu::L2Cache l2(force_scalar ? "rdna4_true16_class_seq_scalar_l2"
                                    : "rdna4_true16_class_seq_simd_l2");

    amdgpu::ComputeUnitCore::Config cfg{};
    cfg.arch = ROCJITSU_CODE_ARCH_RDNA4;
    cfg.num_wf_slots = 1;
    cfg.sgprs_per_wf = 106;
    cfg.vgprs_per_wf = 256;
    cfg.lds_size_kb = 64;

    auto cu = amdgpu::ComputeUnitCore::create("rdna4", cfg, &gpu_mem, &l2);
    ASSERT_NE(cu, nullptr);

    auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_RDNA4);
    ASSERT_NE(decoder, nullptr);

    auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
    ASSERT_NE(wf, nullptr);
    ASSERT_EQ(wf->wf_size(), 32u);

    const uint32_t vb = wf->vgpr_alloc().base;
    const uint32_t sb = wf->sgpr_alloc().base;

    // gfx1201 helper sequence from hip-fpsan's amdgcn_classh lowering:
    //   v_mov_b16_e32 v0.h, v1.l
    //   v_cmp_class_f16_e64 s0, v0.l, v0.h op_sel:[0,1,0]
    //   v_cndmask_b32_e64 v0, 0, 1, s0
    const uint32_t mov_words[] = {0x7F003901U, 0};
    const uint32_t cmp_words[] = {0xD47D1000U, 0x02020100U};
    const uint32_t select_words[] = {0xD5010000U, 0x00010280U};
    std::unique_ptr<Instruction> mov(decoder->decode(mov_words));
    ASSERT_NE(mov, nullptr);
    ASSERT_EQ(std::string_view(mov->mnemonic()), "v_mov_b16_e32");
    std::unique_ptr<Instruction> cmp(decoder->decode(cmp_words));
    ASSERT_NE(cmp, nullptr);
    ASSERT_EQ(std::string_view(cmp->mnemonic()), "v_cmp_class_f16");
    std::unique_ptr<Instruction> select(decoder->decode(select_words));
    ASSERT_NE(select, nullptr);
    ASSERT_EQ(std::string_view(select->mnemonic()), "v_cndmask_b32");

    uint64_t active_mask = 0;
    uint64_t expected_mask = 0;
    for (const auto &test : cases) {
      active_mask |= (1ULL << test.lane);
      if (test.expected)
        expected_mask |= (1ULL << test.lane);
      cu->write_vgpr(vb + 0, test.lane, pack16(test.value, 0xCAFEu));
      cu->write_vgpr(vb + 1, test.lane, pack16(test.mask, 0x1234u));
    }
    wf->set_exec(active_mask);
    cu->write_sgpr(sb + 0, 0);
    cu->write_sgpr(sb + 1, 0);

    cu->execute_instruction(mov.get(), *wf);
    for (const auto &test : cases) {
      EXPECT_EQ(cu->read_vgpr(vb + 0, test.lane), pack16(test.value, test.mask))
          << "lane " << test.lane;
    }

    cu->execute_instruction(cmp.get(), *wf);
    EXPECT_EQ(cu->read_sgpr(sb + 0), static_cast<uint32_t>(expected_mask));
    EXPECT_EQ(cu->read_sgpr(sb + 1), static_cast<uint32_t>(expected_mask >> 32));

    cu->execute_instruction(select.get(), *wf);
    for (const auto &test : cases) {
      EXPECT_EQ(cu->read_vgpr(vb + 0, test.lane), test.expected ? 1u : 0u) << "lane " << test.lane;
    }

    if (!wf->is_halted())
      wf->halt();
  }
}

TEST(Rdna4True16Vop3Test, RcpF16WritesSelectedHighDestinationHalf) {
  for (bool force_scalar : {false, true}) {
    ForceScalarGuard guard(force_scalar);
    amdgpu::GpuMemory gpu_mem(force_scalar ? "rdna4_true16_rcp_scalar_mem"
                                           : "rdna4_true16_rcp_simd_mem");
    amdgpu::L2Cache l2(force_scalar ? "rdna4_true16_rcp_scalar_l2" : "rdna4_true16_rcp_simd_l2");

    amdgpu::ComputeUnitCore::Config cfg{};
    cfg.arch = ROCJITSU_CODE_ARCH_RDNA4;
    cfg.num_wf_slots = 1;
    cfg.sgprs_per_wf = 106;
    cfg.vgprs_per_wf = 256;
    cfg.lds_size_kb = 64;

    auto cu = amdgpu::ComputeUnitCore::create("rdna4", cfg, &gpu_mem, &l2);
    ASSERT_NE(cu, nullptr);

    auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_RDNA4);
    ASSERT_NE(decoder, nullptr);

    auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
    ASSERT_NE(wf, nullptr);
    ASSERT_EQ(wf->wf_size(), 32u);
    wf->set_exec(0x1);

    const uint32_t vb = wf->vgpr_alloc().base;
    cu->write_vgpr(vb + 0, 0, 0xCAFE4000u); // low=2.0h

    // v_rcp_f16_e64 v0.h, -v0.l op_sel:[0,1]
    const uint32_t words[] = {0xD5D44000U, 0x22010100U};
    std::unique_ptr<Instruction> inst(decoder->decode(words));
    ASSERT_NE(inst, nullptr);
    ASSERT_EQ(std::string_view(inst->mnemonic()), "v_rcp_f16");
    cu->execute_instruction(inst.get(), *wf);

    EXPECT_EQ(cu->read_vgpr(vb + 0, 0), 0xB8004000u); // high=-0.5h, low preserved

    if (!wf->is_halted())
      wf->halt();
  }
}

TEST(Rdna4True16Vop3Test, CvtF32F16AppliesAbsToSelectedSourceHalf) {
  for (bool force_scalar : {false, true}) {
    ForceScalarGuard guard(force_scalar);
    amdgpu::GpuMemory gpu_mem(force_scalar ? "rdna4_true16_cvt_scalar_mem"
                                           : "rdna4_true16_cvt_simd_mem");
    amdgpu::L2Cache l2(force_scalar ? "rdna4_true16_cvt_scalar_l2" : "rdna4_true16_cvt_simd_l2");

    amdgpu::ComputeUnitCore::Config cfg{};
    cfg.arch = ROCJITSU_CODE_ARCH_RDNA4;
    cfg.num_wf_slots = 1;
    cfg.sgprs_per_wf = 106;
    cfg.vgprs_per_wf = 256;
    cfg.lds_size_kb = 64;

    auto cu = amdgpu::ComputeUnitCore::create("rdna4", cfg, &gpu_mem, &l2);
    ASSERT_NE(cu, nullptr);

    auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_RDNA4);
    ASSERT_NE(decoder, nullptr);

    auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
    ASSERT_NE(wf, nullptr);
    ASSERT_EQ(wf->wf_size(), 32u);
    wf->set_exec(0x1);

    const uint32_t vb = wf->vgpr_alloc().base;
    cu->write_vgpr(vb + 2, 0, 0x1234C400u); // low=-4.0h, high is ignored

    // v_cvt_f32_f16_e64 v4, |v2.l|
    const uint32_t words[] = {0xD58B0104U, 0x02010102U};
    std::unique_ptr<Instruction> inst(decoder->decode(words));
    ASSERT_NE(inst, nullptr);
    ASSERT_EQ(std::string_view(inst->mnemonic()), "v_cvt_f32_f16");
    cu->execute_instruction(inst.get(), *wf);

    EXPECT_EQ(cu->read_vgpr(vb + 4, 0), 0x40800000u);

    if (!wf->is_halted())
      wf->halt();
  }
}

TEST(Rdna4True16Vop3Test, AddF16UsesSelectedHalvesAndPreservesDestinationHalf) {
  for (bool force_scalar : {false, true}) {
    SCOPED_TRACE(force_scalar ? "scalar" : "simd");
    ForceScalarGuard guard(force_scalar);
    amdgpu::GpuMemory gpu_mem(force_scalar ? "rdna4_true16_add_f16_scalar_mem"
                                           : "rdna4_true16_add_f16_simd_mem");
    amdgpu::L2Cache l2(force_scalar ? "rdna4_true16_add_f16_scalar_l2"
                                    : "rdna4_true16_add_f16_simd_l2");

    amdgpu::ComputeUnitCore::Config cfg{};
    cfg.arch = ROCJITSU_CODE_ARCH_RDNA4;
    cfg.num_wf_slots = 1;
    cfg.sgprs_per_wf = 106;
    cfg.vgprs_per_wf = 256;
    cfg.lds_size_kb = 64;

    auto cu = amdgpu::ComputeUnitCore::create("rdna4", cfg, &gpu_mem, &l2);
    ASSERT_NE(cu, nullptr);

    auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_RDNA4);
    ASSERT_NE(decoder, nullptr);

    auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
    ASSERT_NE(wf, nullptr);
    ASSERT_EQ(wf->wf_size(), 32u);
    wf->set_exec(0x3);

    const uint32_t vb = wf->vgpr_alloc().base;
    cu->write_vgpr(vb + 0, 0, pack16(util::f32_to_f16(100.0f), util::f32_to_f16(2.0f)));
    cu->write_vgpr(vb + 1, 0, pack16(util::f32_to_f16(10.0f), util::f32_to_f16(3.0f)));
    cu->write_vgpr(vb + 0, 1, pack16(util::f32_to_f16(8.0f), util::f32_to_f16(-1.0f)));
    cu->write_vgpr(vb + 1, 1, pack16(util::f32_to_f16(4.0f), util::f32_to_f16(0.5f)));
    cu->write_vgpr(vb + 2, 0, pack16(0xBEEFu, 0xCAFEu));
    cu->write_vgpr(vb + 2, 1, pack16(0x1234u, 0x5678u));

    constexpr auto words = encode_vop3(/*op=*/0x132, /*vdst=*/2, /*src0=*/256,
                                       /*src1=*/257, /*src2=*/0, /*abs=*/0,
                                       /*opsel=*/0xB);
    std::unique_ptr<Instruction> inst(decoder->decode(words.data()));
    ASSERT_NE(inst, nullptr);
    ASSERT_EQ(std::string_view(inst->mnemonic()), "v_add_f16");
    cu->execute_instruction(inst.get(), *wf);

    EXPECT_EQ(cu->read_vgpr(vb + 2, 0), pack16(0xBEEFu, util::f32_to_f16(5.0f)));
    EXPECT_EQ(cu->read_vgpr(vb + 2, 1), pack16(0x1234u, util::f32_to_f16(-0.5f)));

    if (!wf->is_halted())
      wf->halt();
  }
}

TEST(Rdna4True16Vop3Test, AddF16DppPreservesMaskedDestinationHalf) {
  ForceScalarGuard guard(false);
  amdgpu::GpuMemory gpu_mem("rdna4_true16_add_f16_dpp_mem");
  amdgpu::L2Cache l2("rdna4_true16_add_f16_dpp_l2");

  amdgpu::ComputeUnitCore::Config cfg{};
  cfg.arch = ROCJITSU_CODE_ARCH_RDNA4;
  cfg.num_wf_slots = 1;
  cfg.sgprs_per_wf = 106;
  cfg.vgprs_per_wf = 256;
  cfg.lds_size_kb = 64;

  auto cu = amdgpu::ComputeUnitCore::create("rdna4_true16_add_f16_dpp", cfg, &gpu_mem, &l2);
  ASSERT_NE(cu, nullptr);

  auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
  ASSERT_NE(wf, nullptr);
  ASSERT_EQ(wf->wf_size(), 32u);
  wf->set_exec((1ULL << wf->wf_size()) - 1ULL);

  const uint32_t vb = wf->vgpr_alloc().base;
  constexpr uint32_t kSrc0 = 0;
  constexpr uint32_t kSrc1 = 1;
  constexpr uint32_t kDst = 2;

  for (uint32_t lane = 0; lane < wf->wf_size(); ++lane) {
    cu->write_vgpr(vb + kSrc0, lane,
                   pack16(0x1111u, util::f32_to_f16(static_cast<float>(lane + 10))));
    cu->write_vgpr(vb + kSrc1, lane, pack16(0x2222u, util::f32_to_f16(0.5f)));
    cu->write_vgpr(
        vb + kDst, lane,
        pack16(static_cast<uint16_t>(0x4000u + lane), static_cast<uint16_t>(0x7000u + lane)));
  }

  rdna4::Vop3VopDpp16MachineInst raw{};
  raw.vdst = kDst;
  raw.opsel = 0xBu;
  raw.op = 0x132u;
  raw.encoding = 0x35u;
  raw.src0 = amdgpu::SRC_DPP;
  raw.src1 = 256u + kSrc1;
  raw.vsrc0 = kSrc0;
  raw.dpp_ctrl = amdgpu::dpp::ROW_SHR1;
  raw.fi = 1;
  raw.bound_ctrl = 0;
  raw.bank_mask = 0xFu;
  raw.row_mask = 0x1u;

  rdna4::VAddF16Vop3 inst(reinterpret_cast<const rdna4::MachineInst *>(&raw));
  ASSERT_EQ(std::string_view(inst.mnemonic()), "v_add_f16");
  inst.execute_impl(*wf);

  EXPECT_EQ(cu->read_vgpr(vb + kDst, 0), pack16(0x4000u, 0x7000u));
  EXPECT_EQ(cu->read_vgpr(vb + kDst, 1), pack16(0x4001u, util::f32_to_f16(10.5f)));
  EXPECT_EQ(cu->read_vgpr(vb + kDst, 2), pack16(0x4002u, util::f32_to_f16(11.5f)));
  EXPECT_EQ(cu->read_vgpr(vb + kDst, 16), pack16(0x4010u, 0x7010u));

  if (!wf->is_halted())
    wf->halt();
}

TEST(Rdna4True16Vop3Test, UnaryDpp16ScalarAndSimdMatchMaskedLaneRouting) {
  if constexpr (!util::has_stdx_simd) {
    GTEST_SKIP() << "<experimental/simd> unavailable";
  }

  constexpr uint32_t kWaveSize = 32;
  constexpr uint32_t kSrc = 0;
  constexpr uint32_t kDst = 2;
  constexpr uint32_t kDppCtrl = amdgpu::dpp::ROW_SHR1;
  constexpr uint32_t kRowMask = 0b0101;
  constexpr uint32_t kBankMask = 0b1010;
  constexpr uint64_t kExec = (uint64_t{1} << kWaveSize) - 1;
  std::array<std::array<uint32_t, kWaveSize>, 2> outputs{};

  for (uint32_t mode = 0; mode < outputs.size(); ++mode) {
    const bool force_scalar = mode == 0;
    SCOPED_TRACE(force_scalar ? "scalar" : "simd");
    ForceScalarGuard guard(force_scalar);
    amdgpu::GpuMemory gpu_mem(force_scalar ? "rdna4_true16_unary_dpp_scalar_mem"
                                           : "rdna4_true16_unary_dpp_simd_mem");
    amdgpu::L2Cache l2(force_scalar ? "rdna4_true16_unary_dpp_scalar_l2"
                                    : "rdna4_true16_unary_dpp_simd_l2");

    amdgpu::ComputeUnitCore::Config cfg{};
    cfg.arch = ROCJITSU_CODE_ARCH_RDNA4;
    cfg.num_wf_slots = 1;
    cfg.sgprs_per_wf = 106;
    cfg.vgprs_per_wf = 256;
    cfg.lds_size_kb = 64;
    auto cu = amdgpu::ComputeUnitCore::create("rdna4_true16_unary_dpp", cfg, &gpu_mem, &l2);
    ASSERT_NE(cu, nullptr);
    auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
    ASSERT_NE(wf, nullptr);
    ASSERT_EQ(wf->wf_size(), kWaveSize);
    wf->set_exec(kExec);

    const uint32_t vb = wf->vgpr_alloc().base;
    for (uint32_t lane = 0; lane < kWaveSize; ++lane) {
      const float value = static_cast<float>(uint32_t{1} << (lane % 4));
      cu->write_vgpr(vb + kSrc, lane, pack16(0xCAFEu, util::f32_to_f16(value)));
      cu->write_vgpr(
          vb + kDst, lane,
          pack16(static_cast<uint16_t>(0x5000u + lane), static_cast<uint16_t>(0x1000u + lane)));
    }

    rdna4::Vop3VopDpp16MachineInst raw{};
    raw.vdst = kDst;
    raw.opsel = 0x9u; // Select high src0 and high destination halves.
    raw.op = rdna4::kVRcpF16Vop3;
    raw.encoding = 0x35u;
    raw.src0 = amdgpu::SRC_DPP;
    raw.vsrc0 = kSrc;
    raw.dpp_ctrl = kDppCtrl;
    raw.fi = 1;
    raw.bound_ctrl = 0;
    raw.bank_mask = kBankMask;
    raw.row_mask = kRowMask;

    static_assert(sizeof(raw) == 3 * sizeof(uint32_t));
    auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_RDNA4);
    std::unique_ptr<Instruction> inst(decoder->decode(reinterpret_cast<const uint32_t *>(&raw)));
    ASSERT_NE(inst, nullptr);
    ASSERT_EQ(std::string_view(inst->mnemonic()), "v_rcp_f16");
    cu->execute_instruction(inst.get(), *wf);
    for (uint32_t lane = 0; lane < kWaveSize; ++lane)
      outputs[mode][lane] = cu->read_vgpr(vb + kDst, lane);

    if (!wf->is_halted())
      wf->halt();
  }

  EXPECT_EQ(outputs[0], outputs[1]);
  const auto plan = amdgpu::dpp::make_dpp_access_plan(kWaveSize, kDppCtrl, kRowMask, kBankMask,
                                                      /*bound_ctrl=*/0, /*fi=*/1, kExec);
  for (uint32_t lane = 0; lane < kWaveSize; ++lane) {
    const int source_lane = plan.source_lane_for_destination[lane];
    const uint16_t expected_high =
        source_lane == amdgpu::dpp::DppAccessPlan::kNoSourceLane
            ? static_cast<uint16_t>(0x1000u + lane)
            : util::f32_to_f16(1.0f / static_cast<float>(uint32_t{1} << (source_lane % 4)));
    EXPECT_EQ(outputs[1][lane], pack16(static_cast<uint16_t>(0x5000u + lane), expected_high))
        << "lane " << lane;
  }
}

TEST(Rdna4True16Vop3Test, VopcDpp16ScalarAndSimdMatchMaskedLaneRouting) {
  if constexpr (!util::has_stdx_simd) {
    GTEST_SKIP() << "<experimental/simd> unavailable";
  }

  constexpr uint32_t kWaveSize = 32;
  constexpr uint32_t kSrc0 = 0;
  constexpr uint32_t kSrc1 = 1;
  constexpr uint32_t kDppCtrl = amdgpu::dpp::ROW_SHR1;
  constexpr uint32_t kRowMask = 0b0101;
  constexpr uint32_t kBankMask = 0b1010;
  constexpr uint64_t kExec = (uint64_t{1} << kWaveSize) - 1;
  std::array<uint64_t, 2> outputs{};

  for (uint32_t mode = 0; mode < outputs.size(); ++mode) {
    const bool force_scalar = mode == 0;
    SCOPED_TRACE(force_scalar ? "scalar" : "simd");
    ForceScalarGuard guard(force_scalar);
    amdgpu::GpuMemory gpu_mem(force_scalar ? "rdna4_true16_vopc_dpp_scalar_mem"
                                           : "rdna4_true16_vopc_dpp_simd_mem");
    amdgpu::L2Cache l2(force_scalar ? "rdna4_true16_vopc_dpp_scalar_l2"
                                    : "rdna4_true16_vopc_dpp_simd_l2");

    amdgpu::ComputeUnitCore::Config cfg{};
    cfg.arch = ROCJITSU_CODE_ARCH_RDNA4;
    cfg.num_wf_slots = 1;
    cfg.sgprs_per_wf = 106;
    cfg.vgprs_per_wf = 256;
    cfg.lds_size_kb = 64;
    auto cu = amdgpu::ComputeUnitCore::create("rdna4_true16_vopc_dpp", cfg, &gpu_mem, &l2);
    ASSERT_NE(cu, nullptr);
    auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
    ASSERT_NE(wf, nullptr);
    ASSERT_EQ(wf->wf_size(), kWaveSize);
    wf->set_exec(kExec);

    const uint32_t vb = wf->vgpr_alloc().base;
    const uint32_t sb = wf->sgpr_alloc().base;
    for (uint32_t lane = 0; lane < kWaveSize; ++lane) {
      cu->write_vgpr(vb + kSrc0, lane,
                     pack16(0xCAFEu, util::f32_to_f16(static_cast<float>(lane + 1))));
      const float threshold = static_cast<float>(lane + ((lane & 1u) ? 2u : 0u));
      cu->write_vgpr(vb + kSrc1, lane, pack16(0xBEEFu, util::f32_to_f16(threshold)));
    }
    cu->write_sgpr(sb, 0xA5A5A5A5u);
    cu->write_sgpr(sb + 1, 0xDEADBEEFu);

    rdna4::Vop3VopDpp16MachineInst raw{};
    raw.vdst = 0;
    raw.opsel = 0x3u; // Select high halves from both true16 sources.
    raw.op = rdna4::kVCmpGeF16Vop3;
    raw.encoding = 0x35u;
    raw.src0 = amdgpu::SRC_DPP;
    raw.src1 = vgpr_src(kSrc1);
    raw.vsrc0 = kSrc0;
    raw.dpp_ctrl = kDppCtrl;
    raw.fi = 1;
    raw.bound_ctrl = 0;
    raw.bank_mask = kBankMask;
    raw.row_mask = kRowMask;

    static_assert(sizeof(raw) == 3 * sizeof(uint32_t));
    auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_RDNA4);
    std::unique_ptr<Instruction> inst(decoder->decode(reinterpret_cast<const uint32_t *>(&raw)));
    ASSERT_NE(inst, nullptr);
    ASSERT_EQ(std::string_view(inst->mnemonic()), "v_cmp_ge_f16");
    cu->execute_instruction(inst.get(), *wf);
    outputs[mode] = static_cast<uint64_t>(cu->read_sgpr(sb)) |
                    (static_cast<uint64_t>(cu->read_sgpr(sb + 1)) << 32);

    if (!wf->is_halted())
      wf->halt();
  }

  EXPECT_EQ(outputs[0], outputs[1]);
  const auto plan = amdgpu::dpp::make_dpp_access_plan(kWaveSize, kDppCtrl, kRowMask, kBankMask,
                                                      /*bound_ctrl=*/0, /*fi=*/1, kExec);
  uint32_t expected_low = 0;
  for (uint32_t lane = 0; lane < kWaveSize; ++lane) {
    const int source_lane = plan.source_lane_for_destination[lane];
    if (source_lane == amdgpu::dpp::DppAccessPlan::kNoSourceLane)
      continue;
    const float source = static_cast<float>(source_lane + 1);
    const float threshold = static_cast<float>(lane + ((lane & 1u) ? 2u : 0u));
    if (source >= threshold)
      expected_low |= uint32_t{1} << lane;
  }
  EXPECT_EQ(outputs[1], static_cast<uint64_t>(expected_low) | (uint64_t{0xDEADBEEF} << 32));
}

TEST(Rdna4True16Vop3Test, UnaryDpp8ScalarAndSimdMatchPermutedLanes) {
  if constexpr (!util::has_stdx_simd) {
    GTEST_SKIP() << "<experimental/simd> unavailable";
  }

  constexpr uint32_t kWaveSize = 32;
  constexpr uint32_t kSrc = 0;
  constexpr uint32_t kDst = 2;
  constexpr uint64_t kExec = (uint64_t{1} << kWaveSize) - 1;
  constexpr std::array<uint32_t, 8> kLaneSelectors{7, 0, 5, 2, 3, 6, 1, 4};
  std::array<std::array<uint32_t, kWaveSize>, 2> outputs{};

  for (uint32_t mode = 0; mode < outputs.size(); ++mode) {
    const bool force_scalar = mode == 0;
    SCOPED_TRACE(force_scalar ? "scalar" : "simd");
    ForceScalarGuard guard(force_scalar);
    amdgpu::GpuMemory gpu_mem(force_scalar ? "rdna4_true16_unary_dpp8_scalar_mem"
                                           : "rdna4_true16_unary_dpp8_simd_mem");
    amdgpu::L2Cache l2(force_scalar ? "rdna4_true16_unary_dpp8_scalar_l2"
                                    : "rdna4_true16_unary_dpp8_simd_l2");

    amdgpu::ComputeUnitCore::Config cfg{};
    cfg.arch = ROCJITSU_CODE_ARCH_RDNA4;
    cfg.num_wf_slots = 1;
    cfg.sgprs_per_wf = 106;
    cfg.vgprs_per_wf = 256;
    cfg.lds_size_kb = 64;
    auto cu = amdgpu::ComputeUnitCore::create("rdna4_true16_unary_dpp8", cfg, &gpu_mem, &l2);
    ASSERT_NE(cu, nullptr);
    auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
    ASSERT_NE(wf, nullptr);
    ASSERT_EQ(wf->wf_size(), kWaveSize);
    wf->set_exec(kExec);

    const uint32_t vb = wf->vgpr_alloc().base;
    for (uint32_t lane = 0; lane < kWaveSize; ++lane) {
      const float value = static_cast<float>(uint32_t{1} << (lane % 4));
      cu->write_vgpr(vb + kSrc, lane, pack16(util::f32_to_f16(value), 0xCAFEu));
      cu->write_vgpr(
          vb + kDst, lane,
          pack16(static_cast<uint16_t>(0x2000u + lane), static_cast<uint16_t>(0x6000u + lane)));
    }

    rdna4::Vop3VopDpp8MachineInst raw{};
    raw.vdst = kDst;
    raw.op = rdna4::kVRcpF16Vop3;
    raw.encoding = 0x35u;
    raw.src0 = amdgpu::SRC_DPP8_FI_1;
    raw.vsrc0 = kSrc;
    raw.lane_sel_0 = kLaneSelectors[0];
    raw.lane_sel_1 = kLaneSelectors[1];
    raw.lane_sel_2 = kLaneSelectors[2];
    raw.lane_sel_3 = kLaneSelectors[3];
    raw.lane_sel_4 = kLaneSelectors[4];
    raw.lane_sel_5 = kLaneSelectors[5];
    raw.lane_sel_6 = kLaneSelectors[6];
    raw.lane_sel_7 = kLaneSelectors[7];

    static_assert(sizeof(raw) == 3 * sizeof(uint32_t));
    auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_RDNA4);
    std::unique_ptr<Instruction> inst(decoder->decode(reinterpret_cast<const uint32_t *>(&raw)));
    ASSERT_NE(inst, nullptr);
    ASSERT_EQ(std::string_view(inst->mnemonic()), "v_rcp_f16");
    cu->execute_instruction(inst.get(), *wf);
    for (uint32_t lane = 0; lane < kWaveSize; ++lane)
      outputs[mode][lane] = cu->read_vgpr(vb + kDst, lane);

    if (!wf->is_halted())
      wf->halt();
  }

  EXPECT_EQ(outputs[0], outputs[1]);
  uint32_t lane_select_bits = 0;
  for (uint32_t lane = 0; lane < kLaneSelectors.size(); ++lane)
    lane_select_bits |= kLaneSelectors[lane] << (lane * 3u);
  for (uint32_t lane = 0; lane < kWaveSize; ++lane) {
    const uint32_t source_lane = amdgpu::dpp::dpp8_src_lane(lane, lane_select_bits);
    const uint16_t expected_low =
        util::f32_to_f16(1.0f / static_cast<float>(uint32_t{1} << (source_lane % 4)));
    EXPECT_EQ(outputs[1][lane], pack16(expected_low, static_cast<uint16_t>(0x6000u + lane)))
        << "lane " << lane;
  }
}

TEST(Rdna4True16Vop3Test, FmacF16UsesSelectedAccumulatorHalfAndPreservesDestinationHalf) {
  for (bool force_scalar : {false, true}) {
    SCOPED_TRACE(force_scalar ? "scalar" : "simd");
    ForceScalarGuard guard(force_scalar);
    amdgpu::GpuMemory gpu_mem(force_scalar ? "rdna4_true16_fmac_scalar_mem"
                                           : "rdna4_true16_fmac_simd_mem");
    amdgpu::L2Cache l2(force_scalar ? "rdna4_true16_fmac_scalar_l2" : "rdna4_true16_fmac_simd_l2");

    amdgpu::ComputeUnitCore::Config cfg{};
    cfg.arch = ROCJITSU_CODE_ARCH_RDNA4;
    cfg.num_wf_slots = 1;
    cfg.sgprs_per_wf = 106;
    cfg.vgprs_per_wf = 256;
    cfg.lds_size_kb = 64;

    auto cu = amdgpu::ComputeUnitCore::create("rdna4", cfg, &gpu_mem, &l2);
    ASSERT_NE(cu, nullptr);

    auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_RDNA4);
    ASSERT_NE(decoder, nullptr);

    auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
    ASSERT_NE(wf, nullptr);
    ASSERT_EQ(wf->wf_size(), 32u);
    wf->set_exec(0x3);

    const uint32_t vb = wf->vgpr_alloc().base;
    cu->write_vgpr(vb + 0, 0, pack16(util::f32_to_f16(100.0f), util::f32_to_f16(2.0f)));
    cu->write_vgpr(vb + 1, 0, pack16(util::f32_to_f16(10.0f), util::f32_to_f16(3.0f)));
    cu->write_vgpr(vb + 0, 1, pack16(util::f32_to_f16(8.0f), util::f32_to_f16(-2.0f)));
    cu->write_vgpr(vb + 1, 1, pack16(util::f32_to_f16(4.0f), util::f32_to_f16(0.5f)));
    cu->write_vgpr(vb + 3, 0, pack16(0xBEEFu, util::f32_to_f16(4.0f)));
    cu->write_vgpr(vb + 3, 1, pack16(0x1234u, util::f32_to_f16(1.5f)));

    constexpr auto words = encode_vop3(/*op=*/0x136, /*vdst=*/3, /*src0=*/256,
                                       /*src1=*/257, /*src2=*/0, /*abs=*/0,
                                       /*opsel=*/0xB);
    std::unique_ptr<Instruction> inst(decoder->decode(words.data()));
    ASSERT_NE(inst, nullptr);
    ASSERT_EQ(std::string_view(inst->mnemonic()), "v_fmac_f16");
    cu->execute_instruction(inst.get(), *wf);

    EXPECT_EQ(cu->read_vgpr(vb + 3, 0), pack16(0xBEEFu, util::f32_to_f16(10.0f)));
    EXPECT_EQ(cu->read_vgpr(vb + 3, 1), pack16(0x1234u, util::f32_to_f16(0.5f)));

    if (!wf->is_halted())
      wf->halt();
  }
}

TEST(Rdna4True16Vop3Test, TernaryI16U16UsesSelectedHalvesAndPreservesDestinationHalf) {
  for (bool force_scalar : {false, true}) {
    SCOPED_TRACE(force_scalar ? "scalar" : "simd");
    ForceScalarGuard guard(force_scalar);
    amdgpu::GpuMemory gpu_mem(force_scalar ? "rdna4_true16_ternary_scalar_mem"
                                           : "rdna4_true16_ternary_simd_mem");
    amdgpu::L2Cache l2(force_scalar ? "rdna4_true16_ternary_scalar_l2"
                                    : "rdna4_true16_ternary_simd_l2");

    amdgpu::ComputeUnitCore::Config cfg{};
    cfg.arch = ROCJITSU_CODE_ARCH_RDNA4;
    cfg.num_wf_slots = 1;
    cfg.sgprs_per_wf = 106;
    cfg.vgprs_per_wf = 256;
    cfg.lds_size_kb = 64;

    auto cu = amdgpu::ComputeUnitCore::create("rdna4", cfg, &gpu_mem, &l2);
    ASSERT_NE(cu, nullptr);

    auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_RDNA4);
    ASSERT_NE(decoder, nullptr);

    auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
    ASSERT_NE(wf, nullptr);
    ASSERT_EQ(wf->wf_size(), 32u);
    wf->set_exec(0x1);

    const uint32_t vb = wf->vgpr_alloc().base;
    cu->write_vgpr(vb + 0, 0, pack16(100u, 0xFFF9u)); // high=-7
    cu->write_vgpr(vb + 1, 0, pack16(50u, 3u));
    cu->write_vgpr(vb + 2, 0, pack16(75u, 0xFFFEu)); // high=-2
    cu->write_vgpr(vb + 4, 0, pack16(1u, 7u));
    cu->write_vgpr(vb + 5, 0, pack16(2u, 3u));
    cu->write_vgpr(vb + 6, 0, pack16(3u, 5u));

    auto execute = [&](uint32_t op, uint32_t dst, uint32_t src0, uint32_t src1, uint32_t src2,
                       std::string_view mnemonic, uint16_t expected_high) {
      cu->write_vgpr(vb + dst, 0, pack16(static_cast<uint16_t>(0xA000u | dst), 0xCAFEu));
      const auto words = encode_vop3(op, dst, src0, src1, src2, /*abs=*/0, /*opsel=*/0xF);
      std::unique_ptr<Instruction> inst(decoder->decode(words.data()));
      ASSERT_NE(inst, nullptr);
      ASSERT_EQ(std::string_view(inst->mnemonic()), mnemonic);
      cu->execute_instruction(inst.get(), *wf);
      EXPECT_EQ(cu->read_vgpr(vb + dst, 0),
                pack16(static_cast<uint16_t>(0xA000u | dst), expected_high))
          << mnemonic;
    };

    execute(/*op=*/0x24A, /*dst=*/10, /*src0=*/256, /*src1=*/257, /*src2=*/258, "v_min3_i16",
            0xFFF9u);
    execute(/*op=*/0x24D, /*dst=*/11, /*src0=*/256, /*src1=*/257, /*src2=*/258, "v_max3_i16", 3u);
    execute(/*op=*/0x250, /*dst=*/12, /*src0=*/256, /*src1=*/257, /*src2=*/258, "v_med3_i16",
            0xFFFEu);
    execute(/*op=*/0x24B, /*dst=*/13, /*src0=*/260, /*src1=*/261, /*src2=*/262, "v_min3_u16", 3u);
    execute(/*op=*/0x24E, /*dst=*/14, /*src0=*/260, /*src1=*/261, /*src2=*/262, "v_max3_u16", 7u);
    execute(/*op=*/0x251, /*dst=*/15, /*src0=*/260, /*src1=*/261, /*src2=*/262, "v_med3_u16", 5u);

    if (!wf->is_halted())
      wf->halt();
  }
}

TEST(Rdna4True16Vop3Test, SpecialVop3OpsUseSelectedHalves) {
  for (bool force_scalar : {false, true}) {
    ForceScalarGuard guard(force_scalar);
    amdgpu::GpuMemory gpu_mem(force_scalar ? "rdna4_true16_special_scalar_mem"
                                           : "rdna4_true16_special_simd_mem");
    amdgpu::L2Cache l2(force_scalar ? "rdna4_true16_special_scalar_l2"
                                    : "rdna4_true16_special_simd_l2");

    amdgpu::ComputeUnitCore::Config cfg{};
    cfg.arch = ROCJITSU_CODE_ARCH_RDNA4;
    cfg.num_wf_slots = 1;
    cfg.sgprs_per_wf = 106;
    cfg.vgprs_per_wf = 256;
    cfg.lds_size_kb = 64;

    auto cu = amdgpu::ComputeUnitCore::create("rdna4", cfg, &gpu_mem, &l2);
    ASSERT_NE(cu, nullptr);

    auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_RDNA4);
    ASSERT_NE(decoder, nullptr);

    auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
    ASSERT_NE(wf, nullptr);
    ASSERT_EQ(wf->wf_size(), 32u);
    wf->set_exec(0x1);

    const uint32_t vb = wf->vgpr_alloc().base;
    auto execute = [&](const std::array<uint32_t, 2> &words, std::string_view mnemonic) {
      std::unique_ptr<Instruction> inst(decoder->decode(words.data()));
      ASSERT_NE(inst, nullptr);
      ASSERT_EQ(std::string_view(inst->mnemonic()), mnemonic);
      cu->execute_instruction(inst.get(), *wf);
    };

    cu->write_vgpr(vb + 0, 0, pack16(2u, 7u));
    cu->write_vgpr(vb + 1, 0, pack16(3u, 11u));
    cu->write_vgpr(vb + 2, 0, 0x12340064u);
    execute(encode_vop3(/*op=*/601, /*vdst=*/3, /*src0=*/256, /*src1=*/257,
                        /*src2=*/258, /*abs=*/0, /*opsel=*/0x3),
            "v_mad_u32_u16");
    EXPECT_EQ(cu->read_vgpr(vb + 3, 0), 0x123400B1u);

    cu->write_vgpr(vb + 4, 0, pack16(0xFFFEu, 0xFFF9u)); // low=-2, high=-7
    cu->write_vgpr(vb + 5, 0, pack16(5u, 0xFFFDu));      // low=5, high=-3
    cu->write_vgpr(vb + 6, 0, 0x12340032u);
    execute(encode_vop3(/*op=*/602, /*vdst=*/7, /*src0=*/260, /*src1=*/261,
                        /*src2=*/262, /*abs=*/0, /*opsel=*/0x3),
            "v_mad_i32_i16");
    EXPECT_EQ(cu->read_vgpr(vb + 7, 0), 0x12340047u);

    cu->write_vgpr(vb + 0, 0, pack16(0x1111u, 0x2222u));
    cu->write_vgpr(vb + 1, 0, pack16(0x3333u, 0x4444u));
    execute(encode_vop3(/*op=*/785, /*vdst=*/8, /*src0=*/256, /*src1=*/257,
                        /*src2=*/0, /*abs=*/0, /*opsel=*/0x3),
            "v_pack_b32_f16");
    EXPECT_EQ(cu->read_vgpr(vb + 8, 0), 0x44442222u);

    cu->write_vgpr(vb + 0, 0, pack16(util::f32_to_f16(0.0f), util::f32_to_f16(0.5f)));
    cu->write_vgpr(vb + 1, 0, pack16(util::f32_to_f16(0.0f), util::f32_to_f16(1.0f)));
    execute(encode_vop3(/*op=*/786, /*vdst=*/9, /*src0=*/256, /*src1=*/257,
                        /*src2=*/0, /*abs=*/0, /*opsel=*/0x3),
            "v_cvt_pk_norm_i16_f16");
    EXPECT_EQ(cu->read_vgpr(vb + 9, 0), 0x7FFF3FFFu);

    execute(encode_vop3(/*op=*/787, /*vdst=*/10, /*src0=*/256, /*src1=*/257,
                        /*src2=*/0, /*abs=*/0, /*opsel=*/0x3),
            "v_cvt_pk_norm_u16_f16");
    EXPECT_EQ(cu->read_vgpr(vb + 10, 0), 0xFFFF7FFFu);

    cu->write_vgpr(vb + 0, 0, pack16(util::f32_to_f16(0.25f), util::f32_to_f16(2.0f)));
    cu->write_vgpr(vb + 1, 0, pack16(util::f32_to_f16(1.0f), util::f32_to_f16(3.0f)));
    cu->write_vgpr(vb + 2, 0, pack16(util::f32_to_f16(1.0f), util::f32_to_f16(4.0f)));
    cu->write_vgpr(vb + 12, 0, pack16(0xBEEFu, 0xCAFEu));
    execute(encode_vop3(/*op=*/596, /*vdst=*/12, /*src0=*/256, /*src1=*/257,
                        /*src2=*/258, /*abs=*/0, /*opsel=*/0xF),
            "v_div_fixup_f16");
    EXPECT_EQ(cu->read_vgpr(vb + 12, 0), pack16(0xBEEFu, util::f32_to_f16(2.0f)));

    if (!wf->is_halted())
      wf->halt();
  }
}

TEST(Rdna4Vop3CndmaskTest, B32AppliesNegModifierBeforeSelect) {
  for (bool force_scalar : {false, true}) {
    ForceScalarGuard guard(force_scalar);
    amdgpu::GpuMemory gpu_mem(force_scalar ? "rdna4_cndmask_b32_scalar_mem"
                                           : "rdna4_cndmask_b32_simd_mem");
    amdgpu::L2Cache l2(force_scalar ? "rdna4_cndmask_b32_scalar_l2" : "rdna4_cndmask_b32_simd_l2");

    amdgpu::ComputeUnitCore::Config cfg{};
    cfg.arch = ROCJITSU_CODE_ARCH_RDNA4;
    cfg.num_wf_slots = 1;
    cfg.sgprs_per_wf = 106;
    cfg.vgprs_per_wf = 256;
    cfg.lds_size_kb = 64;

    auto cu = amdgpu::ComputeUnitCore::create("rdna4", cfg, &gpu_mem, &l2);
    ASSERT_NE(cu, nullptr);

    auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_RDNA4);
    ASSERT_NE(decoder, nullptr);

    auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
    ASSERT_NE(wf, nullptr);
    ASSERT_EQ(wf->wf_size(), 32u);
    wf->set_exec(0x3);

    const uint32_t vb = wf->vgpr_alloc().base;
    const uint32_t sb = wf->sgpr_alloc().base;
    cu->write_sgpr(sb + 0, 0x1u);
    cu->write_sgpr(sb + 1, 0xDEADBEEFu);
    cu->write_vgpr(vb + 0, 0, 0x3F800000u);
    cu->write_vgpr(vb + 0, 1, 0x3F000000u);
    cu->write_vgpr(vb + 1, 0, 0x11223344u);
    cu->write_vgpr(vb + 1, 1, 0x55667788u);
    cu->write_vgpr(vb + 2, 0, 0xDEADCAFEu);
    cu->write_vgpr(vb + 2, 1, 0xDEADCAFEu);

    // v_cndmask_b32 v2, -v0, v1, s[0:1]
    const uint32_t words[] = {0xD5010002U, 0x20020300U};
    std::unique_ptr<Instruction> inst(decoder->decode(words));
    ASSERT_NE(inst, nullptr);
    ASSERT_EQ(std::string_view(inst->mnemonic()), "v_cndmask_b32");
    cu->execute_instruction(inst.get(), *wf);

    EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0x11223344u);
    EXPECT_EQ(cu->read_vgpr(vb + 2, 1), 0xBF000000u);

    if (!wf->is_halted())
      wf->halt();
  }
}

TEST(Rdna4Atan2F16Test, SignedHalfCompareFeedsQuadrantSelect) {
  for (bool force_scalar : {false, true}) {
    ForceScalarGuard guard(force_scalar);
    amdgpu::GpuMemory gpu_mem(force_scalar ? "rdna4_atan2_f16_scalar_mem"
                                           : "rdna4_atan2_f16_simd_mem");
    amdgpu::L2Cache l2(force_scalar ? "rdna4_atan2_f16_scalar_l2" : "rdna4_atan2_f16_simd_l2");

    amdgpu::ComputeUnitCore::Config cfg{};
    cfg.arch = ROCJITSU_CODE_ARCH_RDNA4;
    cfg.num_wf_slots = 1;
    cfg.sgprs_per_wf = 106;
    cfg.vgprs_per_wf = 256;
    cfg.lds_size_kb = 64;

    auto cu = amdgpu::ComputeUnitCore::create("rdna4", cfg, &gpu_mem, &l2);
    ASSERT_NE(cu, nullptr);

    auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_RDNA4);
    ASSERT_NE(decoder, nullptr);

    auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
    ASSERT_NE(wf, nullptr);
    ASSERT_EQ(wf->wf_size(), 32u);
    wf->set_exec(0x3);

    const uint32_t vb = wf->vgpr_alloc().base;
    cu->write_vgpr(vb + 2, 0, 0x0000BC00u); // low=-1.0h, raw sign bit set
    cu->write_vgpr(vb + 2, 1, 0x00003C00u); // low=+1.0h
    cu->write_vgpr(vb + 6, 0, 0xDEADBEEFu);
    cu->write_vgpr(vb + 6, 1, 0xDEADBEEFu);

    // v_cmp_gt_i16_e32 vcc_lo, 0, v2.l
    const uint32_t i16_cmp_words[] = {0x7C680480U, 0};
    std::unique_ptr<Instruction> i16_cmp(decoder->decode(i16_cmp_words));
    ASSERT_NE(i16_cmp, nullptr);
    ASSERT_EQ(std::string_view(i16_cmp->mnemonic()), "v_cmp_gt_i16_e32");
    cu->execute_instruction(i16_cmp.get(), *wf);
    EXPECT_EQ(wf->vcc() & 0x3u, 0x1u);

    // v_cndmask_b32_e64 v6, 0, 0x40490fdb, vcc_lo
    const uint32_t pi_select_words[] = {0xD5010006U, 0x01A9FE80U, 0x40490FDBU};
    std::unique_ptr<Instruction> pi_select(decoder->decode(pi_select_words));
    ASSERT_NE(pi_select, nullptr);
    ASSERT_EQ(std::string_view(pi_select->mnemonic()), "v_cndmask_b32");
    cu->execute_instruction(pi_select.get(), *wf);
    EXPECT_EQ(cu->read_vgpr(vb + 6, 0), 0x40490FDBu);
    EXPECT_EQ(cu->read_vgpr(vb + 6, 1), 0x00000000u);

    // v_cmp_gt_f16_e32 vcc_lo, 0, v2.l
    const uint32_t f16_cmp_words[] = {0x7C080480U, 0};
    std::unique_ptr<Instruction> f16_cmp(decoder->decode(f16_cmp_words));
    ASSERT_NE(f16_cmp, nullptr);
    ASSERT_EQ(std::string_view(f16_cmp->mnemonic()), "v_cmp_gt_f16_e32");
    cu->execute_instruction(f16_cmp.get(), *wf);
    EXPECT_EQ(wf->vcc() & 0x3u, 0x1u);

    if (!wf->is_halted())
      wf->halt();
  }
}

TEST(Rdna4Atan2F16Test, DualCndmaskReadsOldPairedSlotSource) {
  amdgpu::GpuMemory gpu_mem("rdna4_atan2_f16_vopd_mem");
  amdgpu::L2Cache l2("rdna4_atan2_f16_vopd_l2");

  amdgpu::ComputeUnitCore::Config cfg{};
  cfg.arch = ROCJITSU_CODE_ARCH_RDNA4;
  cfg.num_wf_slots = 1;
  cfg.sgprs_per_wf = 106;
  cfg.vgprs_per_wf = 256;
  cfg.lds_size_kb = 64;

  auto cu = amdgpu::ComputeUnitCore::create("rdna4", cfg, &gpu_mem, &l2);
  ASSERT_NE(cu, nullptr);

  auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_RDNA4);
  ASSERT_NE(decoder, nullptr);

  auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
  ASSERT_NE(wf, nullptr);
  ASSERT_EQ(wf->wf_size(), 32u);
  wf->set_exec(0x3);
  wf->set_vcc(0x1);

  const uint32_t vb = wf->vgpr_alloc().base;
  cu->write_vgpr(vb + 4, 0, 0x3F490FDBu); // pi/4 approximation
  cu->write_vgpr(vb + 4, 1, 0x3F490FDBu);
  cu->write_vgpr(vb + 5, 0, 0x4016CBE4u); // 3*pi/4 approximation
  cu->write_vgpr(vb + 5, 1, 0x4016CBE4u);

  // v_dual_cndmask_b32 v4, v4, v5 :: v_dual_mov_b32 v5, 0x4016cbe4
  const uint32_t words[] = {0xCA500B04U, 0x040400FFU, 0x4016CBE4U};
  std::unique_ptr<Instruction> inst(decoder->decode(words));
  ASSERT_NE(inst, nullptr);
  ASSERT_EQ(std::string_view(inst->mnemonic()), "v_dual_cndmask_b32 :: v_dual_mov_b32");
  cu->execute_instruction(inst.get(), *wf);

  EXPECT_EQ(cu->read_vgpr(vb + 4, 0), 0x4016CBE4u);
  EXPECT_EQ(cu->read_vgpr(vb + 4, 1), 0x3F490FDBu);
  EXPECT_EQ(cu->read_vgpr(vb + 5, 0), 0x4016CBE4u);
  EXPECT_EQ(cu->read_vgpr(vb + 5, 1), 0x4016CBE4u);

  if (!wf->is_halted())
    wf->halt();
}

TEST(Rdna4Atan2F16Test, TailSequenceKeepsNegativeXQuadrant) {
  amdgpu::GpuMemory gpu_mem("rdna4_atan2_f16_tail_mem");
  amdgpu::L2Cache l2("rdna4_atan2_f16_tail_l2");

  amdgpu::ComputeUnitCore::Config cfg{};
  cfg.arch = ROCJITSU_CODE_ARCH_RDNA4;
  cfg.num_wf_slots = 1;
  cfg.sgprs_per_wf = 106;
  cfg.vgprs_per_wf = 256;
  cfg.lds_size_kb = 64;

  auto cu = amdgpu::ComputeUnitCore::create("rdna4", cfg, &gpu_mem, &l2);
  ASSERT_NE(cu, nullptr);

  auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_RDNA4);
  ASSERT_NE(decoder, nullptr);

  auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
  ASSERT_NE(wf, nullptr);
  ASSERT_EQ(wf->wf_size(), 32u);
  wf->set_exec(0x1);

  const uint32_t vb = wf->vgpr_alloc().base;
  const uint32_t sb = wf->sgpr_alloc().base;
  cu->write_sgpr(sb + 0, 0); // input_x == 0 mask
  cu->write_sgpr(sb + 1, 0); // NaN/class mask
  cu->write_sgpr(sb + 2, 0);
  cu->write_vgpr(vb + 1, 0, 0x0000C400u); // input_x=-4.0h, sign kept by final bfi
  cu->write_vgpr(vb + 2, 0, 0x0000C400u); // input_y=-4.0h
  cu->write_vgpr(vb + 4, 0, 0x3F490FDBu); // base angle pi/4
  cu->write_vgpr(vb + 5, 0, 0x3F490FDBu);

  const std::array<std::array<uint32_t, 3>, 13> program{{
      {{0x7C680480U, 0, 0}},                     // v_cmp_gt_i16_e32
      {{0x080A08FFU, 0x40490FDBU, 0}},           // v_sub_f32_e32 v5, pi, v4
      {{0xD5010006U, 0x01A9FE80U, 0x40490FDBU}}, // v_cndmask v6, 0, pi
      {{0x7C080480U, 0, 0}},                     // v_cmp_gt_f16_e32
      {{0xCA500B04U, 0x040400FFU, 0x4016CBE4U}}, // v_dual_cndmask/mov
      {{0xD5010003U, 0x00020D04U, 0}},           // v_cndmask v3, v4, v6, s0
      {{0x02080AFFU, 0x3F490FDBU, 0}},           // v_cndmask v4, pi/4, v5
      {{0x8B6A0102U, 0, 0}},                     // s_and_b32 vcc_lo, s2, s1
      {{0x02060903U, 0, 0}},                     // v_cndmask v3, v3, v4
      {{0x7C0E0302U, 0, 0}},                     // v_cmp_o_f16_e32
      {{0x020406FFU, 0x7FC00000U, 0}},           // v_cndmask v2, qnan, v3
      {{0x7E041502U, 0, 0}},                     // v_cvt_f16_f32_e32
      {{0xD6120001U, 0x040604FFU, 0x00007FFFU}}, // v_bfi_b32
  }};

  for (const auto &words : program) {
    std::unique_ptr<Instruction> inst(decoder->decode(words.data()));
    ASSERT_NE(inst, nullptr);
    cu->execute_instruction(inst.get(), *wf);
  }

  EXPECT_EQ(cu->read_vgpr(vb + 1, 0) & 0xFFFFu, 0xC0B6u);

  if (!wf->is_halted())
    wf->halt();
}

TEST(Rdna4Vop3pFmaMixTest, F32AppliesAbsModifiersFromNegHi) {
  for (bool force_scalar : {false, true}) {
    ForceScalarGuard guard(force_scalar);
    amdgpu::GpuMemory gpu_mem(force_scalar ? "rdna4_fma_mix_abs_scalar_mem"
                                           : "rdna4_fma_mix_abs_simd_mem");
    amdgpu::L2Cache l2(force_scalar ? "rdna4_fma_mix_abs_scalar_l2" : "rdna4_fma_mix_abs_simd_l2");

    amdgpu::ComputeUnitCore::Config cfg{};
    cfg.arch = ROCJITSU_CODE_ARCH_RDNA4;
    cfg.num_wf_slots = 1;
    cfg.sgprs_per_wf = 106;
    cfg.vgprs_per_wf = 256;
    cfg.lds_size_kb = 64;

    auto cu = amdgpu::ComputeUnitCore::create("rdna4", cfg, &gpu_mem, &l2);
    ASSERT_NE(cu, nullptr);

    auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_RDNA4);
    ASSERT_NE(decoder, nullptr);

    auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
    ASSERT_NE(wf, nullptr);
    ASSERT_EQ(wf->wf_size(), 32u);
    wf->set_exec(0x1);

    const uint32_t vb = wf->vgpr_alloc().base;
    const uint32_t sb = wf->sgpr_alloc().base;
    cu->write_sgpr(sb + 0, 0x3F800000u); // 1.0f
    cu->write_vgpr(vb + 0, 0, 0xDEADCAFEu);
    cu->write_vgpr(vb + 3, 0, 0xCAFECC00u); // low=-16.0h

    // v_fma_mix_f32 v0, |v3|, s0, neg(0) op_sel_hi:[1,0,0]
    const uint32_t abs_src0_words[] = {0xCC200100U, 0x8A000103U};
    std::unique_ptr<Instruction> abs_src0_inst(decoder->decode(abs_src0_words));
    ASSERT_NE(abs_src0_inst, nullptr);
    ASSERT_EQ(std::string_view(abs_src0_inst->mnemonic()), "v_fma_mix_f32");
    cu->execute_instruction(abs_src0_inst.get(), *wf);
    EXPECT_EQ(cu->read_vgpr(vb + 0, 0), 0x41800000u); // 16.0f, not -16.0f

    cu->write_sgpr(sb + 0, 0x40400000u); // 3.0f
    cu->write_vgpr(vb + 0, 0, 0xDEADCAFEu);
    cu->write_vgpr(vb + 3, 0, 0xCAFECC00u); // low=-16.0h
    cu->write_vgpr(vb + 4, 0, 0x40000000u); // 2.0f

    // v_fma_mix_f32 v0, v4, s0, |v3| op_sel_hi:[0,0,1]
    const uint32_t abs_src2_words[] = {0xCC204400U, 0x040C0104U};
    std::unique_ptr<Instruction> abs_src2_inst(decoder->decode(abs_src2_words));
    ASSERT_NE(abs_src2_inst, nullptr);
    ASSERT_EQ(std::string_view(abs_src2_inst->mnemonic()), "v_fma_mix_f32");
    cu->execute_instruction(abs_src2_inst.get(), *wf);
    EXPECT_EQ(cu->read_vgpr(vb + 0, 0), 0x41B00000u); // 22.0f, not -10.0f

    cu->write_vgpr(vb + 0, 0, 0x40000000u); // old src2/dst = 2.0f
    cu->write_vgpr(vb + 2, 0, 0xCAFE4C00u); // low=16.0h

    // v_fma_mix_f32 v0, v2, 1.0, v0 op_sel_hi:[1,1,0]
    const uint32_t inline_one_f32_words[] = {0xCC200000U, 0x1C01E502U};
    std::unique_ptr<Instruction> inline_one_f32_inst(decoder->decode(inline_one_f32_words));
    ASSERT_NE(inline_one_f32_inst, nullptr);
    ASSERT_EQ(std::string_view(inline_one_f32_inst->mnemonic()), "v_fma_mix_f32");
    cu->execute_instruction(inline_one_f32_inst.get(), *wf);
    EXPECT_EQ(cu->read_vgpr(vb + 0, 0), 0x41900000u); // 16.0f * 1.0f + 2.0f

    cu->write_vgpr(vb + 0, 0, 0xBEEF0000u);
    cu->write_vgpr(vb + 2, 0, 0xCAFE4C00u); // low=16.0h

    // v_fma_mixlo_f16 v0, v2, 1.0, 0 op_sel_hi:[1,1,0]
    const uint32_t inline_one_f16_words[] = {0xCC210000U, 0x1A01E502U};
    std::unique_ptr<Instruction> inline_one_f16_inst(decoder->decode(inline_one_f16_words));
    ASSERT_NE(inline_one_f16_inst, nullptr);
    ASSERT_EQ(std::string_view(inline_one_f16_inst->mnemonic()), "v_fma_mixlo_f16");
    cu->execute_instruction(inline_one_f16_inst.get(), *wf);
    EXPECT_EQ(cu->read_vgpr(vb + 0, 0), 0xBEEF4C00u); // high half preserved

    cu->write_sgpr(sb + 0, 0x3F800000u); // 1.0f
    cu->write_vgpr(vb + 0, 0, 0xDEADCAFEu);
    cu->write_vgpr(vb + 2, 0, 0xCAFEBA00u); // low=-0.75h

    // v_fma_mix_f32 v0, |v2|, -1.0, s0 op_sel_hi:[1,1,0]
    const uint32_t atanh_den_words[] = {0xCC200100U, 0x1801E702U};
    std::unique_ptr<Instruction> atanh_den_inst(decoder->decode(atanh_den_words));
    ASSERT_NE(atanh_den_inst, nullptr);
    ASSERT_EQ(std::string_view(atanh_den_inst->mnemonic()), "v_fma_mix_f32");
    cu->execute_instruction(atanh_den_inst.get(), *wf);
    EXPECT_EQ(cu->read_vgpr(vb + 0, 0), 0x3E800000u); // abs(-0.75h) * -1.0f + 1.0f

    cu->write_vgpr(vb + 3, 0, 0xDEADCAFEu);

    // v_fma_mix_f32 v3, |v2|, 1.0, s0 op_sel_hi:[1,1,0]
    const uint32_t atanh_num_words[] = {0xCC200103U, 0x1801E502U};
    std::unique_ptr<Instruction> atanh_num_inst(decoder->decode(atanh_num_words));
    ASSERT_NE(atanh_num_inst, nullptr);
    ASSERT_EQ(std::string_view(atanh_num_inst->mnemonic()), "v_fma_mix_f32");
    cu->execute_instruction(atanh_num_inst.get(), *wf);
    EXPECT_EQ(cu->read_vgpr(vb + 3, 0), 0x3FE00000u); // abs(-0.75h) * 1.0f + 1.0f

    cu->write_sgpr(sb + 0, 0x3EB17218u); // 0.5 * ln(2)
    cu->write_vgpr(vb + 0, 0, 0xDEADBEEFu);
    cu->write_vgpr(vb + 3, 0, 0xC033ABB4u); // log2(0.25 / 1.75)

    // v_fma_mixhi_f16 v0, v3, s0, 0
    const uint32_t atanh_scale_words[] = {0xCC220000U, 0x02000103U};
    std::unique_ptr<Instruction> atanh_scale_inst(decoder->decode(atanh_scale_words));
    ASSERT_NE(atanh_scale_inst, nullptr);
    ASSERT_EQ(std::string_view(atanh_scale_inst->mnemonic()), "v_fma_mixhi_f16");
    cu->execute_instruction(atanh_scale_inst.get(), *wf);
    EXPECT_EQ(cu->read_vgpr(vb + 0, 0), 0xBBC9BEEFu);

    if (!wf->is_halted())
      wf->halt();
  }
}

TEST(Rdna4True16Vop3Test, B16BitwiseLiteralPreservesInputSignBit) {
  for (bool force_scalar : {false, true}) {
    ForceScalarGuard guard(force_scalar);
    amdgpu::GpuMemory gpu_mem(force_scalar ? "rdna4_b16_bitwise_scalar_mem"
                                           : "rdna4_b16_bitwise_simd_mem");
    amdgpu::L2Cache l2(force_scalar ? "rdna4_b16_bitwise_scalar_l2" : "rdna4_b16_bitwise_simd_l2");

    amdgpu::ComputeUnitCore::Config cfg{};
    cfg.arch = ROCJITSU_CODE_ARCH_RDNA4;
    cfg.num_wf_slots = 1;
    cfg.sgprs_per_wf = 106;
    cfg.vgprs_per_wf = 256;
    cfg.lds_size_kb = 64;

    auto cu = amdgpu::ComputeUnitCore::create("rdna4", cfg, &gpu_mem, &l2);
    ASSERT_NE(cu, nullptr);

    auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_RDNA4);
    ASSERT_NE(decoder, nullptr);

    auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
    ASSERT_NE(wf, nullptr);
    ASSERT_EQ(wf->wf_size(), 32u);
    wf->set_exec(0x1);

    const uint32_t vb = wf->vgpr_alloc().base;
    cu->write_vgpr(vb + 0, 0, 0xCAFE34D0u); // positive result half
    cu->write_vgpr(vb + 1, 0, 0x11110000u);
    cu->write_vgpr(vb + 3, 0, 0xCAFECC00u); // low=-16.0h

    // v_and_b16 v1.l, 0x8000, v3.l
    const uint32_t and_words[] = {0xD7620001U, 0x020206FFU, 0xFFFF8000U};
    std::unique_ptr<Instruction> and_inst(decoder->decode(and_words));
    ASSERT_NE(and_inst, nullptr);
    ASSERT_EQ(std::string_view(and_inst->mnemonic()), "v_and_b16");
    ASSERT_NE(and_inst->src_operand(0), nullptr);
    EXPECT_EQ(static_cast<uint32_t>(and_inst->src_operand(0)->encoding_value()), 0xFFFF8000u);
    EXPECT_EQ(and_inst->src_operand(0)->name(), "0x8000");
    EXPECT_NE(and_inst->disassemble().find("0x8000"), std::string::npos);
    EXPECT_EQ(and_inst->disassemble().find("0x-"), std::string::npos);
    cu->execute_instruction(and_inst.get(), *wf);
    EXPECT_EQ(cu->read_vgpr(vb + 1, 0), 0x11118000u);

    cu->write_vgpr(vb + 1, 0, 0x22220000u);

    // v_and_b16 v1.l, 0x8000, v3.l op_sel:[1,0,0]
    const uint32_t and_high_words[] = {0xD7620801U, 0x020206FFU, 0x80000000U};
    std::unique_ptr<Instruction> and_high_inst(decoder->decode(and_high_words));
    ASSERT_NE(and_high_inst, nullptr);
    ASSERT_EQ(std::string_view(and_high_inst->mnemonic()), "v_and_b16");
    ASSERT_NE(and_high_inst->src_operand(0), nullptr);
    EXPECT_EQ(static_cast<uint32_t>(and_high_inst->src_operand(0)->encoding_value()), 0x80000000u);
    EXPECT_EQ(and_high_inst->src_operand(0)->name(), "0x8000");
    EXPECT_NE(and_high_inst->disassemble().find("0x8000"), std::string::npos);
    EXPECT_EQ(and_high_inst->disassemble().find("0x-"), std::string::npos);
    cu->execute_instruction(and_high_inst.get(), *wf);
    EXPECT_EQ(cu->read_vgpr(vb + 1, 0), 0x22228000u);

    cu->write_vgpr(vb + 1, 0, 0x33330000u);

    // v_and_b16 v1.l, v3.l, 0x8000 op_sel:[0,1,0]
    const uint32_t and_src1_high_words[] = {0xD7621001U, 0x0201FF03U, 0x80000000U};
    std::unique_ptr<Instruction> and_src1_high_inst(decoder->decode(and_src1_high_words));
    ASSERT_NE(and_src1_high_inst, nullptr);
    ASSERT_EQ(std::string_view(and_src1_high_inst->mnemonic()), "v_and_b16");
    ASSERT_NE(and_src1_high_inst->src_operand(1), nullptr);
    EXPECT_EQ(static_cast<uint32_t>(and_src1_high_inst->src_operand(1)->encoding_value()),
              0x80000000u);
    EXPECT_EQ(and_src1_high_inst->src_operand(1)->name(), "0x8000");
    EXPECT_NE(and_src1_high_inst->disassemble().find("0x8000"), std::string::npos);
    EXPECT_EQ(and_src1_high_inst->disassemble().find("0x-"), std::string::npos);
    cu->execute_instruction(and_src1_high_inst.get(), *wf);
    EXPECT_EQ(cu->read_vgpr(vb + 1, 0), 0x33338000u);

    // v_xor_b16 v0.l, v1.l, v0.l
    const uint32_t xor_words[] = {0xD7640000U, 0x02020101U};
    std::unique_ptr<Instruction> xor_inst(decoder->decode(xor_words));
    ASSERT_NE(xor_inst, nullptr);
    ASSERT_EQ(std::string_view(xor_inst->mnemonic()), "v_xor_b16");
    cu->execute_instruction(xor_inst.get(), *wf);
    EXPECT_EQ(cu->read_vgpr(vb + 0, 0), 0xCAFEB4D0u);

    if (!wf->is_halted())
      wf->halt();
  }
}

TEST(Gfx1250True16Vop3Test, CndmaskB16UsesSelectedSourceHalfAndPreservesDestinationHalf) {
  amdgpu::GpuMemory gpu_mem("gfx1250_true16_cndmask_mem");
  amdgpu::L2Cache l2("gfx1250_true16_cndmask_l2");

  amdgpu::ComputeUnitCore::Config cfg{};
  cfg.arch = ROCJITSU_CODE_ARCH_GFX1250;
  cfg.num_wf_slots = 1;
  cfg.sgprs_per_wf = 106;
  cfg.vgprs_per_wf = 256;
  cfg.lds_size_kb = 64;

  auto cu = amdgpu::ComputeUnitCore::create("gfx1250", cfg, &gpu_mem, &l2);
  ASSERT_NE(cu, nullptr);

  auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
  ASSERT_NE(decoder, nullptr);

  auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
  ASSERT_NE(wf, nullptr);
  ASSERT_EQ(wf->wf_size(), 32u);
  wf->set_exec(0x3);

  const uint32_t vb = wf->vgpr_alloc().base;
  const uint32_t sb = wf->sgpr_alloc().base;
  cu->write_sgpr(sb + 0, 0x1u);
  cu->write_sgpr(sb + 1, 0xDEADBEEFu);
  for (uint32_t lane = 0; lane < wf->wf_size(); ++lane) {
    cu->write_vgpr(vb + 0, lane, 0xAAAA1234u);
    cu->write_vgpr(vb + 1, lane, 0x5678BBBBu);
    cu->write_vgpr(vb + 2, lane, 0xDEADCAFEu);
  }

  // v_cndmask_b16 v2.h, v0.l, v1.h, s[0:1] op_sel:[0,1,0,1]
  const uint32_t words[] = {0xD65D5002U, 0x00020300U};
  std::unique_ptr<Instruction> inst(decoder->decode(words));
  ASSERT_NE(inst, nullptr);
  ASSERT_EQ(std::string_view(inst->mnemonic()), "v_cndmask_b16");
  cu->execute_instruction(inst.get(), *wf);

  EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0x5678CAFEu);
  EXPECT_EQ(cu->read_vgpr(vb + 2, 1), 0x1234CAFEu);
  EXPECT_EQ(cu->read_vgpr(vb + 2, 2), 0xDEADCAFEu);

  if (!wf->is_halted())
    wf->halt();
}

TEST(Gfx1250True16Vop3Test, LshrrevB16UsesSelectedSourceHalfAndPreservesDestinationHalf) {
  amdgpu::GpuMemory gpu_mem("gfx1250_true16_lshrrev_mem");
  amdgpu::L2Cache l2("gfx1250_true16_lshrrev_l2");

  amdgpu::ComputeUnitCore::Config cfg{};
  cfg.arch = ROCJITSU_CODE_ARCH_GFX1250;
  cfg.num_wf_slots = 1;
  cfg.sgprs_per_wf = 106;
  cfg.vgprs_per_wf = 256;
  cfg.lds_size_kb = 64;

  auto cu = amdgpu::ComputeUnitCore::create("gfx1250", cfg, &gpu_mem, &l2);
  ASSERT_NE(cu, nullptr);

  auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
  ASSERT_NE(decoder, nullptr);

  auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
  ASSERT_NE(wf, nullptr);
  wf->set_exec(1);

  const uint32_t vb = wf->vgpr_alloc().base;
  cu->write_vgpr(vb + 16, 0, 0xAAAA5555u);
  cu->write_vgpr(vb + 17, 0, 0xBEEF1234u);

  // v_lshrrev_b16 v16.h, 8, v17.l op_sel:[0,0,1]
  const uint32_t words[] = {0xD7394010U, 0x02022288U};
  std::unique_ptr<Instruction> inst(decoder->decode(words));
  ASSERT_NE(inst, nullptr);
  ASSERT_EQ(std::string_view(inst->mnemonic()), "v_lshrrev_b16");
  cu->execute_instruction(inst.get(), *wf);

  EXPECT_EQ(cu->read_vgpr(vb + 16, 0), 0x00125555u);

  if (!wf->is_halted())
    wf->halt();
}

TEST(Gfx1250True16Vop3Test, Bitop3B16UsesSelectedSourceHalfAndPreservesDestinationHalf) {
  amdgpu::GpuMemory gpu_mem("gfx1250_true16_bitop_mem");
  amdgpu::L2Cache l2("gfx1250_true16_bitop_l2");

  amdgpu::ComputeUnitCore::Config cfg{};
  cfg.arch = ROCJITSU_CODE_ARCH_GFX1250;
  cfg.num_wf_slots = 1;
  cfg.sgprs_per_wf = 106;
  cfg.vgprs_per_wf = 256;
  cfg.lds_size_kb = 64;

  auto cu = amdgpu::ComputeUnitCore::create("gfx1250", cfg, &gpu_mem, &l2);
  ASSERT_NE(cu, nullptr);

  auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
  ASSERT_NE(decoder, nullptr);

  auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
  ASSERT_NE(wf, nullptr);
  ASSERT_EQ(wf->wf_size(), 32u);
  wf->set_exec((1ULL << wf->wf_size()) - 1ULL);

  const uint32_t v2 = wf->vgpr_alloc().base + 2;
  const uint32_t v3 = wf->vgpr_alloc().base + 3;
  const uint32_t v8 = wf->vgpr_alloc().base + 8;
  for (uint32_t lane = 0; lane < wf->wf_size(); ++lane) {
    cu->write_vgpr(v2, lane, 0xA5A50020u);
    cu->write_vgpr(v3, lane, 0x00F00000u);
    cu->write_vgpr(v8, lane, 0x55550014u);
  }

  auto execute = [&](const uint32_t(&words)[2], std::string_view mnemonic) {
    std::unique_ptr<Instruction> inst(decoder->decode(words));
    ASSERT_NE(inst, nullptr);
    ASSERT_EQ(std::string_view(inst->mnemonic()), mnemonic);
    cu->execute_instruction(inst.get(), *wf);
  };

  // v_or_b16 v2.l, v8.l, 1
  const uint32_t or_words[] = {0xD7630002U, 0x02010308U};
  execute(or_words, "v_or_b16");
  for (uint32_t lane = 0; lane < wf->wf_size(); ++lane)
    EXPECT_EQ(cu->read_vgpr(v2, lane), 0xA5A50015u);

  auto bitop3 = [](uint16_t a, uint16_t b, uint16_t c, uint8_t table) {
    uint16_t result = 0;
    for (int i = 0; i < 16; ++i) {
      uint32_t idx = (((a >> i) & 1u) << 2) | (((b >> i) & 1u) << 1) | ((c >> i) & 1u);
      result |= static_cast<uint16_t>(((table >> idx) & 1u) << i);
    }
    return result;
  };

  // v_bitop3_b16 v2.l, v2.l, v3.h, -2 bitop3:0xec op_sel:[0,1,0,0]
  const uint32_t bitop_words[] = {0xD6331502U, 0x9B0A0702U};
  execute(bitop_words, "v_bitop3_b16");
  uint32_t expected = bitop3(0x0015u, 0x00F0u, 0xFFFEu, 0xECu);
  for (uint32_t lane = 0; lane < wf->wf_size(); ++lane)
    EXPECT_EQ(cu->read_vgpr(v2, lane), 0xA5A50000u | expected);

  if (!wf->is_halted())
    wf->halt();
}

TEST(Gfx1250True16Vop1Test, MovB16HighDestinationPreservesLowHalf) {
  amdgpu::GpuMemory gpu_mem("gfx1250_true16_vop1_mem");
  amdgpu::L2Cache l2("gfx1250_true16_vop1_l2");

  amdgpu::ComputeUnitCore::Config cfg{};
  cfg.arch = ROCJITSU_CODE_ARCH_GFX1250;
  cfg.num_wf_slots = 1;
  cfg.sgprs_per_wf = 106;
  cfg.vgprs_per_wf = 256;
  cfg.lds_size_kb = 64;

  auto cu = amdgpu::ComputeUnitCore::create("gfx1250", cfg, &gpu_mem, &l2);
  ASSERT_NE(cu, nullptr);

  auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
  ASSERT_NE(decoder, nullptr);

  auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
  ASSERT_NE(wf, nullptr);
  wf->set_exec(1);

  const uint32_t vb = wf->vgpr_alloc().base;
  cu->write_vgpr(vb + 0, 0, 0x00001234u);
  cu->write_vgpr(vb + 2, 0, 0xAAAA5555u);

  // v_mov_b16_e32 v2.h, v0.l
  const uint32_t words[] = {0x7F043900U, 0x00000000U};
  std::unique_ptr<Instruction> inst(decoder->decode(words));
  ASSERT_NE(inst, nullptr);
  ASSERT_EQ(std::string_view(inst->mnemonic()), "v_mov_b16_e32");
  EXPECT_NE(inst->disassemble().find("v2.h"), std::string::npos);
  cu->execute_instruction(inst.get(), *wf);

  EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0x12345555u);

  if (!wf->is_halted())
    wf->halt();
}

TEST(Rdna3True16Vop1Test, MovB16HighDestinationPreservesLowHalf) {
  amdgpu::GpuMemory gpu_mem("rdna3_true16_vop1_mem");
  amdgpu::L2Cache l2("rdna3_true16_vop1_l2");

  amdgpu::ComputeUnitCore::Config cfg{};
  cfg.arch = ROCJITSU_CODE_ARCH_RDNA3;
  cfg.num_wf_slots = 1;
  cfg.sgprs_per_wf = 106;
  cfg.vgprs_per_wf = 256;
  cfg.lds_size_kb = 64;

  auto cu = amdgpu::ComputeUnitCore::create("rdna3", cfg, &gpu_mem, &l2);
  ASSERT_NE(cu, nullptr);

  auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_RDNA3);
  ASSERT_NE(decoder, nullptr);

  auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
  ASSERT_NE(wf, nullptr);
  wf->set_exec(1);

  const uint32_t vb = wf->vgpr_alloc().base;
  cu->write_vgpr(vb + 0, 0, 0x00001234u);
  cu->write_vgpr(vb + 2, 0, 0xAAAA5555u);

  // v_mov_b16_e32 v2.h, v0.l
  const uint32_t words[] = {0x7F043900U, 0x00000000U};
  std::unique_ptr<Instruction> inst(decoder->decode(words));
  ASSERT_NE(inst, nullptr);
  ASSERT_EQ(std::string_view(inst->mnemonic()), "v_mov_b16_e32");
  EXPECT_NE(inst->disassemble().find("v2.h"), std::string::npos);
  cu->execute_instruction(inst.get(), *wf);

  EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0x12345555u);

  if (!wf->is_halted())
    wf->halt();
}

TEST(Rdna3DppTest, VAddF32RowShrMatchesWaveReduceSequence) {
  const float input[] = {-5.0f, 5.0f,  7.0f,  -2.0f, -3.0f, -6.0f, 7.0f,  -2.0f, -4.0f, 6.0f, 3.0f,
                         -2.0f, -7.0f, -1.0f, 7.0f,  -7.0f, 2.0f,  -3.0f, 0.0f,  -6.0f, 7.0f, -3.0f,
                         1.0f,  -3.0f, -1.0f, 4.0f,  -6.0f, -4.0f, 3.0f,  4.0f,  5.0f,  2.0f};

  const uint32_t dpp_adds[][2] = {
      {0x060606FAU, 0xFF011103U}, // v_add_f32_dpp v3, v3, v3 row_shr:1
      {0x060606FAU, 0xFF011203U}, // v_add_f32_dpp v3, v3, v3 row_shr:2
      {0x060606FAU, 0xFF011403U}, // v_add_f32_dpp v3, v3, v3 row_shr:4
      {0x060606FAU, 0xFF011803U}, // v_add_f32_dpp v3, v3, v3 row_shr:8
  };

  for (bool force_scalar : {false, true}) {
    ForceScalarGuard guard(force_scalar);
    amdgpu::GpuMemory gpu_mem(force_scalar ? "rdna3_dpp_scalar_mem" : "rdna3_dpp_simd_mem");
    amdgpu::L2Cache l2(force_scalar ? "rdna3_dpp_scalar_l2" : "rdna3_dpp_simd_l2");

    amdgpu::ComputeUnitCore::Config cfg{};
    cfg.arch = ROCJITSU_CODE_ARCH_RDNA3;
    cfg.num_wf_slots = 1;
    cfg.sgprs_per_wf = 106;
    cfg.vgprs_per_wf = 256;
    cfg.lds_size_kb = 64;

    auto cu = amdgpu::ComputeUnitCore::create("rdna3", cfg, &gpu_mem, &l2);
    ASSERT_NE(cu, nullptr);

    auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_RDNA3);
    ASSERT_NE(decoder, nullptr);

    auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
    ASSERT_NE(wf, nullptr);
    ASSERT_EQ(wf->wf_size(), 32u);
    wf->set_exec((1ULL << wf->wf_size()) - 1ULL);

    const uint32_t vb = wf->vgpr_alloc().base;
    for (uint32_t lane = 0; lane < wf->wf_size(); ++lane)
      cu->write_vgpr(vb + 3, lane, std::bit_cast<uint32_t>(input[lane]));

    for (const auto &words : dpp_adds) {
      std::unique_ptr<Instruction> inst(decoder->decode(words));
      ASSERT_NE(inst, nullptr);
      ASSERT_EQ(std::string_view(inst->mnemonic()), "v_add_f32_e32");
      cu->execute_instruction(inst.get(), *wf);
    }

    EXPECT_EQ(cu->read_vgpr(vb + 3, 15), std::bit_cast<uint32_t>(-4.0f));
    EXPECT_EQ(cu->read_vgpr(vb + 3, 31), std::bit_cast<uint32_t>(2.0f));

    const uint32_t swizzle_words[] = {0xD8D401E0U, 0x04000003U};
    std::unique_ptr<Instruction> swizzle_inst(decoder->decode(swizzle_words));
    ASSERT_NE(swizzle_inst, nullptr);
    ASSERT_EQ(std::string_view(swizzle_inst->mnemonic()), "ds_swizzle_b32");
    cu->execute_instruction(swizzle_inst.get(), *wf);
    EXPECT_EQ(cu->read_vgpr(vb + 4, 31), std::bit_cast<uint32_t>(-4.0f));

    const uint32_t final_add_words[] = {0xD5030003U, 0x02020903U};
    std::unique_ptr<Instruction> final_add_inst(decoder->decode(final_add_words));
    ASSERT_NE(final_add_inst, nullptr);
    ASSERT_EQ(std::string_view(final_add_inst->mnemonic()), "v_add_f32");
    cu->execute_instruction(final_add_inst.get(), *wf);
    EXPECT_EQ(cu->read_vgpr(vb + 3, 31), std::bit_cast<uint32_t>(-2.0f));

    if (!wf->is_halted())
      wf->halt();
  }
}

TEST(Rdna3ScalarOperandTest, NullSdstDoesNotClobberM0) {
  amdgpu::GpuMemory gpu_mem("rdna3_null_sdst_mem");
  amdgpu::L2Cache l2("rdna3_null_sdst_l2");

  amdgpu::ComputeUnitCore::Config cfg{};
  cfg.arch = ROCJITSU_CODE_ARCH_RDNA3;
  cfg.num_wf_slots = 1;
  cfg.sgprs_per_wf = 106;
  cfg.vgprs_per_wf = 256;
  cfg.lds_size_kb = 64;

  auto cu = amdgpu::ComputeUnitCore::create("rdna3", cfg, &gpu_mem, &l2);
  ASSERT_NE(cu, nullptr);

  auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_RDNA3);
  ASSERT_NE(decoder, nullptr);

  auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
  ASSERT_NE(wf, nullptr);
  ASSERT_EQ(wf->wf_size(), 32u);
  wf->set_exec(0x3);
  wf->set_m0(0xDEADBEEFu);

  const uint32_t vb = wf->vgpr_alloc().base;
  const uint32_t sb = wf->sgpr_alloc().base;
  cu->write_sgpr(sb + 9, 0x1u);
  cu->write_sgpr(sb + 10, 0xA5A5A5A5u);
  for (uint32_t lane = 0; lane < wf->wf_size(); ++lane)
    cu->write_vgpr(vb + 5, lane, 0xCAFECAFEu);

  // v_add_co_ci_u32_e64 v5, null, 0, 0, s9
  const uint32_t words[] = {0xD5207C05U, 0x00250080U};
  std::unique_ptr<Instruction> inst(decoder->decode(words));
  ASSERT_NE(inst, nullptr);
  ASSERT_EQ(std::string_view(inst->mnemonic()), "v_add_co_ci_u32");
  EXPECT_NE(inst->disassemble().find("null"), std::string::npos);
  cu->execute_instruction(inst.get(), *wf);

  EXPECT_EQ(wf->m0(), 0xDEADBEEFu);
  EXPECT_EQ(cu->read_vgpr(vb + 5, 0), 1u);
  EXPECT_EQ(cu->read_vgpr(vb + 5, 1), 0u);
  EXPECT_EQ(cu->read_vgpr(vb + 5, 2), 0xCAFECAFEu);

  if (!wf->is_halted())
    wf->halt();
}

TEST(Gfx1250True16Vop1Test, CvtF16F32HighDestinationPreservesLowHalf) {
  amdgpu::GpuMemory gpu_mem("gfx1250_true16_cvt_vop1_mem");
  amdgpu::L2Cache l2("gfx1250_true16_cvt_vop1_l2");

  amdgpu::ComputeUnitCore::Config cfg{};
  cfg.arch = ROCJITSU_CODE_ARCH_GFX1250;
  cfg.num_wf_slots = 1;
  cfg.sgprs_per_wf = 106;
  cfg.vgprs_per_wf = 256;
  cfg.lds_size_kb = 64;

  auto cu = amdgpu::ComputeUnitCore::create("gfx1250", cfg, &gpu_mem, &l2);
  ASSERT_NE(cu, nullptr);

  auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
  ASSERT_NE(decoder, nullptr);

  auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
  ASSERT_NE(wf, nullptr);
  wf->set_exec(1);

  const uint32_t vb = wf->vgpr_alloc().base;
  cu->write_vgpr(vb + 0, 0, 0xAAAA5555u);
  cu->write_vgpr(vb + 2, 0, std::bit_cast<uint32_t>(1.0f));

  // v_cvt_f16_f32_e32 v0.h, v2
  const uint32_t words[] = {0x7F001502U, 0x00000000U};
  std::unique_ptr<Instruction> inst(decoder->decode(words));
  ASSERT_NE(inst, nullptr);
  ASSERT_EQ(std::string_view(inst->mnemonic()), "v_cvt_f16_f32_e32");
  EXPECT_NE(inst->disassemble().find("v0.h"), std::string::npos);
  cu->execute_instruction(inst.get(), *wf);

  EXPECT_EQ(cu->read_vgpr(vb + 0, 0), 0x3C005555u);

  if (!wf->is_halted())
    wf->halt();
}

TEST(Gfx1250CvtFp8Test, F16DecodeVop3UsesSelectedByte) {
  amdgpu::GpuMemory gpu_mem("gfx1250_cvt_fp8_bytesel_mem");
  amdgpu::L2Cache l2("gfx1250_cvt_fp8_bytesel_l2");

  amdgpu::ComputeUnitCore::Config cfg{};
  cfg.arch = ROCJITSU_CODE_ARCH_GFX1250;
  cfg.num_wf_slots = 1;
  cfg.sgprs_per_wf = 106;
  cfg.vgprs_per_wf = 256;
  cfg.lds_size_kb = 64;

  auto cu = amdgpu::ComputeUnitCore::create("gfx1250", cfg, &gpu_mem, &l2);
  ASSERT_NE(cu, nullptr);

  auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
  ASSERT_NE(decoder, nullptr);

  auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
  ASSERT_NE(wf, nullptr);
  wf->set_exec(1);

  const uint32_t vb = wf->vgpr_alloc().base;
  cu->write_vgpr(vb + 0, 0, 0x00380000u);
  cu->write_vgpr(vb + 1, 0, 0xBEEF5555u);

  // v_cvt_f16_fp8 v1.l, v0 byte_sel:2
  const uint32_t fp8_words[] = {0xD5F70801U, 0x00000100U};
  std::unique_ptr<Instruction> fp8_inst(decoder->decode(fp8_words));
  ASSERT_NE(fp8_inst, nullptr);
  ASSERT_EQ(std::string_view(fp8_inst->mnemonic()), "v_cvt_f16_fp8");
  cu->execute_instruction(fp8_inst.get(), *wf);
  EXPECT_EQ(cu->read_vgpr(vb + 1, 0), 0xBEEF3C00u);

  cu->write_vgpr(vb + 0, 0, 0x3C000000u);
  cu->write_vgpr(vb + 1, 0, 0xBEEF5555u);

  // v_cvt_f16_bf8 v1.l, v0 byte_sel:3
  const uint32_t bf8_words[] = {0xD5F81801U, 0x00000100U};
  std::unique_ptr<Instruction> bf8_inst(decoder->decode(bf8_words));
  ASSERT_NE(bf8_inst, nullptr);
  ASSERT_EQ(std::string_view(bf8_inst->mnemonic()), "v_cvt_f16_bf8");
  cu->execute_instruction(bf8_inst.get(), *wf);
  EXPECT_EQ(cu->read_vgpr(vb + 1, 0), 0xBEEF3C00u);

  if (!wf->is_halted())
    wf->halt();
}

TEST(Gfx1250CvtFp8Test, F16E5M3OverflowHonorsFp16OvflMode) {
  amdgpu::GpuMemory gpu_mem("gfx1250_cvt_fp8_f16_e5m3_overflow_mem");
  amdgpu::L2Cache l2("gfx1250_cvt_fp8_f16_e5m3_overflow_l2");

  amdgpu::ComputeUnitCore::Config cfg{};
  cfg.arch = ROCJITSU_CODE_ARCH_GFX1250;
  cfg.num_wf_slots = 1;
  cfg.sgprs_per_wf = 106;
  cfg.vgprs_per_wf = 256;
  cfg.lds_size_kb = 64;

  auto cu = amdgpu::ComputeUnitCore::create("gfx1250", cfg, &gpu_mem, &l2);
  ASSERT_NE(cu, nullptr);

  auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
  ASSERT_NE(decoder, nullptr);

  auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
  ASSERT_NE(wf, nullptr);
  wf->set_exec(1);

  const uint32_t vb = wf->vgpr_alloc().base;
  cu->write_vgpr(vb + 0, 0, 0x00FE0000u);

  // v_cvt_f16_fp8 v1.l, v0 byte_sel:2 clamp. On gfx1250, clamp selects the
  // unsigned E5M3 FP8 decode; 0xFE is finite but overflows FP16.
  const uint32_t fp8_words[] = {0xD5F78801U, 0x00000100U};
  std::unique_ptr<Instruction> fp8_inst(decoder->decode(fp8_words));
  ASSERT_NE(fp8_inst, nullptr);
  ASSERT_EQ(std::string_view(fp8_inst->mnemonic()), "v_cvt_f16_fp8");

  wf->set_mode_raw(0);
  cu->write_vgpr(vb + 1, 0, 0xBEEF5555u);
  cu->execute_instruction(fp8_inst.get(), *wf);
  EXPECT_EQ(cu->read_vgpr(vb + 1, 0), 0xBEEF7C00u);

  wf->set_mode_raw(amdgpu::Wavefront::FP16_OVFL_BIT);
  cu->write_vgpr(vb + 1, 0, 0xBEEF5555u);
  cu->execute_instruction(fp8_inst.get(), *wf);
  EXPECT_EQ(cu->read_vgpr(vb + 1, 0), 0xBEEF7BFFu);

  // Same conversion with OPSEL[3]=1 writes the high destination half.
  const uint32_t fp8_high_words[] = {0xD5F7C801U, 0x00000100U};
  std::unique_ptr<Instruction> fp8_high_inst(decoder->decode(fp8_high_words));
  ASSERT_NE(fp8_high_inst, nullptr);
  ASSERT_EQ(std::string_view(fp8_high_inst->mnemonic()), "v_cvt_f16_fp8");
  cu->write_vgpr(vb + 1, 0, 0x5555BEEFu);
  cu->execute_instruction(fp8_high_inst.get(), *wf);
  EXPECT_EQ(cu->read_vgpr(vb + 1, 0), 0x7BFFBEEFu);

  if (!wf->is_halted())
    wf->halt();
}

TEST(Gfx1250CvtFp8Test, F32DecodeVop3UsesSelectedByte) {
  amdgpu::GpuMemory gpu_mem("gfx1250_cvt_fp8_f32_bytesel_mem");
  amdgpu::L2Cache l2("gfx1250_cvt_fp8_f32_bytesel_l2");

  amdgpu::ComputeUnitCore::Config cfg{};
  cfg.arch = ROCJITSU_CODE_ARCH_GFX1250;
  cfg.num_wf_slots = 1;
  cfg.sgprs_per_wf = 106;
  cfg.vgprs_per_wf = 256;
  cfg.lds_size_kb = 64;

  auto cu = amdgpu::ComputeUnitCore::create("gfx1250", cfg, &gpu_mem, &l2);
  ASSERT_NE(cu, nullptr);

  auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
  ASSERT_NE(decoder, nullptr);

  auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
  ASSERT_NE(wf, nullptr);
  wf->set_exec(1);

  const uint32_t vb = wf->vgpr_alloc().base;
  cu->write_vgpr(vb + 0, 0, 0x00380000u);

  // v_cvt_f32_fp8 v1, v0 byte_sel:2
  const uint32_t fp8_words[] = {0xD5EC0801U, 0x00000100U};
  std::unique_ptr<Instruction> fp8_inst(decoder->decode(fp8_words));
  ASSERT_NE(fp8_inst, nullptr);
  ASSERT_EQ(std::string_view(fp8_inst->mnemonic()), "v_cvt_f32_fp8");
  cu->execute_instruction(fp8_inst.get(), *wf);
  EXPECT_EQ(cu->read_vgpr(vb + 1, 0), std::bit_cast<uint32_t>(1.0f));

  cu->write_vgpr(vb + 0, 0, 0x3C000000u);

  // v_cvt_f32_bf8 v1, v0 byte_sel:3
  const uint32_t bf8_words[] = {0xD5ED1801U, 0x00000100U};
  std::unique_ptr<Instruction> bf8_inst(decoder->decode(bf8_words));
  ASSERT_NE(bf8_inst, nullptr);
  ASSERT_EQ(std::string_view(bf8_inst->mnemonic()), "v_cvt_f32_bf8");
  cu->execute_instruction(bf8_inst.get(), *wf);
  EXPECT_EQ(cu->read_vgpr(vb + 1, 0), std::bit_cast<uint32_t>(1.0f));

  if (!wf->is_halted())
    wf->halt();
}

TEST(Gfx1250CvtFp8Test, E4M3OverflowHonorsFp16OvflMode) {
  amdgpu::GpuMemory gpu_mem("gfx1250_cvt_fp8_overflow_mem");
  amdgpu::L2Cache l2("gfx1250_cvt_fp8_overflow_l2");

  amdgpu::ComputeUnitCore::Config cfg{};
  cfg.arch = ROCJITSU_CODE_ARCH_GFX1250;
  cfg.num_wf_slots = 1;
  cfg.sgprs_per_wf = 106;
  cfg.vgprs_per_wf = 256;
  cfg.lds_size_kb = 64;

  auto cu = amdgpu::ComputeUnitCore::create("gfx1250", cfg, &gpu_mem, &l2);
  ASSERT_NE(cu, nullptr);

  auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
  ASSERT_NE(decoder, nullptr);

  auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
  ASSERT_NE(wf, nullptr);
  wf->set_exec(1);

  const uint32_t vb = wf->vgpr_alloc().base;
  cu->write_vgpr(vb + 5, 0, std::bit_cast<uint32_t>(768.0f));
  cu->write_vgpr(vb + 6, 0, std::bit_cast<uint32_t>(-768.0f));
  cu->write_vgpr(vb + 2, 0, 0xA5A5BEEFu);

  // v_cvt_pk_fp8_f32 v2.l, v5, v6
  const auto words = encode_vop3(gfx1250::kVCvtPkFp8F32Vop3, 2, vgpr_src(5), vgpr_src(6), 0);
  std::unique_ptr<Instruction> inst(decoder->decode(words.data()));
  ASSERT_NE(inst, nullptr);
  ASSERT_EQ(std::string_view(inst->mnemonic()), "v_cvt_pk_fp8_f32");
  cu->execute_instruction(inst.get(), *wf);
  EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0xA5A5FF7Fu);

  wf->set_mode_raw(amdgpu::Wavefront::FP16_OVFL_BIT);
  cu->write_vgpr(vb + 2, 0, 0xA5A5BEEFu);
  cu->execute_instruction(inst.get(), *wf);
  EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0xA5A5FE7Eu);

  if (!wf->is_halted())
    wf->halt();
}

TEST(Gfx1250CvtFp8Test, Bf8OverflowHonorsFp16OvflMode) {
  amdgpu::GpuMemory gpu_mem("gfx1250_cvt_bf8_overflow_mem");
  amdgpu::L2Cache l2("gfx1250_cvt_bf8_overflow_l2");

  amdgpu::ComputeUnitCore::Config cfg{};
  cfg.arch = ROCJITSU_CODE_ARCH_GFX1250;
  cfg.num_wf_slots = 1;
  cfg.sgprs_per_wf = 106;
  cfg.vgprs_per_wf = 256;
  cfg.lds_size_kb = 64;

  auto cu = amdgpu::ComputeUnitCore::create("gfx1250", cfg, &gpu_mem, &l2);
  ASSERT_NE(cu, nullptr);

  auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
  ASSERT_NE(decoder, nullptr);

  auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
  ASSERT_NE(wf, nullptr);
  wf->set_exec(1);

  const uint32_t vb = wf->vgpr_alloc().base;
  cu->write_vgpr(vb + 5, 0, std::bit_cast<uint32_t>(std::numeric_limits<float>::infinity()));
  cu->write_vgpr(vb + 6, 0, std::bit_cast<uint32_t>(-std::numeric_limits<float>::infinity()));
  cu->write_vgpr(vb + 2, 0, 0xA5A5BEEFu);

  // v_cvt_pk_bf8_f32 v2.l, v5, v6
  const auto words = encode_vop3(gfx1250::kVCvtPkBf8F32Vop3, 2, vgpr_src(5), vgpr_src(6), 0);
  std::unique_ptr<Instruction> inst(decoder->decode(words.data()));
  ASSERT_NE(inst, nullptr);
  ASSERT_EQ(std::string_view(inst->mnemonic()), "v_cvt_pk_bf8_f32");
  cu->execute_instruction(inst.get(), *wf);
  EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0xA5A5FC7Cu);

  wf->set_mode_raw(amdgpu::Wavefront::FP16_OVFL_BIT);
  cu->write_vgpr(vb + 2, 0, 0xA5A5BEEFu);
  cu->execute_instruction(inst.get(), *wf);
  EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0xA5A5FB7Bu);

  if (!wf->is_halted())
    wf->halt();
}

TEST(Gfx1250CvtF16Test, SrPackedF16ConsumesAdvancedSeedAndFp16OvflMode) {
  amdgpu::GpuMemory gpu_mem("gfx1250_cvt_sr_pk_f16_mem");
  amdgpu::L2Cache l2("gfx1250_cvt_sr_pk_f16_l2");

  amdgpu::ComputeUnitCore::Config cfg{};
  cfg.arch = ROCJITSU_CODE_ARCH_GFX1250;
  cfg.num_wf_slots = 1;
  cfg.sgprs_per_wf = 106;
  cfg.vgprs_per_wf = 256;
  cfg.lds_size_kb = 64;

  auto cu = amdgpu::ComputeUnitCore::create("gfx1250", cfg, &gpu_mem, &l2);
  ASSERT_NE(cu, nullptr);

  auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
  ASSERT_NE(decoder, nullptr);

  const auto words =
      encode_vop3(gfx1250::kVCvtSrPkF16F32Vop3, 2, vgpr_src(5), vgpr_src(6), vgpr_src(7));
  std::unique_ptr<Instruction> inst(decoder->decode(words.data()));
  ASSERT_NE(inst, nullptr);
  ASSERT_EQ(std::string_view(inst->mnemonic()), "v_cvt_sr_pk_f16_f32");

  auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
  ASSERT_NE(wf, nullptr);
  wf->set_exec(1);

  const uint32_t vb = wf->vgpr_alloc().base;
  constexpr uint32_t kSeed = 0x40000000u;
  const float sr_value = std::bit_cast<float>(0x3F801000u);
  const uint16_t expected_lo = util::f32_to_f16_sr_mode(sr_value, kSeed, false);
  const uint16_t expected_hi = util::f32_to_f16_sr_mode(sr_value, util::prng_advance(kSeed), false);
  ASSERT_NE(expected_lo, expected_hi);

  wf->set_mode_raw(0);
  cu->write_vgpr(vb + 5, 0, std::bit_cast<uint32_t>(sr_value));
  cu->write_vgpr(vb + 6, 0, std::bit_cast<uint32_t>(sr_value));
  cu->write_vgpr(vb + 7, 0, kSeed);
  cu->execute_instruction(inst.get(), *wf);
  EXPECT_EQ(cu->read_vgpr(vb + 2, 0), pack16(expected_lo, expected_hi));

  cu->write_vgpr(vb + 5, 0, std::bit_cast<uint32_t>(70000.0f));
  cu->write_vgpr(vb + 6, 0, std::bit_cast<uint32_t>(-70000.0f));
  cu->write_vgpr(vb + 7, 0, 0xFFFFFFFFu);
  cu->execute_instruction(inst.get(), *wf);
  EXPECT_EQ(cu->read_vgpr(vb + 2, 0), pack16(0x7C00u, 0xFC00u));

  wf->set_mode_raw(amdgpu::Wavefront::FP16_OVFL_BIT);
  cu->execute_instruction(inst.get(), *wf);
  EXPECT_EQ(cu->read_vgpr(vb + 2, 0), pack16(0x7BFFu, 0xFBFFu));

  if (!wf->is_halted())
    wf->halt();
}

TEST(Rdna4CvtFp8Test, E4M3OverflowHonorsFp16OvflMode) {
  amdgpu::GpuMemory gpu_mem("rdna4_cvt_fp8_overflow_mem");
  amdgpu::L2Cache l2("rdna4_cvt_fp8_overflow_l2");

  amdgpu::ComputeUnitCore::Config cfg{};
  cfg.arch = ROCJITSU_CODE_ARCH_RDNA4;
  cfg.num_wf_slots = 1;
  cfg.sgprs_per_wf = 106;
  cfg.vgprs_per_wf = 256;
  cfg.lds_size_kb = 64;

  auto cu = amdgpu::ComputeUnitCore::create("rdna4", cfg, &gpu_mem, &l2);
  ASSERT_NE(cu, nullptr);

  auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_RDNA4);
  ASSERT_NE(decoder, nullptr);

  auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
  ASSERT_NE(wf, nullptr);
  wf->set_exec(1);

  const uint32_t vb = wf->vgpr_alloc().base;
  cu->write_vgpr(vb + 5, 0, std::bit_cast<uint32_t>(768.0f));
  cu->write_vgpr(vb + 6, 0, std::bit_cast<uint32_t>(-768.0f));
  cu->write_vgpr(vb + 2, 0, 0xA5A5BEEFu);

  // v_cvt_pk_fp8_f32 v2.l, v5, v6
  const auto words = encode_vop3(rdna4::kVCvtPkFp8F32Vop3, 2, vgpr_src(5), vgpr_src(6), 0);
  std::unique_ptr<Instruction> inst(decoder->decode(words.data()));
  ASSERT_NE(inst, nullptr);
  ASSERT_EQ(std::string_view(inst->mnemonic()), "v_cvt_pk_fp8_f32");
  cu->execute_instruction(inst.get(), *wf);
  EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0xA5A5FF7Fu);

  wf->set_mode_raw(amdgpu::Wavefront::FP16_OVFL_BIT);
  cu->write_vgpr(vb + 2, 0, 0xA5A5BEEFu);
  cu->execute_instruction(inst.get(), *wf);
  EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0xA5A5FE7Eu);

  if (!wf->is_halted())
    wf->halt();
}

TEST(Rdna4CvtFp8Test, Bf8OverflowHonorsFp16OvflMode) {
  amdgpu::GpuMemory gpu_mem("rdna4_cvt_bf8_overflow_mem");
  amdgpu::L2Cache l2("rdna4_cvt_bf8_overflow_l2");

  amdgpu::ComputeUnitCore::Config cfg{};
  cfg.arch = ROCJITSU_CODE_ARCH_RDNA4;
  cfg.num_wf_slots = 1;
  cfg.sgprs_per_wf = 106;
  cfg.vgprs_per_wf = 256;
  cfg.lds_size_kb = 64;

  auto cu = amdgpu::ComputeUnitCore::create("rdna4", cfg, &gpu_mem, &l2);
  ASSERT_NE(cu, nullptr);

  auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_RDNA4);
  ASSERT_NE(decoder, nullptr);

  auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
  ASSERT_NE(wf, nullptr);
  wf->set_exec(1);

  const uint32_t vb = wf->vgpr_alloc().base;
  cu->write_vgpr(vb + 5, 0, std::bit_cast<uint32_t>(70000.0f));
  cu->write_vgpr(vb + 6, 0, std::bit_cast<uint32_t>(-70000.0f));
  cu->write_vgpr(vb + 2, 0, 0xA5A5BEEFu);

  // v_cvt_pk_bf8_f32 v2.l, v5, v6
  const auto packed_words = encode_vop3(rdna4::kVCvtPkBf8F32Vop3, 2, vgpr_src(5), vgpr_src(6), 0);
  std::unique_ptr<Instruction> packed_inst(decoder->decode(packed_words.data()));
  ASSERT_NE(packed_inst, nullptr);
  ASSERT_EQ(std::string_view(packed_inst->mnemonic()), "v_cvt_pk_bf8_f32");

  wf->set_mode_raw(0);
  cu->execute_instruction(packed_inst.get(), *wf);
  EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0xA5A5FC7Cu);

  wf->set_mode_raw(amdgpu::Wavefront::FP16_OVFL_BIT);
  cu->write_vgpr(vb + 2, 0, 0xA5A5BEEFu);
  cu->execute_instruction(packed_inst.get(), *wf);
  EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0xA5A5FB7Bu);

  cu->write_vgpr(vb + 6, 0, 0u);
  cu->write_vgpr(vb + 2, 0, 0xA5A5BEEFu);

  // v_cvt_sr_bf8_f32 v2, v5, v6 byte_sel:0
  const auto sr_words = encode_vop3(rdna4::kVCvtSrBf8F32Vop3, 2, vgpr_src(5), vgpr_src(6), 0);
  std::unique_ptr<Instruction> sr_inst(decoder->decode(sr_words.data()));
  ASSERT_NE(sr_inst, nullptr);
  ASSERT_EQ(std::string_view(sr_inst->mnemonic()), "v_cvt_sr_bf8_f32");

  wf->set_mode_raw(0);
  cu->execute_instruction(sr_inst.get(), *wf);
  EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0xA5A5BE7Cu);

  wf->set_mode_raw(amdgpu::Wavefront::FP16_OVFL_BIT);
  cu->write_vgpr(vb + 2, 0, 0xA5A5BEEFu);
  cu->execute_instruction(sr_inst.get(), *wf);
  EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0xA5A5BE7Bu);

  if (!wf->is_halted())
    wf->halt();
}

TEST(Rdna4CvtF16Test, F32OverflowHonorsFp16OvflMode) {
  amdgpu::GpuMemory gpu_mem("rdna4_cvt_f16_overflow_mem");
  amdgpu::L2Cache l2("rdna4_cvt_f16_overflow_l2");

  amdgpu::ComputeUnitCore::Config cfg{};
  cfg.arch = ROCJITSU_CODE_ARCH_RDNA4;
  cfg.num_wf_slots = 1;
  cfg.sgprs_per_wf = 106;
  cfg.vgprs_per_wf = 256;
  cfg.lds_size_kb = 64;

  auto cu = amdgpu::ComputeUnitCore::create("rdna4", cfg, &gpu_mem, &l2);
  ASSERT_NE(cu, nullptr);

  auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_RDNA4);
  ASSERT_NE(decoder, nullptr);

  auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
  ASSERT_NE(wf, nullptr);
  wf->set_exec(1);

  const uint32_t vb = wf->vgpr_alloc().base;
  cu->write_vgpr(vb + 5, 0, std::bit_cast<uint32_t>(70000.0f));
  cu->write_vgpr(vb + 2, 0, 0xA5A5BEEFu);

  // v_cvt_f16_f32 v2.l, v5
  const auto words = encode_vop3(rdna4::kVCvtF16F32Vop3, 2, vgpr_src(5), 0, 0);
  std::unique_ptr<Instruction> inst(decoder->decode(words.data()));
  ASSERT_NE(inst, nullptr);
  ASSERT_EQ(std::string_view(inst->mnemonic()), "v_cvt_f16_f32");
  cu->execute_instruction(inst.get(), *wf);
  EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0xA5A57C00u);

  wf->set_mode_raw(amdgpu::Wavefront::FP16_OVFL_BIT);
  cu->write_vgpr(vb + 2, 0, 0xA5A5BEEFu);
  cu->execute_instruction(inst.get(), *wf);
  EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0xA5A57BFFu);

  if (!wf->is_halted())
    wf->halt();
}

TEST(Rdna4CvtF16Test, U16OverflowHonorsFp16OvflModeInScalarAndSimdPaths) {
  for (bool force_scalar : {false, true}) {
    SCOPED_TRACE(force_scalar ? "scalar" : "simd");
    ForceScalarGuard guard(force_scalar);
    amdgpu::GpuMemory gpu_mem(force_scalar ? "rdna4_cvt_f16_u16_ovfl_scalar_mem"
                                           : "rdna4_cvt_f16_u16_ovfl_simd_mem");
    amdgpu::L2Cache l2(force_scalar ? "rdna4_cvt_f16_u16_ovfl_scalar_l2"
                                    : "rdna4_cvt_f16_u16_ovfl_simd_l2");

    amdgpu::ComputeUnitCore::Config cfg{};
    cfg.arch = ROCJITSU_CODE_ARCH_RDNA4;
    cfg.num_wf_slots = 1;
    cfg.sgprs_per_wf = 106;
    cfg.vgprs_per_wf = 256;
    cfg.lds_size_kb = 64;

    auto cu = amdgpu::ComputeUnitCore::create("rdna4", cfg, &gpu_mem, &l2);
    ASSERT_NE(cu, nullptr);

    auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_RDNA4);
    ASSERT_NE(decoder, nullptr);

    auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
    ASSERT_NE(wf, nullptr);
    wf->set_exec(1);

    const uint32_t vb = wf->vgpr_alloc().base;
    cu->write_vgpr(vb + 5, 0, 0xffffu);
    cu->write_vgpr(vb + 2, 0, 0xA5A5BEEFu);

    // v_cvt_f16_u16 v2.l, v5
    const auto words = encode_vop3(rdna4::kVCvtF16U16Vop3, 2, vgpr_src(5), 0, 0);
    std::unique_ptr<Instruction> inst(decoder->decode(words.data()));
    ASSERT_NE(inst, nullptr);
    ASSERT_EQ(std::string_view(inst->mnemonic()), "v_cvt_f16_u16");
    cu->execute_instruction(inst.get(), *wf);
    EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0xA5A57C00u);

    wf->set_mode_raw(amdgpu::Wavefront::FP16_OVFL_BIT);
    cu->write_vgpr(vb + 2, 0, 0xA5A5BEEFu);
    cu->execute_instruction(inst.get(), *wf);
    EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0xA5A57BFFu);

    if (!wf->is_halted())
      wf->halt();
  }
}

TEST(Rdna4Fp16ValuTest, AddOverflowHonorsFp16OvflModeInScalarAndSimdPaths) {
  for (bool force_scalar : {false, true}) {
    SCOPED_TRACE(force_scalar ? "scalar" : "simd");
    ForceScalarGuard guard(force_scalar);
    amdgpu::GpuMemory gpu_mem(force_scalar ? "rdna4_add_f16_ovfl_scalar_mem"
                                           : "rdna4_add_f16_ovfl_simd_mem");
    amdgpu::L2Cache l2(force_scalar ? "rdna4_add_f16_ovfl_scalar_l2"
                                    : "rdna4_add_f16_ovfl_simd_l2");

    amdgpu::ComputeUnitCore::Config cfg{};
    cfg.arch = ROCJITSU_CODE_ARCH_RDNA4;
    cfg.num_wf_slots = 1;
    cfg.sgprs_per_wf = 106;
    cfg.vgprs_per_wf = 256;
    cfg.lds_size_kb = 64;

    auto cu = amdgpu::ComputeUnitCore::create("rdna4", cfg, &gpu_mem, &l2);
    ASSERT_NE(cu, nullptr);

    auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_RDNA4);
    ASSERT_NE(decoder, nullptr);

    auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
    ASSERT_NE(wf, nullptr);
    wf->set_exec(1);

    const uint32_t vb = wf->vgpr_alloc().base;
    cu->write_vgpr(vb + 0, 0, pack16(0x7BFFu, 0));
    cu->write_vgpr(vb + 1, 0, pack16(0x7BFFu, 0));
    cu->write_vgpr(vb + 2, 0, 0xA5A5BEEFu);

    const auto words = encode_vop3(rdna4::kVAddF16Vop3, 2, vgpr_src(0), vgpr_src(1), 0);
    std::unique_ptr<Instruction> inst(decoder->decode(words.data()));
    ASSERT_NE(inst, nullptr);
    ASSERT_EQ(std::string_view(inst->mnemonic()), "v_add_f16");
    cu->execute_instruction(inst.get(), *wf);
    EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0xA5A57C00u);

    wf->set_mode_raw(amdgpu::Wavefront::FP16_OVFL_BIT);
    cu->write_vgpr(vb + 2, 0, 0xA5A5BEEFu);
    cu->execute_instruction(inst.get(), *wf);
    EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0xA5A57BFFu);

    if (!wf->is_halted())
      wf->halt();
  }
}

TEST(Cdna4CvtSrTest, SingleResultF16Bf16ConsumesSeedAndFp16OvflMode) {
  amdgpu::GpuMemory gpu_mem("cdna4_cvt_sr_f16_bf16_mem");
  amdgpu::L2Cache l2("cdna4_cvt_sr_f16_bf16_l2");

  amdgpu::ComputeUnitCore::Config cfg{};
  cfg.arch = ROCJITSU_CODE_ARCH_CDNA4;
  cfg.num_wf_slots = 1;
  cfg.sgprs_per_wf = 106;
  cfg.vgprs_per_wf = 256;
  cfg.lds_size_kb = 64;

  auto cu = amdgpu::ComputeUnitCore::create("cdna4", cfg, &gpu_mem, &l2);
  ASSERT_NE(cu, nullptr);

  auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_CDNA4);
  ASSERT_NE(decoder, nullptr);

  const auto f16_words = encode_cdna_vop3(cdna4::kVCvtSrF16F32Vop3, 2, vgpr_src(5), vgpr_src(6), 0);
  std::unique_ptr<Instruction> f16_inst(decoder->decode(f16_words.data()));
  ASSERT_NE(f16_inst, nullptr);
  ASSERT_EQ(std::string_view(f16_inst->mnemonic()), "v_cvt_sr_f16_f32");

  const auto f16_hi_words =
      encode_cdna_vop3(cdna4::kVCvtSrF16F32Vop3, 2, vgpr_src(5), vgpr_src(6), 0, 0x8);
  std::unique_ptr<Instruction> f16_hi_inst(decoder->decode(f16_hi_words.data()));
  ASSERT_NE(f16_hi_inst, nullptr);
  ASSERT_EQ(std::string_view(f16_hi_inst->mnemonic()), "v_cvt_sr_f16_f32");

  const auto bf16_words =
      encode_cdna_vop3(cdna4::kVCvtSrBf16F32Vop3, 3, vgpr_src(5), vgpr_src(6), 0);
  std::unique_ptr<Instruction> bf16_inst(decoder->decode(bf16_words.data()));
  ASSERT_NE(bf16_inst, nullptr);
  ASSERT_EQ(std::string_view(bf16_inst->mnemonic()), "v_cvt_sr_bf16_f32");

  const auto bf16_hi_words =
      encode_cdna_vop3(cdna4::kVCvtSrBf16F32Vop3, 3, vgpr_src(5), vgpr_src(6), 0, 0x8);
  std::unique_ptr<Instruction> bf16_hi_inst(decoder->decode(bf16_hi_words.data()));
  ASSERT_NE(bf16_hi_inst, nullptr);
  ASSERT_EQ(std::string_view(bf16_hi_inst->mnemonic()), "v_cvt_sr_bf16_f32");

  auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
  ASSERT_NE(wf, nullptr);
  wf->set_exec(1);

  const uint32_t vb = wf->vgpr_alloc().base;
  constexpr uint32_t kSeedLo = 0u;
  constexpr uint32_t kSeedHi = 0xFFFFFFFFu;

  const float f16_value = std::bit_cast<float>(0x3F801000u);
  const uint16_t f16_seed_lo = util::f32_to_f16_sr_mode(f16_value, kSeedLo, false);
  const uint16_t f16_seed_hi = util::f32_to_f16_sr_mode(f16_value, kSeedHi, false);
  ASSERT_NE(f16_seed_lo, f16_seed_hi);

  wf->set_mode_raw(0);
  cu->write_vgpr(vb + 5, 0, std::bit_cast<uint32_t>(f16_value));
  cu->write_vgpr(vb + 6, 0, kSeedLo);
  cu->write_vgpr(vb + 2, 0, 0xA5A50000u);
  cu->execute_instruction(f16_inst.get(), *wf);
  EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0xA5A50000u | f16_seed_lo);
  cu->write_vgpr(vb + 6, 0, kSeedHi);
  cu->write_vgpr(vb + 2, 0, 0x5A5A0000u);
  cu->execute_instruction(f16_inst.get(), *wf);
  EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0x5A5A0000u | f16_seed_hi);

  cu->write_vgpr(vb + 6, 0, kSeedLo);
  cu->write_vgpr(vb + 2, 0, 0x0000BEEFu);
  cu->execute_instruction(f16_hi_inst.get(), *wf);
  EXPECT_EQ(cu->read_vgpr(vb + 2, 0), (static_cast<uint32_t>(f16_seed_lo) << 16) | 0xBEEFu);

  constexpr uint32_t kOverflowSeed = 0xFFFFFFFFu;
  const float f16_overflow = 70000.0f;
  const uint16_t f16_clear = util::f32_to_f16_sr_mode(f16_overflow, kOverflowSeed, false);
  const uint16_t f16_set = util::f32_to_f16_sr_mode(f16_overflow, kOverflowSeed, true);
  ASSERT_NE(f16_clear, f16_set);

  cu->write_vgpr(vb + 5, 0, std::bit_cast<uint32_t>(f16_overflow));
  cu->write_vgpr(vb + 6, 0, kOverflowSeed);
  wf->set_mode_raw(0);
  cu->write_vgpr(vb + 2, 0, 0xA5A50000u);
  cu->execute_instruction(f16_inst.get(), *wf);
  EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0xA5A50000u | f16_clear);
  wf->set_mode_raw(amdgpu::Wavefront::FP16_OVFL_BIT);
  cu->write_vgpr(vb + 2, 0, 0x5A5A0000u);
  cu->execute_instruction(f16_inst.get(), *wf);
  EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0x5A5A0000u | f16_set);

  const float bf16_value = std::bit_cast<float>(0x3F808000u);
  const uint16_t bf16_seed_lo = util::f32_to_bf16_sr_mode(bf16_value, kSeedLo, false);
  const uint16_t bf16_seed_hi = util::f32_to_bf16_sr_mode(bf16_value, kSeedHi, false);
  ASSERT_NE(bf16_seed_lo, bf16_seed_hi);

  cu->write_vgpr(vb + 5, 0, std::bit_cast<uint32_t>(bf16_value));
  cu->write_vgpr(vb + 6, 0, kSeedLo);
  cu->write_vgpr(vb + 3, 0, 0xA5A50000u);
  cu->execute_instruction(bf16_inst.get(), *wf);
  EXPECT_EQ(cu->read_vgpr(vb + 3, 0), 0xA5A50000u | bf16_seed_lo);
  cu->write_vgpr(vb + 6, 0, kSeedHi);
  cu->write_vgpr(vb + 3, 0, 0x5A5A0000u);
  cu->execute_instruction(bf16_inst.get(), *wf);
  EXPECT_EQ(cu->read_vgpr(vb + 3, 0), 0x5A5A0000u | bf16_seed_hi);

  cu->write_vgpr(vb + 6, 0, kSeedLo);
  cu->write_vgpr(vb + 3, 0, 0x0000BEEFu);
  cu->execute_instruction(bf16_hi_inst.get(), *wf);
  EXPECT_EQ(cu->read_vgpr(vb + 3, 0), (static_cast<uint32_t>(bf16_seed_lo) << 16) | 0xBEEFu);

  const float bf16_overflow = std::numeric_limits<float>::max();
  const uint16_t bf16_clear = util::f32_to_bf16_sr_mode(bf16_overflow, kOverflowSeed, false);
  const uint16_t bf16_set = util::f32_to_bf16_sr_mode(bf16_overflow, kOverflowSeed, true);
  ASSERT_NE(bf16_clear, bf16_set);

  cu->write_vgpr(vb + 5, 0, std::bit_cast<uint32_t>(bf16_overflow));
  cu->write_vgpr(vb + 6, 0, kOverflowSeed);
  wf->set_mode_raw(0);
  cu->write_vgpr(vb + 3, 0, 0xA5A50000u);
  cu->execute_instruction(bf16_inst.get(), *wf);
  EXPECT_EQ(cu->read_vgpr(vb + 3, 0), 0xA5A50000u | bf16_clear);
  wf->set_mode_raw(amdgpu::Wavefront::FP16_OVFL_BIT);
  cu->write_vgpr(vb + 3, 0, 0x5A5A0000u);
  cu->execute_instruction(bf16_inst.get(), *wf);
  EXPECT_EQ(cu->read_vgpr(vb + 3, 0), 0x5A5A0000u | bf16_set);

  if (!wf->is_halted())
    wf->halt();
}

TEST(Cdna4CvtPkTest, F16Bf16F32OverflowHonorsFp16OvflMode) {
  amdgpu::GpuMemory gpu_mem("cdna4_cvt_pk_f16_bf16_f32_overflow_mem");
  amdgpu::L2Cache l2("cdna4_cvt_pk_f16_bf16_f32_overflow_l2");

  amdgpu::ComputeUnitCore::Config cfg{};
  cfg.arch = ROCJITSU_CODE_ARCH_CDNA4;
  cfg.num_wf_slots = 1;
  cfg.sgprs_per_wf = 106;
  cfg.vgprs_per_wf = 256;
  cfg.lds_size_kb = 64;

  auto cu = amdgpu::ComputeUnitCore::create("cdna4", cfg, &gpu_mem, &l2);
  ASSERT_NE(cu, nullptr);

  auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_CDNA4);
  ASSERT_NE(decoder, nullptr);

  auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
  ASSERT_NE(wf, nullptr);
  wf->set_exec(1);

  const uint32_t vb = wf->vgpr_alloc().base;

  // v_cvt_pk_f16_f32 v2, v5, v6
  const auto f16_words = encode_cdna_vop3(cdna4::kVCvtPkF16F32Vop3, 2, vgpr_src(5), vgpr_src(6), 0);
  std::unique_ptr<Instruction> f16_inst(decoder->decode(f16_words.data()));
  ASSERT_NE(f16_inst, nullptr);
  ASSERT_EQ(std::string_view(f16_inst->mnemonic()), "v_cvt_pk_f16_f32");

  cu->write_vgpr(vb + 5, 0, std::bit_cast<uint32_t>(70000.0f));
  cu->write_vgpr(vb + 6, 0, std::bit_cast<uint32_t>(-70000.0f));

  wf->set_mode_raw(0);
  cu->execute_instruction(f16_inst.get(), *wf);
  EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0xFC007C00u);

  wf->set_mode_raw(amdgpu::Wavefront::FP16_OVFL_BIT);
  cu->execute_instruction(f16_inst.get(), *wf);
  EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0xFBFF7BFFu);

  // v_cvt_pk_bf16_f32 v2, v5, v6
  const auto bf16_words =
      encode_cdna_vop3(cdna4::kVCvtPkBf16F32Vop3, 2, vgpr_src(5), vgpr_src(6), 0);
  std::unique_ptr<Instruction> bf16_inst(decoder->decode(bf16_words.data()));
  ASSERT_NE(bf16_inst, nullptr);
  ASSERT_EQ(std::string_view(bf16_inst->mnemonic()), "v_cvt_pk_bf16_f32");

  cu->write_vgpr(vb + 5, 0, std::bit_cast<uint32_t>(std::numeric_limits<float>::max()));
  cu->write_vgpr(vb + 6, 0, std::bit_cast<uint32_t>(-std::numeric_limits<float>::max()));

  wf->set_mode_raw(0);
  cu->execute_instruction(bf16_inst.get(), *wf);
  EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0xFF807F80u);

  wf->set_mode_raw(amdgpu::Wavefront::FP16_OVFL_BIT);
  cu->execute_instruction(bf16_inst.get(), *wf);
  EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0xFF7F7F7Fu);

  cu->write_vgpr(vb + 5, 0, std::bit_cast<uint32_t>(std::numeric_limits<float>::infinity()));
  cu->write_vgpr(vb + 6, 0, std::bit_cast<uint32_t>(-std::numeric_limits<float>::infinity()));
  cu->execute_instruction(bf16_inst.get(), *wf);
  EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0xFF807F80u);

  if (!wf->is_halted())
    wf->halt();
}

TEST(CdnaLegacyCvtF16Test, SetregImm32ModeControlsOverflowingF32ToF16) {
  struct CdnaLegacyCase {
    rj_code_arch_t arch;
    const char *name;
    uint32_t setreg_imm_op;
    uint32_t cvt_op;
  };

  const CdnaLegacyCase cases[] = {
      {ROCJITSU_CODE_ARCH_CDNA1, "cdna1", cdna1::kSSetregImm32B32Sopk, cdna1::kVCvtF16F32Vop3},
      {ROCJITSU_CODE_ARCH_CDNA2, "cdna2", cdna2::kSSetregImm32B32Sopk, cdna2::kVCvtF16F32Vop3},
  };

  for (const CdnaLegacyCase &arch_case : cases) {
    SCOPED_TRACE(arch_case.name);
    amdgpu::GpuMemory gpu_mem(std::string("cdna_legacy_cvt_f16_mode_mem_") + arch_case.name);
    amdgpu::L2Cache l2(std::string("cdna_legacy_cvt_f16_mode_l2_") + arch_case.name);

    amdgpu::ComputeUnitCore::Config cfg{};
    cfg.arch = arch_case.arch;
    cfg.num_wf_slots = 1;
    cfg.sgprs_per_wf = 106;
    cfg.vgprs_per_wf = 256;
    cfg.lds_size_kb = 64;

    auto cu = amdgpu::ComputeUnitCore::create(arch_case.name, cfg, &gpu_mem, &l2);
    ASSERT_NE(cu, nullptr);

    auto decoder = Decoder::create(arch_case.arch);
    ASSERT_NE(decoder, nullptr);

    auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
    ASSERT_NE(wf, nullptr);
    wf->set_exec(1);

    const uint32_t vb = wf->vgpr_alloc().base;
    cu->write_vgpr(vb + 5, 0, std::bit_cast<uint32_t>(70000.0f));

    const auto cvt_words = encode_cdna_vop3(arch_case.cvt_op, 2, vgpr_src(5), 0, 0);
    std::unique_ptr<Instruction> cvt_inst(decoder->decode(cvt_words.data()));
    ASSERT_NE(cvt_inst, nullptr);
    ASSERT_EQ(std::string_view(cvt_inst->mnemonic()), "v_cvt_f16_f32");

    constexpr uint32_t kHwregFp16Ovfl = encode_hwreg(1, 23, 1);
    const auto setreg_words = encode_sopk(arch_case.setreg_imm_op, 0, kHwregFp16Ovfl, 1);
    std::unique_ptr<Instruction> setreg_inst(decoder->decode(setreg_words.data()));
    ASSERT_NE(setreg_inst, nullptr);
    ASSERT_EQ(std::string_view(setreg_inst->mnemonic()), "s_setreg_imm32_b32");

    wf->set_mode_raw(0);
    cu->execute_instruction(cvt_inst.get(), *wf);
    EXPECT_EQ(cu->read_vgpr(vb + 2, 0) & 0xFFFFu, 0x7C00u);

    cu->write_vgpr(vb + 2, 0, 0);
    cu->execute_instruction(setreg_inst.get(), *wf);
    EXPECT_TRUE(wf->fp16_ovfl());
    cu->execute_instruction(cvt_inst.get(), *wf);
    EXPECT_EQ(cu->read_vgpr(vb + 2, 0) & 0xFFFFu, 0x7BFFu);

    if (!wf->is_halted())
      wf->halt();
  }
}

TEST(Rdna4ScalarF16Test, AddDoesNotConsumeFp16OvflMode) {
  amdgpu::GpuMemory gpu_mem("rdna4_scalar_f16_add_mode_mem");
  amdgpu::L2Cache l2("rdna4_scalar_f16_add_mode_l2");

  amdgpu::ComputeUnitCore::Config cfg{};
  cfg.arch = ROCJITSU_CODE_ARCH_RDNA4;
  cfg.num_wf_slots = 1;
  cfg.sgprs_per_wf = 106;
  cfg.vgprs_per_wf = 256;
  cfg.lds_size_kb = 64;

  auto cu = amdgpu::ComputeUnitCore::create("rdna4", cfg, &gpu_mem, &l2);
  ASSERT_NE(cu, nullptr);

  auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_RDNA4);
  ASSERT_NE(decoder, nullptr);

  auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
  ASSERT_NE(wf, nullptr);

  const uint32_t sb = wf->sgpr_alloc().base;
  cu->write_sgpr(sb + 0, util::f32_to_f16(65504.0f));
  cu->write_sgpr(sb + 1, util::f32_to_f16(65504.0f));

  const auto words = encode_sop2(rdna4::kSAddF16Sop2, 2, 0, 1);
  std::unique_ptr<Instruction> inst(decoder->decode(words.data()));
  ASSERT_NE(inst, nullptr);
  ASSERT_EQ(std::string_view(inst->mnemonic()), "s_add_f16");

  wf->set_mode_raw(amdgpu::Wavefront::FP16_OVFL_BIT);
  cu->execute_instruction(inst.get(), *wf);
  EXPECT_EQ(cu->read_sgpr(sb + 2) & 0xFFFFu, 0x7C00u);

  if (!wf->is_halted())
    wf->halt();
}

TEST(Rdna4ScalarF16Test, CvtF16F32ConsumesFp16OvflMode) {
  amdgpu::GpuMemory gpu_mem("rdna4_scalar_cvt_f16_mode_mem");
  amdgpu::L2Cache l2("rdna4_scalar_cvt_f16_mode_l2");

  amdgpu::ComputeUnitCore::Config cfg{};
  cfg.arch = ROCJITSU_CODE_ARCH_RDNA4;
  cfg.num_wf_slots = 1;
  cfg.sgprs_per_wf = 106;
  cfg.vgprs_per_wf = 256;
  cfg.lds_size_kb = 64;

  auto cu = amdgpu::ComputeUnitCore::create("rdna4", cfg, &gpu_mem, &l2);
  ASSERT_NE(cu, nullptr);

  auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_RDNA4);
  ASSERT_NE(decoder, nullptr);

  auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
  ASSERT_NE(wf, nullptr);

  const uint32_t sb = wf->sgpr_alloc().base;
  cu->write_sgpr(sb + 0, std::bit_cast<uint32_t>(70000.0f));

  const auto words = encode_sop1(rdna4::kSCvtF16F32Sop1, 2, 0);
  std::unique_ptr<Instruction> inst(decoder->decode(words.data()));
  ASSERT_NE(inst, nullptr);
  ASSERT_EQ(std::string_view(inst->mnemonic()), "s_cvt_f16_f32");

  wf->set_mode_raw(0);
  cu->execute_instruction(inst.get(), *wf);
  EXPECT_EQ(cu->read_sgpr(sb + 2) & 0xFFFFu, 0x7C00u);

  wf->set_mode_raw(amdgpu::Wavefront::FP16_OVFL_BIT);
  cu->execute_instruction(inst.get(), *wf);
  EXPECT_EQ(cu->read_sgpr(sb + 2) & 0xFFFFu, 0x7BFFu);

  if (!wf->is_halted())
    wf->halt();
}

TEST(Cdna3CvtFp8Test, SetregImm32ModeChangeRecomputesFp16OvflRounding) {
  amdgpu::GpuMemory gpu_mem("cdna3_cvt_fp8_mode_change_mem");
  amdgpu::L2Cache l2("cdna3_cvt_fp8_mode_change_l2");

  amdgpu::ComputeUnitCore::Config cfg{};
  cfg.arch = ROCJITSU_CODE_ARCH_CDNA3;
  cfg.num_wf_slots = 1;
  cfg.sgprs_per_wf = 106;
  cfg.vgprs_per_wf = 256;
  cfg.lds_size_kb = 64;

  auto cu = amdgpu::ComputeUnitCore::create("cdna3", cfg, &gpu_mem, &l2);
  ASSERT_NE(cu, nullptr);

  auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_CDNA3);
  ASSERT_NE(decoder, nullptr);

  auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
  ASSERT_NE(wf, nullptr);
  wf->set_exec(1);

  const uint32_t vb = wf->vgpr_alloc().base;
  cu->write_vgpr(vb + 5, 0, std::bit_cast<uint32_t>(300.0f));
  cu->write_vgpr(vb + 6, 0, std::bit_cast<uint32_t>(-300.0f));
  cu->write_vgpr(vb + 2, 0, 0xA5A5BEEFu);

  // v_cvt_pk_fp8_f32 v2.l, v5, v6. CDNA3 uses FNUZ FP8 encodings.
  const auto words = encode_cdna_vop3(cdna3::kVCvtPkFp8F32Vop3, 2, vgpr_src(5), vgpr_src(6), 0);
  std::unique_ptr<Instruction> inst(decoder->decode(words.data()));
  ASSERT_NE(inst, nullptr);
  ASSERT_EQ(std::string_view(inst->mnemonic()), "v_cvt_pk_fp8_f32");

  constexpr uint32_t kHwregFp16Ovfl = encode_hwreg(1, 23, 1);
  const auto setreg_words = encode_sopk(cdna3::kSSetregImm32B32Sopk, 0, kHwregFp16Ovfl, 1);
  std::unique_ptr<Instruction> setreg_inst(decoder->decode(setreg_words.data()));
  ASSERT_NE(setreg_inst, nullptr);
  ASSERT_EQ(std::string_view(setreg_inst->mnemonic()), "s_setreg_imm32_b32");

  // CDNA3 FNUZ rounded overflow toggles these deeper values between the FNUZ
  // NaN code and signed max finite.
  // Recompute through s_setreg_imm32_b32 so the conversion observes shader MODE updates end to end.
  wf->set_mode_raw(0);
  cu->execute_instruction(inst.get(), *wf);
  EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0x00008080u);

  cu->write_vgpr(vb + 2, 0, 0xA5A5BEEFu);
  cu->execute_instruction(setreg_inst.get(), *wf);
  EXPECT_EQ(wf->mode_raw(), amdgpu::Wavefront::FP16_OVFL_BIT);
  EXPECT_TRUE(wf->fp16_ovfl());
  cu->execute_instruction(inst.get(), *wf);
  EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0x0000FF7Fu);

  if (!wf->is_halted())
    wf->halt();
}

TEST(Gfx1250CvtFp8Test, F16SourceFp8Bf8ConversionsIgnoreFp16OvflMode) {
  amdgpu::GpuMemory gpu_mem("gfx1250_cvt_fp8_f16_source_mode_mem");
  amdgpu::L2Cache l2("gfx1250_cvt_fp8_f16_source_mode_l2");

  amdgpu::ComputeUnitCore::Config cfg{};
  cfg.arch = ROCJITSU_CODE_ARCH_GFX1250;
  cfg.num_wf_slots = 1;
  cfg.sgprs_per_wf = 106;
  cfg.vgprs_per_wf = 256;
  cfg.lds_size_kb = 64;

  auto cu = amdgpu::ComputeUnitCore::create("gfx1250", cfg, &gpu_mem, &l2);
  ASSERT_NE(cu, nullptr);

  auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
  ASSERT_NE(decoder, nullptr);

  auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
  ASSERT_NE(wf, nullptr);
  wf->set_exec(1);

  const uint32_t vb = wf->vgpr_alloc().base;
  const uint32_t f16_fp8_src = pack16(util::f32_to_f16(512.0f), util::f32_to_f16(-512.0f));
  const uint32_t f16_bf8_src = pack16(util::f32_to_f16(65504.0f), util::f32_to_f16(-65504.0f));

  auto run_packed_case = [&](uint32_t op, std::string_view mnemonic, uint32_t src,
                             uint32_t expected) {
    const auto words = encode_vop3(op, 2, vgpr_src(5), 0, 0);
    std::unique_ptr<Instruction> inst(decoder->decode(words.data()));
    ASSERT_NE(inst, nullptr);
    ASSERT_EQ(std::string_view(inst->mnemonic()), mnemonic);

    cu->write_vgpr(vb + 5, 0, src);
    wf->set_mode_raw(0);
    cu->write_vgpr(vb + 2, 0, 0xA5A5BEEFu);
    cu->execute_instruction(inst.get(), *wf);
    EXPECT_EQ(cu->read_vgpr(vb + 2, 0), expected);

    wf->set_mode_raw(amdgpu::Wavefront::FP16_OVFL_BIT);
    cu->write_vgpr(vb + 2, 0, 0x5A5ABEEFu);
    cu->execute_instruction(inst.get(), *wf);
    EXPECT_EQ(cu->read_vgpr(vb + 2, 0), (0x5A5A0000u | (expected & 0xFFFFu)));
  };

  auto run_sr_case = [&](uint32_t op, std::string_view mnemonic, uint32_t src, uint32_t seed,
                         uint32_t expected_byte) {
    const auto words = encode_vop3(op, 2, vgpr_src(5), vgpr_src(6), 0);
    std::unique_ptr<Instruction> inst(decoder->decode(words.data()));
    ASSERT_NE(inst, nullptr);
    ASSERT_EQ(std::string_view(inst->mnemonic()), mnemonic);

    cu->write_vgpr(vb + 5, 0, src);
    cu->write_vgpr(vb + 6, 0, seed);
    wf->set_mode_raw(0);
    cu->write_vgpr(vb + 2, 0, 0xA5A5BEEFu);
    cu->execute_instruction(inst.get(), *wf);
    EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0xA5A5BE00u | expected_byte);

    wf->set_mode_raw(amdgpu::Wavefront::FP16_OVFL_BIT);
    cu->write_vgpr(vb + 2, 0, 0x5A5ABEEFu);
    cu->execute_instruction(inst.get(), *wf);
    EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0x5A5ABE00u | expected_byte);
  };

  run_packed_case(gfx1250::kVCvtPkFp8F16Vop3, "v_cvt_pk_fp8_f16", f16_fp8_src, 0xA5A5FF7Fu);
  run_packed_case(gfx1250::kVCvtPkBf8F16Vop3, "v_cvt_pk_bf8_f16", f16_bf8_src, 0xA5A5FC7Cu);
  run_sr_case(gfx1250::kVCvtSrFp8F16Vop3, "v_cvt_sr_fp8_f16", f16_fp8_src, 0, 0x7Fu);
  run_sr_case(gfx1250::kVCvtSrBf8F16Vop3, "v_cvt_sr_bf8_f16", f16_bf8_src, 0xFFFFFFFFu, 0x7Cu);

  if (!wf->is_halted())
    wf->halt();
}

TEST(Cdna3CvtFp8Test, Bf8AndStochasticPathsHonorFp16OvflMode) {
  amdgpu::GpuMemory gpu_mem("cdna3_cvt_fp8_bf8_sr_overflow_mem");
  amdgpu::L2Cache l2("cdna3_cvt_fp8_bf8_sr_overflow_l2");

  amdgpu::ComputeUnitCore::Config cfg{};
  cfg.arch = ROCJITSU_CODE_ARCH_CDNA3;
  cfg.num_wf_slots = 1;
  cfg.sgprs_per_wf = 106;
  cfg.vgprs_per_wf = 256;
  cfg.lds_size_kb = 64;

  auto cu = amdgpu::ComputeUnitCore::create("cdna3", cfg, &gpu_mem, &l2);
  ASSERT_NE(cu, nullptr);

  auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_CDNA3);
  ASSERT_NE(decoder, nullptr);

  auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
  ASSERT_NE(wf, nullptr);
  wf->set_exec(1);

  const uint32_t vb = wf->vgpr_alloc().base;

  cu->write_vgpr(vb + 5, 0, std::bit_cast<uint32_t>(70000.0f));
  cu->write_vgpr(vb + 6, 0, std::bit_cast<uint32_t>(-70000.0f));
  cu->write_vgpr(vb + 2, 0, 0xA5A5BEEFu);

  // v_cvt_pk_bf8_f32 v2.l, v5, v6
  const auto bf8_words = encode_cdna_vop3(cdna3::kVCvtPkBf8F32Vop3, 2, vgpr_src(5), vgpr_src(6), 0);
  std::unique_ptr<Instruction> bf8_inst(decoder->decode(bf8_words.data()));
  ASSERT_NE(bf8_inst, nullptr);
  ASSERT_EQ(std::string_view(bf8_inst->mnemonic()), "v_cvt_pk_bf8_f32");

  wf->set_mode_raw(0);
  cu->execute_instruction(bf8_inst.get(), *wf);
  EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0x00008080u);

  wf->set_mode_raw(amdgpu::Wavefront::FP16_OVFL_BIT);
  cu->write_vgpr(vb + 2, 0, 0xA5A5BEEFu);
  cu->execute_instruction(bf8_inst.get(), *wf);
  EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0x0000FF7Fu);

  cu->write_vgpr(vb + 5, 0, std::bit_cast<uint32_t>(300.0f));
  cu->write_vgpr(vb + 6, 0, 0u);
  cu->write_vgpr(vb + 2, 0, 0xA5A5BEEFu);

  // v_cvt_sr_fp8_f32 v2, v5, v6 byte_sel:0
  const auto sr_fp8_words =
      encode_cdna_vop3(cdna3::kVCvtSrFp8F32Vop3, 2, vgpr_src(5), vgpr_src(6), 0);
  std::unique_ptr<Instruction> sr_fp8_inst(decoder->decode(sr_fp8_words.data()));
  ASSERT_NE(sr_fp8_inst, nullptr);
  ASSERT_EQ(std::string_view(sr_fp8_inst->mnemonic()), "v_cvt_sr_fp8_f32");

  wf->set_mode_raw(0);
  cu->execute_instruction(sr_fp8_inst.get(), *wf);
  EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0xA5A5BE80u);

  wf->set_mode_raw(amdgpu::Wavefront::FP16_OVFL_BIT);
  cu->write_vgpr(vb + 2, 0, 0xA5A5BEEFu);
  cu->execute_instruction(sr_fp8_inst.get(), *wf);
  EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0xA5A5BE7Fu);

  cu->write_vgpr(vb + 5, 0, std::bit_cast<uint32_t>(70000.0f));
  cu->write_vgpr(vb + 6, 0, 0u);
  cu->write_vgpr(vb + 2, 0, 0xA5A5BEEFu);

  // v_cvt_sr_bf8_f32 v2, v5, v6 byte_sel:0
  const auto sr_bf8_words =
      encode_cdna_vop3(cdna3::kVCvtSrBf8F32Vop3, 2, vgpr_src(5), vgpr_src(6), 0);
  std::unique_ptr<Instruction> sr_bf8_inst(decoder->decode(sr_bf8_words.data()));
  ASSERT_NE(sr_bf8_inst, nullptr);
  ASSERT_EQ(std::string_view(sr_bf8_inst->mnemonic()), "v_cvt_sr_bf8_f32");

  wf->set_mode_raw(0);
  cu->execute_instruction(sr_bf8_inst.get(), *wf);
  EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0xA5A5BE80u);

  wf->set_mode_raw(amdgpu::Wavefront::FP16_OVFL_BIT);
  cu->write_vgpr(vb + 2, 0, 0xA5A5BEEFu);
  cu->execute_instruction(sr_bf8_inst.get(), *wf);
  EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0xA5A5BE7Fu);

  if (!wf->is_halted())
    wf->halt();
}

TEST(Cdna4CvtScaleTest, ScaledFp8Bf8OverflowHonorsFp16OvflMode) {
  amdgpu::GpuMemory gpu_mem("cdna4_cvt_scale_fp8_bf8_overflow_mem");
  amdgpu::L2Cache l2("cdna4_cvt_scale_fp8_bf8_overflow_l2");

  amdgpu::ComputeUnitCore::Config cfg{};
  cfg.arch = ROCJITSU_CODE_ARCH_CDNA4;
  cfg.num_wf_slots = 1;
  cfg.sgprs_per_wf = 106;
  cfg.vgprs_per_wf = 256;
  cfg.lds_size_kb = 64;

  auto cu = amdgpu::ComputeUnitCore::create("cdna4", cfg, &gpu_mem, &l2);
  ASSERT_NE(cu, nullptr);

  auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_CDNA4);
  ASSERT_NE(decoder, nullptr);

  auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
  ASSERT_NE(wf, nullptr);
  wf->set_exec(1);

  const uint32_t vb = wf->vgpr_alloc().base;

  cu->write_vgpr(vb + 5, 0, std::bit_cast<uint32_t>(500.0f));
  cu->write_vgpr(vb + 6, 0, std::bit_cast<uint32_t>(-500.0f));
  cu->write_vgpr(vb + 7, 0, std::bit_cast<uint32_t>(1.0f));
  cu->write_vgpr(vb + 2, 0, 0xA5A5BEEFu);

  // v_cvt_scalef32_pk_fp8_f32 v2.l, v5, v6, v7
  const auto scaled_fp8_words =
      encode_cdna_vop3(cdna4::kVCvtScalef32PkFp8F32Vop3, 2, vgpr_src(5), vgpr_src(6), vgpr_src(7));
  std::unique_ptr<Instruction> scaled_fp8_inst(decoder->decode(scaled_fp8_words.data()));
  ASSERT_NE(scaled_fp8_inst, nullptr);
  ASSERT_EQ(std::string_view(scaled_fp8_inst->mnemonic()), "v_cvt_scalef32_pk_fp8_f32");

  wf->set_mode_raw(0);
  cu->execute_instruction(scaled_fp8_inst.get(), *wf);
  EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0xA5A5FF7Fu);

  wf->set_mode_raw(amdgpu::Wavefront::FP16_OVFL_BIT);
  cu->write_vgpr(vb + 2, 0, 0xA5A5BEEFu);
  cu->execute_instruction(scaled_fp8_inst.get(), *wf);
  EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0xA5A5FE7Eu);

  cu->write_vgpr(vb + 5, 0, std::bit_cast<uint32_t>(std::numeric_limits<float>::infinity()));
  cu->write_vgpr(vb + 6, 0, std::bit_cast<uint32_t>(-std::numeric_limits<float>::infinity()));
  cu->write_vgpr(vb + 7, 0, std::bit_cast<uint32_t>(1.0f));
  cu->write_vgpr(vb + 2, 0, 0xA5A5BEEFu);

  // v_cvt_scalef32_pk_bf8_f32 v2.l, v5, v6, v7
  const auto scaled_bf8_words =
      encode_cdna_vop3(cdna4::kVCvtScalef32PkBf8F32Vop3, 2, vgpr_src(5), vgpr_src(6), vgpr_src(7));
  std::unique_ptr<Instruction> scaled_bf8_inst(decoder->decode(scaled_bf8_words.data()));
  ASSERT_NE(scaled_bf8_inst, nullptr);
  ASSERT_EQ(std::string_view(scaled_bf8_inst->mnemonic()), "v_cvt_scalef32_pk_bf8_f32");

  wf->set_mode_raw(0);
  cu->execute_instruction(scaled_bf8_inst.get(), *wf);
  EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0xA5A5FC7Cu);

  wf->set_mode_raw(amdgpu::Wavefront::FP16_OVFL_BIT);
  cu->write_vgpr(vb + 2, 0, 0xA5A5BEEFu);
  cu->execute_instruction(scaled_bf8_inst.get(), *wf);
  EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0xA5A5FB7Bu);

  auto run_scaled_f16_dst_case = [&](uint32_t op, std::string_view mnemonic, uint32_t packed_src,
                                     float scale) {
    const auto low_words = encode_cdna_vop3(op, 2, vgpr_src(5), vgpr_src(7), 0);
    std::unique_ptr<Instruction> low_inst(decoder->decode(low_words.data()));
    ASSERT_NE(low_inst, nullptr);
    ASSERT_EQ(std::string_view(low_inst->mnemonic()), mnemonic);

    const auto high_words = encode_cdna_vop3(op, 2, vgpr_src(5), vgpr_src(7), 0, 0x8);
    std::unique_ptr<Instruction> high_inst(decoder->decode(high_words.data()));
    ASSERT_NE(high_inst, nullptr);
    ASSERT_EQ(std::string_view(high_inst->mnemonic()), mnemonic);

    cu->write_vgpr(vb + 5, 0, packed_src);
    cu->write_vgpr(vb + 7, 0, std::bit_cast<uint32_t>(scale));

    wf->set_mode_raw(0);
    cu->write_vgpr(vb + 2, 0, 0xA5A5BEEFu);
    cu->execute_instruction(low_inst.get(), *wf);
    EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0xA5A57C00u);

    wf->set_mode_raw(amdgpu::Wavefront::FP16_OVFL_BIT);
    cu->write_vgpr(vb + 2, 0, 0xA5A5BEEFu);
    cu->execute_instruction(low_inst.get(), *wf);
    EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0xA5A57BFFu);

    cu->write_vgpr(vb + 2, 0, 0x0000BEEFu);
    cu->execute_instruction(high_inst.get(), *wf);
    EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0x7BFFBEEFu);
  };

  run_scaled_f16_dst_case(cdna4::kVCvtScalef32F16Fp8Vop3, "v_cvt_scalef32_f16_fp8", 0x0000007Eu,
                          256.0f);
  run_scaled_f16_dst_case(cdna4::kVCvtScalef32F16Bf8Vop3, "v_cvt_scalef32_f16_bf8", 0x0000007Bu,
                          2.0f);

  auto run_scaled_packed_case = [&](uint32_t op, std::string_view mnemonic, uint32_t packed_src,
                                    uint32_t expected_clear, uint32_t expected_set) {
    const auto words = encode_cdna_vop3(op, 2, vgpr_src(5), vgpr_src(7), 0);
    std::unique_ptr<Instruction> inst(decoder->decode(words.data()));
    ASSERT_NE(inst, nullptr);
    ASSERT_EQ(std::string_view(inst->mnemonic()), mnemonic);

    cu->write_vgpr(vb + 5, 0, packed_src);
    cu->write_vgpr(vb + 7, 0, std::bit_cast<uint32_t>(1.0f));
    cu->write_vgpr(vb + 2, 0, 0xA5A5BEEFu);

    wf->set_mode_raw(0);
    cu->execute_instruction(inst.get(), *wf);
    EXPECT_EQ(cu->read_vgpr(vb + 2, 0), expected_clear);

    wf->set_mode_raw(amdgpu::Wavefront::FP16_OVFL_BIT);
    cu->write_vgpr(vb + 2, 0, 0xA5A5BEEFu);
    cu->execute_instruction(inst.get(), *wf);
    EXPECT_EQ(cu->read_vgpr(vb + 2, 0), expected_set);
  };

  const uint32_t f16_fp8_src = pack16(util::f32_to_f16(512.0f), util::f32_to_f16(-512.0f));
  run_scaled_packed_case(cdna4::kVCvtScalef32PkFp8F16Vop3, "v_cvt_scalef32_pk_fp8_f16", f16_fp8_src,
                         0xA5A5FF7Fu, 0xA5A5FE7Eu);

  const uint32_t f16_bf8_src = pack16(util::f32_to_f16(65504.0f), util::f32_to_f16(-65504.0f));
  run_scaled_packed_case(cdna4::kVCvtScalef32PkBf8F16Vop3, "v_cvt_scalef32_pk_bf8_f16", f16_bf8_src,
                         0xA5A5FC7Cu, 0xA5A5FB7Bu);

  const uint32_t bf16_fp8_src = pack16(util::f32_to_bf16(512.0f), util::f32_to_bf16(-512.0f));
  run_scaled_packed_case(cdna4::kVCvtScalef32PkFp8Bf16Vop3, "v_cvt_scalef32_pk_fp8_bf16",
                         bf16_fp8_src, 0xA5A5FF7Fu, 0xA5A5FE7Eu);

  const uint32_t bf16_bf8_src = pack16(util::f32_to_bf16(65536.0f), util::f32_to_bf16(-65536.0f));
  run_scaled_packed_case(cdna4::kVCvtScalef32PkBf8Bf16Vop3, "v_cvt_scalef32_pk_bf8_bf16",
                         bf16_bf8_src, 0xA5A5FC7Cu, 0xA5A5FB7Bu);

  if (!wf->is_halted())
    wf->halt();
}

TEST(Cdna4CvtScaleTest, WideFp6ToF16ConsumesFp16OvflMode) {
  amdgpu::GpuMemory gpu_mem("cdna4_cvt_scale_wide_fp6_f16_overflow_mem");
  amdgpu::L2Cache l2("cdna4_cvt_scale_wide_fp6_f16_overflow_l2");

  amdgpu::ComputeUnitCore::Config cfg{};
  cfg.arch = ROCJITSU_CODE_ARCH_CDNA4;
  cfg.num_wf_slots = 1;
  cfg.sgprs_per_wf = 106;
  cfg.vgprs_per_wf = 256;
  cfg.lds_size_kb = 64;

  auto cu = amdgpu::ComputeUnitCore::create("cdna4", cfg, &gpu_mem, &l2);
  ASSERT_NE(cu, nullptr);

  auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_CDNA4);
  ASSERT_NE(decoder, nullptr);

  auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
  ASSERT_NE(wf, nullptr);
  wf->set_exec(1);

  const uint32_t vb = wf->vgpr_alloc().base;
  uint8_t fp6_values[32]{};
  for (uint32_t i = 0; i < 32; ++i)
    fp6_values[i] = (i & 1u) ? 0x3Fu : 0x1Fu;

  uint32_t packed_fp6[6]{};
  util::pack_6bit(fp6_values, packed_fp6);
  for (uint32_t i = 0; i < 6; ++i)
    cu->write_vgpr(vb + 20 + i, 0, packed_fp6[i]);
  cu->write_vgpr(vb + 30, 0, std::bit_cast<uint32_t>(16384.0f));

  // v_cvt_scalef32_pk32_f16_fp6 v[2:17], v[20:25], v30
  const auto words =
      encode_cdna_vop3(cdna4::kVCvtScalef32Pk32F16Fp6Vop3, 2, vgpr_src(20), vgpr_src(30), 0);
  std::unique_ptr<Instruction> inst(decoder->decode(words.data()));
  ASSERT_NE(inst, nullptr);
  ASSERT_EQ(std::string_view(inst->mnemonic()), "v_cvt_scalef32_pk32_f16_fp6");

  wf->set_mode_raw(0);
  cu->execute_instruction(inst.get(), *wf);
  for (uint32_t i = 0; i < 16; ++i)
    EXPECT_EQ(cu->read_vgpr(vb + 2 + i, 0), 0xFC007C00u) << "dst dword " << i;

  wf->set_mode_raw(amdgpu::Wavefront::FP16_OVFL_BIT);
  cu->execute_instruction(inst.get(), *wf);
  for (uint32_t i = 0; i < 16; ++i)
    EXPECT_EQ(cu->read_vgpr(vb + 2 + i, 0), 0xFBFF7BFFu) << "dst dword " << i;

  if (!wf->is_halted())
    wf->halt();
}

TEST(Gfx1250CvtFp8Test, E5M3ClampSelectsUnsignedFp8Format) {
  amdgpu::GpuMemory gpu_mem("gfx1250_cvt_fp8_e5m3_mem");
  amdgpu::L2Cache l2("gfx1250_cvt_fp8_e5m3_l2");

  amdgpu::ComputeUnitCore::Config cfg{};
  cfg.arch = ROCJITSU_CODE_ARCH_GFX1250;
  cfg.num_wf_slots = 1;
  cfg.sgprs_per_wf = 106;
  cfg.vgprs_per_wf = 256;
  cfg.lds_size_kb = 64;

  auto cu = amdgpu::ComputeUnitCore::create("gfx1250", cfg, &gpu_mem, &l2);
  ASSERT_NE(cu, nullptr);

  auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
  ASSERT_NE(decoder, nullptr);

  auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
  ASSERT_NE(wf, nullptr);
  wf->set_exec(1);

  const uint32_t vb = wf->vgpr_alloc().base;
  const uint32_t sb = wf->sgpr_alloc().base;
  const uint32_t e5m3_0x38 = std::bit_cast<uint32_t>(std::ldexp(1.0f, -8));
  const uint32_t e5m3_0x08 = std::bit_cast<uint32_t>(std::ldexp(1.0f, -14));

  cu->write_vgpr(vb + 0, 0, 0x00380000u);

  // v_cvt_f32_fp8 v1, v0 byte_sel:2 clamp
  const uint32_t decode_words[] = {0xD5EC8801U, 0x00000100U};
  std::unique_ptr<Instruction> decode_inst(decoder->decode(decode_words));
  ASSERT_NE(decode_inst, nullptr);
  ASSERT_EQ(std::string_view(decode_inst->mnemonic()), "v_cvt_f32_fp8");
  cu->execute_instruction(decode_inst.get(), *wf);
  EXPECT_EQ(cu->read_vgpr(vb + 1, 0), e5m3_0x38);

  cu->write_sgpr(sb + 5, e5m3_0x38);
  cu->write_sgpr(sb + 6, e5m3_0x08);
  cu->write_vgpr(vb + 2, 0, 0xA5A5BEEFu);

  // v_cvt_pk_fp8_f32 v2.l, s5, s6 clamp
  const uint32_t pack_lo_words[] = {0xD7698002U, 0x02000C05U};
  std::unique_ptr<Instruction> pack_lo_inst(decoder->decode(pack_lo_words));
  ASSERT_NE(pack_lo_inst, nullptr);
  ASSERT_EQ(std::string_view(pack_lo_inst->mnemonic()), "v_cvt_pk_fp8_f32");
  cu->execute_instruction(pack_lo_inst.get(), *wf);
  EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0xA5A50838u);

  cu->write_vgpr(vb + 2, 0, 0xDEADBEEFu);

  // v_cvt_pk_fp8_f32 v2.h, s5, s6 op_sel:[0,0,1] clamp
  const uint32_t pack_hi_words[] = {0xD769C002U, 0x02000C05U};
  std::unique_ptr<Instruction> pack_hi_inst(decoder->decode(pack_hi_words));
  ASSERT_NE(pack_hi_inst, nullptr);
  ASSERT_EQ(std::string_view(pack_hi_inst->mnemonic()), "v_cvt_pk_fp8_f32");
  cu->execute_instruction(pack_hi_inst.get(), *wf);
  EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0x0838BEEFu);

  cu->write_vgpr(vb + 2, 0, 0xDEADBEEFu);

  // v_cvt_sr_fp8_f32 v2, s5, s6 byte_sel:2 clamp
  const uint32_t sr_words[] = {0xD76BC002U, 0x02000C05U};
  std::unique_ptr<Instruction> sr_inst(decoder->decode(sr_words));
  ASSERT_NE(sr_inst, nullptr);
  ASSERT_EQ(std::string_view(sr_inst->mnemonic()), "v_cvt_sr_fp8_f32");
  cu->execute_instruction(sr_inst.get(), *wf);
  EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0xDE38BEEFu);

  cu->write_sgpr(sb + 5, util::f32_to_f16(std::ldexp(1.0f, -8)));
  cu->write_sgpr(sb + 6, 0);
  cu->write_vgpr(vb + 2, 0, 0xDEADBEEFu);

  // v_cvt_sr_fp8_f16 v2, s5, s6 byte_sel:2 clamp
  const uint32_t sr_f16_words[] = {0xD774C002U, 0x02000C05U};
  std::unique_ptr<Instruction> sr_f16_inst(decoder->decode(sr_f16_words));
  ASSERT_NE(sr_f16_inst, nullptr);
  ASSERT_EQ(std::string_view(sr_f16_inst->mnemonic()), "v_cvt_sr_fp8_f16");
  cu->execute_instruction(sr_f16_inst.get(), *wf);
  EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0xDE38BEEFu);

  if (!wf->is_halted())
    wf->halt();
}

TEST(Gfx1250CvtScaleTest, UnpackUsesSelectedE8M0ScaleByte) {
  amdgpu::GpuMemory gpu_mem("gfx1250_cvt_scale_e8m0_mem");
  amdgpu::L2Cache l2("gfx1250_cvt_scale_e8m0_l2");

  amdgpu::ComputeUnitCore::Config cfg{};
  cfg.arch = ROCJITSU_CODE_ARCH_GFX1250;
  cfg.num_wf_slots = 1;
  cfg.sgprs_per_wf = 106;
  cfg.vgprs_per_wf = 256;
  cfg.lds_size_kb = 64;

  auto cu = amdgpu::ComputeUnitCore::create("gfx1250", cfg, &gpu_mem, &l2);
  ASSERT_NE(cu, nullptr);

  auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
  ASSERT_NE(decoder, nullptr);

  auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
  ASSERT_NE(wf, nullptr);
  wf->set_exec(1);

  const uint32_t vb = wf->vgpr_alloc().base;
  const uint32_t e8m0_scales = 0x817E807Fu; // 1.0, 2.0, 0.5, 4.0

  cu->write_vgpr(vb + 0, 0, 0x38383838u);
  cu->write_vgpr(vb + 1, 0, 0x38383838u);
  cu->write_vgpr(vb + 10, 0, e8m0_scales);

  // v_cvt_scale_pk8_f32_fp8 v[2:9], v[0:1], v10 scale_sel:2
  const uint32_t pk8_words[] = {0xD6AA1002U, 0x02021500U};
  std::unique_ptr<Instruction> pk8_inst(decoder->decode(pk8_words));
  ASSERT_NE(pk8_inst, nullptr);
  ASSERT_EQ(std::string_view(pk8_inst->mnemonic()), "v_cvt_scale_pk8_f32_fp8");
  cu->execute_instruction(pk8_inst.get(), *wf);
  for (uint32_t i = 2; i <= 9; ++i)
    EXPECT_EQ(cu->read_vgpr(vb + i, 0), std::bit_cast<uint32_t>(0.5f)) << "vgpr=" << i;

  cu->write_vgpr(vb + 0, 0, 0x08208208u);
  cu->write_vgpr(vb + 1, 0, 0x82082082u);
  cu->write_vgpr(vb + 2, 0, 0x20820820u);
  cu->write_vgpr(vb + 10, 0, e8m0_scales);

  // v_cvt_scale_pk16_f32_fp6 v[18:33], v[0:2], v10 scale_sel:1
  const uint32_t pk16_words[] = {0xD6C90812U, 0x02021500U};
  std::unique_ptr<Instruction> pk16_inst(decoder->decode(pk16_words));
  ASSERT_NE(pk16_inst, nullptr);
  ASSERT_EQ(std::string_view(pk16_inst->mnemonic()), "v_cvt_scale_pk16_f32_fp6");
  cu->execute_instruction(pk16_inst.get(), *wf);
  for (uint32_t i = 18; i <= 33; ++i)
    EXPECT_EQ(cu->read_vgpr(vb + i, 0), std::bit_cast<uint32_t>(2.0f)) << "vgpr=" << i;

  if (!wf->is_halted())
    wf->halt();
}

TEST(Gfx1250DsSwizzleTest, VdsBroadcastReadsAddrSource) {
  amdgpu::GpuMemory gpu_mem("gfx1250_ds_swizzle_vds_mem");
  amdgpu::L2Cache l2("gfx1250_ds_swizzle_vds_l2");

  amdgpu::ComputeUnitCore::Config cfg{};
  cfg.arch = ROCJITSU_CODE_ARCH_GFX1250;
  cfg.num_wf_slots = 1;
  cfg.sgprs_per_wf = 106;
  cfg.vgprs_per_wf = 256;
  cfg.lds_size_kb = 64;

  auto cu = amdgpu::ComputeUnitCore::create("gfx1250", cfg, &gpu_mem, &l2);
  ASSERT_NE(cu, nullptr);

  auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
  ASSERT_NE(decoder, nullptr);

  auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
  ASSERT_NE(wf, nullptr);
  ASSERT_EQ(wf->wf_size(), 32u);
  wf->set_exec((1ULL << wf->wf_size()) - 1ULL);

  const uint32_t vb = wf->vgpr_alloc().base;
  for (uint32_t lane = 0; lane < wf->wf_size(); ++lane) {
    cu->write_vgpr(vb + 0, lane, 0u);
    cu->write_vgpr(vb + 1, lane, 0x1000u + lane);
    cu->write_vgpr(vb + 2, lane, 0xDEADBEEFu);
  }

  // ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
  // gfx1250 encodes this as VDS, where the sole source VGPR is the addr field.
  const uint32_t swizzle_words[] = {0xD8D401E0U, 0x02000001U};
  std::unique_ptr<Instruction> swizzle_inst(decoder->decode(swizzle_words));
  ASSERT_NE(swizzle_inst, nullptr);
  ASSERT_EQ(std::string_view(swizzle_inst->mnemonic()), "ds_swizzle_b32");
  cu->execute_instruction(swizzle_inst.get(), *wf);

  for (uint32_t lane = 0; lane < wf->wf_size(); ++lane)
    EXPECT_EQ(cu->read_vgpr(vb + 2, lane), 0x100Fu) << "lane=" << lane;

  for (uint32_t lane = 0; lane < wf->wf_size(); ++lane)
    cu->write_vgpr(vb + 2, lane, 0xDEADBEEFu);

  // ds_swizzle_b32 v2, v1 offset:0x80b1, QDMode quad permutation (1, 0, 3, 2)
  const uint32_t quad_perm_words[] = {0xD8D480B1U, 0x02000001U};
  std::unique_ptr<Instruction> quad_perm_inst(decoder->decode(quad_perm_words));
  ASSERT_NE(quad_perm_inst, nullptr);
  ASSERT_EQ(std::string_view(quad_perm_inst->mnemonic()), "ds_swizzle_b32");
  cu->execute_instruction(quad_perm_inst.get(), *wf);

  constexpr uint32_t qd_src[4] = {1, 0, 3, 2};
  for (uint32_t lane = 0; lane < wf->wf_size(); ++lane) {
    const uint32_t src_lane = (lane & ~0x3u) | qd_src[lane & 0x3u];
    EXPECT_EQ(cu->read_vgpr(vb + 2, lane), 0x1000u + src_lane) << "lane=" << lane;
  }

  if (!wf->is_halted())
    wf->halt();
}

TEST(Gfx1250AtomicReturnTest, GlobalAtomicAddF32ReturnsOldValue) {
  amdgpu::GpuMemory gpu_mem("gfx1250_atomic_add_mem");
  amdgpu::L2Cache l2("gfx1250_atomic_add_l2");

  amdgpu::ComputeUnitCore::Config cfg{};
  cfg.arch = ROCJITSU_CODE_ARCH_GFX1250;
  cfg.num_wf_slots = 1;
  cfg.sgprs_per_wf = 106;
  cfg.vgprs_per_wf = 256;
  cfg.lds_size_kb = 64;

  Gfx1250MemoryTestCu cu("gfx1250", cfg, &gpu_mem, &l2);

  auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
  ASSERT_NE(decoder, nullptr);

  auto *wf = cu.dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
  ASSERT_NE(wf, nullptr);
  wf->set_exec(1);

  constexpr uint64_t kAddr = 0x1000;
  constexpr float kOld = 2.0f;
  constexpr float kAdd = 5.0f;
  gpu_mem.write32(kAddr, std::bit_cast<uint32_t>(kOld));
  cu.write_sgpr(wf->sgpr_alloc().base + 4, static_cast<uint32_t>(kAddr));
  cu.write_sgpr(wf->sgpr_alloc().base + 5, static_cast<uint32_t>(kAddr >> 32));
  cu.write_vgpr(wf->vgpr_alloc().base + 1, 0, std::bit_cast<uint32_t>(kAdd));
  cu.write_vgpr(wf->vgpr_alloc().base + 2, 0, 0);

  // global_atomic_add_f32 v0, v2, v1, s[4:5] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
  const uint32_t words[] = {0xEE158004U, 0x00980000U, 0x00000002U};
  std::unique_ptr<Instruction> inst(decoder->decode(words));
  ASSERT_NE(inst, nullptr);
  ASSERT_EQ(std::string_view(inst->mnemonic()), "global_atomic_add_f32");
  cu.execute_and_route(inst.release(), *wf);

  EXPECT_EQ(cu.read_vgpr(wf->vgpr_alloc().base + 0, 0), std::bit_cast<uint32_t>(kOld));
  EXPECT_EQ(gpu_mem.read32(kAddr), std::bit_cast<uint32_t>(kOld + kAdd));

  if (!wf->is_halted())
    wf->halt();
}

TEST(Gfx1250AtomicReturnTest, GlobalAtomicAddF32AllowsDstSrcAlias) {
  amdgpu::GpuMemory gpu_mem("gfx1250_atomic_add_alias_mem");
  amdgpu::L2Cache l2("gfx1250_atomic_add_alias_l2");

  amdgpu::ComputeUnitCore::Config cfg{};
  cfg.arch = ROCJITSU_CODE_ARCH_GFX1250;
  cfg.num_wf_slots = 1;
  cfg.sgprs_per_wf = 106;
  cfg.vgprs_per_wf = 256;
  cfg.lds_size_kb = 64;

  Gfx1250MemoryTestCu cu("gfx1250", cfg, &gpu_mem, &l2);

  auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
  ASSERT_NE(decoder, nullptr);

  auto *wf = cu.dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
  ASSERT_NE(wf, nullptr);
  wf->set_exec(1);

  constexpr uint64_t kAddr = 0x1080;
  constexpr float kOld = 0.0f;
  constexpr float kAdd = 5.0f;
  gpu_mem.write32(kAddr, std::bit_cast<uint32_t>(kOld));
  cu.write_sgpr(wf->sgpr_alloc().base + 4, static_cast<uint32_t>(kAddr));
  cu.write_sgpr(wf->sgpr_alloc().base + 5, static_cast<uint32_t>(kAddr >> 32));
  cu.write_vgpr(wf->vgpr_alloc().base + 1, 0, std::bit_cast<uint32_t>(kAdd));
  cu.write_vgpr(wf->vgpr_alloc().base + 2, 0, 0);

  // global_atomic_add_f32 v1, v2, v1, s[4:5] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
  const uint32_t words[] = {0xEE158004U, 0x00980001U, 0x00000002U};
  std::unique_ptr<Instruction> inst(decoder->decode(words));
  ASSERT_NE(inst, nullptr);
  ASSERT_EQ(std::string_view(inst->mnemonic()), "global_atomic_add_f32");
  cu.execute_and_route(inst.release(), *wf);

  EXPECT_EQ(cu.read_vgpr(wf->vgpr_alloc().base + 1, 0), std::bit_cast<uint32_t>(kOld));
  EXPECT_EQ(gpu_mem.read32(kAddr), std::bit_cast<uint32_t>(kOld + kAdd));

  if (!wf->is_halted())
    wf->halt();
}

TEST(Gfx1250AtomicReturnTest, GlobalAtomicAddU64CopiesHighSourceDword) {
  amdgpu::GpuMemory gpu_mem("gfx1250_atomic_add_u64_mem");
  amdgpu::L2Cache l2("gfx1250_atomic_add_u64_l2");

  amdgpu::ComputeUnitCore::Config cfg{};
  cfg.arch = ROCJITSU_CODE_ARCH_GFX1250;
  cfg.num_wf_slots = 1;
  cfg.sgprs_per_wf = 106;
  cfg.vgprs_per_wf = 256;
  cfg.lds_size_kb = 64;

  Gfx1250MemoryTestCu cu("gfx1250", cfg, &gpu_mem, &l2);

  auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
  ASSERT_NE(decoder, nullptr);

  auto *wf = cu.dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
  ASSERT_NE(wf, nullptr);
  wf->set_exec(1);

  constexpr uint64_t kAddr = 0x10C0;
  constexpr uint64_t kOld = 0x0000000200000003ull;
  constexpr uint64_t kAdd = 0x0000000400000005ull;
  gpu_mem.write64(kAddr, kOld);
  cu.write_sgpr(wf->sgpr_alloc().base + 4, static_cast<uint32_t>(kAddr));
  cu.write_sgpr(wf->sgpr_alloc().base + 5, static_cast<uint32_t>(kAddr >> 32));
  cu.write_vgpr(wf->vgpr_alloc().base + 0, 0, static_cast<uint32_t>(kAdd));
  cu.write_vgpr(wf->vgpr_alloc().base + 1, 0, static_cast<uint32_t>(kAdd >> 32));
  cu.write_vgpr(wf->vgpr_alloc().base + 2, 0, 0);

  // global_atomic_add_u64 v2, v[0:1], s[4:5] scope:SCOPE_DEV
  const uint32_t words[] = {0xEE10C004U, 0x00080000U, 0x00000002U};
  std::unique_ptr<Instruction> inst(decoder->decode(words));
  ASSERT_NE(inst, nullptr);
  ASSERT_EQ(std::string_view(inst->mnemonic()), "global_atomic_add_u64");
  cu.execute_and_route(inst.release(), *wf);

  EXPECT_EQ(gpu_mem.read64(kAddr), kOld + kAdd);

  if (!wf->is_halted())
    wf->halt();
}

TEST(Gfx1250AtomicReturnTest, GlobalAtomicCmpswapB32ReturnsOldValue) {
  amdgpu::GpuMemory gpu_mem("gfx1250_atomic_cmpswap_mem");
  amdgpu::L2Cache l2("gfx1250_atomic_cmpswap_l2");

  amdgpu::ComputeUnitCore::Config cfg{};
  cfg.arch = ROCJITSU_CODE_ARCH_GFX1250;
  cfg.num_wf_slots = 1;
  cfg.sgprs_per_wf = 106;
  cfg.vgprs_per_wf = 256;
  cfg.lds_size_kb = 64;

  Gfx1250MemoryTestCu cu("gfx1250", cfg, &gpu_mem, &l2);

  auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
  ASSERT_NE(decoder, nullptr);

  auto *wf = cu.dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
  ASSERT_NE(wf, nullptr);
  wf->set_exec(1);

  constexpr uint64_t kAddr = 0x1040;
  constexpr uint32_t kOld = 0x11111111u;
  constexpr uint32_t kNew = 0x22222222u;
  gpu_mem.write32(kAddr, kOld);
  cu.write_sgpr(wf->sgpr_alloc().base + 4, static_cast<uint32_t>(kAddr));
  cu.write_sgpr(wf->sgpr_alloc().base + 5, static_cast<uint32_t>(kAddr >> 32));
  cu.write_vgpr(wf->vgpr_alloc().base + 2, 0, 0);
  cu.write_vgpr(wf->vgpr_alloc().base + 4, 0, kNew);
  cu.write_vgpr(wf->vgpr_alloc().base + 5, 0, kOld);

  // global_atomic_cmpswap_b32 v0, v2, v[4:5], s[4:5] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
  const uint32_t words[] = {0xEE0D0004U, 0x02180000U, 0x00000002U};
  std::unique_ptr<Instruction> inst(decoder->decode(words));
  ASSERT_NE(inst, nullptr);
  ASSERT_EQ(std::string_view(inst->mnemonic()), "global_atomic_cmpswap_b32");
  cu.execute_and_route(inst.release(), *wf);

  EXPECT_EQ(cu.read_vgpr(wf->vgpr_alloc().base + 0, 0), kOld);
  EXPECT_EQ(gpu_mem.read32(kAddr), kNew);

  if (!wf->is_halted())
    wf->halt();
}

TEST(HwregTest, GetregReadsModeAndStatusTargetIds) {
  for (const HwregArchCase &arch_case : kHwregArchCases) {
    SCOPED_TRACE(arch_case.name);
    amdgpu::GpuMemory gpu_mem(std::string("hwreg_getreg_mem_") + arch_case.name);
    amdgpu::L2Cache l2(std::string("hwreg_getreg_l2_") + arch_case.name);

    amdgpu::ComputeUnitCore::Config cfg{};
    cfg.arch = arch_case.arch;
    cfg.num_wf_slots = 1;
    cfg.sgprs_per_wf = 106;
    cfg.vgprs_per_wf = 256;
    cfg.lds_size_kb = 64;

    auto cu = amdgpu::ComputeUnitCore::create(arch_case.name, cfg, &gpu_mem, &l2);
    ASSERT_NE(cu, nullptr);

    auto decoder = Decoder::create(arch_case.arch);
    ASSERT_NE(decoder, nullptr);

    constexpr uint32_t kHwregMode32 = encode_hwreg(1);
    constexpr uint32_t kHwregStatus32 = encode_hwreg(2);
    constexpr uint32_t kHwregUnknown32 = encode_hwreg(63);
    const std::array<uint32_t, 2> mode_words = encode_sopk(arch_case.getreg_op, 4, kHwregMode32);
    const std::array<uint32_t, 2> status_words =
        encode_sopk(arch_case.getreg_op, 5, kHwregStatus32);
    const std::array<uint32_t, 2> unknown_words =
        encode_sopk(arch_case.getreg_op, 6, kHwregUnknown32);

    std::unique_ptr<Instruction> mode_inst(decoder->decode(mode_words.data()));
    ASSERT_NE(mode_inst, nullptr);
    EXPECT_EQ(std::string_view(mode_inst->mnemonic()), "s_getreg_b32");
    std::unique_ptr<Instruction> status_inst(decoder->decode(status_words.data()));
    ASSERT_NE(status_inst, nullptr);
    EXPECT_EQ(std::string_view(status_inst->mnemonic()), "s_getreg_b32");
    std::unique_ptr<Instruction> unknown_inst(decoder->decode(unknown_words.data()));
    ASSERT_NE(unknown_inst, nullptr);
    EXPECT_EQ(std::string_view(unknown_inst->mnemonic()), "s_getreg_b32");

    auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
    ASSERT_NE(wf, nullptr);

    constexpr uint32_t kMode = 0x87654321u;
    constexpr uint32_t kStatus = 0x12345678u;
    wf->set_mode_raw(kMode);
    wf->set_status_raw(kStatus);

    cu->execute_instruction(mode_inst.get(), *wf);
    cu->execute_instruction(status_inst.get(), *wf);
    cu->execute_instruction(unknown_inst.get(), *wf);

    const uint32_t sb = wf->sgpr_alloc().base;
    EXPECT_EQ(cu->read_sgpr(sb + 4), kMode);
    EXPECT_EQ(cu->read_sgpr(sb + 5), kStatus);
    EXPECT_EQ(cu->read_sgpr(sb + 6), 0u);

    if (!wf->is_halted())
      wf->halt();
  }
}

TEST(HwregHelperTest, ReportsReadWriteResultContracts) {
  amdgpu::GpuMemory gpu_mem("hwreg_helper_contract_mem");
  amdgpu::L2Cache l2("hwreg_helper_contract_l2");

  amdgpu::ComputeUnitCore::Config cfg{};
  cfg.arch = ROCJITSU_CODE_ARCH_RDNA4;
  cfg.num_wf_slots = 1;
  cfg.sgprs_per_wf = 106;
  cfg.vgprs_per_wf = 256;
  cfg.lds_size_kb = 64;

  auto cu = amdgpu::ComputeUnitCore::create("rdna4", cfg, &gpu_mem, &l2);
  ASSERT_NE(cu, nullptr);

  auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
  ASSERT_NE(wf, nullptr);

  wf->set_mode_raw(amdgpu::Wavefront::FP16_OVFL_BIT);
  uint32_t value = 0;
  EXPECT_EQ(amdgpu::read_hwreg_field(*wf, encode_hwreg(1, 23, 1), value),
            amdgpu::HwregAccessResult::Success);
  EXPECT_EQ(value, 1u);

  EXPECT_EQ(amdgpu::write_hwreg_field(*wf, encode_hwreg(2), 0xDEADBEEFu),
            amdgpu::HwregAccessResult::ReadOnly);
  EXPECT_EQ(amdgpu::write_hwreg_field(*wf, encode_hwreg(4), 1u),
            amdgpu::HwregAccessResult::Privileged);
  EXPECT_EQ(amdgpu::write_hwreg_field(*wf, encode_hwreg(18), 1u),
            amdgpu::HwregAccessResult::Unsupported);
  EXPECT_EQ(amdgpu::read_hwreg_field(*wf, encode_hwreg(63), value),
            amdgpu::HwregAccessResult::Unsupported);
  EXPECT_EQ(value, 0u);

  if (!wf->is_halted())
    wf->halt();
}

TEST(HwregHelperTest, Gfx1250PreservesWaveSchedModeHwreg) {
  amdgpu::GpuMemory gpu_mem("hwreg_helper_gfx1250_wave_sched_mem");
  amdgpu::L2Cache l2("hwreg_helper_gfx1250_wave_sched_l2");

  amdgpu::ComputeUnitCore::Config cfg{};
  cfg.arch = ROCJITSU_CODE_ARCH_GFX1250;
  cfg.num_wf_slots = 1;
  cfg.sgprs_per_wf = 106;
  cfg.vgprs_per_wf = 256;
  cfg.lds_size_kb = 64;

  auto cu = amdgpu::ComputeUnitCore::create("gfx1250", cfg, &gpu_mem, &l2);
  ASSERT_NE(cu, nullptr);

  auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
  ASSERT_NE(wf, nullptr);

  constexpr uint32_t kWaveSchedMode = 0xA5A55A5Au;
  uint32_t value = 0;
  wf->set_wave_sched_mode_raw(kWaveSchedMode);

  EXPECT_STREQ(amdgpu::hwreg_name(*wf, encode_hwreg(26)), "WAVE_SCHED_MODE");
  EXPECT_EQ(amdgpu::read_hwreg_field(*wf, encode_hwreg(26), value),
            amdgpu::HwregAccessResult::Success);
  EXPECT_EQ(value, kWaveSchedMode);
  EXPECT_EQ(amdgpu::read_hwreg_field(*wf, encode_hwreg(26, 8, 8), value),
            amdgpu::HwregAccessResult::Success);
  EXPECT_EQ(value, 0x5Au);

  EXPECT_EQ(amdgpu::write_hwreg_field(*wf, encode_hwreg(26, 4, 4), 0x3u),
            amdgpu::HwregAccessResult::Success);
  EXPECT_EQ(wf->wave_sched_mode_raw(), 0xA5A55A3Au);

  if (!wf->is_halted())
    wf->halt();
}

TEST(HwregHelperTest, Gfx1250UsesManualHwregIdsInsteadOfLegacyAliases) {
  amdgpu::GpuMemory gpu_mem("hwreg_helper_gfx1250_hwreg_ids_mem");
  amdgpu::L2Cache l2("hwreg_helper_gfx1250_hwreg_ids_l2");

  amdgpu::ComputeUnitCore::Config cfg{};
  cfg.arch = ROCJITSU_CODE_ARCH_GFX1250;
  cfg.num_wf_slots = 1;
  cfg.sgprs_per_wf = 106;
  cfg.vgprs_per_wf = 256;
  cfg.lds_size_kb = 64;

  auto cu = amdgpu::ComputeUnitCore::create("gfx1250", cfg, &gpu_mem, &l2);
  ASSERT_NE(cu, nullptr);

  auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
  ASSERT_NE(wf, nullptr);

  EXPECT_STREQ(amdgpu::hwreg_name(*wf, encode_hwreg(4)), "WAVE_STATE_PRIV");
  EXPECT_EQ(amdgpu::write_hwreg_field(*wf, encode_hwreg(4), 1u),
            amdgpu::HwregAccessResult::Privileged);
  EXPECT_STREQ(amdgpu::hwreg_name(*wf, encode_hwreg(5)), "WAVE_GPR_ALLOC");
  EXPECT_STREQ(amdgpu::hwreg_name(*wf, encode_hwreg(6)), "WAVE_LDS_ALLOC");
  EXPECT_STREQ(amdgpu::hwreg_name(*wf, encode_hwreg(7)), "IB_STS");
  EXPECT_EQ(amdgpu::write_hwreg_field(*wf, encode_hwreg(7), 1u),
            amdgpu::HwregAccessResult::ReadOnly);
  EXPECT_STREQ(amdgpu::hwreg_name(*wf, encode_hwreg(14)), "FLUSH_IB");
  EXPECT_EQ(amdgpu::write_hwreg_field(*wf, encode_hwreg(14), 1u),
            amdgpu::HwregAccessResult::Unsupported);
  EXPECT_STREQ(amdgpu::hwreg_name(*wf, encode_hwreg(23)), "WAVE_HW_ID1");
  EXPECT_STREQ(amdgpu::hwreg_name(*wf, encode_hwreg(24)), "WAVE_HW_ID2");

  if (!wf->is_halted())
    wf->halt();
}

TEST(HwregHelperTest, Gfx1250ReadsModeledIbStsCounters) {
  amdgpu::GpuMemory gpu_mem("hwreg_helper_gfx1250_ib_sts_mem");
  amdgpu::L2Cache l2("hwreg_helper_gfx1250_ib_sts_l2");

  amdgpu::ComputeUnitCore::Config cfg{};
  cfg.arch = ROCJITSU_CODE_ARCH_GFX1250;
  cfg.num_wf_slots = 1;
  cfg.sgprs_per_wf = 106;
  cfg.vgprs_per_wf = 256;
  cfg.lds_size_kb = 64;

  auto cu = amdgpu::ComputeUnitCore::create("gfx1250", cfg, &gpu_mem, &l2);
  ASSERT_NE(cu, nullptr);

  auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
  ASSERT_NE(wf, nullptr);

  wf->wait_counters().dscnt = 5;
  wf->wait_counters().vmcnt = 6;
  wf->wait_counters().vscnt = 7;

  uint32_t value = 0;
  EXPECT_STREQ(amdgpu::hwreg_name(*wf, encode_hwreg(7)), "IB_STS");
  EXPECT_EQ(amdgpu::read_hwreg_field(*wf, encode_hwreg(7), value),
            amdgpu::HwregAccessResult::Success);
  EXPECT_EQ(value, (5u << 3) | (6u << 9) | (7u << 24));
  EXPECT_EQ(amdgpu::read_hwreg_field(*wf, encode_hwreg(7, 3, 6), value),
            amdgpu::HwregAccessResult::Success);
  EXPECT_EQ(value, 5u);
  EXPECT_EQ(amdgpu::read_hwreg_field(*wf, encode_hwreg(7, 9, 6), value),
            amdgpu::HwregAccessResult::Success);
  EXPECT_EQ(value, 6u);
  EXPECT_EQ(amdgpu::read_hwreg_field(*wf, encode_hwreg(7, 24, 6), value),
            amdgpu::HwregAccessResult::Success);
  EXPECT_EQ(value, 7u);

  if (!wf->is_halted())
    wf->halt();
}

TEST(HwregHelperTest, Gfx1250ReadsModeledIbSts2Counters) {
  amdgpu::GpuMemory gpu_mem("hwreg_helper_gfx1250_ib_sts2_mem");
  amdgpu::L2Cache l2("hwreg_helper_gfx1250_ib_sts2_l2");

  amdgpu::ComputeUnitCore::Config cfg{};
  cfg.arch = ROCJITSU_CODE_ARCH_GFX1250;
  cfg.num_wf_slots = 1;
  cfg.sgprs_per_wf = 106;
  cfg.vgprs_per_wf = 256;
  cfg.lds_size_kb = 64;

  auto cu = amdgpu::ComputeUnitCore::create("gfx1250", cfg, &gpu_mem, &l2);
  ASSERT_NE(cu, nullptr);

  auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
  ASSERT_NE(wf, nullptr);

  wf->wait_counters().kmcnt = 17;
  wf->wait_counters().asynccnt = 42;
  wf->wait_counters().tensorcnt = 4;
  wf->set_cluster_info(/*rank=*/6, /*size=*/8);

  uint32_t value = 0;
  EXPECT_STREQ(amdgpu::hwreg_name(*wf, encode_hwreg(28)), "IB_STS2");
  EXPECT_EQ(amdgpu::read_hwreg_field(*wf, encode_hwreg(28), value),
            amdgpu::HwregAccessResult::Success);
  EXPECT_EQ(value, 17u | (1u << 6) | (42u << 12) | (3u << 18) | (6u << 21) | (3u << 28));
  EXPECT_EQ(amdgpu::read_hwreg_field(*wf, encode_hwreg(28, 0, 5), value),
            amdgpu::HwregAccessResult::Success);
  EXPECT_EQ(value, 17u);
  EXPECT_EQ(amdgpu::read_hwreg_field(*wf, encode_hwreg(28, 6, 4), value),
            amdgpu::HwregAccessResult::Success);
  EXPECT_EQ(value, 1u);
  EXPECT_EQ(amdgpu::read_hwreg_field(*wf, encode_hwreg(28, 12, 6), value),
            amdgpu::HwregAccessResult::Success);
  EXPECT_EQ(value, 42u);
  EXPECT_EQ(amdgpu::read_hwreg_field(*wf, encode_hwreg(28, 18, 2), value),
            amdgpu::HwregAccessResult::Success);
  EXPECT_EQ(value, 3u);
  EXPECT_EQ(amdgpu::read_hwreg_field(*wf, encode_hwreg(28, 21, 4), value),
            amdgpu::HwregAccessResult::Success);
  EXPECT_EQ(value, 6u);
  EXPECT_EQ(amdgpu::read_hwreg_field(*wf, encode_hwreg(28, 28, 2), value),
            amdgpu::HwregAccessResult::Success);
  EXPECT_EQ(value, 3u);

  if (!wf->is_halted())
    wf->halt();
}

TEST(HwregHelperTest, ModeBit27OnlyEnablesGprIndexingWhereArchitected) {
  struct GprIdxModeCase {
    rj_code_arch_t arch;
    const char *name;
    bool mode_has_gpr_idx_en;
  };

  const GprIdxModeCase cases[] = {
      {ROCJITSU_CODE_ARCH_CDNA1, "cdna1", true},  {ROCJITSU_CODE_ARCH_CDNA2, "cdna2", true},
      {ROCJITSU_CODE_ARCH_CDNA3, "cdna3", true},  {ROCJITSU_CODE_ARCH_CDNA4, "cdna4", true},
      {ROCJITSU_CODE_ARCH_RDNA1, "rdna1", false}, {ROCJITSU_CODE_ARCH_RDNA2, "rdna2", false},
      {ROCJITSU_CODE_ARCH_RDNA3, "rdna3", false}, {ROCJITSU_CODE_ARCH_RDNA3_5, "rdna3_5", false},
      {ROCJITSU_CODE_ARCH_RDNA4, "rdna4", false}, {ROCJITSU_CODE_ARCH_GFX1250, "gfx1250", true},
  };

  for (const GprIdxModeCase &arch_case : cases) {
    SCOPED_TRACE(arch_case.name);
    amdgpu::GpuMemory gpu_mem(std::string("hwreg_gpr_idx_mode_mem_") + arch_case.name);
    amdgpu::L2Cache l2(std::string("hwreg_gpr_idx_mode_l2_") + arch_case.name);

    amdgpu::ComputeUnitCore::Config cfg{};
    cfg.arch = arch_case.arch;
    cfg.num_wf_slots = 1;
    cfg.sgprs_per_wf = 106;
    cfg.vgprs_per_wf = 256;
    cfg.lds_size_kb = 64;

    auto cu = amdgpu::ComputeUnitCore::create(arch_case.name, cfg, &gpu_mem, &l2);
    ASSERT_NE(cu, nullptr);

    auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
    ASSERT_NE(wf, nullptr);

    wf->set_mode_raw(amdgpu::Wavefront::GPR_IDX_EN_BIT);
    EXPECT_EQ(wf->mode_raw() & amdgpu::Wavefront::GPR_IDX_EN_BIT,
              amdgpu::Wavefront::GPR_IDX_EN_BIT);
    EXPECT_EQ(wf->gpr_idx_en(), arch_case.mode_has_gpr_idx_en);

    if (!wf->is_halted())
      wf->halt();
  }
}

TEST(HwregHelperTest, ReadsGprAllocFromWaveState) {
  enum class GprAllocLayout {
    Gfx9_10,
    Cdna3_4,
  };

  struct GprAllocCase {
    rj_code_arch_t arch;
    const char *name;
    GprAllocLayout layout;
  };

  const GprAllocCase cases[] = {
      {ROCJITSU_CODE_ARCH_CDNA1, "cdna1", GprAllocLayout::Gfx9_10},
      {ROCJITSU_CODE_ARCH_CDNA2, "cdna2", GprAllocLayout::Gfx9_10},
      {ROCJITSU_CODE_ARCH_CDNA3, "cdna3", GprAllocLayout::Cdna3_4},
      {ROCJITSU_CODE_ARCH_CDNA4, "cdna4", GprAllocLayout::Cdna3_4},
      {ROCJITSU_CODE_ARCH_RDNA1, "rdna1", GprAllocLayout::Gfx9_10},
      {ROCJITSU_CODE_ARCH_RDNA2, "rdna2", GprAllocLayout::Gfx9_10},
  };

  auto blocks_minus_one = [](uint32_t count, uint32_t block_size) {
    return ((count + block_size - 1u) / block_size) - 1u;
  };

  for (const GprAllocCase &arch_case : cases) {
    SCOPED_TRACE(arch_case.name);
    amdgpu::GpuMemory gpu_mem(std::string("hwreg_gpr_alloc_mem_") + arch_case.name);
    amdgpu::L2Cache l2(std::string("hwreg_gpr_alloc_l2_") + arch_case.name);

    amdgpu::ComputeUnitCore::Config cfg{};
    cfg.arch = arch_case.arch;
    cfg.num_wf_slots = 2;
    cfg.sgprs_per_wf = 106;
    cfg.vgprs_per_wf = 256;
    cfg.lds_size_kb = 64;

    auto cu = amdgpu::ComputeUnitCore::create(arch_case.name, cfg, &gpu_mem, &l2);
    ASSERT_NE(cu, nullptr);

    auto *first_wf = cu->dispatch_wf(0, 0, 32, 16);
    ASSERT_NE(first_wf, nullptr);
    auto *wf = cu->dispatch_wf(0, 1, 80, 40);
    ASSERT_NE(wf, nullptr);

    const amdgpu::RegAllocation &vgpr = wf->vgpr_alloc();
    const amdgpu::RegAllocation &sgpr = wf->sgpr_alloc();
    uint32_t expected = 0;
    uint32_t expected_sgpr_size = 0;
    switch (arch_case.layout) {
    case GprAllocLayout::Gfx9_10:
      expected_sgpr_size = blocks_minus_one(sgpr.count, 8);
      expected = ((vgpr.base >> 2) & 0x3Fu) | ((blocks_minus_one(vgpr.count, 4) & 0x3Fu) << 8) |
                 (((sgpr.base >> 3) & 0x3Fu) << 16) | ((expected_sgpr_size & 0xFu) << 24);
      break;
    case GprAllocLayout::Cdna3_4:
      expected_sgpr_size = blocks_minus_one(sgpr.count, 16);
      expected = ((vgpr.base >> 2) & 0x3Fu) | ((blocks_minus_one(vgpr.count, 4) & 0x3Fu) << 6) |
                 (((sgpr.base >> 3) & 0x3Fu) << 18) | ((expected_sgpr_size & 0xFu) << 24);
      break;
    }

    uint32_t value = 0;
    if (arch_case.layout == GprAllocLayout::Gfx9_10) {
      EXPECT_EQ(amdgpu::read_hwreg_field(*wf, encode_hwreg(5), value),
                amdgpu::HwregAccessResult::Success);
      EXPECT_EQ(value, expected);
    } else {
      EXPECT_EQ(amdgpu::read_hwreg_field(*wf, encode_hwreg(5), value),
                amdgpu::HwregAccessResult::Unsupported);
      EXPECT_EQ(value, 0u);
    }

    EXPECT_EQ(amdgpu::read_hwreg_field(*wf, encode_hwreg(5, 0, 6), value),
              amdgpu::HwregAccessResult::Success);
    EXPECT_EQ(value, (vgpr.base >> 2) & 0x3Fu);

    EXPECT_EQ(amdgpu::read_hwreg_field(*wf, encode_hwreg(5, 24, 4), value),
              amdgpu::HwregAccessResult::Success);
    EXPECT_EQ(value, expected_sgpr_size);
    if (arch_case.layout == GprAllocLayout::Cdna3_4) {
      EXPECT_EQ(amdgpu::read_hwreg_field(*wf, encode_hwreg(5, 12, 6), value),
                amdgpu::HwregAccessResult::Unsupported);
      EXPECT_EQ(value, 0u);
    }
    EXPECT_EQ(amdgpu::write_hwreg_field(*wf, encode_hwreg(5), 0),
              amdgpu::HwregAccessResult::ReadOnly);

    if (!first_wf->is_halted())
      first_wf->halt();
    if (!wf->is_halted())
      wf->halt();
  }
}

TEST(HwregHelperTest, Rdna3TablesNameXmlHwregIds) {
  struct RdnaHwregCase {
    rj_code_arch_t arch;
    const char *name;
  };

  const RdnaHwregCase cases[] = {
      {ROCJITSU_CODE_ARCH_RDNA3, "rdna3"},
      {ROCJITSU_CODE_ARCH_RDNA3_5, "rdna3_5"},
  };

  for (const RdnaHwregCase &arch_case : cases) {
    SCOPED_TRACE(arch_case.name);
    amdgpu::GpuMemory gpu_mem(std::string("hwreg_helper_rdna3_mem_") + arch_case.name);
    amdgpu::L2Cache l2(std::string("hwreg_helper_rdna3_l2_") + arch_case.name);

    amdgpu::ComputeUnitCore::Config cfg{};
    cfg.arch = arch_case.arch;
    cfg.num_wf_slots = 1;
    cfg.sgprs_per_wf = 106;
    cfg.vgprs_per_wf = 256;
    cfg.lds_size_kb = 64;

    auto cu = amdgpu::ComputeUnitCore::create(arch_case.name, cfg, &gpu_mem, &l2);
    ASSERT_NE(cu, nullptr);

    auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
    ASSERT_NE(wf, nullptr);

    EXPECT_STREQ(amdgpu::hwreg_name(*wf, encode_hwreg(3)), "TRAPSTS");
    EXPECT_EQ(amdgpu::write_hwreg_field(*wf, encode_hwreg(3), 1u),
              amdgpu::HwregAccessResult::Unsupported);
    EXPECT_STREQ(amdgpu::hwreg_name(*wf, encode_hwreg(14)), "FLUSH_IB");
    EXPECT_EQ(amdgpu::write_hwreg_field(*wf, encode_hwreg(14), 1u),
              amdgpu::HwregAccessResult::Unsupported);
    EXPECT_STREQ(amdgpu::hwreg_name(*wf, encode_hwreg(20)), "SHADER_FLAT_SCRATCH_LO");
    EXPECT_EQ(amdgpu::write_hwreg_field(*wf, encode_hwreg(20), 1u),
              amdgpu::HwregAccessResult::ReadOnly);
    EXPECT_STREQ(amdgpu::hwreg_name(*wf, encode_hwreg(23)), "HW_ID1");
    EXPECT_EQ(amdgpu::write_hwreg_field(*wf, encode_hwreg(23), 1u),
              amdgpu::HwregAccessResult::ReadOnly);
    EXPECT_STREQ(amdgpu::hwreg_name(*wf, encode_hwreg(63)), "unknown");

    if (!wf->is_halted())
      wf->halt();
  }
}

TEST(HwregTest, GetregExtractsPartialModeAndStatusFields) {
  for (const HwregArchCase &arch_case : kHwregArchCases) {
    SCOPED_TRACE(arch_case.name);
    amdgpu::GpuMemory gpu_mem(std::string("hwreg_getreg_partial_mem_") + arch_case.name);
    amdgpu::L2Cache l2(std::string("hwreg_getreg_partial_l2_") + arch_case.name);

    amdgpu::ComputeUnitCore::Config cfg{};
    cfg.arch = arch_case.arch;
    cfg.num_wf_slots = 1;
    cfg.sgprs_per_wf = 106;
    cfg.vgprs_per_wf = 256;
    cfg.lds_size_kb = 64;

    auto cu = amdgpu::ComputeUnitCore::create(arch_case.name, cfg, &gpu_mem, &l2);
    ASSERT_NE(cu, nullptr);

    auto decoder = Decoder::create(arch_case.arch);
    ASSERT_NE(decoder, nullptr);

    constexpr uint32_t kHwregModeFp16Ovfl = encode_hwreg(1, 23, 1);
    constexpr uint32_t kHwregStatusNibble = encode_hwreg(2, 8, 4);
    const std::array<uint32_t, 2> mode_words =
        encode_sopk(arch_case.getreg_op, 4, kHwregModeFp16Ovfl);
    const std::array<uint32_t, 2> status_words =
        encode_sopk(arch_case.getreg_op, 5, kHwregStatusNibble);

    std::unique_ptr<Instruction> mode_inst(decoder->decode(mode_words.data()));
    ASSERT_NE(mode_inst, nullptr);
    EXPECT_EQ(std::string_view(mode_inst->mnemonic()), "s_getreg_b32");
    std::unique_ptr<Instruction> status_inst(decoder->decode(status_words.data()));
    ASSERT_NE(status_inst, nullptr);
    EXPECT_EQ(std::string_view(status_inst->mnemonic()), "s_getreg_b32");

    auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
    ASSERT_NE(wf, nullptr);

    wf->set_mode_raw(amdgpu::Wavefront::FP16_OVFL_BIT);
    wf->set_status_raw(0x12345678u);

    cu->execute_instruction(mode_inst.get(), *wf);
    cu->execute_instruction(status_inst.get(), *wf);

    const uint32_t sb = wf->sgpr_alloc().base;
    EXPECT_EQ(cu->read_sgpr(sb + 4), 1u);
    EXPECT_EQ(cu->read_sgpr(sb + 5), 0x6u);

    if (!wf->is_halted())
      wf->halt();
  }
}

/// @brief Targeted test for s_setreg_imm32_b32: verifies that STATUS is not
/// writable through shader HWREG writes without modeled privileged semantics.
TEST(HwregTest, SetregImm32StatusIsReadOnly) {
  for (const HwregArchCase &arch_case : kHwregArchCases) {
    SCOPED_TRACE(arch_case.name);
    amdgpu::GpuMemory gpu_mem(std::string("hwreg_status_readonly_mem_") + arch_case.name);
    amdgpu::L2Cache l2(std::string("hwreg_status_readonly_l2_") + arch_case.name);

    amdgpu::ComputeUnitCore::Config cfg{};
    cfg.arch = arch_case.arch;
    cfg.num_wf_slots = 1;
    cfg.sgprs_per_wf = 106;
    cfg.vgprs_per_wf = 256;
    cfg.lds_size_kb = 64;

    auto cu = amdgpu::ComputeUnitCore::create(arch_case.name, cfg, &gpu_mem, &l2);
    ASSERT_NE(cu, nullptr);

    auto decoder = Decoder::create(arch_case.arch);
    ASSERT_NE(decoder, nullptr);

    constexpr uint32_t kHwregStatus32 = encode_hwreg(2);
    constexpr uint32_t kImm32Value = 0xDEADBEEF;
    const std::array<uint32_t, 2> words =
        encode_sopk(arch_case.setreg_imm_op, 0, kHwregStatus32, kImm32Value);

    std::unique_ptr<Instruction> inst(decoder->decode(words.data()));
    ASSERT_NE(inst, nullptr) << "Failed to decode s_setreg_imm32_b32";
    EXPECT_EQ(std::string_view(inst->mnemonic()).substr(0, 16), "s_setreg_imm32_b");

    auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
    ASSERT_NE(wf, nullptr);

    wf->set_mode_raw(0);
    wf->set_status_raw(0x12345678u);
    cu->execute_instruction(inst.get(), *wf);
    EXPECT_EQ(wf->status_raw(), 0x12345678u);
    EXPECT_EQ(wf->mode_raw(), 0u);

    if (!wf->is_halted())
      wf->halt();
  }
}

TEST(HwregTest, SetregImm32WritesModeFp16Ovfl) {
  for (const HwregArchCase &arch_case : kHwregArchCases) {
    SCOPED_TRACE(arch_case.name);
    amdgpu::GpuMemory gpu_mem(std::string("hwreg_mode_mem_") + arch_case.name);
    amdgpu::L2Cache l2(std::string("hwreg_mode_l2_") + arch_case.name);

    amdgpu::ComputeUnitCore::Config cfg{};
    cfg.arch = arch_case.arch;
    cfg.num_wf_slots = 1;
    cfg.sgprs_per_wf = 106;
    cfg.vgprs_per_wf = 256;
    cfg.lds_size_kb = 64;

    auto cu = amdgpu::ComputeUnitCore::create(arch_case.name, cfg, &gpu_mem, &l2);
    ASSERT_NE(cu, nullptr);

    auto decoder = Decoder::create(arch_case.arch);
    ASSERT_NE(decoder, nullptr);

    constexpr uint32_t kHwregMode32 = encode_hwreg(1);
    constexpr uint32_t kImm32Value = amdgpu::Wavefront::FP16_OVFL_BIT;
    const std::array<uint32_t, 2> words =
        encode_sopk(arch_case.setreg_imm_op, 0, kHwregMode32, kImm32Value);

    std::unique_ptr<Instruction> inst(decoder->decode(words.data()));
    ASSERT_NE(inst, nullptr) << "Failed to decode s_setreg_imm32_b32";

    auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
    ASSERT_NE(wf, nullptr);

    wf->set_mode_raw(0);
    wf->set_status_raw(0x12345678u);
    cu->execute_instruction(inst.get(), *wf);
    EXPECT_EQ(wf->mode_raw(), amdgpu::Wavefront::FP16_OVFL_BIT);
    EXPECT_TRUE(wf->fp16_ovfl());
    EXPECT_EQ(wf->status_raw(), 0x12345678u);

    if (!wf->is_halted())
      wf->halt();
  }
}

TEST(HwregTest, SetregB32WritesModeFp16OvflFromSgpr) {
  for (const HwregArchCase &arch_case : kHwregArchCases) {
    SCOPED_TRACE(arch_case.name);
    amdgpu::GpuMemory gpu_mem(std::string("hwreg_setreg_mode_mem_") + arch_case.name);
    amdgpu::L2Cache l2(std::string("hwreg_setreg_mode_l2_") + arch_case.name);

    amdgpu::ComputeUnitCore::Config cfg{};
    cfg.arch = arch_case.arch;
    cfg.num_wf_slots = 1;
    cfg.sgprs_per_wf = 106;
    cfg.vgprs_per_wf = 256;
    cfg.lds_size_kb = 64;

    auto cu = amdgpu::ComputeUnitCore::create(arch_case.name, cfg, &gpu_mem, &l2);
    ASSERT_NE(cu, nullptr);

    auto decoder = Decoder::create(arch_case.arch);
    ASSERT_NE(decoder, nullptr);

    constexpr uint32_t kHwregMode32 = encode_hwreg(1);
    const std::array<uint32_t, 2> words = encode_sopk(arch_case.setreg_op, 4, kHwregMode32);
    std::unique_ptr<Instruction> inst(decoder->decode(words.data()));
    ASSERT_NE(inst, nullptr) << "Failed to decode s_setreg_b32";
    EXPECT_EQ(std::string_view(inst->mnemonic()), "s_setreg_b32");

    auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
    ASSERT_NE(wf, nullptr);

    const uint32_t sb = wf->sgpr_alloc().base;
    wf->set_mode_raw(0);
    wf->set_status_raw(0x12345678u);
    cu->write_sgpr(sb + 4, amdgpu::Wavefront::FP16_OVFL_BIT);
    cu->execute_instruction(inst.get(), *wf);
    EXPECT_EQ(wf->mode_raw(), amdgpu::Wavefront::FP16_OVFL_BIT);
    EXPECT_TRUE(wf->fp16_ovfl());
    EXPECT_EQ(wf->status_raw(), 0x12345678u);

    if (!wf->is_halted())
      wf->halt();
  }
}

/// @brief Targeted test for s_setreg_imm32_b32 with partial MODE bitfield write.
TEST(HwregTest, SetregImm32PartialBitfield) {
  for (const HwregArchCase &arch_case : kHwregArchCases) {
    SCOPED_TRACE(arch_case.name);
    amdgpu::GpuMemory gpu_mem(std::string("hwreg_partial_mem_") + arch_case.name);
    amdgpu::L2Cache l2(std::string("hwreg_partial_l2_") + arch_case.name);

    amdgpu::ComputeUnitCore::Config cfg{};
    cfg.arch = arch_case.arch;
    cfg.num_wf_slots = 1;
    cfg.sgprs_per_wf = 106;
    cfg.vgprs_per_wf = 256;
    cfg.lds_size_kb = 64;

    auto cu = amdgpu::ComputeUnitCore::create(arch_case.name, cfg, &gpu_mem, &l2);
    ASSERT_NE(cu, nullptr);

    auto decoder = Decoder::create(arch_case.arch);
    ASSERT_NE(decoder, nullptr);

    // Write MODE.FP_DENORM bits[7:4].
    constexpr uint32_t kHwreg = encode_hwreg(1, 4, 4);
    constexpr uint32_t kImm32 = 0xFFFFFFFF;
    const std::array<uint32_t, 2> words = encode_sopk(arch_case.setreg_imm_op, 0, kHwreg, kImm32);

    std::unique_ptr<Instruction> inst(decoder->decode(words.data()));
    ASSERT_NE(inst, nullptr);

    auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
    ASSERT_NE(wf, nullptr);

    wf->set_mode_raw(0xAAAAAAAAu);
    wf->set_status_raw(0x13579BDFu);
    cu->execute_instruction(inst.get(), *wf);
    EXPECT_EQ(wf->mode_raw(), 0xAAAAAAFAu) << "Partial bitfield write to MODE incorrect";
    EXPECT_EQ(wf->status_raw(), 0x13579BDFu);

    if (!wf->is_halted())
      wf->halt();
  }
}

} // namespace