derek.holloway
7502bf64a4
CodeQL / Analyze (actions) (push) Canceled after 0s
CodeQL / Analyze (python) (push) Canceled after 0s
pre-commit / pre-commit (push) Canceled after 0s
Multi-Arch CI / setup (push) Canceled after 0s
Publish build_manylinux_x86_64 images / publish_build_manylinux_x86_64 (push) Canceled after 0s
Setup (Multi-Arch) / setup (push) Canceled after 0s
Multi-Arch CI / Linux::${{ fromJSON(needs.setup.outputs.linux_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubi10 image / publish_no_rocm_image_ubi10 (push) Canceled after 0s
Multi-Arch CI / Windows::${{ fromJSON(needs.setup.outputs.windows_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Multi-Arch CI / Manifest Diff (push) Canceled after 0s
TheRock Manifest Diff Report / Generate Manifest Diff Report (push) Canceled after 0s
Multi-Arch CI / CI Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_ocl_rt images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_rocgdb images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Unit Tests / Unit Tests :: ubuntu-24.04 (push) Canceled after 0s
Unit Tests / Unit Tests :: windows-2022 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_media (push) Canceled after 0s
Unit Tests / Unit Tests Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_openmpi (push) Canceled after 0s
6031 lines
228 KiB
C++
6031 lines
228 KiB
C++
// Copyright (c) 2026 Advanced Micro Devices, Inc.
|
|
// SPDX-License-Identifier: MIT
|
|
|
|
/// @file instruction_execution_harness_test.cpp
|
|
/// @brief Phase E parameterized test: execute safe scalar encodings on every ISA.
|
|
///
|
|
/// For each ISA, iterates auto-generated encodings that are safe to execute on
|
|
/// a zeroed wavefront, decodes them, and calls execute(). Decode failures are
|
|
/// rejected, and UnimplementedInst results must exactly match the explicit
|
|
/// expectation for that ISA.
|
|
|
|
#include "rocjitsu/base/rj_compiler.h"
|
|
#include "rocjitsu/code/rj_code.h"
|
|
#include "rocjitsu/isa/arch/amdgpu/cdna1/opcodes.h"
|
|
#include "rocjitsu/isa/arch/amdgpu/cdna2/opcodes.h"
|
|
#include "rocjitsu/isa/arch/amdgpu/cdna3/opcodes.h"
|
|
#include "rocjitsu/isa/arch/amdgpu/cdna4/opcodes.h"
|
|
#include "rocjitsu/isa/arch/amdgpu/gfx1250/isa.h"
|
|
#include "rocjitsu/isa/arch/amdgpu/gfx1250/opcodes.h"
|
|
#include "rocjitsu/isa/arch/amdgpu/rdna1/opcodes.h"
|
|
#include "rocjitsu/isa/arch/amdgpu/rdna2/opcodes.h"
|
|
#include "rocjitsu/isa/arch/amdgpu/rdna3/opcodes.h"
|
|
#include "rocjitsu/isa/arch/amdgpu/rdna3_5/opcodes.h"
|
|
#include "rocjitsu/isa/arch/amdgpu/rdna4/opcodes.h"
|
|
#include "rocjitsu/isa/arch/amdgpu/rdna4/vop3.h"
|
|
#include "rocjitsu/isa/arch/amdgpu/shared/dpp_sdwa_ops.h"
|
|
#include "rocjitsu/isa/arch/amdgpu/shared/mma_exec.h"
|
|
#include "rocjitsu/isa/decoder.h"
|
|
#include "rocjitsu/isa/instruction.h"
|
|
#include "rocjitsu/vm/amdgpu/compute_unit.h"
|
|
#include "rocjitsu/vm/amdgpu/gpu_memory.h"
|
|
#include "rocjitsu/vm/amdgpu/hwreg.h"
|
|
#include "rocjitsu/vm/amdgpu/l2_cache.h"
|
|
#include "rocjitsu/vm/amdgpu/wavefront.h"
|
|
#include "util/data_types.h"
|
|
#include "util/except.h"
|
|
#include "util/simd.h"
|
|
#include "util/simd_test_hooks.h"
|
|
|
|
// Per-ISA test encoding tables (auto-generated by amdisa codegen).
|
|
#include "rocjitsu/isa/arch/amdgpu/cdna4/builders.h"
|
|
#include "rocjitsu/isa/arch/amdgpu/cdna4/opcodes.h"
|
|
|
|
#include "rocjitsu/isa/arch/amdgpu/cdna1/test_encodings.h"
|
|
#include "rocjitsu/isa/arch/amdgpu/cdna2/test_encodings.h"
|
|
#include "rocjitsu/isa/arch/amdgpu/cdna3/test_encodings.h"
|
|
#include "rocjitsu/isa/arch/amdgpu/cdna4/test_encodings.h"
|
|
#include "rocjitsu/isa/arch/amdgpu/gfx1250/test_encodings.h"
|
|
#include "rocjitsu/isa/arch/amdgpu/rdna1/test_encodings.h"
|
|
#include "rocjitsu/isa/arch/amdgpu/rdna2/test_encodings.h"
|
|
#include "rocjitsu/isa/arch/amdgpu/rdna3/test_encodings.h"
|
|
#include "rocjitsu/isa/arch/amdgpu/rdna3_5/test_encodings.h"
|
|
#include "rocjitsu/isa/arch/amdgpu/rdna4/test_encodings.h"
|
|
|
|
#include <gtest/gtest.h>
|
|
|
|
#include <algorithm>
|
|
#include <array>
|
|
#include <bit>
|
|
#include <cmath>
|
|
#include <cstdint>
|
|
#include <exception>
|
|
#include <limits>
|
|
#include <memory>
|
|
#include <span>
|
|
#include <string>
|
|
#include <string_view>
|
|
#include <utility>
|
|
#include <vector>
|
|
|
|
namespace {
|
|
|
|
using namespace rocjitsu;
|
|
|
|
/// @brief Common test encoding entry (matches all ISA test_encodings.h).
|
|
struct TestEncEntry {
|
|
std::string_view mnemonic;
|
|
std::array<uint32_t, 2> words;
|
|
};
|
|
|
|
constexpr uint32_t pack16(uint16_t lo, uint16_t hi) {
|
|
return static_cast<uint32_t>(lo) | (static_cast<uint32_t>(hi) << 16);
|
|
}
|
|
|
|
constexpr uint32_t vgpr_src(uint32_t reg) { return 256u + reg; }
|
|
|
|
constexpr std::array<uint32_t, 2> encode_vop3(uint32_t op, uint32_t vdst, uint32_t src0,
|
|
uint32_t src1, uint32_t src2, uint32_t abs = 0,
|
|
uint32_t opsel = 0, uint32_t clamp = 0,
|
|
uint32_t omod = 0, uint32_t neg = 0) {
|
|
return {vdst | ((abs & 0x7u) << 8) | ((opsel & 0xFu) << 11) | ((clamp & 0x1u) << 15) |
|
|
((op & 0x3FFu) << 16) | (0x35u << 26),
|
|
(src0 & 0x1FFu) | ((src1 & 0x1FFu) << 9) | ((src2 & 0x1FFu) << 18) |
|
|
((omod & 0x3u) << 27) | ((neg & 0x7u) << 29)};
|
|
}
|
|
|
|
constexpr std::array<uint32_t, 2> encode_cdna_vop3(uint32_t op, uint32_t vdst, uint32_t src0,
|
|
uint32_t src1, uint32_t src2,
|
|
uint32_t opsel = 0) {
|
|
return {vdst | ((opsel & 0xFu) << 11) | ((op & 0x3FFu) << 16) | (0x34u << 26),
|
|
(src0 & 0x1FFu) | ((src1 & 0x1FFu) << 9) | ((src2 & 0x1FFu) << 18)};
|
|
}
|
|
|
|
constexpr std::array<uint32_t, 2> encode_vop2(uint32_t op, uint32_t vdst, uint32_t src0,
|
|
uint32_t vsrc1) {
|
|
return {(src0 & 0x1FFu) | ((vsrc1 & 0xFFu) << 9) | ((vdst & 0xFFu) << 17) | ((op & 0x3Fu) << 25),
|
|
0u};
|
|
}
|
|
|
|
constexpr std::array<uint32_t, 2> encode_sop1(uint32_t op, uint32_t sdst, uint32_t ssrc0) {
|
|
return {(ssrc0 & 0xFFu) | ((op & 0xFFu) << 8) | ((sdst & 0x7Fu) << 16) | 0xBE800000u, 0u};
|
|
}
|
|
|
|
constexpr std::array<uint32_t, 2> encode_sop2(uint32_t op, uint32_t sdst, uint32_t ssrc0,
|
|
uint32_t ssrc1) {
|
|
return {(ssrc0 & 0xFFu) | ((ssrc1 & 0xFFu) << 8) | ((sdst & 0x7Fu) << 16) | ((op & 0x7Fu) << 23) |
|
|
(0x2u << 30),
|
|
0u};
|
|
}
|
|
|
|
constexpr uint32_t encode_hwreg(uint32_t id, uint32_t offset = 0, uint32_t size = 32) {
|
|
return (id & 0x3Fu) | ((offset & 0x1Fu) << 6) | (((size - 1u) & 0x1Fu) << 11);
|
|
}
|
|
|
|
constexpr std::array<uint32_t, 2> encode_sopk(uint32_t op, uint32_t sdst, uint32_t simm16,
|
|
uint32_t literal = 0) {
|
|
return {(0xBu << 28) | ((op & 0x1Fu) << 23) | ((sdst & 0x7Fu) << 16) | (simm16 & 0xFFFFu),
|
|
literal};
|
|
}
|
|
|
|
struct HwregArchCase {
|
|
rj_code_arch_t arch;
|
|
const char *name;
|
|
uint32_t getreg_op;
|
|
uint32_t setreg_op;
|
|
uint32_t setreg_imm_op;
|
|
};
|
|
|
|
constexpr HwregArchCase kHwregArchCases[] = {
|
|
{ROCJITSU_CODE_ARCH_CDNA1, "cdna1", cdna1::kSGetregB32Sopk, cdna1::kSSetregB32Sopk,
|
|
cdna1::kSSetregImm32B32Sopk},
|
|
{ROCJITSU_CODE_ARCH_CDNA2, "cdna2", cdna2::kSGetregB32Sopk, cdna2::kSSetregB32Sopk,
|
|
cdna2::kSSetregImm32B32Sopk},
|
|
{ROCJITSU_CODE_ARCH_CDNA3, "cdna3", cdna3::kSGetregB32Sopk, cdna3::kSSetregB32Sopk,
|
|
cdna3::kSSetregImm32B32Sopk},
|
|
{ROCJITSU_CODE_ARCH_CDNA4, "cdna4", cdna4::kSGetregB32Sopk, cdna4::kSSetregB32Sopk,
|
|
cdna4::kSSetregImm32B32Sopk},
|
|
{ROCJITSU_CODE_ARCH_RDNA1, "rdna1", rdna1::kSGetregB32Sopk, rdna1::kSSetregB32Sopk,
|
|
rdna1::kSSetregImm32B32Sopk},
|
|
{ROCJITSU_CODE_ARCH_RDNA2, "rdna2", rdna2::kSGetregB32Sopk, rdna2::kSSetregB32Sopk,
|
|
rdna2::kSSetregImm32B32Sopk},
|
|
{ROCJITSU_CODE_ARCH_RDNA3, "rdna3", rdna3::kSGetregB32Sopk, rdna3::kSSetregB32Sopk,
|
|
rdna3::kSSetregImm32B32Sopk},
|
|
{ROCJITSU_CODE_ARCH_RDNA3_5, "rdna3_5", rdna3_5::kSGetregB32Sopk, rdna3_5::kSSetregB32Sopk,
|
|
rdna3_5::kSSetregImm32B32Sopk},
|
|
{ROCJITSU_CODE_ARCH_RDNA4, "rdna4", rdna4::kSGetregB32Sopk, rdna4::kSSetregB32Sopk,
|
|
rdna4::kSSetregImm32B32Sopk},
|
|
{ROCJITSU_CODE_ARCH_GFX1250, "gfx1250", gfx1250::kSGetregB32Sopk, gfx1250::kSSetregB32Sopk,
|
|
gfx1250::kSSetregImm32B32Sopk},
|
|
};
|
|
|
|
struct ForceScalarGuard {
|
|
explicit ForceScalarGuard(bool force_scalar) : old_force_scalar(util::force_scalar()) {
|
|
util::set_force_scalar_for_testing(force_scalar);
|
|
}
|
|
~ForceScalarGuard() { util::set_force_scalar_for_testing(old_force_scalar); }
|
|
|
|
bool old_force_scalar;
|
|
};
|
|
|
|
class Gfx1250MemoryTestCu
|
|
: public amdgpu::IsaExecComputeUnit<simdojo::ExecMode::FUNCTIONAL, gfx1250::Isa> {
|
|
public:
|
|
using Base = amdgpu::IsaExecComputeUnit<simdojo::ExecMode::FUNCTIONAL, gfx1250::Isa>;
|
|
|
|
Gfx1250MemoryTestCu(std::string name, const amdgpu::ComputeUnitCore::Config &config,
|
|
amdgpu::GpuMemory *memory, amdgpu::L2Cache *l2)
|
|
: Base(std::move(name), config, memory, l2) {
|
|
if (l2)
|
|
l2->set_backing_memory(memory);
|
|
set_memory(memory);
|
|
set_l2(l2);
|
|
}
|
|
|
|
void execute_and_route(Instruction *inst, amdgpu::Wavefront &wf) {
|
|
execute_instruction(inst, wf);
|
|
if (inst->is_memory_op())
|
|
route_memory_inst(inst, wf);
|
|
else
|
|
delete inst;
|
|
}
|
|
};
|
|
|
|
/// @brief Check if a mnemonic should be skipped in the execution harness.
|
|
///
|
|
/// Memory instructions require valid addresses and will crash on zeroed state.
|
|
/// Control flow instructions (branch, endpgm, barrier, waitcnt, trap) may
|
|
/// halt, loop, or enter wait states that hang the test.
|
|
inline bool should_skip_inst(std::string_view mn) {
|
|
// Only execute scalar ALU instructions (s_* that don't touch memory or
|
|
// control flow). Vector instructions may crash because the synthesized
|
|
// encoding has src0=0 which maps to SGPR[0] for scalar source, but in
|
|
// VOP encodings src0=0 is VGPR[0] and VGPR reading with zeroed register
|
|
// allocation can segfault.
|
|
//
|
|
// Allowlist approach: only s_* instructions that are pure ALU.
|
|
if (mn.empty() || mn[0] != 's' || mn[1] != '_')
|
|
return true; // Skip all non-scalar instructions.
|
|
static constexpr std::string_view SKIP_PREFIXES[] = {
|
|
// Memory instructions
|
|
"s_load_",
|
|
"s_store_",
|
|
"s_buffer_",
|
|
"s_dcache_",
|
|
"s_atomic_",
|
|
// Control flow that halts or waits
|
|
"s_endpgm",
|
|
"s_branch",
|
|
"s_cbranch",
|
|
"s_setpc",
|
|
"s_swappc",
|
|
"s_call",
|
|
"s_waitcnt",
|
|
"s_wait_",
|
|
"s_barrier",
|
|
"s_trap",
|
|
"s_sleep",
|
|
"s_sethalt",
|
|
"s_sendmsg",
|
|
"s_sendmsghalt",
|
|
"s_icache_inv",
|
|
"s_nop",
|
|
"s_getpc",
|
|
"s_getreg",
|
|
"s_setreg",
|
|
"s_rfe",
|
|
};
|
|
for (auto p : SKIP_PREFIXES)
|
|
if (mn.substr(0, p.size()) == p)
|
|
return true;
|
|
return false;
|
|
}
|
|
|
|
constexpr std::array<std::string_view, 1> EXPECTED_CDNA_UNIMPLEMENTED = {"s_setvskip"};
|
|
constexpr std::array<std::string_view, 3> EXPECTED_RDNA1_UNIMPLEMENTED = {
|
|
"s_subvector_loop_begin", "s_subvector_loop_end", "s_get_waveid_in_workgroup"};
|
|
constexpr std::array<std::string_view, 2> EXPECTED_RDNA2_UNIMPLEMENTED = {"s_subvector_loop_begin",
|
|
"s_subvector_loop_end"};
|
|
constexpr std::array<std::string_view, 0> EXPECTED_NO_UNIMPLEMENTED = {};
|
|
|
|
struct HarnessExpectation {
|
|
std::string_view arch_name;
|
|
std::span<const std::string_view> unimplemented;
|
|
};
|
|
|
|
constexpr HarnessExpectation HARNESS_EXPECTATIONS[] = {
|
|
{"cdna1", EXPECTED_CDNA_UNIMPLEMENTED}, {"cdna2", EXPECTED_CDNA_UNIMPLEMENTED},
|
|
{"cdna3", EXPECTED_CDNA_UNIMPLEMENTED}, {"cdna4", EXPECTED_CDNA_UNIMPLEMENTED},
|
|
{"rdna1", EXPECTED_RDNA1_UNIMPLEMENTED}, {"rdna2", EXPECTED_RDNA2_UNIMPLEMENTED},
|
|
{"rdna3", EXPECTED_NO_UNIMPLEMENTED}, {"rdna3_5", EXPECTED_NO_UNIMPLEMENTED},
|
|
{"rdna4", EXPECTED_NO_UNIMPLEMENTED}, {"gfx1250", EXPECTED_NO_UNIMPLEMENTED},
|
|
};
|
|
|
|
const HarnessExpectation *harness_expectation(std::string_view arch_name) {
|
|
for (const auto &expectation : HARNESS_EXPECTATIONS)
|
|
if (expectation.arch_name == arch_name)
|
|
return ℰ
|
|
return nullptr;
|
|
}
|
|
|
|
/// @brief Helper to run all test encodings for a given ISA.
|
|
void run_execution_harness(rj_code_arch_t arch, std::string_view arch_name,
|
|
const TestEncEntry *encodings, size_t num_encodings) {
|
|
// Set up minimal CU with zeroed memory.
|
|
amdgpu::GpuMemory gpu_mem(std::string(arch_name) + "_mem");
|
|
amdgpu::L2Cache l2(std::string(arch_name) + "_l2");
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = arch;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106; // max across all ISAs
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
auto cu = amdgpu::ComputeUnitCore::create(std::string(arch_name), cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr) << "Failed to create CU for " << arch_name;
|
|
|
|
auto decoder = Decoder::create(arch);
|
|
ASSERT_NE(decoder, nullptr) << "Failed to create decoder for " << arch_name;
|
|
|
|
size_t total = num_encodings;
|
|
size_t decoded = 0;
|
|
size_t executed = 0;
|
|
size_t skipped = 0;
|
|
std::vector<std::string_view> unimpl_list;
|
|
std::vector<std::string> decode_fail_list;
|
|
std::vector<std::string> mnemonic_mismatch_list;
|
|
std::vector<std::string> execution_fail_list;
|
|
|
|
for (size_t i = 0; i < total; ++i) {
|
|
const auto &te = encodings[i];
|
|
|
|
// Skip memory instructions — they require valid addresses and will
|
|
// crash or hang when executed on zeroed wavefront/memory state.
|
|
if (should_skip_inst(te.mnemonic)) {
|
|
++skipped;
|
|
continue;
|
|
}
|
|
|
|
// Decode the instruction.
|
|
std::unique_ptr<Instruction> inst;
|
|
try {
|
|
inst.reset(decoder->decode(te.words.data()));
|
|
} catch (const util::InvalidInst &) {
|
|
decode_fail_list.emplace_back(te.mnemonic);
|
|
continue;
|
|
} catch (...) {
|
|
decode_fail_list.emplace_back(te.mnemonic);
|
|
continue;
|
|
}
|
|
if (!inst) {
|
|
decode_fail_list.emplace_back(te.mnemonic);
|
|
continue;
|
|
}
|
|
|
|
const std::string_view decoded_mnemonic = inst->mnemonic();
|
|
if (decoded_mnemonic != te.mnemonic) {
|
|
mnemonic_mismatch_list.emplace_back(
|
|
std::string(te.mnemonic).append(" decoded as ").append(decoded_mnemonic));
|
|
continue;
|
|
}
|
|
++decoded;
|
|
|
|
// Dispatch a fresh wavefront for each instruction to avoid state leaks. The
|
|
// previous iteration halted its wave (freeing the slot), so this always
|
|
// succeeds on the single-slot scratch CU.
|
|
amdgpu::Wavefront *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr) << "Failed to dispatch WF for " << te.mnemonic;
|
|
|
|
// Execute and catch UnimplementedInst.
|
|
try {
|
|
cu->execute_instruction(inst.get(), *wf);
|
|
++executed;
|
|
} catch (const util::UnimplementedInst &) {
|
|
unimpl_list.emplace_back(te.mnemonic);
|
|
} catch (const std::exception &error) {
|
|
execution_fail_list.emplace_back(std::string(te.mnemonic).append(": ").append(error.what()));
|
|
} catch (...) {
|
|
execution_fail_list.emplace_back(te.mnemonic);
|
|
}
|
|
|
|
// Reset the wavefront for the next instruction.
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
|
|
size_t testable = total - skipped;
|
|
std::printf("\n %.*s: %zu/%zu testable encodings executed, %zu unimplemented, "
|
|
"%zu decode fail, %zu execution fail, %zu skipped\n",
|
|
static_cast<int>(arch_name.size()), arch_name.data(), executed, testable,
|
|
unimpl_list.size(), decode_fail_list.size(), execution_fail_list.size(), skipped);
|
|
|
|
if (!unimpl_list.empty()) {
|
|
std::printf(" Unimplemented (%zu):", unimpl_list.size());
|
|
for (size_t i = 0; i < std::min(unimpl_list.size(), size_t{20}); ++i) {
|
|
const auto mnemonic = unimpl_list[i];
|
|
std::printf(" %.*s", static_cast<int>(mnemonic.size()), mnemonic.data());
|
|
}
|
|
if (unimpl_list.size() > 20)
|
|
std::printf(" ... +%zu more", unimpl_list.size() - 20);
|
|
std::printf("\n");
|
|
}
|
|
|
|
// Every encoding admitted by should_skip_inst must decode. Known
|
|
// UnimplementedInst results are named explicitly so failures identify the
|
|
// exact instruction that regressed instead of only crossing a numeric
|
|
// coverage threshold.
|
|
EXPECT_GT(decoded, 0u) << "No instructions decoded for " << arch_name;
|
|
EXPECT_EQ(decode_fail_list.size(), 0u) << arch_name << " gained decode failures";
|
|
EXPECT_TRUE(mnemonic_mismatch_list.empty())
|
|
<< arch_name << " encodings decoded as a different instruction: "
|
|
<< ::testing::PrintToString(mnemonic_mismatch_list);
|
|
EXPECT_EQ(execution_fail_list.size(), 0u)
|
|
<< arch_name << " failed to execute implemented instructions";
|
|
|
|
const auto *expectation = harness_expectation(arch_name);
|
|
ASSERT_NE(expectation, nullptr) << "Missing harness expectation for " << arch_name;
|
|
std::sort(unimpl_list.begin(), unimpl_list.end());
|
|
std::vector<std::string_view> expected(expectation->unimplemented.begin(),
|
|
expectation->unimplemented.end());
|
|
std::sort(expected.begin(), expected.end());
|
|
EXPECT_EQ(unimpl_list, expected) << arch_name << " unimplemented instruction set changed";
|
|
}
|
|
|
|
// --- Parameterized tests per ISA ---
|
|
|
|
// Helper macro: cast the ISA-specific TestEncoding array to the common layout.
|
|
// The struct layout (string_view + array<uint32_t,2>) is identical across all ISAs.
|
|
#define RUN_HARNESS(ISA_NS, ARCH_ENUM, ARCH_STR) \
|
|
run_execution_harness(ARCH_ENUM, ARCH_STR, \
|
|
reinterpret_cast<const TestEncEntry *>(ISA_NS::test_data::ENCODINGS), \
|
|
ISA_NS::test_data::NUM_ENCODINGS)
|
|
|
|
TEST(InstructionExecutionHarness, Cdna1) { RUN_HARNESS(cdna1, ROCJITSU_CODE_ARCH_CDNA1, "cdna1"); }
|
|
TEST(InstructionExecutionHarness, Cdna2) { RUN_HARNESS(cdna2, ROCJITSU_CODE_ARCH_CDNA2, "cdna2"); }
|
|
TEST(InstructionExecutionHarness, Cdna3) { RUN_HARNESS(cdna3, ROCJITSU_CODE_ARCH_CDNA3, "cdna3"); }
|
|
TEST(InstructionExecutionHarness, Cdna4) { RUN_HARNESS(cdna4, ROCJITSU_CODE_ARCH_CDNA4, "cdna4"); }
|
|
TEST(InstructionExecutionHarness, Rdna1) { RUN_HARNESS(rdna1, ROCJITSU_CODE_ARCH_RDNA1, "rdna1"); }
|
|
TEST(InstructionExecutionHarness, Rdna2) { RUN_HARNESS(rdna2, ROCJITSU_CODE_ARCH_RDNA2, "rdna2"); }
|
|
TEST(InstructionExecutionHarness, Rdna3) { RUN_HARNESS(rdna3, ROCJITSU_CODE_ARCH_RDNA3, "rdna3"); }
|
|
TEST(InstructionExecutionHarness, Rdna3_5) {
|
|
RUN_HARNESS(rdna3_5, ROCJITSU_CODE_ARCH_RDNA3_5, "rdna3_5");
|
|
}
|
|
TEST(InstructionExecutionHarness, Rdna4) { RUN_HARNESS(rdna4, ROCJITSU_CODE_ARCH_RDNA4, "rdna4"); }
|
|
|
|
TEST(Rdna4ExecMaskTest, Wave32ExecHiRemainsAvailableAsScalarScratch) {
|
|
amdgpu::GpuMemory gpu_mem("rdna4_exec_hi_mem");
|
|
amdgpu::L2Cache l2("rdna4_exec_hi_l2");
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = ROCJITSU_CODE_ARCH_RDNA4;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 128;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
auto cu = amdgpu::ComputeUnitCore::create("rdna4_exec_hi", cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
ASSERT_EQ(wf->wf_size(), 32u);
|
|
|
|
auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_RDNA4);
|
|
ASSERT_NE(decoder, nullptr);
|
|
cu->write_sgpr(wf->sgpr_alloc().base, 0xffffffffu);
|
|
wf->set_exec(0xffffffffu);
|
|
|
|
constexpr uint32_t kInline16 = 128u + 16u;
|
|
const std::array<std::pair<uint32_t, std::string_view>, 3> sequence{{
|
|
{encode_sop2(/*s_lshr_b32=*/10, /*exec_lo=*/126, /*s0=*/0, kInline16)[0], "s_lshr_b32"},
|
|
{encode_sop2(/*s_lshl_b32=*/8, /*exec_hi=*/127, /*s0=*/0, kInline16)[0], "s_lshl_b32"},
|
|
{encode_sop2(/*s_or_b32=*/24, /*exec_lo=*/126, /*exec_lo=*/126, /*exec_hi=*/127)[0],
|
|
"s_or_b32"},
|
|
}};
|
|
|
|
for (size_t i = 0; i < sequence.size(); ++i) {
|
|
const uint32_t words[] = {sequence[i].first, 0};
|
|
std::unique_ptr<Instruction> inst(decoder->decode(words));
|
|
ASSERT_NE(inst, nullptr);
|
|
ASSERT_EQ(std::string_view(inst->mnemonic()), sequence[i].second);
|
|
cu->execute_instruction(inst.get(), *wf);
|
|
if (i == 1) {
|
|
EXPECT_EQ(wf->exec(), 0x0000ffffu);
|
|
EXPECT_EQ(wf->exec_raw(), 0xffff0000'0000ffffULL);
|
|
}
|
|
}
|
|
|
|
EXPECT_EQ(wf->exec(), 0xffffffffu);
|
|
EXPECT_EQ(wf->exec_raw(), 0xffff0000'ffffffffULL);
|
|
|
|
constexpr uint64_t kRawExec = 0x2468ace0'13579bdfULL;
|
|
cu->write_sgpr(wf->sgpr_alloc().base, static_cast<uint32_t>(kRawExec));
|
|
cu->write_sgpr(wf->sgpr_alloc().base + 1, static_cast<uint32_t>(kRawExec >> 32));
|
|
wf->set_exec_raw(0xdeadbeef'ffffffffULL);
|
|
|
|
const auto mov_b64 = encode_sop1(/*s_mov_b64=*/1, /*exec=*/126, /*s0=*/0);
|
|
std::unique_ptr<Instruction> inst(decoder->decode(mov_b64.data()));
|
|
ASSERT_NE(inst, nullptr);
|
|
ASSERT_EQ(std::string_view(inst->mnemonic()), "s_mov_b64");
|
|
cu->execute_instruction(inst.get(), *wf);
|
|
|
|
EXPECT_EQ(wf->exec_raw(), kRawExec);
|
|
EXPECT_EQ(wf->exec(), static_cast<uint32_t>(kRawExec));
|
|
}
|
|
TEST(InstructionExecutionHarness, Gfx1250) {
|
|
RUN_HARNESS(gfx1250, ROCJITSU_CODE_ARCH_GFX1250, "gfx1250");
|
|
}
|
|
|
|
#undef RUN_HARNESS
|
|
|
|
TEST(Gfx1250MemoryExecutionHarness, ExecutesRepresentativeValidAddressStores) {
|
|
amdgpu::GpuMemory gpu_mem("gfx1250_memory_harness_mem");
|
|
amdgpu::L2Cache l2("gfx1250_memory_harness_l2");
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = ROCJITSU_CODE_ARCH_GFX1250;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
Gfx1250MemoryTestCu cu("gfx1250", cfg, &gpu_mem, &l2);
|
|
|
|
auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
|
|
ASSERT_NE(decoder, nullptr);
|
|
|
|
auto *wf = cu.dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
wf->set_exec(0x3u);
|
|
|
|
constexpr uint64_t kGlobalAddr = 0x2000;
|
|
const uint32_t sb = wf->sgpr_alloc().base;
|
|
const uint32_t vb = wf->vgpr_alloc().base;
|
|
|
|
cu.write_sgpr(sb + 4, static_cast<uint32_t>(kGlobalAddr));
|
|
cu.write_sgpr(sb + 5, static_cast<uint32_t>(kGlobalAddr >> 32));
|
|
for (uint32_t lane = 0; lane < 2; ++lane) {
|
|
gpu_mem.write32(kGlobalAddr + lane * sizeof(uint32_t), 0u);
|
|
cu.write_vgpr(vb + 0, lane, lane * sizeof(uint32_t));
|
|
cu.write_vgpr(vb + 1, lane, 0x11000000u + lane);
|
|
}
|
|
|
|
// global_store_b32 v0, v1, s[4:5]
|
|
const uint32_t global_store_words[] = {0xEE068004u, 0x00800000u, 0x00000000u};
|
|
std::unique_ptr<Instruction> global_store(decoder->decode(global_store_words));
|
|
ASSERT_NE(global_store, nullptr);
|
|
ASSERT_EQ(std::string_view(global_store->mnemonic()), "global_store_b32");
|
|
cu.execute_and_route(global_store.release(), *wf);
|
|
cu.flush_all();
|
|
|
|
EXPECT_EQ(gpu_mem.read32(kGlobalAddr), 0x11000000u);
|
|
EXPECT_EQ(gpu_mem.read32(kGlobalAddr + sizeof(uint32_t)), 0x11000001u);
|
|
|
|
constexpr uint64_t kBufferAddr = 0x3000;
|
|
cu.write_sgpr(sb + 4, static_cast<uint32_t>(kBufferAddr));
|
|
cu.write_sgpr(sb + 5, static_cast<uint32_t>(kBufferAddr >> 32));
|
|
cu.write_sgpr(sb + 6, 0x1000u);
|
|
cu.write_sgpr(sb + 7, 0u);
|
|
wf->set_m0(16u);
|
|
for (uint32_t lane = 0; lane < 2; ++lane) {
|
|
gpu_mem.write32(kBufferAddr + lane * 32, 0u);
|
|
gpu_mem.write32(kBufferAddr + 16 + lane * 32, 0u);
|
|
cu.write_vgpr(vb + 0, lane, 0x22000000u + lane);
|
|
cu.write_vgpr(vb + 5, lane, lane * 32);
|
|
}
|
|
|
|
// buffer_store_b32 v0, v5, s[4:7], m0 offen
|
|
const uint32_t buffer_store_words[] = {0xC406807Du, 0x40800800u, 0x00000005u};
|
|
std::unique_ptr<Instruction> buffer_store(decoder->decode(buffer_store_words));
|
|
ASSERT_NE(buffer_store, nullptr);
|
|
ASSERT_EQ(std::string_view(buffer_store->mnemonic()), "buffer_store_b32");
|
|
cu.execute_and_route(buffer_store.release(), *wf);
|
|
cu.flush_all();
|
|
|
|
EXPECT_EQ(gpu_mem.read32(kBufferAddr), 0u);
|
|
EXPECT_EQ(gpu_mem.read32(kBufferAddr + 16), 0x22000000u);
|
|
EXPECT_EQ(gpu_mem.read32(kBufferAddr + 32), 0u);
|
|
EXPECT_EQ(gpu_mem.read32(kBufferAddr + 48), 0x22000001u);
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
|
|
TEST(Rdna4ScalarSccTest, AddSubCoI32UseSignedOverflow) {
|
|
amdgpu::GpuMemory gpu_mem("rdna4_scc_i32_overflow_mem");
|
|
amdgpu::L2Cache l2("rdna4_scc_i32_overflow_l2");
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = ROCJITSU_CODE_ARCH_RDNA4;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
auto cu = amdgpu::ComputeUnitCore::create("rdna4", cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
|
|
auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_RDNA4);
|
|
ASSERT_NE(decoder, nullptr);
|
|
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
|
|
const uint32_t sb = wf->sgpr_alloc().base;
|
|
auto run = [&](const std::array<uint32_t, 2> &words, std::string_view mnemonic, uint32_t lhs,
|
|
uint32_t rhs, uint32_t expected_result, bool expected_scc) {
|
|
std::unique_ptr<Instruction> inst(decoder->decode(words.data()));
|
|
ASSERT_NE(inst, nullptr);
|
|
ASSERT_EQ(std::string_view(inst->mnemonic()), mnemonic);
|
|
|
|
cu->write_sgpr(sb + 0, lhs);
|
|
cu->write_sgpr(sb + 1, rhs);
|
|
wf->write_scc(!expected_scc);
|
|
cu->execute_instruction(inst.get(), *wf);
|
|
|
|
EXPECT_EQ(cu->read_sgpr(sb + 2), expected_result) << mnemonic;
|
|
EXPECT_EQ(wf->read_scc(), expected_scc) << mnemonic;
|
|
};
|
|
|
|
const auto add = encode_sop2(/*op=*/2, /*sdst=*/2, /*ssrc0=*/0, /*ssrc1=*/1);
|
|
const auto sub = encode_sop2(/*op=*/3, /*sdst=*/2, /*ssrc0=*/0, /*ssrc1=*/1);
|
|
|
|
run(add, "s_add_co_i32", 0x7FFFFFFFu, 1u, 0x80000000u, true);
|
|
run(add, "s_add_co_i32", 0x7FFFFFFFu, 0u, 0x7FFFFFFFu, false);
|
|
run(sub, "s_sub_co_i32", 0x80000000u, 1u, 0x7FFFFFFFu, true);
|
|
run(sub, "s_sub_co_i32", 5u, 3u, 2u, false);
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
|
|
struct ScalarCvtSccCase {
|
|
uint32_t op;
|
|
const char *mnemonic;
|
|
uint32_t src;
|
|
uint32_t expected;
|
|
};
|
|
|
|
void run_scalar_cvt_preserves_scc(rj_code_arch_t arch, std::string_view arch_name,
|
|
const ScalarCvtSccCase *cases, size_t num_cases) {
|
|
amdgpu::GpuMemory gpu_mem(std::string(arch_name) + "_s_cvt_scc_mem");
|
|
amdgpu::L2Cache l2(std::string(arch_name) + "_s_cvt_scc_l2");
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = arch;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
auto cu = amdgpu::ComputeUnitCore::create(std::string(arch_name), cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr) << arch_name;
|
|
|
|
auto decoder = Decoder::create(arch);
|
|
ASSERT_NE(decoder, nullptr) << arch_name;
|
|
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr) << arch_name;
|
|
|
|
const uint32_t sb = wf->sgpr_alloc().base;
|
|
|
|
for (size_t i = 0; i < num_cases; ++i) {
|
|
const auto &tc = cases[i];
|
|
const auto words = encode_sop1(tc.op, /*sdst=*/1, /*ssrc0=*/0);
|
|
for (bool initial_scc : std::array<bool, 2>{false, true}) {
|
|
std::unique_ptr<Instruction> inst(decoder->decode(words.data()));
|
|
ASSERT_NE(inst, nullptr);
|
|
ASSERT_EQ(std::string_view(inst->mnemonic()), tc.mnemonic) << arch_name;
|
|
|
|
cu->write_sgpr(sb + 0, tc.src);
|
|
cu->write_sgpr(sb + 1, 0u);
|
|
wf->write_scc(initial_scc);
|
|
cu->execute_instruction(inst.get(), *wf);
|
|
|
|
EXPECT_EQ(cu->read_sgpr(sb + 1), tc.expected) << arch_name << " " << tc.mnemonic;
|
|
EXPECT_EQ(wf->read_scc(), initial_scc)
|
|
<< arch_name << " " << tc.mnemonic << " initial_scc=" << initial_scc;
|
|
}
|
|
}
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
|
|
TEST(ScalarSccTest, ScalarCvtPreservesScc) {
|
|
const uint32_t one_f32 = std::bit_cast<uint32_t>(1.0f);
|
|
const uint32_t one_f16 = util::f32_to_f16(1.0f);
|
|
const uint32_t qnan_f32 = 0x7FC00000u;
|
|
const uint32_t i32_overflow_f32 = std::bit_cast<uint32_t>(2147483648.0f);
|
|
const uint32_t below_i32_min_f32 = std::bit_cast<uint32_t>(-2147483904.0f);
|
|
const uint32_t u32_overflow_f32 = std::bit_cast<uint32_t>(4294967296.0f);
|
|
|
|
const std::array<ScalarCvtSccCase, 13> cases{{
|
|
{0x64u, "s_cvt_f32_i32", 1u, one_f32},
|
|
{0x65u, "s_cvt_f32_u32", 1u, one_f32},
|
|
{0x66u, "s_cvt_i32_f32", one_f32, 1u},
|
|
{0x66u, "s_cvt_i32_f32", qnan_f32, 0u},
|
|
{0x66u, "s_cvt_i32_f32", i32_overflow_f32, 0x7FFFFFFFu},
|
|
{0x66u, "s_cvt_i32_f32", below_i32_min_f32, 0x80000000u},
|
|
{0x67u, "s_cvt_u32_f32", one_f32, 1u},
|
|
{0x67u, "s_cvt_u32_f32", qnan_f32, 0u},
|
|
{0x67u, "s_cvt_u32_f32", std::bit_cast<uint32_t>(-1.0f), 0u},
|
|
{0x67u, "s_cvt_u32_f32", u32_overflow_f32, 0xFFFFFFFFu},
|
|
{0x68u, "s_cvt_f16_f32", one_f32, one_f16},
|
|
{0x69u, "s_cvt_f32_f16", one_f16, one_f32},
|
|
{0x6Au, "s_cvt_hi_f32_f16", one_f16 << 16, one_f32},
|
|
}};
|
|
|
|
run_scalar_cvt_preserves_scc(ROCJITSU_CODE_ARCH_RDNA3_5, "rdna3_5", cases.data(), cases.size());
|
|
run_scalar_cvt_preserves_scc(ROCJITSU_CODE_ARCH_RDNA4, "rdna4", cases.data(), cases.size());
|
|
run_scalar_cvt_preserves_scc(ROCJITSU_CODE_ARCH_GFX1250, "gfx1250", cases.data(), cases.size());
|
|
}
|
|
|
|
TEST(Cdna4Vop3Test, CmpClassF16WritesWave64UpperMaskDword) {
|
|
struct Case {
|
|
uint32_t lane;
|
|
uint16_t value;
|
|
uint16_t mask;
|
|
bool expected;
|
|
};
|
|
|
|
constexpr Case cases[] = {
|
|
{0, 0x0000u, 0x0040u, true}, // +0
|
|
{5, 0x7C01u, 0x0002u, false}, // sNaN is not qNaN
|
|
{32, 0x0000u, 0x0040u, true}, // upper SGPR dword low bit
|
|
{45, 0x7E00u, 0x0001u, false}, // qNaN is not sNaN
|
|
{63, 0x7C00u, 0x0200u, true}, // upper SGPR dword high bit
|
|
};
|
|
|
|
for (bool force_scalar : {false, true}) {
|
|
ForceScalarGuard guard(force_scalar);
|
|
amdgpu::GpuMemory gpu_mem(force_scalar ? "cdna4_true16_class64_scalar_mem"
|
|
: "cdna4_true16_class64_simd_mem");
|
|
amdgpu::L2Cache l2(force_scalar ? "cdna4_true16_class64_scalar_l2"
|
|
: "cdna4_true16_class64_simd_l2");
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = ROCJITSU_CODE_ARCH_CDNA4;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
auto cu = amdgpu::ComputeUnitCore::create("cdna4", cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
|
|
auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_CDNA4);
|
|
ASSERT_NE(decoder, nullptr);
|
|
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
ASSERT_EQ(wf->wf_size(), 64u);
|
|
|
|
const uint32_t vb = wf->vgpr_alloc().base;
|
|
const uint32_t sb = wf->sgpr_alloc().base;
|
|
auto &src0 = cu->raw_vgpr_reg<64>(vb + 0);
|
|
auto &src1 = cu->raw_vgpr_reg<64>(vb + 1);
|
|
|
|
uint64_t active_mask = 0;
|
|
uint64_t expected_mask = 0;
|
|
for (const auto &test : cases) {
|
|
active_mask |= (1ULL << test.lane);
|
|
if (test.expected)
|
|
expected_mask |= (1ULL << test.lane);
|
|
src0[test.lane] = pack16(test.value, 0xCAFEu);
|
|
src1[test.lane] = pack16(test.mask, 0x1234u);
|
|
}
|
|
wf->set_exec(active_mask);
|
|
cu->write_sgpr(sb + 0, 0xFFFFFFFFu);
|
|
cu->write_sgpr(sb + 1, 0xDEADBEEFu);
|
|
|
|
// CDNA4 v_cmp_class_f16 s0, v0, v1.
|
|
const uint32_t words[] = {0xD0140000U, 0x00020300U};
|
|
std::unique_ptr<Instruction> inst(decoder->decode(words));
|
|
ASSERT_NE(inst, nullptr);
|
|
ASSERT_EQ(std::string_view(inst->mnemonic()), "v_cmp_class_f16");
|
|
cu->execute_instruction(inst.get(), *wf);
|
|
|
|
EXPECT_EQ(cu->read_sgpr(sb + 0), static_cast<uint32_t>(expected_mask));
|
|
EXPECT_EQ(cu->read_sgpr(sb + 1), static_cast<uint32_t>(expected_mask >> 32));
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
}
|
|
|
|
TEST(CdnaVop3True16Test, B16I16U16OpsUseOpSelAndCdnaDestinationPolicy) {
|
|
struct ArchCase {
|
|
rj_code_arch_t arch;
|
|
std::string_view name;
|
|
};
|
|
|
|
constexpr ArchCase arch_cases[] = {
|
|
{ROCJITSU_CODE_ARCH_CDNA1, "cdna1"},
|
|
{ROCJITSU_CODE_ARCH_CDNA2, "cdna2"},
|
|
{ROCJITSU_CODE_ARCH_CDNA3, "cdna3"},
|
|
{ROCJITSU_CODE_ARCH_CDNA4, "cdna4"},
|
|
};
|
|
|
|
for (const auto &arch : arch_cases) {
|
|
for (bool force_scalar : {false, true}) {
|
|
ForceScalarGuard guard(force_scalar);
|
|
const std::string suffix = force_scalar ? "_scalar" : "_simd";
|
|
amdgpu::GpuMemory gpu_mem(std::string(arch.name) + "_true16_vop3_mem" + suffix);
|
|
amdgpu::L2Cache l2(std::string(arch.name) + "_true16_vop3_l2" + suffix);
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = arch.arch;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
auto cu = amdgpu::ComputeUnitCore::create(std::string(arch.name), cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
|
|
auto decoder = Decoder::create(arch.arch);
|
|
ASSERT_NE(decoder, nullptr);
|
|
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
ASSERT_EQ(wf->wf_size(), 64u);
|
|
wf->set_exec(~uint64_t{0});
|
|
|
|
const uint32_t vb = wf->vgpr_alloc().base;
|
|
|
|
auto execute = [&](const std::array<uint32_t, 2> &words, std::string_view mnemonic) {
|
|
std::unique_ptr<Instruction> inst(decoder->decode(words.data()));
|
|
ASSERT_NE(inst, nullptr) << arch.name << " " << mnemonic;
|
|
ASSERT_EQ(std::string_view(inst->mnemonic()), mnemonic) << arch.name;
|
|
cu->execute_instruction(inst.get(), *wf);
|
|
};
|
|
|
|
for (uint32_t lane = 0; lane < wf->wf_size(); ++lane) {
|
|
cu->write_vgpr(vb + 0, lane, pack16(1u, 4u));
|
|
cu->write_vgpr(vb + 1, lane, pack16(0x8888u, 0x1234u));
|
|
cu->write_vgpr(vb + 16, lane, 0xBEEFCAFEu);
|
|
}
|
|
// v_lshrrev_b16 v16.h, v0.h, v1.h op_sel:[1,1,0,1]
|
|
execute(encode_cdna_vop3(/*op=*/0x12B, /*vdst=*/16, /*src0=*/256, /*src1=*/257,
|
|
/*src2=*/0, /*opsel=*/0xBu),
|
|
"v_lshrrev_b16");
|
|
for (uint32_t lane = 0; lane < wf->wf_size(); ++lane) {
|
|
EXPECT_EQ(cu->read_vgpr(vb + 16, lane), pack16(0xCAFEu, 0x0123u))
|
|
<< arch.name << " force_scalar=" << force_scalar << " lane " << lane;
|
|
}
|
|
|
|
for (uint32_t lane = 0; lane < wf->wf_size(); ++lane) {
|
|
cu->write_vgpr(vb + 0, lane, pack16(0x0011u, 2u));
|
|
cu->write_vgpr(vb + 1, lane, pack16(0x0022u, 3u));
|
|
cu->write_vgpr(vb + 2, lane, pack16(0x0033u, 4u));
|
|
cu->write_vgpr(vb + 4, lane, 0xBEEFCAFEu);
|
|
}
|
|
// v_mad_u16 v4.l, v0.h, v1.h, v2.h op_sel:[1,1,1,0]
|
|
execute(encode_cdna_vop3(/*op=*/0x204, /*vdst=*/4, /*src0=*/256, /*src1=*/257,
|
|
/*src2=*/258, /*opsel=*/0x7u),
|
|
"v_mad_u16");
|
|
for (uint32_t lane = 0; lane < wf->wf_size(); ++lane) {
|
|
EXPECT_EQ(cu->read_vgpr(vb + 4, lane), 10u)
|
|
<< arch.name << " force_scalar=" << force_scalar << " lane " << lane;
|
|
}
|
|
|
|
for (uint32_t lane = 0; lane < wf->wf_size(); ++lane) {
|
|
cu->write_vgpr(vb + 0, lane, pack16(0x0002u, 0xFFFEu));
|
|
cu->write_vgpr(vb + 1, lane, pack16(0x0004u, 3u));
|
|
cu->write_vgpr(vb + 2, lane, pack16(0x0006u, 5u));
|
|
cu->write_vgpr(vb + 5, lane, 0xCAFE1234u);
|
|
}
|
|
// v_mad_i16 v5.h, v0.h, v1.h, v2.h op_sel:[1,1,1,1]
|
|
execute(encode_cdna_vop3(/*op=*/0x205, /*vdst=*/5, /*src0=*/256, /*src1=*/257,
|
|
/*src2=*/258, /*opsel=*/0xFu),
|
|
"v_mad_i16");
|
|
for (uint32_t lane = 0; lane < wf->wf_size(); ++lane) {
|
|
EXPECT_EQ(cu->read_vgpr(vb + 5, lane), 0xFFFF1234u)
|
|
<< arch.name << " force_scalar=" << force_scalar << " lane " << lane;
|
|
}
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
}
|
|
}
|
|
|
|
TEST(Rdna3Dot2ExecutionTest, Vop2Dot2accF32F16AccumulatesDst) {
|
|
amdgpu::GpuMemory gpu_mem("rdna3_dot2acc_vop2_mem");
|
|
amdgpu::L2Cache l2("rdna3_dot2acc_vop2_l2");
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = ROCJITSU_CODE_ARCH_RDNA3;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
auto cu = amdgpu::ComputeUnitCore::create("rdna3", cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
|
|
auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_RDNA3);
|
|
ASSERT_NE(decoder, nullptr);
|
|
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
wf->set_exec(0x1u);
|
|
|
|
const uint32_t vb = wf->vgpr_alloc().base;
|
|
cu->write_vgpr(vb + 0, 0, pack16(util::f32_to_f16(1.0f), util::f32_to_f16(2.0f)));
|
|
cu->write_vgpr(vb + 1, 0, pack16(util::f32_to_f16(3.0f), util::f32_to_f16(4.0f)));
|
|
cu->write_vgpr(vb + 2, 0, std::bit_cast<uint32_t>(5.0f));
|
|
|
|
const auto words = encode_vop2(/*op=*/2, /*vdst=*/2, /*src0=*/256, /*vsrc1=*/1);
|
|
std::unique_ptr<Instruction> inst(decoder->decode(words.data()));
|
|
ASSERT_NE(inst, nullptr);
|
|
ASSERT_EQ(std::string_view(inst->mnemonic()), "v_dot2acc_f32_f16_e32");
|
|
cu->execute_instruction(inst.get(), *wf);
|
|
|
|
EXPECT_EQ(cu->read_vgpr(vb + 2, 0), std::bit_cast<uint32_t>(16.0f));
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
|
|
TEST(Rdna4Dot2True16ExecutionTest, F16AppliesVop3ModifiersAndSelectedHalves) {
|
|
amdgpu::GpuMemory gpu_mem("rdna4_dot2_true16_f16_mem");
|
|
amdgpu::L2Cache l2("rdna4_dot2_true16_f16_l2");
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = ROCJITSU_CODE_ARCH_RDNA4;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
auto cu = amdgpu::ComputeUnitCore::create("rdna4", cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
|
|
auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_RDNA4);
|
|
ASSERT_NE(decoder, nullptr);
|
|
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
wf->set_exec(0x1u);
|
|
|
|
const uint32_t vb = wf->vgpr_alloc().base;
|
|
cu->write_vgpr(vb + 0, 0, pack16(util::f32_to_f16(-2.0f), util::f32_to_f16(3.0f)));
|
|
cu->write_vgpr(vb + 1, 0, pack16(util::f32_to_f16(-4.0f), util::f32_to_f16(-5.0f)));
|
|
cu->write_vgpr(vb + 2, 0, pack16(util::f32_to_f16(0.25f), util::f32_to_f16(0.5f)));
|
|
cu->write_vgpr(vb + 3, 0, 0xCAFEBEEFu);
|
|
|
|
const auto words = encode_vop3(/*op=*/0x266, /*vdst=*/3, /*src0=*/256, /*src1=*/257,
|
|
/*src2=*/258, /*abs=*/0x1, /*opsel=*/0xCu, /*clamp=*/1,
|
|
/*omod=*/3, /*neg=*/0x2);
|
|
std::unique_ptr<Instruction> inst(decoder->decode(words.data()));
|
|
ASSERT_NE(inst, nullptr);
|
|
ASSERT_EQ(std::string_view(inst->mnemonic()), "v_dot2_f16_f16");
|
|
cu->execute_instruction(inst.get(), *wf);
|
|
|
|
EXPECT_EQ(cu->read_vgpr(vb + 3, 0), pack16(0xBEEFu, util::f32_to_f16(1.0f)));
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
|
|
TEST(Rdna4Dot2True16ExecutionTest, Bf16UsesSelectedAccumulatorAndDestinationHalf) {
|
|
amdgpu::GpuMemory gpu_mem("rdna4_dot2_true16_bf16_mem");
|
|
amdgpu::L2Cache l2("rdna4_dot2_true16_bf16_l2");
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = ROCJITSU_CODE_ARCH_RDNA4;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
auto cu = amdgpu::ComputeUnitCore::create("rdna4", cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
|
|
auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_RDNA4);
|
|
ASSERT_NE(decoder, nullptr);
|
|
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
wf->set_exec(0x1u);
|
|
|
|
const uint32_t vb = wf->vgpr_alloc().base;
|
|
cu->write_vgpr(vb + 0, 0, pack16(util::f32_to_bf16(1.0f), util::f32_to_bf16(2.0f)));
|
|
cu->write_vgpr(vb + 1, 0, pack16(util::f32_to_bf16(3.0f), util::f32_to_bf16(4.0f)));
|
|
cu->write_vgpr(vb + 2, 0, pack16(util::f32_to_bf16(0.5f), util::f32_to_bf16(5.0f)));
|
|
cu->write_vgpr(vb + 3, 0, 0xCAFEBEEFu);
|
|
|
|
const auto words = encode_vop3(/*op=*/0x267, /*vdst=*/3, /*src0=*/256, /*src1=*/257,
|
|
/*src2=*/258, /*abs=*/0, /*opsel=*/0x4u);
|
|
std::unique_ptr<Instruction> inst(decoder->decode(words.data()));
|
|
ASSERT_NE(inst, nullptr);
|
|
ASSERT_EQ(std::string_view(inst->mnemonic()), "v_dot2_bf16_bf16");
|
|
cu->execute_instruction(inst.get(), *wf);
|
|
|
|
EXPECT_EQ(cu->read_vgpr(vb + 3, 0), pack16(util::f32_to_bf16(16.0f), 0xCAFEu));
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
|
|
// VOP3P v_dot2_f32_bf16 must widen each 16-bit half as BF16 (8-bit exponent),
|
|
// not F16 (5-bit exponent). Values like 3.0/4.0 have BF16 and F16 bit patterns
|
|
// that decode to completely different floats, so a wrong widening yields a wildly
|
|
// wrong dot product rather than a small rounding difference. This asserts the
|
|
// exact result against hand-computed BF16 arithmetic. Runs both the SIMD fast
|
|
// path and the forced-scalar path so a divergence in either is caught.
|
|
void run_cdna4_dot2_f32_bf16(bool force_scalar) {
|
|
ForceScalarGuard guard(force_scalar);
|
|
amdgpu::GpuMemory gpu_mem("cdna4_dot2_f32_bf16_mem");
|
|
amdgpu::L2Cache l2("cdna4_dot2_f32_bf16_l2");
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = ROCJITSU_CODE_ARCH_CDNA4;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
auto cu = amdgpu::ComputeUnitCore::create("cdna4", cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_CDNA4);
|
|
ASSERT_NE(decoder, nullptr);
|
|
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
wf->set_exec(0x1u);
|
|
|
|
const uint32_t vb = wf->vgpr_alloc().base;
|
|
// src0 = {1.0, 2.0}, src1 = {3.0, 4.0}, acc = 0.5.
|
|
cu->write_vgpr(vb + 0, 0, pack16(util::f32_to_bf16(1.0f), util::f32_to_bf16(2.0f)));
|
|
cu->write_vgpr(vb + 1, 0, pack16(util::f32_to_bf16(3.0f), util::f32_to_bf16(4.0f)));
|
|
cu->write_vgpr(vb + 2, 0, std::bit_cast<uint32_t>(0.5f));
|
|
|
|
// Default packing (op_sel = 0, op_sel_hi = 3).
|
|
const auto words =
|
|
cdna4::build_vop3p(cdna4::kVDot2F32Bf16Vop3p, {.vdst = static_cast<uint8_t>(3),
|
|
.op_sel_hi_2 = 1,
|
|
.src0 = static_cast<uint16_t>(256),
|
|
.src1 = static_cast<uint16_t>(257),
|
|
.src2 = static_cast<uint16_t>(258),
|
|
.op_sel_hi = 3});
|
|
std::unique_ptr<Instruction> inst(decoder->decode(words.data()));
|
|
ASSERT_NE(inst, nullptr);
|
|
ASSERT_EQ(std::string_view(inst->mnemonic()), "v_dot2_f32_bf16");
|
|
cu->execute_instruction(inst.get(), *wf);
|
|
|
|
// 1.0*3.0 + 2.0*4.0 + 0.5 = 11.5, computed in exact f32 (all inputs represent
|
|
// exactly in BF16). A wrong f16 widening would not produce this.
|
|
EXPECT_EQ(cu->read_vgpr(vb + 3, 0), std::bit_cast<uint32_t>(11.5f));
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
|
|
TEST(Cdna4Dot2Vop3pExecutionTest, Bf16WidensAsBf16Simd) {
|
|
run_cdna4_dot2_f32_bf16(/*force_scalar=*/false);
|
|
}
|
|
|
|
TEST(Cdna4Dot2Vop3pExecutionTest, Bf16WidensAsBf16Scalar) {
|
|
run_cdna4_dot2_f32_bf16(/*force_scalar=*/true);
|
|
}
|
|
|
|
// v_permlane16_swap_b32 on a wave64 swaps ALL FOUR 16-lane groups: it exchanges
|
|
// vdst[16..31] with src0[0..15] AND vdst[48..63] with src0[32..47]. An earlier
|
|
// simulator only swapped the first block (lanes 0-31), silently leaving lanes
|
|
// 32-63 untouched. This test writes a distinct value per lane and asserts the
|
|
// swap happened in BOTH 32-lane halves, so a regression that drops the upper
|
|
// half is caught.
|
|
TEST(Cdna4Permlane16SwapExecutionTest, Wave64SwapsAllFourGroups) {
|
|
amdgpu::GpuMemory gpu_mem("cdna4_permlane16_swap_mem");
|
|
amdgpu::L2Cache l2("cdna4_permlane16_swap_l2");
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = ROCJITSU_CODE_ARCH_CDNA4;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
auto cu = amdgpu::ComputeUnitCore::create("cdna4", cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_CDNA4);
|
|
ASSERT_NE(decoder, nullptr);
|
|
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
ASSERT_EQ(wf->wf_size(), 64u);
|
|
wf->set_exec(~0ULL);
|
|
|
|
const uint32_t vb = wf->vgpr_alloc().base;
|
|
// Distinct, decodable per-lane values: vdst[lane] = 0xD0000000 | lane,
|
|
// src0[lane] = 0x50000000 | lane.
|
|
const uint32_t vdst_reg = 0;
|
|
const uint32_t vsrc_reg = 1;
|
|
for (uint32_t lane = 0; lane < 64; ++lane) {
|
|
cu->write_vgpr(vb + vdst_reg, lane, 0xD0000000u | lane);
|
|
cu->write_vgpr(vb + vsrc_reg, lane, 0x50000000u | lane);
|
|
}
|
|
|
|
const auto words = cdna4::build_vop1(
|
|
cdna4::kVPermlane16SwapB32Vop1,
|
|
{.src0 = static_cast<uint16_t>(256 + vsrc_reg), .vdst = static_cast<uint8_t>(vdst_reg)});
|
|
std::unique_ptr<Instruction> inst(decoder->decode(words.data()));
|
|
ASSERT_NE(inst, nullptr);
|
|
ASSERT_EQ(std::string_view(inst->mnemonic()), "v_permlane16_swap_b32_e32");
|
|
cu->execute_instruction(inst.get(), *wf);
|
|
|
|
for (uint32_t base = 0; base < 64; base += 32) {
|
|
for (uint32_t i = 0; i < 16; ++i) {
|
|
const uint32_t lo = base + i; // group L of this 32-lane block
|
|
const uint32_t hi = base + 16u + i; // group H of this 32-lane block
|
|
// src0[lo] gets the old vdst[hi]; vdst[hi] gets the old src0[lo].
|
|
EXPECT_EQ(cu->read_vgpr(vb + vsrc_reg, lo), 0xD0000000u | hi)
|
|
<< "src0 lane " << lo << " should hold old vdst lane " << hi;
|
|
EXPECT_EQ(cu->read_vgpr(vb + vdst_reg, hi), 0x50000000u | lo)
|
|
<< "vdst lane " << hi << " should hold old src0 lane " << lo;
|
|
// The untouched halves of each block keep their originals.
|
|
EXPECT_EQ(cu->read_vgpr(vb + vdst_reg, lo), 0xD0000000u | lo);
|
|
EXPECT_EQ(cu->read_vgpr(vb + vsrc_reg, hi), 0x50000000u | hi);
|
|
}
|
|
}
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
|
|
uint32_t wmma64_ab_lane(uint32_t row_or_col, uint32_t k) {
|
|
return row_or_col + 16u * ((k >> 2) & 1u);
|
|
}
|
|
|
|
uint32_t wmma64_ab_index(uint32_t k) {
|
|
const uint32_t reg =
|
|
((k >> 1) & 1u) + 2u * ((k >> 3) & 1u) + 4u * ((k >> 4) & 1u) + 8u * ((k >> 5) & 1u);
|
|
return 2u * reg + (k & 1u);
|
|
}
|
|
|
|
void write_packed_byte(amdgpu::ComputeUnitCore &cu, uint32_t reg, uint32_t lane, uint32_t byte,
|
|
uint8_t value) {
|
|
const uint32_t shift = 8u * byte;
|
|
const uint32_t old = cu.read_vgpr(reg, lane);
|
|
cu.write_vgpr(reg, lane, (old & ~(0xFFu << shift)) | (static_cast<uint32_t>(value) << shift));
|
|
}
|
|
|
|
void write_packed_half(amdgpu::ComputeUnitCore &cu, uint32_t reg, uint32_t lane, uint32_t half,
|
|
uint16_t value) {
|
|
const uint32_t shift = 16u * half;
|
|
const uint32_t old = cu.read_vgpr(reg, lane);
|
|
cu.write_vgpr(reg, lane, (old & ~(0xFFFFu << shift)) | (static_cast<uint32_t>(value) << shift));
|
|
}
|
|
|
|
float wmma_test_a(uint32_t row, uint32_t k) {
|
|
return static_cast<float>(static_cast<int>((row + k) % 5u) - 2);
|
|
}
|
|
|
|
float wmma_test_b(uint32_t k, uint32_t col) {
|
|
return static_cast<float>(static_cast<int>((col + 3u * k) % 5u) - 2);
|
|
}
|
|
|
|
float wmma_test_c(uint32_t row, uint32_t col) {
|
|
return static_cast<float>(static_cast<int>((2u * row + col) % 7u) - 3);
|
|
}
|
|
|
|
TEST(Gfx1250WmmaTest, F16Fp8K64MatchesReferenceLayout) {
|
|
amdgpu::GpuMemory gpu_mem("gfx1250_wmma_f16_fp8_k64_mem");
|
|
amdgpu::L2Cache l2("gfx1250_wmma_f16_fp8_k64_l2");
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = ROCJITSU_CODE_ARCH_GFX1250;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
auto cu = amdgpu::ComputeUnitCore::create("gfx1250", cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
ASSERT_EQ(wf->wf_size(), 32u);
|
|
wf->set_exec((1ULL << wf->wf_size()) - 1ULL);
|
|
|
|
auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
|
|
ASSERT_NE(decoder, nullptr);
|
|
|
|
const uint32_t vb = wf->vgpr_alloc().base;
|
|
const uint32_t a_base = vb + 10;
|
|
const uint32_t b_base = vb + 20;
|
|
const uint32_t c_base = vb + 30;
|
|
const uint32_t d_base = vb + 40;
|
|
|
|
for (uint32_t reg = 0; reg < 8; ++reg) {
|
|
for (uint32_t lane = 0; lane < wf->wf_size(); ++lane) {
|
|
cu->write_vgpr(a_base + reg, lane, 0);
|
|
cu->write_vgpr(b_base + reg, lane, 0);
|
|
}
|
|
}
|
|
for (uint32_t reg = 0; reg < 4; ++reg) {
|
|
for (uint32_t lane = 0; lane < wf->wf_size(); ++lane) {
|
|
cu->write_vgpr(c_base + reg, lane, 0);
|
|
cu->write_vgpr(d_base + reg, lane, 0xDEADBEEFu);
|
|
}
|
|
}
|
|
|
|
for (uint32_t row = 0; row < 16; ++row) {
|
|
for (uint32_t k = 0; k < 64; ++k) {
|
|
const uint32_t idx = wmma64_ab_index(k);
|
|
write_packed_byte(*cu, a_base + idx / 4u, wmma64_ab_lane(row, k), idx % 4u,
|
|
util::f32_to_fp8_e4m3_rne(wmma_test_a(row, k)));
|
|
}
|
|
}
|
|
for (uint32_t k = 0; k < 64; ++k) {
|
|
for (uint32_t col = 0; col < 16; ++col) {
|
|
const uint32_t idx = wmma64_ab_index(k);
|
|
write_packed_byte(*cu, b_base + idx / 4u, wmma64_ab_lane(col, k), idx % 4u,
|
|
util::f32_to_fp8_e4m3_rne(wmma_test_b(k, col)));
|
|
}
|
|
}
|
|
for (uint32_t row = 0; row < 16; ++row) {
|
|
for (uint32_t col = 0; col < 16; ++col) {
|
|
const auto out = amdgpu::wmma_output_loc_16(16, 16, row, col);
|
|
write_packed_half(*cu, c_base + out.reg, out.lane, out.sub_element,
|
|
util::f32_to_f16(wmma_test_c(row, col)));
|
|
}
|
|
}
|
|
|
|
amdgpu::exec_wmma_f16(*cu, 16, 16, 64, 8, d_base, a_base, b_base, c_base, amdgpu::extract_fp8,
|
|
amdgpu::extract_fp8, amdgpu::ACC_FROM_VGPR);
|
|
|
|
auto expect_reference = [&] {
|
|
for (uint32_t row = 0; row < 16; ++row) {
|
|
for (uint32_t col = 0; col < 16; ++col) {
|
|
float ref = wmma_test_c(row, col);
|
|
for (uint32_t k = 0; k < 64; ++k)
|
|
ref += wmma_test_a(row, k) * wmma_test_b(k, col);
|
|
const auto out = amdgpu::wmma_output_loc_16(16, 16, row, col);
|
|
const uint32_t raw = cu->read_vgpr(d_base + out.reg, out.lane);
|
|
const auto actual = static_cast<uint16_t>((raw >> (16u * out.sub_element)) & 0xFFFFu);
|
|
EXPECT_EQ(actual, util::f32_to_f16(ref)) << "row=" << row << " col=" << col;
|
|
}
|
|
}
|
|
};
|
|
expect_reference();
|
|
|
|
for (uint32_t reg = 0; reg < 4; ++reg)
|
|
for (uint32_t lane = 0; lane < wf->wf_size(); ++lane)
|
|
cu->write_vgpr(d_base + reg, lane, 0xDEADBEEFu);
|
|
|
|
const uint32_t wmma_words[] = {
|
|
0xCC6E0000U | 40u,
|
|
(256u + 10u) | ((256u + 20u) << 9u) | ((256u + 30u) << 18u),
|
|
};
|
|
std::unique_ptr<Instruction> inst(decoder->decode(wmma_words));
|
|
ASSERT_NE(inst, nullptr);
|
|
ASSERT_EQ(std::string_view(inst->mnemonic()), "v_wmma_f16_16x16x64_fp8_fp8");
|
|
cu->execute_instruction(inst.get(), *wf);
|
|
expect_reference();
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
|
|
TEST(Gfx1250WmmaTest, CModifierAppliesAbsBeforeNegate) {
|
|
EXPECT_EQ(amdgpu::wmma_c_modifier(0u, 0u), 0u);
|
|
EXPECT_EQ(amdgpu::wmma_c_modifier(1u << 2u, 0u), 1u);
|
|
EXPECT_EQ(amdgpu::wmma_c_modifier(0u, 1u << 2u), 2u);
|
|
EXPECT_EQ(amdgpu::wmma_c_modifier(1u << 2u, 1u << 2u), 3u);
|
|
|
|
EXPECT_EQ(amdgpu::apply_wmma_c_modifier(-3.0f, 0u), -3.0f);
|
|
EXPECT_EQ(amdgpu::apply_wmma_c_modifier(-3.0f, 1u), 3.0f);
|
|
EXPECT_EQ(amdgpu::apply_wmma_c_modifier(-3.0f, 2u), 3.0f);
|
|
EXPECT_EQ(amdgpu::apply_wmma_c_modifier(-3.0f, 3u), -3.0f);
|
|
}
|
|
|
|
TEST(Gfx1250WmmaTest, E8M0ScaleZeroMatchesUtilContract) {
|
|
EXPECT_EQ(std::bit_cast<uint32_t>(amdgpu::decode_wmma_scale_byte(0x00, 0)),
|
|
std::bit_cast<uint32_t>(util::e8m0_to_f32(0x00)));
|
|
EXPECT_TRUE(std::isnan(amdgpu::decode_wmma_scale_byte(0xff, 0)));
|
|
}
|
|
|
|
TEST(Gfx1250WmmaTest, Bf16F32K32Bf16PreservesAccumulatorLayout) {
|
|
amdgpu::GpuMemory gpu_mem("gfx1250_wmma_bf16f32_k32_bf16_mem");
|
|
amdgpu::L2Cache l2("gfx1250_wmma_bf16f32_k32_bf16_l2");
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = ROCJITSU_CODE_ARCH_GFX1250;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
auto cu = amdgpu::ComputeUnitCore::create("gfx1250", cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
ASSERT_EQ(wf->wf_size(), 32u);
|
|
wf->set_exec((1ULL << wf->wf_size()) - 1ULL);
|
|
|
|
const uint32_t vb = wf->vgpr_alloc().base;
|
|
const uint32_t a_base = vb + 10;
|
|
const uint32_t b_base = vb + 20;
|
|
const uint32_t c_base = vb + 30;
|
|
const uint32_t d_base = vb + 40;
|
|
|
|
for (uint32_t reg = 0; reg < 8; ++reg) {
|
|
for (uint32_t lane = 0; lane < wf->wf_size(); ++lane) {
|
|
cu->write_vgpr(a_base + reg, lane, 0);
|
|
cu->write_vgpr(b_base + reg, lane, 0);
|
|
cu->write_vgpr(c_base + reg, lane, 0);
|
|
}
|
|
}
|
|
for (uint32_t reg = 0; reg < 4; ++reg)
|
|
for (uint32_t lane = 0; lane < wf->wf_size(); ++lane)
|
|
cu->write_vgpr(d_base + reg, lane, 0xDEADBEEFu);
|
|
|
|
for (uint32_t row = 0; row < 16; ++row) {
|
|
for (uint32_t k = 0; k < 32; ++k) {
|
|
const auto in = amdgpu::wmma_input_loc(16, 32, row, k, 16);
|
|
write_packed_half(*cu, a_base + in.vgpr_offset, in.lane, in.sub_element,
|
|
util::f32_to_bf16(wmma_test_a(row, k)));
|
|
}
|
|
}
|
|
for (uint32_t k = 0; k < 32; ++k) {
|
|
for (uint32_t col = 0; col < 16; ++col) {
|
|
const auto in = amdgpu::wmma_input_loc(16, 32, col, k, 16);
|
|
write_packed_half(*cu, b_base + in.vgpr_offset, in.lane, in.sub_element,
|
|
util::f32_to_bf16(wmma_test_b(k, col)));
|
|
}
|
|
}
|
|
for (uint32_t row = 0; row < 16; ++row) {
|
|
for (uint32_t col = 0; col < 16; ++col) {
|
|
const auto out = amdgpu::wmma_output_loc_32(16, 16, row, col);
|
|
cu->write_vgpr(c_base + out.reg, out.lane, std::bit_cast<uint32_t>(wmma_test_c(row, col)));
|
|
}
|
|
}
|
|
|
|
amdgpu::exec_wmma_bf16f32_16x16x32_bf16(*cu, d_base, a_base, b_base, c_base);
|
|
|
|
for (uint32_t row = 0; row < 16; ++row) {
|
|
for (uint32_t col = 0; col < 16; ++col) {
|
|
float ref = wmma_test_c(row, col);
|
|
for (uint32_t k = 0; k < 32; ++k)
|
|
ref += wmma_test_a(row, k) * wmma_test_b(k, col);
|
|
const auto out = amdgpu::wmma_output_loc_16(16, 16, row, col);
|
|
const uint32_t raw = cu->read_vgpr(d_base + out.reg, out.lane);
|
|
const auto actual = static_cast<uint16_t>((raw >> (16u * out.sub_element)) & 0xFFFFu);
|
|
EXPECT_EQ(actual, util::f32_to_bf16(ref)) << "row=" << row << " col=" << col;
|
|
}
|
|
}
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
|
|
TEST(Gfx1250Dpp8Test, Vop2AddF16UsesPermutedSourceLanes) {
|
|
amdgpu::GpuMemory gpu_mem("gfx1250_dpp8_add_f16_mem");
|
|
amdgpu::L2Cache l2("gfx1250_dpp8_add_f16_l2");
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = ROCJITSU_CODE_ARCH_GFX1250;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
auto cu = amdgpu::ComputeUnitCore::create("gfx1250", cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
|
|
auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
|
|
ASSERT_NE(decoder, nullptr);
|
|
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
ASSERT_EQ(wf->wf_size(), 32u);
|
|
wf->set_exec((1ULL << wf->wf_size()) - 1ULL);
|
|
|
|
const uint32_t vb = wf->vgpr_alloc().base;
|
|
constexpr uint32_t kSrc0 = 2;
|
|
constexpr uint32_t kSrc1 = 3;
|
|
constexpr uint32_t kDst = 4;
|
|
const uint32_t lane_sel = (7u << 0u) | (0u << 3u) | (3u << 6u) | (2u << 9u) | (5u << 12u) |
|
|
(4u << 15u) | (1u << 18u) | (6u << 21u);
|
|
|
|
for (uint32_t lane = 0; lane < wf->wf_size(); ++lane) {
|
|
cu->write_vgpr(vb + kSrc0, lane, util::f32_to_f16(static_cast<float>(lane + 1)));
|
|
cu->write_vgpr(vb + kSrc1, lane, util::f32_to_f16(0.0f));
|
|
cu->write_vgpr(vb + kDst, lane, 0xDEAD0000u);
|
|
}
|
|
|
|
const uint32_t add_f16_dpp8_words[] = {
|
|
0x64000000u | amdgpu::SRC_DPP8_FI_0 | (kSrc1 << 9u) | (kDst << 17u),
|
|
kSrc0 | (lane_sel << 8u),
|
|
};
|
|
std::unique_ptr<Instruction> inst(decoder->decode(add_f16_dpp8_words));
|
|
ASSERT_NE(inst, nullptr);
|
|
ASSERT_EQ(std::string_view(inst->mnemonic()), "v_add_f16_e32");
|
|
cu->execute_instruction(inst.get(), *wf);
|
|
|
|
for (uint32_t lane = 0; lane < wf->wf_size(); ++lane) {
|
|
const uint32_t src_lane = amdgpu::dpp::dpp8_src_lane(lane, lane_sel);
|
|
const uint32_t raw = cu->read_vgpr(vb + kDst, lane);
|
|
EXPECT_EQ(raw >> 16, 0xDEADu) << "lane=" << lane;
|
|
EXPECT_EQ(raw & 0xFFFFu, util::f32_to_f16(static_cast<float>(src_lane + 1))) << "lane=" << lane;
|
|
}
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
|
|
TEST(Gfx1250ExecMaskTest, CmpxNeqAndScalarOrRestoreExec) {
|
|
amdgpu::GpuMemory gpu_mem("gfx1250_exec_mask_mem");
|
|
amdgpu::L2Cache l2("gfx1250_exec_mask_l2");
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = ROCJITSU_CODE_ARCH_GFX1250;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
auto cu = amdgpu::ComputeUnitCore::create("gfx1250", cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
|
|
auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
|
|
ASSERT_NE(decoder, nullptr);
|
|
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
ASSERT_LE(wf->wf_size(), 64u);
|
|
const uint64_t all_lanes = wf->wf_size() == 64 ? ~0ULL : ((1ULL << wf->wf_size()) - 1ULL);
|
|
wf->set_exec(all_lanes);
|
|
|
|
// s_mov_b32 s1, exec_lo
|
|
const uint32_t save_exec_words[] = {0xBE81007EU, 0};
|
|
std::unique_ptr<Instruction> save_exec(decoder->decode(save_exec_words));
|
|
ASSERT_NE(save_exec, nullptr);
|
|
ASSERT_EQ(std::string_view(save_exec->mnemonic()), "s_mov_b32");
|
|
cu->execute_instruction(save_exec.get(), *wf);
|
|
EXPECT_EQ(cu->read_sgpr(wf->sgpr_alloc().base + 1), static_cast<uint32_t>(all_lanes));
|
|
|
|
const uint32_t v4 = wf->vgpr_alloc().base + 4;
|
|
for (uint32_t lane = 0; lane < wf->wf_size(); ++lane) {
|
|
const float value = lane == 4 ? 0.0f : 1.0f;
|
|
cu->write_vgpr(v4, lane, std::bit_cast<uint32_t>(value));
|
|
}
|
|
|
|
// v_cmpx_neq_f32_e32 0, v4
|
|
const uint32_t mask_zero_lane_words[] = {0x7D3A0880U, 0};
|
|
std::unique_ptr<Instruction> mask_zero_lane(decoder->decode(mask_zero_lane_words));
|
|
ASSERT_NE(mask_zero_lane, nullptr);
|
|
ASSERT_EQ(std::string_view(mask_zero_lane->mnemonic()), "v_cmpx_neq_f32_e32");
|
|
cu->execute_instruction(mask_zero_lane.get(), *wf);
|
|
EXPECT_EQ(wf->exec(), all_lanes & ~(1ULL << 4));
|
|
|
|
// s_or_b32 exec_lo, exec_lo, s1
|
|
const uint32_t restore_exec_words[] = {0x8C7E017EU, 0};
|
|
std::unique_ptr<Instruction> restore_exec(decoder->decode(restore_exec_words));
|
|
ASSERT_NE(restore_exec, nullptr);
|
|
ASSERT_EQ(std::string_view(restore_exec->mnemonic()), "s_or_b32");
|
|
cu->execute_instruction(restore_exec.get(), *wf);
|
|
EXPECT_EQ(wf->exec(), all_lanes);
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
|
|
TEST(Gfx1250ExecMaskTest, Vop3CmpxGtI32KeepsInRangeLanesActive) {
|
|
amdgpu::GpuMemory gpu_mem("gfx1250_cmpx_gt_i32_mem");
|
|
amdgpu::L2Cache l2("gfx1250_cmpx_gt_i32_l2");
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = ROCJITSU_CODE_ARCH_GFX1250;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
auto cu = amdgpu::ComputeUnitCore::create("gfx1250", cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
|
|
auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
|
|
ASSERT_NE(decoder, nullptr);
|
|
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
ASSERT_LE(wf->wf_size(), 64u);
|
|
const uint64_t all_lanes = wf->wf_size() == 64 ? ~0ULL : ((1ULL << wf->wf_size()) - 1ULL);
|
|
wf->set_exec(all_lanes);
|
|
|
|
cu->write_sgpr(wf->sgpr_alloc().base + 3, 64);
|
|
const uint32_t v2 = wf->vgpr_alloc().base + 2;
|
|
for (uint32_t lane = 0; lane < wf->wf_size(); ++lane)
|
|
cu->write_vgpr(v2, lane, lane);
|
|
|
|
// v_cmpx_gt_i32_e64 s3, v2
|
|
const uint32_t in_range_words[] = {0xD4C4007EU, 0x02020403U};
|
|
std::unique_ptr<Instruction> in_range(decoder->decode(in_range_words));
|
|
ASSERT_NE(in_range, nullptr);
|
|
ASSERT_EQ(std::string_view(in_range->mnemonic()), "v_cmpx_gt_i32");
|
|
cu->execute_instruction(in_range.get(), *wf);
|
|
EXPECT_EQ(wf->exec(), all_lanes);
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
|
|
TEST(Gfx1250ExecMaskTest, Vop3CmpF32PreservesHighMaskSgprOnWave32) {
|
|
amdgpu::GpuMemory gpu_mem("gfx1250_cmp_f32_mask_mem");
|
|
amdgpu::L2Cache l2("gfx1250_cmp_f32_mask_l2");
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = ROCJITSU_CODE_ARCH_GFX1250;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
auto cu = amdgpu::ComputeUnitCore::create("gfx1250", cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
|
|
auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
|
|
ASSERT_NE(decoder, nullptr);
|
|
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
ASSERT_EQ(wf->wf_size(), 32u);
|
|
const uint64_t all_lanes = (1ULL << wf->wf_size()) - 1ULL;
|
|
wf->set_exec(all_lanes);
|
|
|
|
constexpr uint32_t saved_exec = 0xDEADBEEFu;
|
|
cu->write_sgpr(wf->sgpr_alloc().base + 1, saved_exec);
|
|
|
|
const uint32_t v4 = wf->vgpr_alloc().base + 4;
|
|
for (uint32_t lane = 0; lane < wf->wf_size(); ++lane) {
|
|
const float value = lane == 4 ? 0.0f : 1.0f;
|
|
cu->write_vgpr(v4, lane, std::bit_cast<uint32_t>(value));
|
|
}
|
|
|
|
// v_cmp_eq_f32_e64 s0, 0, v4
|
|
const uint32_t compare_words[] = {0xD4120000U, 0x02020880U};
|
|
std::unique_ptr<Instruction> compare(decoder->decode(compare_words));
|
|
ASSERT_NE(compare, nullptr);
|
|
ASSERT_EQ(std::string_view(compare->mnemonic()), "v_cmp_eq_f32");
|
|
cu->execute_instruction(compare.get(), *wf);
|
|
|
|
EXPECT_EQ(cu->read_sgpr(wf->sgpr_alloc().base + 0), 1u << 4);
|
|
EXPECT_EQ(cu->read_sgpr(wf->sgpr_alloc().base + 1), saved_exec);
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
|
|
TEST(Gfx1250True16Vop3Test, SelectedHalfArithmeticPreservesDestinationHalf) {
|
|
amdgpu::GpuMemory gpu_mem("gfx1250_true16_arith_mem");
|
|
amdgpu::L2Cache l2("gfx1250_true16_arith_l2");
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = ROCJITSU_CODE_ARCH_GFX1250;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
auto cu = amdgpu::ComputeUnitCore::create("gfx1250", cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
|
|
auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
|
|
ASSERT_NE(decoder, nullptr);
|
|
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
ASSERT_EQ(wf->wf_size(), 32u);
|
|
wf->set_exec((1ULL << wf->wf_size()) - 1ULL);
|
|
|
|
const uint32_t v2 = wf->vgpr_alloc().base + 2;
|
|
const uint32_t v3 = wf->vgpr_alloc().base + 3;
|
|
for (uint32_t lane = 0; lane < wf->wf_size(); ++lane) {
|
|
cu->write_vgpr(v2, lane, 0xCAFE0005u);
|
|
cu->write_vgpr(v3, lane, 0xBEEF0007u);
|
|
}
|
|
|
|
auto execute = [&](const uint32_t(&words)[2], std::string_view mnemonic) {
|
|
std::unique_ptr<Instruction> inst(decoder->decode(words));
|
|
ASSERT_NE(inst, nullptr);
|
|
ASSERT_EQ(std::string_view(inst->mnemonic()), mnemonic);
|
|
cu->execute_instruction(inst.get(), *wf);
|
|
};
|
|
|
|
// v_mul_lo_u16 v3.h, v3.l, v2.l op_sel:[0,0,1]
|
|
const uint32_t mul_hi_words[] = {0xD7054003U, 0x02020503U};
|
|
execute(mul_hi_words, "v_mul_lo_u16");
|
|
for (uint32_t lane = 0; lane < wf->wf_size(); ++lane)
|
|
EXPECT_EQ(cu->read_vgpr(v3, lane), 0x00230007u);
|
|
|
|
// v_sub_nc_u16 v3.h, 2, v3.h op_sel:[0,1,1]
|
|
const uint32_t sub_hi_words[] = {0xD7045003U, 0x02020682U};
|
|
execute(sub_hi_words, "v_sub_nc_u16");
|
|
for (uint32_t lane = 0; lane < wf->wf_size(); ++lane)
|
|
EXPECT_EQ(cu->read_vgpr(v3, lane), 0xFFDF0007u);
|
|
|
|
for (uint32_t lane = 0; lane < wf->wf_size(); ++lane) {
|
|
cu->write_vgpr(v2, lane, 0xCAFE0005u);
|
|
cu->write_vgpr(v3, lane, 0x12340007u);
|
|
}
|
|
|
|
// v_mad_u16 v3.h, v3.l, v2.l, v3.h op_sel:[0,0,1,1]
|
|
const uint32_t mad_hi_words[] = {0xD6416003U, 0x040E0503U};
|
|
execute(mad_hi_words, "v_mad_u16");
|
|
for (uint32_t lane = 0; lane < wf->wf_size(); ++lane)
|
|
EXPECT_EQ(cu->read_vgpr(v3, lane), 0x12570007u);
|
|
|
|
for (uint32_t lane = 0; lane < wf->wf_size(); ++lane) {
|
|
cu->write_vgpr(v2, lane, 0xCAFE0005u);
|
|
cu->write_vgpr(v3, lane, 0xBEEF0007u);
|
|
}
|
|
|
|
// v_mul_lo_u16 v3.l, v3.l, v2.l op_sel:[0,0,0]
|
|
const uint32_t mul_lo_words[] = {0xD7050003U, 0x02020503U};
|
|
execute(mul_lo_words, "v_mul_lo_u16");
|
|
for (uint32_t lane = 0; lane < wf->wf_size(); ++lane)
|
|
EXPECT_EQ(cu->read_vgpr(v3, lane), 0xBEEF0023u);
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
|
|
TEST(Gfx1250True16Vop2Test, AddF16UsesSelectedHighVsrc1AndPreservesLowDestination) {
|
|
amdgpu::GpuMemory gpu_mem("gfx1250_true16_vop2_add_mem");
|
|
amdgpu::L2Cache l2("gfx1250_true16_vop2_add_l2");
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = ROCJITSU_CODE_ARCH_GFX1250;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
auto cu = amdgpu::ComputeUnitCore::create("gfx1250", cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
|
|
auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
|
|
ASSERT_NE(decoder, nullptr);
|
|
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
wf->set_exec(1);
|
|
|
|
const uint32_t vb = wf->vgpr_alloc().base;
|
|
cu->write_vgpr(vb + 0, 0, 0x40003C00u); // low=1.0h, high=2.0h
|
|
cu->write_vgpr(vb + 1, 0, 0x44004200u); // low=3.0h, high=4.0h
|
|
cu->write_vgpr(vb + 2, 0, 0xDEADCAFEu);
|
|
|
|
// v_add_f16_e32 v2.h, v0.h, v1.h
|
|
const uint32_t words[] = {0x65050380U, 0x00000000U};
|
|
std::unique_ptr<Instruction> inst(decoder->decode(words));
|
|
ASSERT_NE(inst, nullptr);
|
|
ASSERT_EQ(std::string_view(inst->mnemonic()), "v_add_f16_e32");
|
|
EXPECT_NE(inst->disassemble().find("v2.h"), std::string::npos);
|
|
EXPECT_NE(inst->disassemble().find("v0.h"), std::string::npos);
|
|
EXPECT_NE(inst->disassemble().find("v1.h"), std::string::npos);
|
|
cu->execute_instruction(inst.get(), *wf);
|
|
|
|
EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0x4600CAFEu); // high=6.0h, low preserved
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
|
|
TEST(Gfx1250True16Vop2Test, FmacF16HighDestinationUsesSelectedAccumulatorHalf) {
|
|
amdgpu::GpuMemory gpu_mem("gfx1250_true16_vop2_fmac_mem");
|
|
amdgpu::L2Cache l2("gfx1250_true16_vop2_fmac_l2");
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = ROCJITSU_CODE_ARCH_GFX1250;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
auto cu = amdgpu::ComputeUnitCore::create("gfx1250", cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
|
|
auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
|
|
ASSERT_NE(decoder, nullptr);
|
|
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
wf->set_exec(1);
|
|
|
|
const uint32_t vb = wf->vgpr_alloc().base;
|
|
cu->write_vgpr(vb + 0, 0, 0x40003C00u); // low=1.0h, high=2.0h accumulator
|
|
cu->write_vgpr(vb + 1, 0, 0x42000000u); // high=3.0h
|
|
cu->write_vgpr(vb + 2, 0, 0x44000000u); // high=4.0h
|
|
|
|
// v_fmac_f16_e32 v0.h, v1.h, v2.h
|
|
const uint32_t words[] = {0x6D010581U, 0x00000000U};
|
|
std::unique_ptr<Instruction> inst(decoder->decode(words));
|
|
ASSERT_NE(inst, nullptr);
|
|
ASSERT_EQ(std::string_view(inst->mnemonic()), "v_fmac_f16_e32");
|
|
EXPECT_NE(inst->disassemble().find("v0.h"), std::string::npos);
|
|
EXPECT_NE(inst->disassemble().find("v1.h"), std::string::npos);
|
|
EXPECT_NE(inst->disassemble().find("v2.h"), std::string::npos);
|
|
cu->execute_instruction(inst.get(), *wf);
|
|
|
|
EXPECT_EQ(cu->read_vgpr(vb + 0, 0), 0x4B003C00u); // high=14.0h, low preserved
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
|
|
TEST(Gfx1250True16Vop3Test, MulLoU16UsesSelectedHighSourceHalf) {
|
|
amdgpu::GpuMemory gpu_mem("gfx1250_true16_mul_src_mem");
|
|
amdgpu::L2Cache l2("gfx1250_true16_mul_src_l2");
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = ROCJITSU_CODE_ARCH_GFX1250;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
auto cu = amdgpu::ComputeUnitCore::create("gfx1250", cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
|
|
auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
|
|
ASSERT_NE(decoder, nullptr);
|
|
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
wf->set_exec(1);
|
|
|
|
const uint32_t vb = wf->vgpr_alloc().base;
|
|
cu->write_vgpr(vb + 0, 0, 0x12345678u);
|
|
cu->write_vgpr(vb + 2, 0, 0xAAAA5555u);
|
|
|
|
// v_mul_lo_u16 v2.l, 0x12b3, v0.h op_sel:[0,1,0]
|
|
const uint32_t words[] = {0xD7051002U, 0x020200FFU, 0x000012B3U};
|
|
std::unique_ptr<Instruction> inst(decoder->decode(words));
|
|
ASSERT_NE(inst, nullptr);
|
|
ASSERT_EQ(std::string_view(inst->mnemonic()), "v_mul_lo_u16");
|
|
cu->execute_instruction(inst.get(), *wf);
|
|
|
|
EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0xAAAA625Cu);
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
|
|
TEST(Gfx1250True16Vop3Test, CmpLtI16UsesSelectedHighSourceHalf) {
|
|
amdgpu::GpuMemory gpu_mem("gfx1250_true16_cmp_src_mem");
|
|
amdgpu::L2Cache l2("gfx1250_true16_cmp_src_l2");
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = ROCJITSU_CODE_ARCH_GFX1250;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
auto cu = amdgpu::ComputeUnitCore::create("gfx1250", cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
|
|
auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
|
|
ASSERT_NE(decoder, nullptr);
|
|
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
ASSERT_EQ(wf->wf_size(), 32u);
|
|
wf->set_exec(1);
|
|
|
|
const uint32_t vb = wf->vgpr_alloc().base;
|
|
const uint32_t sb = wf->sgpr_alloc().base;
|
|
cu->write_vgpr(vb + 0, 0, 0xC0003C00u);
|
|
cu->write_sgpr(sb + 0, 0xFFFFFFFFu);
|
|
cu->write_sgpr(sb + 1, 0xDEADBEEFu);
|
|
|
|
// v_cmp_lt_i16 s0, -1, v0.h op_sel:[0,1,0]
|
|
const uint32_t words[] = {0xD4311000U, 0x020200C1U};
|
|
std::unique_ptr<Instruction> inst(decoder->decode(words));
|
|
ASSERT_NE(inst, nullptr);
|
|
ASSERT_EQ(std::string_view(inst->mnemonic()), "v_cmp_lt_i16");
|
|
cu->execute_instruction(inst.get(), *wf);
|
|
|
|
EXPECT_EQ(cu->read_sgpr(sb + 0), 0u);
|
|
EXPECT_EQ(cu->read_sgpr(sb + 1), 0xDEADBEEFu);
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
|
|
TEST(Gfx1250True16Vop3Test, CmpClassF16UsesSelectedMaskHalf) {
|
|
for (bool force_scalar : {false, true}) {
|
|
ForceScalarGuard guard(force_scalar);
|
|
amdgpu::GpuMemory gpu_mem(force_scalar ? "gfx1250_true16_class_scalar_mem"
|
|
: "gfx1250_true16_class_simd_mem");
|
|
amdgpu::L2Cache l2(force_scalar ? "gfx1250_true16_class_scalar_l2"
|
|
: "gfx1250_true16_class_simd_l2");
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = ROCJITSU_CODE_ARCH_GFX1250;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
auto cu = amdgpu::ComputeUnitCore::create("gfx1250", cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
|
|
auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
|
|
ASSERT_NE(decoder, nullptr);
|
|
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
ASSERT_EQ(wf->wf_size(), 32u);
|
|
wf->set_exec(0x3);
|
|
|
|
const uint32_t vb = wf->vgpr_alloc().base;
|
|
const uint32_t sb = wf->sgpr_alloc().base;
|
|
cu->write_vgpr(vb + 0, 0, pack16(0x0000u, 0x0040u)); // +0h, mask=+0
|
|
cu->write_vgpr(vb + 0, 1, pack16(0x7C01u, 0x0002u)); // sNaN, mask=qNaN
|
|
cu->write_sgpr(sb + 0, 0xFFFFFFFFu);
|
|
cu->write_sgpr(sb + 1, 0xDEADBEEFu);
|
|
|
|
constexpr auto words = encode_vop3(/*op=*/125, /*vdst=*/0, /*src0=*/256, /*src1=*/256,
|
|
/*src2=*/0, /*abs=*/0, /*opsel=*/0x2);
|
|
std::unique_ptr<Instruction> inst(decoder->decode(words.data()));
|
|
ASSERT_NE(inst, nullptr);
|
|
ASSERT_EQ(std::string_view(inst->mnemonic()), "v_cmp_class_f16");
|
|
cu->execute_instruction(inst.get(), *wf);
|
|
|
|
EXPECT_EQ(cu->read_sgpr(sb + 0), 0x1u);
|
|
EXPECT_EQ(cu->read_sgpr(sb + 1), 0xDEADBEEFu);
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
}
|
|
|
|
TEST(Gfx1250True16Vop3Test, CmpxClassF16UsesSelectedMaskHalf) {
|
|
for (bool force_scalar : {false, true}) {
|
|
ForceScalarGuard guard(force_scalar);
|
|
amdgpu::GpuMemory gpu_mem(force_scalar ? "gfx1250_true16_cmpx_class_scalar_mem"
|
|
: "gfx1250_true16_cmpx_class_simd_mem");
|
|
amdgpu::L2Cache l2(force_scalar ? "gfx1250_true16_cmpx_class_scalar_l2"
|
|
: "gfx1250_true16_cmpx_class_simd_l2");
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = ROCJITSU_CODE_ARCH_GFX1250;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
auto cu = amdgpu::ComputeUnitCore::create("gfx1250", cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
|
|
auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
|
|
ASSERT_NE(decoder, nullptr);
|
|
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
ASSERT_EQ(wf->wf_size(), 32u);
|
|
wf->set_exec(0x3);
|
|
wf->set_vcc(0xA5A5u);
|
|
|
|
const uint32_t vb = wf->vgpr_alloc().base;
|
|
cu->write_vgpr(vb + 0, 0, pack16(0x0000u, 0x0040u)); // +0h, mask=+0
|
|
cu->write_vgpr(vb + 0, 1, pack16(0x7C01u, 0x0002u)); // sNaN, mask=qNaN
|
|
|
|
constexpr auto words = encode_vop3(/*op=*/253, /*vdst=*/0, /*src0=*/256, /*src1=*/256,
|
|
/*src2=*/0, /*abs=*/0, /*opsel=*/0x2);
|
|
std::unique_ptr<Instruction> inst(decoder->decode(words.data()));
|
|
ASSERT_NE(inst, nullptr);
|
|
ASSERT_EQ(std::string_view(inst->mnemonic()), "v_cmpx_class_f16");
|
|
cu->execute_instruction(inst.get(), *wf);
|
|
|
|
EXPECT_EQ(wf->exec(), 0x1u);
|
|
EXPECT_EQ(wf->vcc(), 0xA5A5u);
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
}
|
|
|
|
TEST(Gfx1250True16Vop3Test, SpecialVop3OpsUseSelectedHalves) {
|
|
for (bool force_scalar : {false, true}) {
|
|
ForceScalarGuard guard(force_scalar);
|
|
amdgpu::GpuMemory gpu_mem(force_scalar ? "gfx1250_true16_special_scalar_mem"
|
|
: "gfx1250_true16_special_simd_mem");
|
|
amdgpu::L2Cache l2(force_scalar ? "gfx1250_true16_special_scalar_l2"
|
|
: "gfx1250_true16_special_simd_l2");
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = ROCJITSU_CODE_ARCH_GFX1250;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
auto cu = amdgpu::ComputeUnitCore::create("gfx1250", cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
|
|
auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
|
|
ASSERT_NE(decoder, nullptr);
|
|
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
ASSERT_EQ(wf->wf_size(), 32u);
|
|
wf->set_exec(0x1);
|
|
|
|
const uint32_t vb = wf->vgpr_alloc().base;
|
|
auto execute = [&](const std::array<uint32_t, 2> &words, std::string_view mnemonic) {
|
|
std::unique_ptr<Instruction> inst(decoder->decode(words.data()));
|
|
ASSERT_NE(inst, nullptr);
|
|
ASSERT_EQ(std::string_view(inst->mnemonic()), mnemonic);
|
|
cu->execute_instruction(inst.get(), *wf);
|
|
};
|
|
|
|
cu->write_vgpr(vb + 0, 0, pack16(2u, 7u));
|
|
cu->write_vgpr(vb + 1, 0, pack16(3u, 11u));
|
|
cu->write_vgpr(vb + 2, 0, 0x12340064u);
|
|
execute(encode_vop3(/*op=*/601, /*vdst=*/3, /*src0=*/256, /*src1=*/257,
|
|
/*src2=*/258, /*abs=*/0, /*opsel=*/0x3),
|
|
"v_mad_u32_u16");
|
|
EXPECT_EQ(cu->read_vgpr(vb + 3, 0), 0x123400B1u);
|
|
|
|
cu->write_vgpr(vb + 4, 0, pack16(0xFFFEu, 0xFFF9u)); // low=-2, high=-7
|
|
cu->write_vgpr(vb + 5, 0, pack16(5u, 0xFFFDu)); // low=5, high=-3
|
|
cu->write_vgpr(vb + 6, 0, 0x12340032u);
|
|
execute(encode_vop3(/*op=*/602, /*vdst=*/7, /*src0=*/260, /*src1=*/261,
|
|
/*src2=*/262, /*abs=*/0, /*opsel=*/0x3),
|
|
"v_mad_i32_i16");
|
|
EXPECT_EQ(cu->read_vgpr(vb + 7, 0), 0x12340047u);
|
|
|
|
cu->write_vgpr(vb + 0, 0, pack16(0x1111u, 0x2222u));
|
|
cu->write_vgpr(vb + 1, 0, pack16(0x3333u, 0x4444u));
|
|
execute(encode_vop3(/*op=*/785, /*vdst=*/8, /*src0=*/256, /*src1=*/257,
|
|
/*src2=*/0, /*abs=*/0, /*opsel=*/0x3),
|
|
"v_pack_b32_f16");
|
|
EXPECT_EQ(cu->read_vgpr(vb + 8, 0), 0x44442222u);
|
|
|
|
cu->write_vgpr(vb + 0, 0, pack16(util::f32_to_f16(0.0f), util::f32_to_f16(0.5f)));
|
|
cu->write_vgpr(vb + 1, 0, pack16(util::f32_to_f16(0.0f), util::f32_to_f16(1.0f)));
|
|
execute(encode_vop3(/*op=*/786, /*vdst=*/9, /*src0=*/256, /*src1=*/257,
|
|
/*src2=*/0, /*abs=*/0, /*opsel=*/0x3),
|
|
"v_cvt_pk_norm_i16_f16");
|
|
EXPECT_EQ(cu->read_vgpr(vb + 9, 0), 0x7FFF3FFFu);
|
|
|
|
execute(encode_vop3(/*op=*/787, /*vdst=*/10, /*src0=*/256, /*src1=*/257,
|
|
/*src2=*/0, /*abs=*/0, /*opsel=*/0x3),
|
|
"v_cvt_pk_norm_u16_f16");
|
|
EXPECT_EQ(cu->read_vgpr(vb + 10, 0), 0xFFFF7FFFu);
|
|
|
|
cu->write_vgpr(vb + 0, 0, pack16(util::f32_to_f16(0.25f), util::f32_to_f16(2.0f)));
|
|
cu->write_vgpr(vb + 1, 0, pack16(util::f32_to_f16(1.0f), util::f32_to_f16(3.0f)));
|
|
cu->write_vgpr(vb + 2, 0, pack16(util::f32_to_f16(1.0f), util::f32_to_f16(4.0f)));
|
|
cu->write_vgpr(vb + 12, 0, pack16(0xBEEFu, 0xCAFEu));
|
|
execute(encode_vop3(/*op=*/596, /*vdst=*/12, /*src0=*/256, /*src1=*/257,
|
|
/*src2=*/258, /*abs=*/0, /*opsel=*/0xF),
|
|
"v_div_fixup_f16");
|
|
EXPECT_EQ(cu->read_vgpr(vb + 12, 0), pack16(0xBEEFu, util::f32_to_f16(2.0f)));
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
}
|
|
|
|
TEST(Gfx1250True16VopcTest, CmpLtI16ReadsPackedHighVsrc1) {
|
|
amdgpu::GpuMemory gpu_mem("gfx1250_true16_vopc_cmp_src_mem");
|
|
amdgpu::L2Cache l2("gfx1250_true16_vopc_cmp_src_l2");
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = ROCJITSU_CODE_ARCH_GFX1250;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
auto cu = amdgpu::ComputeUnitCore::create("gfx1250", cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
|
|
auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
|
|
ASSERT_NE(decoder, nullptr);
|
|
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
ASSERT_EQ(wf->wf_size(), 32u);
|
|
wf->set_exec(0x3);
|
|
wf->set_vcc(0);
|
|
|
|
const uint32_t vb = wf->vgpr_alloc().base;
|
|
cu->write_vgpr(vb + 0, 0, 0xC0003C00u);
|
|
cu->write_vgpr(vb + 0, 1, 0x3C00C000u);
|
|
|
|
// v_cmp_lt_i16_e32 vcc_lo, -1, v0.h
|
|
const uint32_t words[] = {0x7C6300C1U};
|
|
std::unique_ptr<Instruction> inst(decoder->decode(words));
|
|
ASSERT_NE(inst, nullptr);
|
|
ASSERT_EQ(std::string_view(inst->mnemonic()), "v_cmp_lt_i16_e32");
|
|
cu->execute_instruction(inst.get(), *wf);
|
|
|
|
EXPECT_EQ(wf->vcc(), 0x2u);
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
|
|
TEST(Rdna4True16Vop3Test, CmpGeF16UsesSelectedHighSourceHalf) {
|
|
for (bool force_scalar : {false, true}) {
|
|
SCOPED_TRACE(force_scalar ? "scalar" : "simd");
|
|
ForceScalarGuard guard(force_scalar);
|
|
amdgpu::GpuMemory gpu_mem(force_scalar ? "rdna4_true16_cmp_scalar_mem"
|
|
: "rdna4_true16_cmp_simd_mem");
|
|
amdgpu::L2Cache l2(force_scalar ? "rdna4_true16_cmp_scalar_l2" : "rdna4_true16_cmp_simd_l2");
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = ROCJITSU_CODE_ARCH_RDNA4;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
auto cu = amdgpu::ComputeUnitCore::create("rdna4", cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
|
|
auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_RDNA4);
|
|
ASSERT_NE(decoder, nullptr);
|
|
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
ASSERT_EQ(wf->wf_size(), 32u);
|
|
wf->set_exec(0x3);
|
|
|
|
const uint32_t vb = wf->vgpr_alloc().base;
|
|
const uint32_t sb = wf->sgpr_alloc().base;
|
|
cu->write_vgpr(vb + 0, 0, 0x34003C00u); // low=1.0h, high=0.25h
|
|
cu->write_vgpr(vb + 0, 1, 0x38000000u); // low=0.0h, high=0.5h
|
|
cu->write_sgpr(sb + 0, 0xFFFFFFFFu);
|
|
cu->write_sgpr(sb + 1, 0xDEADBEEFu);
|
|
|
|
// v_cmp_ge_f16_e64 s0, |v0.h|, 0.5 op_sel:[1,0,0]
|
|
const uint32_t words[] = {0xD4060900U, 0x0201E100U};
|
|
std::unique_ptr<Instruction> inst(decoder->decode(words));
|
|
ASSERT_NE(inst, nullptr);
|
|
ASSERT_EQ(std::string_view(inst->mnemonic()), "v_cmp_ge_f16");
|
|
cu->execute_instruction(inst.get(), *wf);
|
|
|
|
EXPECT_EQ(cu->read_sgpr(sb + 0), 0x2u);
|
|
EXPECT_EQ(cu->read_sgpr(sb + 1), 0xDEADBEEFu);
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
}
|
|
|
|
TEST(Rdna4True16Vop3Test, CmpClassF16UsesSelectedMaskHalf) {
|
|
for (bool force_scalar : {false, true}) {
|
|
ForceScalarGuard guard(force_scalar);
|
|
amdgpu::GpuMemory gpu_mem(force_scalar ? "rdna4_true16_class_scalar_mem"
|
|
: "rdna4_true16_class_simd_mem");
|
|
amdgpu::L2Cache l2(force_scalar ? "rdna4_true16_class_scalar_l2"
|
|
: "rdna4_true16_class_simd_l2");
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = ROCJITSU_CODE_ARCH_RDNA4;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
auto cu = amdgpu::ComputeUnitCore::create("rdna4", cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
|
|
auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_RDNA4);
|
|
ASSERT_NE(decoder, nullptr);
|
|
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
ASSERT_EQ(wf->wf_size(), 32u);
|
|
wf->set_exec(0x3);
|
|
|
|
const uint32_t vb = wf->vgpr_alloc().base;
|
|
const uint32_t sb = wf->sgpr_alloc().base;
|
|
cu->write_vgpr(vb + 0, 0, pack16(0x0000u, 0x0040u)); // +0h, mask=+0
|
|
cu->write_vgpr(vb + 0, 1, pack16(0x7C01u, 0x0002u)); // sNaN, mask=qNaN
|
|
cu->write_sgpr(sb + 0, 0xFFFFFFFFu);
|
|
cu->write_sgpr(sb + 1, 0xDEADBEEFu);
|
|
|
|
constexpr auto words = encode_vop3(/*op=*/125, /*vdst=*/0, /*src0=*/256, /*src1=*/256,
|
|
/*src2=*/0, /*abs=*/0, /*opsel=*/0x2);
|
|
std::unique_ptr<Instruction> inst(decoder->decode(words.data()));
|
|
ASSERT_NE(inst, nullptr);
|
|
ASSERT_EQ(std::string_view(inst->mnemonic()), "v_cmp_class_f16");
|
|
cu->execute_instruction(inst.get(), *wf);
|
|
|
|
EXPECT_EQ(cu->read_sgpr(sb + 0), 0x1u);
|
|
EXPECT_EQ(cu->read_sgpr(sb + 1), 0xDEADBEEFu);
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
}
|
|
|
|
TEST(Rdna4True16Vop3Test, CmpClassF16InlineConstantUsesF16Broadcast) {
|
|
for (bool force_scalar : {false, true}) {
|
|
SCOPED_TRACE(force_scalar ? "scalar" : "simd");
|
|
ForceScalarGuard guard(force_scalar);
|
|
amdgpu::GpuMemory gpu_mem(force_scalar ? "rdna4_true16_class_inline_scalar_mem"
|
|
: "rdna4_true16_class_inline_simd_mem");
|
|
amdgpu::L2Cache l2(force_scalar ? "rdna4_true16_class_inline_scalar_l2"
|
|
: "rdna4_true16_class_inline_simd_l2");
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = ROCJITSU_CODE_ARCH_RDNA4;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
auto cu = amdgpu::ComputeUnitCore::create("rdna4", cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
|
|
auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_RDNA4);
|
|
ASSERT_NE(decoder, nullptr);
|
|
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
ASSERT_EQ(wf->wf_size(), 32u);
|
|
wf->set_exec(0x3);
|
|
|
|
const uint32_t sb = wf->sgpr_alloc().base;
|
|
cu->write_sgpr(sb + 0, 0xFFFFFFFFu);
|
|
cu->write_sgpr(sb + 1, 0xDEADBEEFu);
|
|
cu->write_sgpr(sb + 4, 0x0100u);
|
|
|
|
constexpr auto words = encode_vop3(/*op=*/125, /*vdst=*/0, /*src0=*/242, /*src1=*/4,
|
|
/*src2=*/0, /*abs=*/0, /*opsel=*/0);
|
|
std::unique_ptr<Instruction> inst(decoder->decode(words.data()));
|
|
ASSERT_NE(inst, nullptr);
|
|
ASSERT_EQ(std::string_view(inst->mnemonic()), "v_cmp_class_f16");
|
|
cu->execute_instruction(inst.get(), *wf);
|
|
|
|
EXPECT_EQ(cu->read_sgpr(sb + 0), 0x3u);
|
|
EXPECT_EQ(cu->read_sgpr(sb + 1), 0xDEADBEEFu);
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
}
|
|
|
|
TEST(Rdna4True16Vop3Test, CmpxClassF16UsesSelectedMaskHalf) {
|
|
for (bool force_scalar : {false, true}) {
|
|
ForceScalarGuard guard(force_scalar);
|
|
amdgpu::GpuMemory gpu_mem(force_scalar ? "rdna4_true16_cmpx_class_scalar_mem"
|
|
: "rdna4_true16_cmpx_class_simd_mem");
|
|
amdgpu::L2Cache l2(force_scalar ? "rdna4_true16_cmpx_class_scalar_l2"
|
|
: "rdna4_true16_cmpx_class_simd_l2");
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = ROCJITSU_CODE_ARCH_RDNA4;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
auto cu = amdgpu::ComputeUnitCore::create("rdna4", cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
|
|
auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_RDNA4);
|
|
ASSERT_NE(decoder, nullptr);
|
|
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
ASSERT_EQ(wf->wf_size(), 32u);
|
|
wf->set_exec(0x3);
|
|
wf->set_vcc(0xA5A5u);
|
|
|
|
const uint32_t vb = wf->vgpr_alloc().base;
|
|
cu->write_vgpr(vb + 0, 0, pack16(0x0000u, 0x0040u)); // +0h, mask=+0
|
|
cu->write_vgpr(vb + 0, 1, pack16(0x7C01u, 0x0002u)); // sNaN, mask=qNaN
|
|
|
|
constexpr auto words = encode_vop3(/*op=*/253, /*vdst=*/0, /*src0=*/256, /*src1=*/256,
|
|
/*src2=*/0, /*abs=*/0, /*opsel=*/0x2);
|
|
std::unique_ptr<Instruction> inst(decoder->decode(words.data()));
|
|
ASSERT_NE(inst, nullptr);
|
|
ASSERT_EQ(std::string_view(inst->mnemonic()), "v_cmpx_class_f16");
|
|
cu->execute_instruction(inst.get(), *wf);
|
|
|
|
EXPECT_EQ(wf->exec(), 0x1u);
|
|
EXPECT_EQ(wf->vcc(), 0xA5A5u);
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
}
|
|
|
|
TEST(Rdna4True16Vop3Test, ClassF16HelperSequenceMovesMaskIntoSelectedHighHalf) {
|
|
struct Case {
|
|
uint32_t lane;
|
|
uint16_t value;
|
|
uint16_t mask;
|
|
bool expected;
|
|
};
|
|
|
|
constexpr Case cases[] = {
|
|
{0, 0x0000u, 0x0040u, true}, // +0
|
|
{1, 0x8000u, 0x0020u, true}, // -0
|
|
{2, 0x3C00u, 0x0100u, true}, // +normal
|
|
{3, 0xBC00u, 0x0008u, true}, // -normal
|
|
{4, 0x0001u, 0x0080u, true}, // +denormal
|
|
{5, 0x8001u, 0x0010u, true}, // -denormal
|
|
{6, 0x7C00u, 0x0200u, true}, // +inf
|
|
{7, 0xFC00u, 0x0004u, true}, // -inf
|
|
{8, 0x7C01u, 0x0001u, true}, // sNaN
|
|
{9, 0x7E00u, 0x0002u, true}, // qNaN
|
|
{10, 0x7C01u, 0x0002u, false}, // sNaN is not qNaN
|
|
};
|
|
|
|
for (bool force_scalar : {false, true}) {
|
|
ForceScalarGuard guard(force_scalar);
|
|
amdgpu::GpuMemory gpu_mem(force_scalar ? "rdna4_true16_class_seq_scalar_mem"
|
|
: "rdna4_true16_class_seq_simd_mem");
|
|
amdgpu::L2Cache l2(force_scalar ? "rdna4_true16_class_seq_scalar_l2"
|
|
: "rdna4_true16_class_seq_simd_l2");
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = ROCJITSU_CODE_ARCH_RDNA4;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
auto cu = amdgpu::ComputeUnitCore::create("rdna4", cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
|
|
auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_RDNA4);
|
|
ASSERT_NE(decoder, nullptr);
|
|
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
ASSERT_EQ(wf->wf_size(), 32u);
|
|
|
|
const uint32_t vb = wf->vgpr_alloc().base;
|
|
const uint32_t sb = wf->sgpr_alloc().base;
|
|
|
|
// gfx1201 helper sequence from hip-fpsan's amdgcn_classh lowering:
|
|
// v_mov_b16_e32 v0.h, v1.l
|
|
// v_cmp_class_f16_e64 s0, v0.l, v0.h op_sel:[0,1,0]
|
|
// v_cndmask_b32_e64 v0, 0, 1, s0
|
|
const uint32_t mov_words[] = {0x7F003901U, 0};
|
|
const uint32_t cmp_words[] = {0xD47D1000U, 0x02020100U};
|
|
const uint32_t select_words[] = {0xD5010000U, 0x00010280U};
|
|
std::unique_ptr<Instruction> mov(decoder->decode(mov_words));
|
|
ASSERT_NE(mov, nullptr);
|
|
ASSERT_EQ(std::string_view(mov->mnemonic()), "v_mov_b16_e32");
|
|
std::unique_ptr<Instruction> cmp(decoder->decode(cmp_words));
|
|
ASSERT_NE(cmp, nullptr);
|
|
ASSERT_EQ(std::string_view(cmp->mnemonic()), "v_cmp_class_f16");
|
|
std::unique_ptr<Instruction> select(decoder->decode(select_words));
|
|
ASSERT_NE(select, nullptr);
|
|
ASSERT_EQ(std::string_view(select->mnemonic()), "v_cndmask_b32");
|
|
|
|
uint64_t active_mask = 0;
|
|
uint64_t expected_mask = 0;
|
|
for (const auto &test : cases) {
|
|
active_mask |= (1ULL << test.lane);
|
|
if (test.expected)
|
|
expected_mask |= (1ULL << test.lane);
|
|
cu->write_vgpr(vb + 0, test.lane, pack16(test.value, 0xCAFEu));
|
|
cu->write_vgpr(vb + 1, test.lane, pack16(test.mask, 0x1234u));
|
|
}
|
|
wf->set_exec(active_mask);
|
|
cu->write_sgpr(sb + 0, 0);
|
|
cu->write_sgpr(sb + 1, 0);
|
|
|
|
cu->execute_instruction(mov.get(), *wf);
|
|
for (const auto &test : cases) {
|
|
EXPECT_EQ(cu->read_vgpr(vb + 0, test.lane), pack16(test.value, test.mask))
|
|
<< "lane " << test.lane;
|
|
}
|
|
|
|
cu->execute_instruction(cmp.get(), *wf);
|
|
EXPECT_EQ(cu->read_sgpr(sb + 0), static_cast<uint32_t>(expected_mask));
|
|
EXPECT_EQ(cu->read_sgpr(sb + 1), static_cast<uint32_t>(expected_mask >> 32));
|
|
|
|
cu->execute_instruction(select.get(), *wf);
|
|
for (const auto &test : cases) {
|
|
EXPECT_EQ(cu->read_vgpr(vb + 0, test.lane), test.expected ? 1u : 0u) << "lane " << test.lane;
|
|
}
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
}
|
|
|
|
TEST(Rdna4True16Vop3Test, RcpF16WritesSelectedHighDestinationHalf) {
|
|
for (bool force_scalar : {false, true}) {
|
|
ForceScalarGuard guard(force_scalar);
|
|
amdgpu::GpuMemory gpu_mem(force_scalar ? "rdna4_true16_rcp_scalar_mem"
|
|
: "rdna4_true16_rcp_simd_mem");
|
|
amdgpu::L2Cache l2(force_scalar ? "rdna4_true16_rcp_scalar_l2" : "rdna4_true16_rcp_simd_l2");
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = ROCJITSU_CODE_ARCH_RDNA4;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
auto cu = amdgpu::ComputeUnitCore::create("rdna4", cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
|
|
auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_RDNA4);
|
|
ASSERT_NE(decoder, nullptr);
|
|
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
ASSERT_EQ(wf->wf_size(), 32u);
|
|
wf->set_exec(0x1);
|
|
|
|
const uint32_t vb = wf->vgpr_alloc().base;
|
|
cu->write_vgpr(vb + 0, 0, 0xCAFE4000u); // low=2.0h
|
|
|
|
// v_rcp_f16_e64 v0.h, -v0.l op_sel:[0,1]
|
|
const uint32_t words[] = {0xD5D44000U, 0x22010100U};
|
|
std::unique_ptr<Instruction> inst(decoder->decode(words));
|
|
ASSERT_NE(inst, nullptr);
|
|
ASSERT_EQ(std::string_view(inst->mnemonic()), "v_rcp_f16");
|
|
cu->execute_instruction(inst.get(), *wf);
|
|
|
|
EXPECT_EQ(cu->read_vgpr(vb + 0, 0), 0xB8004000u); // high=-0.5h, low preserved
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
}
|
|
|
|
TEST(Rdna4True16Vop3Test, CvtF32F16AppliesAbsToSelectedSourceHalf) {
|
|
for (bool force_scalar : {false, true}) {
|
|
ForceScalarGuard guard(force_scalar);
|
|
amdgpu::GpuMemory gpu_mem(force_scalar ? "rdna4_true16_cvt_scalar_mem"
|
|
: "rdna4_true16_cvt_simd_mem");
|
|
amdgpu::L2Cache l2(force_scalar ? "rdna4_true16_cvt_scalar_l2" : "rdna4_true16_cvt_simd_l2");
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = ROCJITSU_CODE_ARCH_RDNA4;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
auto cu = amdgpu::ComputeUnitCore::create("rdna4", cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
|
|
auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_RDNA4);
|
|
ASSERT_NE(decoder, nullptr);
|
|
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
ASSERT_EQ(wf->wf_size(), 32u);
|
|
wf->set_exec(0x1);
|
|
|
|
const uint32_t vb = wf->vgpr_alloc().base;
|
|
cu->write_vgpr(vb + 2, 0, 0x1234C400u); // low=-4.0h, high is ignored
|
|
|
|
// v_cvt_f32_f16_e64 v4, |v2.l|
|
|
const uint32_t words[] = {0xD58B0104U, 0x02010102U};
|
|
std::unique_ptr<Instruction> inst(decoder->decode(words));
|
|
ASSERT_NE(inst, nullptr);
|
|
ASSERT_EQ(std::string_view(inst->mnemonic()), "v_cvt_f32_f16");
|
|
cu->execute_instruction(inst.get(), *wf);
|
|
|
|
EXPECT_EQ(cu->read_vgpr(vb + 4, 0), 0x40800000u);
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
}
|
|
|
|
TEST(Rdna4True16Vop3Test, AddF16UsesSelectedHalvesAndPreservesDestinationHalf) {
|
|
for (bool force_scalar : {false, true}) {
|
|
SCOPED_TRACE(force_scalar ? "scalar" : "simd");
|
|
ForceScalarGuard guard(force_scalar);
|
|
amdgpu::GpuMemory gpu_mem(force_scalar ? "rdna4_true16_add_f16_scalar_mem"
|
|
: "rdna4_true16_add_f16_simd_mem");
|
|
amdgpu::L2Cache l2(force_scalar ? "rdna4_true16_add_f16_scalar_l2"
|
|
: "rdna4_true16_add_f16_simd_l2");
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = ROCJITSU_CODE_ARCH_RDNA4;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
auto cu = amdgpu::ComputeUnitCore::create("rdna4", cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
|
|
auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_RDNA4);
|
|
ASSERT_NE(decoder, nullptr);
|
|
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
ASSERT_EQ(wf->wf_size(), 32u);
|
|
wf->set_exec(0x3);
|
|
|
|
const uint32_t vb = wf->vgpr_alloc().base;
|
|
cu->write_vgpr(vb + 0, 0, pack16(util::f32_to_f16(100.0f), util::f32_to_f16(2.0f)));
|
|
cu->write_vgpr(vb + 1, 0, pack16(util::f32_to_f16(10.0f), util::f32_to_f16(3.0f)));
|
|
cu->write_vgpr(vb + 0, 1, pack16(util::f32_to_f16(8.0f), util::f32_to_f16(-1.0f)));
|
|
cu->write_vgpr(vb + 1, 1, pack16(util::f32_to_f16(4.0f), util::f32_to_f16(0.5f)));
|
|
cu->write_vgpr(vb + 2, 0, pack16(0xBEEFu, 0xCAFEu));
|
|
cu->write_vgpr(vb + 2, 1, pack16(0x1234u, 0x5678u));
|
|
|
|
constexpr auto words = encode_vop3(/*op=*/0x132, /*vdst=*/2, /*src0=*/256,
|
|
/*src1=*/257, /*src2=*/0, /*abs=*/0,
|
|
/*opsel=*/0xB);
|
|
std::unique_ptr<Instruction> inst(decoder->decode(words.data()));
|
|
ASSERT_NE(inst, nullptr);
|
|
ASSERT_EQ(std::string_view(inst->mnemonic()), "v_add_f16");
|
|
cu->execute_instruction(inst.get(), *wf);
|
|
|
|
EXPECT_EQ(cu->read_vgpr(vb + 2, 0), pack16(0xBEEFu, util::f32_to_f16(5.0f)));
|
|
EXPECT_EQ(cu->read_vgpr(vb + 2, 1), pack16(0x1234u, util::f32_to_f16(-0.5f)));
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
}
|
|
|
|
TEST(Rdna4True16Vop3Test, AddF16DppPreservesMaskedDestinationHalf) {
|
|
ForceScalarGuard guard(false);
|
|
amdgpu::GpuMemory gpu_mem("rdna4_true16_add_f16_dpp_mem");
|
|
amdgpu::L2Cache l2("rdna4_true16_add_f16_dpp_l2");
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = ROCJITSU_CODE_ARCH_RDNA4;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
auto cu = amdgpu::ComputeUnitCore::create("rdna4_true16_add_f16_dpp", cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
ASSERT_EQ(wf->wf_size(), 32u);
|
|
wf->set_exec((1ULL << wf->wf_size()) - 1ULL);
|
|
|
|
const uint32_t vb = wf->vgpr_alloc().base;
|
|
constexpr uint32_t kSrc0 = 0;
|
|
constexpr uint32_t kSrc1 = 1;
|
|
constexpr uint32_t kDst = 2;
|
|
|
|
for (uint32_t lane = 0; lane < wf->wf_size(); ++lane) {
|
|
cu->write_vgpr(vb + kSrc0, lane,
|
|
pack16(0x1111u, util::f32_to_f16(static_cast<float>(lane + 10))));
|
|
cu->write_vgpr(vb + kSrc1, lane, pack16(0x2222u, util::f32_to_f16(0.5f)));
|
|
cu->write_vgpr(
|
|
vb + kDst, lane,
|
|
pack16(static_cast<uint16_t>(0x4000u + lane), static_cast<uint16_t>(0x7000u + lane)));
|
|
}
|
|
|
|
rdna4::Vop3VopDpp16MachineInst raw{};
|
|
raw.vdst = kDst;
|
|
raw.opsel = 0xBu;
|
|
raw.op = 0x132u;
|
|
raw.encoding = 0x35u;
|
|
raw.src0 = amdgpu::SRC_DPP;
|
|
raw.src1 = 256u + kSrc1;
|
|
raw.vsrc0 = kSrc0;
|
|
raw.dpp_ctrl = amdgpu::dpp::ROW_SHR1;
|
|
raw.fi = 1;
|
|
raw.bound_ctrl = 0;
|
|
raw.bank_mask = 0xFu;
|
|
raw.row_mask = 0x1u;
|
|
|
|
rdna4::VAddF16Vop3 inst(reinterpret_cast<const rdna4::MachineInst *>(&raw));
|
|
ASSERT_EQ(std::string_view(inst.mnemonic()), "v_add_f16");
|
|
inst.execute_impl(*wf);
|
|
|
|
EXPECT_EQ(cu->read_vgpr(vb + kDst, 0), pack16(0x4000u, 0x7000u));
|
|
EXPECT_EQ(cu->read_vgpr(vb + kDst, 1), pack16(0x4001u, util::f32_to_f16(10.5f)));
|
|
EXPECT_EQ(cu->read_vgpr(vb + kDst, 2), pack16(0x4002u, util::f32_to_f16(11.5f)));
|
|
EXPECT_EQ(cu->read_vgpr(vb + kDst, 16), pack16(0x4010u, 0x7010u));
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
|
|
TEST(Rdna4True16Vop3Test, UnaryDpp16ScalarAndSimdMatchMaskedLaneRouting) {
|
|
if constexpr (!util::has_stdx_simd) {
|
|
GTEST_SKIP() << "<experimental/simd> unavailable";
|
|
}
|
|
|
|
constexpr uint32_t kWaveSize = 32;
|
|
constexpr uint32_t kSrc = 0;
|
|
constexpr uint32_t kDst = 2;
|
|
constexpr uint32_t kDppCtrl = amdgpu::dpp::ROW_SHR1;
|
|
constexpr uint32_t kRowMask = 0b0101;
|
|
constexpr uint32_t kBankMask = 0b1010;
|
|
constexpr uint64_t kExec = (uint64_t{1} << kWaveSize) - 1;
|
|
std::array<std::array<uint32_t, kWaveSize>, 2> outputs{};
|
|
|
|
for (uint32_t mode = 0; mode < outputs.size(); ++mode) {
|
|
const bool force_scalar = mode == 0;
|
|
SCOPED_TRACE(force_scalar ? "scalar" : "simd");
|
|
ForceScalarGuard guard(force_scalar);
|
|
amdgpu::GpuMemory gpu_mem(force_scalar ? "rdna4_true16_unary_dpp_scalar_mem"
|
|
: "rdna4_true16_unary_dpp_simd_mem");
|
|
amdgpu::L2Cache l2(force_scalar ? "rdna4_true16_unary_dpp_scalar_l2"
|
|
: "rdna4_true16_unary_dpp_simd_l2");
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = ROCJITSU_CODE_ARCH_RDNA4;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
auto cu = amdgpu::ComputeUnitCore::create("rdna4_true16_unary_dpp", cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
ASSERT_EQ(wf->wf_size(), kWaveSize);
|
|
wf->set_exec(kExec);
|
|
|
|
const uint32_t vb = wf->vgpr_alloc().base;
|
|
for (uint32_t lane = 0; lane < kWaveSize; ++lane) {
|
|
const float value = static_cast<float>(uint32_t{1} << (lane % 4));
|
|
cu->write_vgpr(vb + kSrc, lane, pack16(0xCAFEu, util::f32_to_f16(value)));
|
|
cu->write_vgpr(
|
|
vb + kDst, lane,
|
|
pack16(static_cast<uint16_t>(0x5000u + lane), static_cast<uint16_t>(0x1000u + lane)));
|
|
}
|
|
|
|
rdna4::Vop3VopDpp16MachineInst raw{};
|
|
raw.vdst = kDst;
|
|
raw.opsel = 0x9u; // Select high src0 and high destination halves.
|
|
raw.op = rdna4::kVRcpF16Vop3;
|
|
raw.encoding = 0x35u;
|
|
raw.src0 = amdgpu::SRC_DPP;
|
|
raw.vsrc0 = kSrc;
|
|
raw.dpp_ctrl = kDppCtrl;
|
|
raw.fi = 1;
|
|
raw.bound_ctrl = 0;
|
|
raw.bank_mask = kBankMask;
|
|
raw.row_mask = kRowMask;
|
|
|
|
static_assert(sizeof(raw) == 3 * sizeof(uint32_t));
|
|
auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_RDNA4);
|
|
std::unique_ptr<Instruction> inst(decoder->decode(reinterpret_cast<const uint32_t *>(&raw)));
|
|
ASSERT_NE(inst, nullptr);
|
|
ASSERT_EQ(std::string_view(inst->mnemonic()), "v_rcp_f16");
|
|
cu->execute_instruction(inst.get(), *wf);
|
|
for (uint32_t lane = 0; lane < kWaveSize; ++lane)
|
|
outputs[mode][lane] = cu->read_vgpr(vb + kDst, lane);
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
|
|
EXPECT_EQ(outputs[0], outputs[1]);
|
|
const auto plan = amdgpu::dpp::make_dpp_access_plan(kWaveSize, kDppCtrl, kRowMask, kBankMask,
|
|
/*bound_ctrl=*/0, /*fi=*/1, kExec);
|
|
for (uint32_t lane = 0; lane < kWaveSize; ++lane) {
|
|
const int source_lane = plan.source_lane_for_destination[lane];
|
|
const uint16_t expected_high =
|
|
source_lane == amdgpu::dpp::DppAccessPlan::kNoSourceLane
|
|
? static_cast<uint16_t>(0x1000u + lane)
|
|
: util::f32_to_f16(1.0f / static_cast<float>(uint32_t{1} << (source_lane % 4)));
|
|
EXPECT_EQ(outputs[1][lane], pack16(static_cast<uint16_t>(0x5000u + lane), expected_high))
|
|
<< "lane " << lane;
|
|
}
|
|
}
|
|
|
|
TEST(Rdna4True16Vop3Test, VopcDpp16ScalarAndSimdMatchMaskedLaneRouting) {
|
|
if constexpr (!util::has_stdx_simd) {
|
|
GTEST_SKIP() << "<experimental/simd> unavailable";
|
|
}
|
|
|
|
constexpr uint32_t kWaveSize = 32;
|
|
constexpr uint32_t kSrc0 = 0;
|
|
constexpr uint32_t kSrc1 = 1;
|
|
constexpr uint32_t kDppCtrl = amdgpu::dpp::ROW_SHR1;
|
|
constexpr uint32_t kRowMask = 0b0101;
|
|
constexpr uint32_t kBankMask = 0b1010;
|
|
constexpr uint64_t kExec = (uint64_t{1} << kWaveSize) - 1;
|
|
std::array<uint64_t, 2> outputs{};
|
|
|
|
for (uint32_t mode = 0; mode < outputs.size(); ++mode) {
|
|
const bool force_scalar = mode == 0;
|
|
SCOPED_TRACE(force_scalar ? "scalar" : "simd");
|
|
ForceScalarGuard guard(force_scalar);
|
|
amdgpu::GpuMemory gpu_mem(force_scalar ? "rdna4_true16_vopc_dpp_scalar_mem"
|
|
: "rdna4_true16_vopc_dpp_simd_mem");
|
|
amdgpu::L2Cache l2(force_scalar ? "rdna4_true16_vopc_dpp_scalar_l2"
|
|
: "rdna4_true16_vopc_dpp_simd_l2");
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = ROCJITSU_CODE_ARCH_RDNA4;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
auto cu = amdgpu::ComputeUnitCore::create("rdna4_true16_vopc_dpp", cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
ASSERT_EQ(wf->wf_size(), kWaveSize);
|
|
wf->set_exec(kExec);
|
|
|
|
const uint32_t vb = wf->vgpr_alloc().base;
|
|
const uint32_t sb = wf->sgpr_alloc().base;
|
|
for (uint32_t lane = 0; lane < kWaveSize; ++lane) {
|
|
cu->write_vgpr(vb + kSrc0, lane,
|
|
pack16(0xCAFEu, util::f32_to_f16(static_cast<float>(lane + 1))));
|
|
const float threshold = static_cast<float>(lane + ((lane & 1u) ? 2u : 0u));
|
|
cu->write_vgpr(vb + kSrc1, lane, pack16(0xBEEFu, util::f32_to_f16(threshold)));
|
|
}
|
|
cu->write_sgpr(sb, 0xA5A5A5A5u);
|
|
cu->write_sgpr(sb + 1, 0xDEADBEEFu);
|
|
|
|
rdna4::Vop3VopDpp16MachineInst raw{};
|
|
raw.vdst = 0;
|
|
raw.opsel = 0x3u; // Select high halves from both true16 sources.
|
|
raw.op = rdna4::kVCmpGeF16Vop3;
|
|
raw.encoding = 0x35u;
|
|
raw.src0 = amdgpu::SRC_DPP;
|
|
raw.src1 = vgpr_src(kSrc1);
|
|
raw.vsrc0 = kSrc0;
|
|
raw.dpp_ctrl = kDppCtrl;
|
|
raw.fi = 1;
|
|
raw.bound_ctrl = 0;
|
|
raw.bank_mask = kBankMask;
|
|
raw.row_mask = kRowMask;
|
|
|
|
static_assert(sizeof(raw) == 3 * sizeof(uint32_t));
|
|
auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_RDNA4);
|
|
std::unique_ptr<Instruction> inst(decoder->decode(reinterpret_cast<const uint32_t *>(&raw)));
|
|
ASSERT_NE(inst, nullptr);
|
|
ASSERT_EQ(std::string_view(inst->mnemonic()), "v_cmp_ge_f16");
|
|
cu->execute_instruction(inst.get(), *wf);
|
|
outputs[mode] = static_cast<uint64_t>(cu->read_sgpr(sb)) |
|
|
(static_cast<uint64_t>(cu->read_sgpr(sb + 1)) << 32);
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
|
|
EXPECT_EQ(outputs[0], outputs[1]);
|
|
const auto plan = amdgpu::dpp::make_dpp_access_plan(kWaveSize, kDppCtrl, kRowMask, kBankMask,
|
|
/*bound_ctrl=*/0, /*fi=*/1, kExec);
|
|
uint32_t expected_low = 0;
|
|
for (uint32_t lane = 0; lane < kWaveSize; ++lane) {
|
|
const int source_lane = plan.source_lane_for_destination[lane];
|
|
if (source_lane == amdgpu::dpp::DppAccessPlan::kNoSourceLane)
|
|
continue;
|
|
const float source = static_cast<float>(source_lane + 1);
|
|
const float threshold = static_cast<float>(lane + ((lane & 1u) ? 2u : 0u));
|
|
if (source >= threshold)
|
|
expected_low |= uint32_t{1} << lane;
|
|
}
|
|
EXPECT_EQ(outputs[1], static_cast<uint64_t>(expected_low) | (uint64_t{0xDEADBEEF} << 32));
|
|
}
|
|
|
|
TEST(Rdna4True16Vop3Test, UnaryDpp8ScalarAndSimdMatchPermutedLanes) {
|
|
if constexpr (!util::has_stdx_simd) {
|
|
GTEST_SKIP() << "<experimental/simd> unavailable";
|
|
}
|
|
|
|
constexpr uint32_t kWaveSize = 32;
|
|
constexpr uint32_t kSrc = 0;
|
|
constexpr uint32_t kDst = 2;
|
|
constexpr uint64_t kExec = (uint64_t{1} << kWaveSize) - 1;
|
|
constexpr std::array<uint32_t, 8> kLaneSelectors{7, 0, 5, 2, 3, 6, 1, 4};
|
|
std::array<std::array<uint32_t, kWaveSize>, 2> outputs{};
|
|
|
|
for (uint32_t mode = 0; mode < outputs.size(); ++mode) {
|
|
const bool force_scalar = mode == 0;
|
|
SCOPED_TRACE(force_scalar ? "scalar" : "simd");
|
|
ForceScalarGuard guard(force_scalar);
|
|
amdgpu::GpuMemory gpu_mem(force_scalar ? "rdna4_true16_unary_dpp8_scalar_mem"
|
|
: "rdna4_true16_unary_dpp8_simd_mem");
|
|
amdgpu::L2Cache l2(force_scalar ? "rdna4_true16_unary_dpp8_scalar_l2"
|
|
: "rdna4_true16_unary_dpp8_simd_l2");
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = ROCJITSU_CODE_ARCH_RDNA4;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
auto cu = amdgpu::ComputeUnitCore::create("rdna4_true16_unary_dpp8", cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
ASSERT_EQ(wf->wf_size(), kWaveSize);
|
|
wf->set_exec(kExec);
|
|
|
|
const uint32_t vb = wf->vgpr_alloc().base;
|
|
for (uint32_t lane = 0; lane < kWaveSize; ++lane) {
|
|
const float value = static_cast<float>(uint32_t{1} << (lane % 4));
|
|
cu->write_vgpr(vb + kSrc, lane, pack16(util::f32_to_f16(value), 0xCAFEu));
|
|
cu->write_vgpr(
|
|
vb + kDst, lane,
|
|
pack16(static_cast<uint16_t>(0x2000u + lane), static_cast<uint16_t>(0x6000u + lane)));
|
|
}
|
|
|
|
rdna4::Vop3VopDpp8MachineInst raw{};
|
|
raw.vdst = kDst;
|
|
raw.op = rdna4::kVRcpF16Vop3;
|
|
raw.encoding = 0x35u;
|
|
raw.src0 = amdgpu::SRC_DPP8_FI_1;
|
|
raw.vsrc0 = kSrc;
|
|
raw.lane_sel_0 = kLaneSelectors[0];
|
|
raw.lane_sel_1 = kLaneSelectors[1];
|
|
raw.lane_sel_2 = kLaneSelectors[2];
|
|
raw.lane_sel_3 = kLaneSelectors[3];
|
|
raw.lane_sel_4 = kLaneSelectors[4];
|
|
raw.lane_sel_5 = kLaneSelectors[5];
|
|
raw.lane_sel_6 = kLaneSelectors[6];
|
|
raw.lane_sel_7 = kLaneSelectors[7];
|
|
|
|
static_assert(sizeof(raw) == 3 * sizeof(uint32_t));
|
|
auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_RDNA4);
|
|
std::unique_ptr<Instruction> inst(decoder->decode(reinterpret_cast<const uint32_t *>(&raw)));
|
|
ASSERT_NE(inst, nullptr);
|
|
ASSERT_EQ(std::string_view(inst->mnemonic()), "v_rcp_f16");
|
|
cu->execute_instruction(inst.get(), *wf);
|
|
for (uint32_t lane = 0; lane < kWaveSize; ++lane)
|
|
outputs[mode][lane] = cu->read_vgpr(vb + kDst, lane);
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
|
|
EXPECT_EQ(outputs[0], outputs[1]);
|
|
uint32_t lane_select_bits = 0;
|
|
for (uint32_t lane = 0; lane < kLaneSelectors.size(); ++lane)
|
|
lane_select_bits |= kLaneSelectors[lane] << (lane * 3u);
|
|
for (uint32_t lane = 0; lane < kWaveSize; ++lane) {
|
|
const uint32_t source_lane = amdgpu::dpp::dpp8_src_lane(lane, lane_select_bits);
|
|
const uint16_t expected_low =
|
|
util::f32_to_f16(1.0f / static_cast<float>(uint32_t{1} << (source_lane % 4)));
|
|
EXPECT_EQ(outputs[1][lane], pack16(expected_low, static_cast<uint16_t>(0x6000u + lane)))
|
|
<< "lane " << lane;
|
|
}
|
|
}
|
|
|
|
TEST(Rdna4True16Vop3Test, FmacF16UsesSelectedAccumulatorHalfAndPreservesDestinationHalf) {
|
|
for (bool force_scalar : {false, true}) {
|
|
SCOPED_TRACE(force_scalar ? "scalar" : "simd");
|
|
ForceScalarGuard guard(force_scalar);
|
|
amdgpu::GpuMemory gpu_mem(force_scalar ? "rdna4_true16_fmac_scalar_mem"
|
|
: "rdna4_true16_fmac_simd_mem");
|
|
amdgpu::L2Cache l2(force_scalar ? "rdna4_true16_fmac_scalar_l2" : "rdna4_true16_fmac_simd_l2");
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = ROCJITSU_CODE_ARCH_RDNA4;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
auto cu = amdgpu::ComputeUnitCore::create("rdna4", cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
|
|
auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_RDNA4);
|
|
ASSERT_NE(decoder, nullptr);
|
|
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
ASSERT_EQ(wf->wf_size(), 32u);
|
|
wf->set_exec(0x3);
|
|
|
|
const uint32_t vb = wf->vgpr_alloc().base;
|
|
cu->write_vgpr(vb + 0, 0, pack16(util::f32_to_f16(100.0f), util::f32_to_f16(2.0f)));
|
|
cu->write_vgpr(vb + 1, 0, pack16(util::f32_to_f16(10.0f), util::f32_to_f16(3.0f)));
|
|
cu->write_vgpr(vb + 0, 1, pack16(util::f32_to_f16(8.0f), util::f32_to_f16(-2.0f)));
|
|
cu->write_vgpr(vb + 1, 1, pack16(util::f32_to_f16(4.0f), util::f32_to_f16(0.5f)));
|
|
cu->write_vgpr(vb + 3, 0, pack16(0xBEEFu, util::f32_to_f16(4.0f)));
|
|
cu->write_vgpr(vb + 3, 1, pack16(0x1234u, util::f32_to_f16(1.5f)));
|
|
|
|
constexpr auto words = encode_vop3(/*op=*/0x136, /*vdst=*/3, /*src0=*/256,
|
|
/*src1=*/257, /*src2=*/0, /*abs=*/0,
|
|
/*opsel=*/0xB);
|
|
std::unique_ptr<Instruction> inst(decoder->decode(words.data()));
|
|
ASSERT_NE(inst, nullptr);
|
|
ASSERT_EQ(std::string_view(inst->mnemonic()), "v_fmac_f16");
|
|
cu->execute_instruction(inst.get(), *wf);
|
|
|
|
EXPECT_EQ(cu->read_vgpr(vb + 3, 0), pack16(0xBEEFu, util::f32_to_f16(10.0f)));
|
|
EXPECT_EQ(cu->read_vgpr(vb + 3, 1), pack16(0x1234u, util::f32_to_f16(0.5f)));
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
}
|
|
|
|
TEST(Rdna4True16Vop3Test, TernaryI16U16UsesSelectedHalvesAndPreservesDestinationHalf) {
|
|
for (bool force_scalar : {false, true}) {
|
|
SCOPED_TRACE(force_scalar ? "scalar" : "simd");
|
|
ForceScalarGuard guard(force_scalar);
|
|
amdgpu::GpuMemory gpu_mem(force_scalar ? "rdna4_true16_ternary_scalar_mem"
|
|
: "rdna4_true16_ternary_simd_mem");
|
|
amdgpu::L2Cache l2(force_scalar ? "rdna4_true16_ternary_scalar_l2"
|
|
: "rdna4_true16_ternary_simd_l2");
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = ROCJITSU_CODE_ARCH_RDNA4;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
auto cu = amdgpu::ComputeUnitCore::create("rdna4", cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
|
|
auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_RDNA4);
|
|
ASSERT_NE(decoder, nullptr);
|
|
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
ASSERT_EQ(wf->wf_size(), 32u);
|
|
wf->set_exec(0x1);
|
|
|
|
const uint32_t vb = wf->vgpr_alloc().base;
|
|
cu->write_vgpr(vb + 0, 0, pack16(100u, 0xFFF9u)); // high=-7
|
|
cu->write_vgpr(vb + 1, 0, pack16(50u, 3u));
|
|
cu->write_vgpr(vb + 2, 0, pack16(75u, 0xFFFEu)); // high=-2
|
|
cu->write_vgpr(vb + 4, 0, pack16(1u, 7u));
|
|
cu->write_vgpr(vb + 5, 0, pack16(2u, 3u));
|
|
cu->write_vgpr(vb + 6, 0, pack16(3u, 5u));
|
|
|
|
auto execute = [&](uint32_t op, uint32_t dst, uint32_t src0, uint32_t src1, uint32_t src2,
|
|
std::string_view mnemonic, uint16_t expected_high) {
|
|
cu->write_vgpr(vb + dst, 0, pack16(static_cast<uint16_t>(0xA000u | dst), 0xCAFEu));
|
|
const auto words = encode_vop3(op, dst, src0, src1, src2, /*abs=*/0, /*opsel=*/0xF);
|
|
std::unique_ptr<Instruction> inst(decoder->decode(words.data()));
|
|
ASSERT_NE(inst, nullptr);
|
|
ASSERT_EQ(std::string_view(inst->mnemonic()), mnemonic);
|
|
cu->execute_instruction(inst.get(), *wf);
|
|
EXPECT_EQ(cu->read_vgpr(vb + dst, 0),
|
|
pack16(static_cast<uint16_t>(0xA000u | dst), expected_high))
|
|
<< mnemonic;
|
|
};
|
|
|
|
execute(/*op=*/0x24A, /*dst=*/10, /*src0=*/256, /*src1=*/257, /*src2=*/258, "v_min3_i16",
|
|
0xFFF9u);
|
|
execute(/*op=*/0x24D, /*dst=*/11, /*src0=*/256, /*src1=*/257, /*src2=*/258, "v_max3_i16", 3u);
|
|
execute(/*op=*/0x250, /*dst=*/12, /*src0=*/256, /*src1=*/257, /*src2=*/258, "v_med3_i16",
|
|
0xFFFEu);
|
|
execute(/*op=*/0x24B, /*dst=*/13, /*src0=*/260, /*src1=*/261, /*src2=*/262, "v_min3_u16", 3u);
|
|
execute(/*op=*/0x24E, /*dst=*/14, /*src0=*/260, /*src1=*/261, /*src2=*/262, "v_max3_u16", 7u);
|
|
execute(/*op=*/0x251, /*dst=*/15, /*src0=*/260, /*src1=*/261, /*src2=*/262, "v_med3_u16", 5u);
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
}
|
|
|
|
TEST(Rdna4True16Vop3Test, SpecialVop3OpsUseSelectedHalves) {
|
|
for (bool force_scalar : {false, true}) {
|
|
ForceScalarGuard guard(force_scalar);
|
|
amdgpu::GpuMemory gpu_mem(force_scalar ? "rdna4_true16_special_scalar_mem"
|
|
: "rdna4_true16_special_simd_mem");
|
|
amdgpu::L2Cache l2(force_scalar ? "rdna4_true16_special_scalar_l2"
|
|
: "rdna4_true16_special_simd_l2");
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = ROCJITSU_CODE_ARCH_RDNA4;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
auto cu = amdgpu::ComputeUnitCore::create("rdna4", cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
|
|
auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_RDNA4);
|
|
ASSERT_NE(decoder, nullptr);
|
|
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
ASSERT_EQ(wf->wf_size(), 32u);
|
|
wf->set_exec(0x1);
|
|
|
|
const uint32_t vb = wf->vgpr_alloc().base;
|
|
auto execute = [&](const std::array<uint32_t, 2> &words, std::string_view mnemonic) {
|
|
std::unique_ptr<Instruction> inst(decoder->decode(words.data()));
|
|
ASSERT_NE(inst, nullptr);
|
|
ASSERT_EQ(std::string_view(inst->mnemonic()), mnemonic);
|
|
cu->execute_instruction(inst.get(), *wf);
|
|
};
|
|
|
|
cu->write_vgpr(vb + 0, 0, pack16(2u, 7u));
|
|
cu->write_vgpr(vb + 1, 0, pack16(3u, 11u));
|
|
cu->write_vgpr(vb + 2, 0, 0x12340064u);
|
|
execute(encode_vop3(/*op=*/601, /*vdst=*/3, /*src0=*/256, /*src1=*/257,
|
|
/*src2=*/258, /*abs=*/0, /*opsel=*/0x3),
|
|
"v_mad_u32_u16");
|
|
EXPECT_EQ(cu->read_vgpr(vb + 3, 0), 0x123400B1u);
|
|
|
|
cu->write_vgpr(vb + 4, 0, pack16(0xFFFEu, 0xFFF9u)); // low=-2, high=-7
|
|
cu->write_vgpr(vb + 5, 0, pack16(5u, 0xFFFDu)); // low=5, high=-3
|
|
cu->write_vgpr(vb + 6, 0, 0x12340032u);
|
|
execute(encode_vop3(/*op=*/602, /*vdst=*/7, /*src0=*/260, /*src1=*/261,
|
|
/*src2=*/262, /*abs=*/0, /*opsel=*/0x3),
|
|
"v_mad_i32_i16");
|
|
EXPECT_EQ(cu->read_vgpr(vb + 7, 0), 0x12340047u);
|
|
|
|
cu->write_vgpr(vb + 0, 0, pack16(0x1111u, 0x2222u));
|
|
cu->write_vgpr(vb + 1, 0, pack16(0x3333u, 0x4444u));
|
|
execute(encode_vop3(/*op=*/785, /*vdst=*/8, /*src0=*/256, /*src1=*/257,
|
|
/*src2=*/0, /*abs=*/0, /*opsel=*/0x3),
|
|
"v_pack_b32_f16");
|
|
EXPECT_EQ(cu->read_vgpr(vb + 8, 0), 0x44442222u);
|
|
|
|
cu->write_vgpr(vb + 0, 0, pack16(util::f32_to_f16(0.0f), util::f32_to_f16(0.5f)));
|
|
cu->write_vgpr(vb + 1, 0, pack16(util::f32_to_f16(0.0f), util::f32_to_f16(1.0f)));
|
|
execute(encode_vop3(/*op=*/786, /*vdst=*/9, /*src0=*/256, /*src1=*/257,
|
|
/*src2=*/0, /*abs=*/0, /*opsel=*/0x3),
|
|
"v_cvt_pk_norm_i16_f16");
|
|
EXPECT_EQ(cu->read_vgpr(vb + 9, 0), 0x7FFF3FFFu);
|
|
|
|
execute(encode_vop3(/*op=*/787, /*vdst=*/10, /*src0=*/256, /*src1=*/257,
|
|
/*src2=*/0, /*abs=*/0, /*opsel=*/0x3),
|
|
"v_cvt_pk_norm_u16_f16");
|
|
EXPECT_EQ(cu->read_vgpr(vb + 10, 0), 0xFFFF7FFFu);
|
|
|
|
cu->write_vgpr(vb + 0, 0, pack16(util::f32_to_f16(0.25f), util::f32_to_f16(2.0f)));
|
|
cu->write_vgpr(vb + 1, 0, pack16(util::f32_to_f16(1.0f), util::f32_to_f16(3.0f)));
|
|
cu->write_vgpr(vb + 2, 0, pack16(util::f32_to_f16(1.0f), util::f32_to_f16(4.0f)));
|
|
cu->write_vgpr(vb + 12, 0, pack16(0xBEEFu, 0xCAFEu));
|
|
execute(encode_vop3(/*op=*/596, /*vdst=*/12, /*src0=*/256, /*src1=*/257,
|
|
/*src2=*/258, /*abs=*/0, /*opsel=*/0xF),
|
|
"v_div_fixup_f16");
|
|
EXPECT_EQ(cu->read_vgpr(vb + 12, 0), pack16(0xBEEFu, util::f32_to_f16(2.0f)));
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
}
|
|
|
|
TEST(Rdna4Vop3CndmaskTest, B32AppliesNegModifierBeforeSelect) {
|
|
for (bool force_scalar : {false, true}) {
|
|
ForceScalarGuard guard(force_scalar);
|
|
amdgpu::GpuMemory gpu_mem(force_scalar ? "rdna4_cndmask_b32_scalar_mem"
|
|
: "rdna4_cndmask_b32_simd_mem");
|
|
amdgpu::L2Cache l2(force_scalar ? "rdna4_cndmask_b32_scalar_l2" : "rdna4_cndmask_b32_simd_l2");
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = ROCJITSU_CODE_ARCH_RDNA4;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
auto cu = amdgpu::ComputeUnitCore::create("rdna4", cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
|
|
auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_RDNA4);
|
|
ASSERT_NE(decoder, nullptr);
|
|
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
ASSERT_EQ(wf->wf_size(), 32u);
|
|
wf->set_exec(0x3);
|
|
|
|
const uint32_t vb = wf->vgpr_alloc().base;
|
|
const uint32_t sb = wf->sgpr_alloc().base;
|
|
cu->write_sgpr(sb + 0, 0x1u);
|
|
cu->write_sgpr(sb + 1, 0xDEADBEEFu);
|
|
cu->write_vgpr(vb + 0, 0, 0x3F800000u);
|
|
cu->write_vgpr(vb + 0, 1, 0x3F000000u);
|
|
cu->write_vgpr(vb + 1, 0, 0x11223344u);
|
|
cu->write_vgpr(vb + 1, 1, 0x55667788u);
|
|
cu->write_vgpr(vb + 2, 0, 0xDEADCAFEu);
|
|
cu->write_vgpr(vb + 2, 1, 0xDEADCAFEu);
|
|
|
|
// v_cndmask_b32 v2, -v0, v1, s[0:1]
|
|
const uint32_t words[] = {0xD5010002U, 0x20020300U};
|
|
std::unique_ptr<Instruction> inst(decoder->decode(words));
|
|
ASSERT_NE(inst, nullptr);
|
|
ASSERT_EQ(std::string_view(inst->mnemonic()), "v_cndmask_b32");
|
|
cu->execute_instruction(inst.get(), *wf);
|
|
|
|
EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0x11223344u);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 2, 1), 0xBF000000u);
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
}
|
|
|
|
TEST(Rdna4Atan2F16Test, SignedHalfCompareFeedsQuadrantSelect) {
|
|
for (bool force_scalar : {false, true}) {
|
|
ForceScalarGuard guard(force_scalar);
|
|
amdgpu::GpuMemory gpu_mem(force_scalar ? "rdna4_atan2_f16_scalar_mem"
|
|
: "rdna4_atan2_f16_simd_mem");
|
|
amdgpu::L2Cache l2(force_scalar ? "rdna4_atan2_f16_scalar_l2" : "rdna4_atan2_f16_simd_l2");
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = ROCJITSU_CODE_ARCH_RDNA4;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
auto cu = amdgpu::ComputeUnitCore::create("rdna4", cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
|
|
auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_RDNA4);
|
|
ASSERT_NE(decoder, nullptr);
|
|
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
ASSERT_EQ(wf->wf_size(), 32u);
|
|
wf->set_exec(0x3);
|
|
|
|
const uint32_t vb = wf->vgpr_alloc().base;
|
|
cu->write_vgpr(vb + 2, 0, 0x0000BC00u); // low=-1.0h, raw sign bit set
|
|
cu->write_vgpr(vb + 2, 1, 0x00003C00u); // low=+1.0h
|
|
cu->write_vgpr(vb + 6, 0, 0xDEADBEEFu);
|
|
cu->write_vgpr(vb + 6, 1, 0xDEADBEEFu);
|
|
|
|
// v_cmp_gt_i16_e32 vcc_lo, 0, v2.l
|
|
const uint32_t i16_cmp_words[] = {0x7C680480U, 0};
|
|
std::unique_ptr<Instruction> i16_cmp(decoder->decode(i16_cmp_words));
|
|
ASSERT_NE(i16_cmp, nullptr);
|
|
ASSERT_EQ(std::string_view(i16_cmp->mnemonic()), "v_cmp_gt_i16_e32");
|
|
cu->execute_instruction(i16_cmp.get(), *wf);
|
|
EXPECT_EQ(wf->vcc() & 0x3u, 0x1u);
|
|
|
|
// v_cndmask_b32_e64 v6, 0, 0x40490fdb, vcc_lo
|
|
const uint32_t pi_select_words[] = {0xD5010006U, 0x01A9FE80U, 0x40490FDBU};
|
|
std::unique_ptr<Instruction> pi_select(decoder->decode(pi_select_words));
|
|
ASSERT_NE(pi_select, nullptr);
|
|
ASSERT_EQ(std::string_view(pi_select->mnemonic()), "v_cndmask_b32");
|
|
cu->execute_instruction(pi_select.get(), *wf);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 6, 0), 0x40490FDBu);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 6, 1), 0x00000000u);
|
|
|
|
// v_cmp_gt_f16_e32 vcc_lo, 0, v2.l
|
|
const uint32_t f16_cmp_words[] = {0x7C080480U, 0};
|
|
std::unique_ptr<Instruction> f16_cmp(decoder->decode(f16_cmp_words));
|
|
ASSERT_NE(f16_cmp, nullptr);
|
|
ASSERT_EQ(std::string_view(f16_cmp->mnemonic()), "v_cmp_gt_f16_e32");
|
|
cu->execute_instruction(f16_cmp.get(), *wf);
|
|
EXPECT_EQ(wf->vcc() & 0x3u, 0x1u);
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
}
|
|
|
|
TEST(Rdna4Atan2F16Test, DualCndmaskReadsOldPairedSlotSource) {
|
|
amdgpu::GpuMemory gpu_mem("rdna4_atan2_f16_vopd_mem");
|
|
amdgpu::L2Cache l2("rdna4_atan2_f16_vopd_l2");
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = ROCJITSU_CODE_ARCH_RDNA4;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
auto cu = amdgpu::ComputeUnitCore::create("rdna4", cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
|
|
auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_RDNA4);
|
|
ASSERT_NE(decoder, nullptr);
|
|
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
ASSERT_EQ(wf->wf_size(), 32u);
|
|
wf->set_exec(0x3);
|
|
wf->set_vcc(0x1);
|
|
|
|
const uint32_t vb = wf->vgpr_alloc().base;
|
|
cu->write_vgpr(vb + 4, 0, 0x3F490FDBu); // pi/4 approximation
|
|
cu->write_vgpr(vb + 4, 1, 0x3F490FDBu);
|
|
cu->write_vgpr(vb + 5, 0, 0x4016CBE4u); // 3*pi/4 approximation
|
|
cu->write_vgpr(vb + 5, 1, 0x4016CBE4u);
|
|
|
|
// v_dual_cndmask_b32 v4, v4, v5 :: v_dual_mov_b32 v5, 0x4016cbe4
|
|
const uint32_t words[] = {0xCA500B04U, 0x040400FFU, 0x4016CBE4U};
|
|
std::unique_ptr<Instruction> inst(decoder->decode(words));
|
|
ASSERT_NE(inst, nullptr);
|
|
ASSERT_EQ(std::string_view(inst->mnemonic()), "v_dual_cndmask_b32 :: v_dual_mov_b32");
|
|
cu->execute_instruction(inst.get(), *wf);
|
|
|
|
EXPECT_EQ(cu->read_vgpr(vb + 4, 0), 0x4016CBE4u);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 4, 1), 0x3F490FDBu);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 5, 0), 0x4016CBE4u);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 5, 1), 0x4016CBE4u);
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
|
|
TEST(Rdna4Atan2F16Test, TailSequenceKeepsNegativeXQuadrant) {
|
|
amdgpu::GpuMemory gpu_mem("rdna4_atan2_f16_tail_mem");
|
|
amdgpu::L2Cache l2("rdna4_atan2_f16_tail_l2");
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = ROCJITSU_CODE_ARCH_RDNA4;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
auto cu = amdgpu::ComputeUnitCore::create("rdna4", cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
|
|
auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_RDNA4);
|
|
ASSERT_NE(decoder, nullptr);
|
|
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
ASSERT_EQ(wf->wf_size(), 32u);
|
|
wf->set_exec(0x1);
|
|
|
|
const uint32_t vb = wf->vgpr_alloc().base;
|
|
const uint32_t sb = wf->sgpr_alloc().base;
|
|
cu->write_sgpr(sb + 0, 0); // input_x == 0 mask
|
|
cu->write_sgpr(sb + 1, 0); // NaN/class mask
|
|
cu->write_sgpr(sb + 2, 0);
|
|
cu->write_vgpr(vb + 1, 0, 0x0000C400u); // input_x=-4.0h, sign kept by final bfi
|
|
cu->write_vgpr(vb + 2, 0, 0x0000C400u); // input_y=-4.0h
|
|
cu->write_vgpr(vb + 4, 0, 0x3F490FDBu); // base angle pi/4
|
|
cu->write_vgpr(vb + 5, 0, 0x3F490FDBu);
|
|
|
|
const std::array<std::array<uint32_t, 3>, 13> program{{
|
|
{{0x7C680480U, 0, 0}}, // v_cmp_gt_i16_e32
|
|
{{0x080A08FFU, 0x40490FDBU, 0}}, // v_sub_f32_e32 v5, pi, v4
|
|
{{0xD5010006U, 0x01A9FE80U, 0x40490FDBU}}, // v_cndmask v6, 0, pi
|
|
{{0x7C080480U, 0, 0}}, // v_cmp_gt_f16_e32
|
|
{{0xCA500B04U, 0x040400FFU, 0x4016CBE4U}}, // v_dual_cndmask/mov
|
|
{{0xD5010003U, 0x00020D04U, 0}}, // v_cndmask v3, v4, v6, s0
|
|
{{0x02080AFFU, 0x3F490FDBU, 0}}, // v_cndmask v4, pi/4, v5
|
|
{{0x8B6A0102U, 0, 0}}, // s_and_b32 vcc_lo, s2, s1
|
|
{{0x02060903U, 0, 0}}, // v_cndmask v3, v3, v4
|
|
{{0x7C0E0302U, 0, 0}}, // v_cmp_o_f16_e32
|
|
{{0x020406FFU, 0x7FC00000U, 0}}, // v_cndmask v2, qnan, v3
|
|
{{0x7E041502U, 0, 0}}, // v_cvt_f16_f32_e32
|
|
{{0xD6120001U, 0x040604FFU, 0x00007FFFU}}, // v_bfi_b32
|
|
}};
|
|
|
|
for (const auto &words : program) {
|
|
std::unique_ptr<Instruction> inst(decoder->decode(words.data()));
|
|
ASSERT_NE(inst, nullptr);
|
|
cu->execute_instruction(inst.get(), *wf);
|
|
}
|
|
|
|
EXPECT_EQ(cu->read_vgpr(vb + 1, 0) & 0xFFFFu, 0xC0B6u);
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
|
|
TEST(Rdna4Vop3pFmaMixTest, F32AppliesAbsModifiersFromNegHi) {
|
|
for (bool force_scalar : {false, true}) {
|
|
ForceScalarGuard guard(force_scalar);
|
|
amdgpu::GpuMemory gpu_mem(force_scalar ? "rdna4_fma_mix_abs_scalar_mem"
|
|
: "rdna4_fma_mix_abs_simd_mem");
|
|
amdgpu::L2Cache l2(force_scalar ? "rdna4_fma_mix_abs_scalar_l2" : "rdna4_fma_mix_abs_simd_l2");
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = ROCJITSU_CODE_ARCH_RDNA4;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
auto cu = amdgpu::ComputeUnitCore::create("rdna4", cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
|
|
auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_RDNA4);
|
|
ASSERT_NE(decoder, nullptr);
|
|
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
ASSERT_EQ(wf->wf_size(), 32u);
|
|
wf->set_exec(0x1);
|
|
|
|
const uint32_t vb = wf->vgpr_alloc().base;
|
|
const uint32_t sb = wf->sgpr_alloc().base;
|
|
cu->write_sgpr(sb + 0, 0x3F800000u); // 1.0f
|
|
cu->write_vgpr(vb + 0, 0, 0xDEADCAFEu);
|
|
cu->write_vgpr(vb + 3, 0, 0xCAFECC00u); // low=-16.0h
|
|
|
|
// v_fma_mix_f32 v0, |v3|, s0, neg(0) op_sel_hi:[1,0,0]
|
|
const uint32_t abs_src0_words[] = {0xCC200100U, 0x8A000103U};
|
|
std::unique_ptr<Instruction> abs_src0_inst(decoder->decode(abs_src0_words));
|
|
ASSERT_NE(abs_src0_inst, nullptr);
|
|
ASSERT_EQ(std::string_view(abs_src0_inst->mnemonic()), "v_fma_mix_f32");
|
|
cu->execute_instruction(abs_src0_inst.get(), *wf);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 0, 0), 0x41800000u); // 16.0f, not -16.0f
|
|
|
|
cu->write_sgpr(sb + 0, 0x40400000u); // 3.0f
|
|
cu->write_vgpr(vb + 0, 0, 0xDEADCAFEu);
|
|
cu->write_vgpr(vb + 3, 0, 0xCAFECC00u); // low=-16.0h
|
|
cu->write_vgpr(vb + 4, 0, 0x40000000u); // 2.0f
|
|
|
|
// v_fma_mix_f32 v0, v4, s0, |v3| op_sel_hi:[0,0,1]
|
|
const uint32_t abs_src2_words[] = {0xCC204400U, 0x040C0104U};
|
|
std::unique_ptr<Instruction> abs_src2_inst(decoder->decode(abs_src2_words));
|
|
ASSERT_NE(abs_src2_inst, nullptr);
|
|
ASSERT_EQ(std::string_view(abs_src2_inst->mnemonic()), "v_fma_mix_f32");
|
|
cu->execute_instruction(abs_src2_inst.get(), *wf);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 0, 0), 0x41B00000u); // 22.0f, not -10.0f
|
|
|
|
cu->write_vgpr(vb + 0, 0, 0x40000000u); // old src2/dst = 2.0f
|
|
cu->write_vgpr(vb + 2, 0, 0xCAFE4C00u); // low=16.0h
|
|
|
|
// v_fma_mix_f32 v0, v2, 1.0, v0 op_sel_hi:[1,1,0]
|
|
const uint32_t inline_one_f32_words[] = {0xCC200000U, 0x1C01E502U};
|
|
std::unique_ptr<Instruction> inline_one_f32_inst(decoder->decode(inline_one_f32_words));
|
|
ASSERT_NE(inline_one_f32_inst, nullptr);
|
|
ASSERT_EQ(std::string_view(inline_one_f32_inst->mnemonic()), "v_fma_mix_f32");
|
|
cu->execute_instruction(inline_one_f32_inst.get(), *wf);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 0, 0), 0x41900000u); // 16.0f * 1.0f + 2.0f
|
|
|
|
cu->write_vgpr(vb + 0, 0, 0xBEEF0000u);
|
|
cu->write_vgpr(vb + 2, 0, 0xCAFE4C00u); // low=16.0h
|
|
|
|
// v_fma_mixlo_f16 v0, v2, 1.0, 0 op_sel_hi:[1,1,0]
|
|
const uint32_t inline_one_f16_words[] = {0xCC210000U, 0x1A01E502U};
|
|
std::unique_ptr<Instruction> inline_one_f16_inst(decoder->decode(inline_one_f16_words));
|
|
ASSERT_NE(inline_one_f16_inst, nullptr);
|
|
ASSERT_EQ(std::string_view(inline_one_f16_inst->mnemonic()), "v_fma_mixlo_f16");
|
|
cu->execute_instruction(inline_one_f16_inst.get(), *wf);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 0, 0), 0xBEEF4C00u); // high half preserved
|
|
|
|
cu->write_sgpr(sb + 0, 0x3F800000u); // 1.0f
|
|
cu->write_vgpr(vb + 0, 0, 0xDEADCAFEu);
|
|
cu->write_vgpr(vb + 2, 0, 0xCAFEBA00u); // low=-0.75h
|
|
|
|
// v_fma_mix_f32 v0, |v2|, -1.0, s0 op_sel_hi:[1,1,0]
|
|
const uint32_t atanh_den_words[] = {0xCC200100U, 0x1801E702U};
|
|
std::unique_ptr<Instruction> atanh_den_inst(decoder->decode(atanh_den_words));
|
|
ASSERT_NE(atanh_den_inst, nullptr);
|
|
ASSERT_EQ(std::string_view(atanh_den_inst->mnemonic()), "v_fma_mix_f32");
|
|
cu->execute_instruction(atanh_den_inst.get(), *wf);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 0, 0), 0x3E800000u); // abs(-0.75h) * -1.0f + 1.0f
|
|
|
|
cu->write_vgpr(vb + 3, 0, 0xDEADCAFEu);
|
|
|
|
// v_fma_mix_f32 v3, |v2|, 1.0, s0 op_sel_hi:[1,1,0]
|
|
const uint32_t atanh_num_words[] = {0xCC200103U, 0x1801E502U};
|
|
std::unique_ptr<Instruction> atanh_num_inst(decoder->decode(atanh_num_words));
|
|
ASSERT_NE(atanh_num_inst, nullptr);
|
|
ASSERT_EQ(std::string_view(atanh_num_inst->mnemonic()), "v_fma_mix_f32");
|
|
cu->execute_instruction(atanh_num_inst.get(), *wf);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 3, 0), 0x3FE00000u); // abs(-0.75h) * 1.0f + 1.0f
|
|
|
|
cu->write_sgpr(sb + 0, 0x3EB17218u); // 0.5 * ln(2)
|
|
cu->write_vgpr(vb + 0, 0, 0xDEADBEEFu);
|
|
cu->write_vgpr(vb + 3, 0, 0xC033ABB4u); // log2(0.25 / 1.75)
|
|
|
|
// v_fma_mixhi_f16 v0, v3, s0, 0
|
|
const uint32_t atanh_scale_words[] = {0xCC220000U, 0x02000103U};
|
|
std::unique_ptr<Instruction> atanh_scale_inst(decoder->decode(atanh_scale_words));
|
|
ASSERT_NE(atanh_scale_inst, nullptr);
|
|
ASSERT_EQ(std::string_view(atanh_scale_inst->mnemonic()), "v_fma_mixhi_f16");
|
|
cu->execute_instruction(atanh_scale_inst.get(), *wf);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 0, 0), 0xBBC9BEEFu);
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
}
|
|
|
|
TEST(Rdna4True16Vop3Test, B16BitwiseLiteralPreservesInputSignBit) {
|
|
for (bool force_scalar : {false, true}) {
|
|
ForceScalarGuard guard(force_scalar);
|
|
amdgpu::GpuMemory gpu_mem(force_scalar ? "rdna4_b16_bitwise_scalar_mem"
|
|
: "rdna4_b16_bitwise_simd_mem");
|
|
amdgpu::L2Cache l2(force_scalar ? "rdna4_b16_bitwise_scalar_l2" : "rdna4_b16_bitwise_simd_l2");
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = ROCJITSU_CODE_ARCH_RDNA4;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
auto cu = amdgpu::ComputeUnitCore::create("rdna4", cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
|
|
auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_RDNA4);
|
|
ASSERT_NE(decoder, nullptr);
|
|
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
ASSERT_EQ(wf->wf_size(), 32u);
|
|
wf->set_exec(0x1);
|
|
|
|
const uint32_t vb = wf->vgpr_alloc().base;
|
|
cu->write_vgpr(vb + 0, 0, 0xCAFE34D0u); // positive result half
|
|
cu->write_vgpr(vb + 1, 0, 0x11110000u);
|
|
cu->write_vgpr(vb + 3, 0, 0xCAFECC00u); // low=-16.0h
|
|
|
|
// v_and_b16 v1.l, 0x8000, v3.l
|
|
const uint32_t and_words[] = {0xD7620001U, 0x020206FFU, 0xFFFF8000U};
|
|
std::unique_ptr<Instruction> and_inst(decoder->decode(and_words));
|
|
ASSERT_NE(and_inst, nullptr);
|
|
ASSERT_EQ(std::string_view(and_inst->mnemonic()), "v_and_b16");
|
|
ASSERT_NE(and_inst->src_operand(0), nullptr);
|
|
EXPECT_EQ(static_cast<uint32_t>(and_inst->src_operand(0)->encoding_value()), 0xFFFF8000u);
|
|
EXPECT_EQ(and_inst->src_operand(0)->name(), "0x8000");
|
|
EXPECT_NE(and_inst->disassemble().find("0x8000"), std::string::npos);
|
|
EXPECT_EQ(and_inst->disassemble().find("0x-"), std::string::npos);
|
|
cu->execute_instruction(and_inst.get(), *wf);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 1, 0), 0x11118000u);
|
|
|
|
cu->write_vgpr(vb + 1, 0, 0x22220000u);
|
|
|
|
// v_and_b16 v1.l, 0x8000, v3.l op_sel:[1,0,0]
|
|
const uint32_t and_high_words[] = {0xD7620801U, 0x020206FFU, 0x80000000U};
|
|
std::unique_ptr<Instruction> and_high_inst(decoder->decode(and_high_words));
|
|
ASSERT_NE(and_high_inst, nullptr);
|
|
ASSERT_EQ(std::string_view(and_high_inst->mnemonic()), "v_and_b16");
|
|
ASSERT_NE(and_high_inst->src_operand(0), nullptr);
|
|
EXPECT_EQ(static_cast<uint32_t>(and_high_inst->src_operand(0)->encoding_value()), 0x80000000u);
|
|
EXPECT_EQ(and_high_inst->src_operand(0)->name(), "0x8000");
|
|
EXPECT_NE(and_high_inst->disassemble().find("0x8000"), std::string::npos);
|
|
EXPECT_EQ(and_high_inst->disassemble().find("0x-"), std::string::npos);
|
|
cu->execute_instruction(and_high_inst.get(), *wf);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 1, 0), 0x22228000u);
|
|
|
|
cu->write_vgpr(vb + 1, 0, 0x33330000u);
|
|
|
|
// v_and_b16 v1.l, v3.l, 0x8000 op_sel:[0,1,0]
|
|
const uint32_t and_src1_high_words[] = {0xD7621001U, 0x0201FF03U, 0x80000000U};
|
|
std::unique_ptr<Instruction> and_src1_high_inst(decoder->decode(and_src1_high_words));
|
|
ASSERT_NE(and_src1_high_inst, nullptr);
|
|
ASSERT_EQ(std::string_view(and_src1_high_inst->mnemonic()), "v_and_b16");
|
|
ASSERT_NE(and_src1_high_inst->src_operand(1), nullptr);
|
|
EXPECT_EQ(static_cast<uint32_t>(and_src1_high_inst->src_operand(1)->encoding_value()),
|
|
0x80000000u);
|
|
EXPECT_EQ(and_src1_high_inst->src_operand(1)->name(), "0x8000");
|
|
EXPECT_NE(and_src1_high_inst->disassemble().find("0x8000"), std::string::npos);
|
|
EXPECT_EQ(and_src1_high_inst->disassemble().find("0x-"), std::string::npos);
|
|
cu->execute_instruction(and_src1_high_inst.get(), *wf);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 1, 0), 0x33338000u);
|
|
|
|
// v_xor_b16 v0.l, v1.l, v0.l
|
|
const uint32_t xor_words[] = {0xD7640000U, 0x02020101U};
|
|
std::unique_ptr<Instruction> xor_inst(decoder->decode(xor_words));
|
|
ASSERT_NE(xor_inst, nullptr);
|
|
ASSERT_EQ(std::string_view(xor_inst->mnemonic()), "v_xor_b16");
|
|
cu->execute_instruction(xor_inst.get(), *wf);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 0, 0), 0xCAFEB4D0u);
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
}
|
|
|
|
TEST(Gfx1250True16Vop3Test, CndmaskB16UsesSelectedSourceHalfAndPreservesDestinationHalf) {
|
|
amdgpu::GpuMemory gpu_mem("gfx1250_true16_cndmask_mem");
|
|
amdgpu::L2Cache l2("gfx1250_true16_cndmask_l2");
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = ROCJITSU_CODE_ARCH_GFX1250;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
auto cu = amdgpu::ComputeUnitCore::create("gfx1250", cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
|
|
auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
|
|
ASSERT_NE(decoder, nullptr);
|
|
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
ASSERT_EQ(wf->wf_size(), 32u);
|
|
wf->set_exec(0x3);
|
|
|
|
const uint32_t vb = wf->vgpr_alloc().base;
|
|
const uint32_t sb = wf->sgpr_alloc().base;
|
|
cu->write_sgpr(sb + 0, 0x1u);
|
|
cu->write_sgpr(sb + 1, 0xDEADBEEFu);
|
|
for (uint32_t lane = 0; lane < wf->wf_size(); ++lane) {
|
|
cu->write_vgpr(vb + 0, lane, 0xAAAA1234u);
|
|
cu->write_vgpr(vb + 1, lane, 0x5678BBBBu);
|
|
cu->write_vgpr(vb + 2, lane, 0xDEADCAFEu);
|
|
}
|
|
|
|
// v_cndmask_b16 v2.h, v0.l, v1.h, s[0:1] op_sel:[0,1,0,1]
|
|
const uint32_t words[] = {0xD65D5002U, 0x00020300U};
|
|
std::unique_ptr<Instruction> inst(decoder->decode(words));
|
|
ASSERT_NE(inst, nullptr);
|
|
ASSERT_EQ(std::string_view(inst->mnemonic()), "v_cndmask_b16");
|
|
cu->execute_instruction(inst.get(), *wf);
|
|
|
|
EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0x5678CAFEu);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 2, 1), 0x1234CAFEu);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 2, 2), 0xDEADCAFEu);
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
|
|
TEST(Gfx1250True16Vop3Test, LshrrevB16UsesSelectedSourceHalfAndPreservesDestinationHalf) {
|
|
amdgpu::GpuMemory gpu_mem("gfx1250_true16_lshrrev_mem");
|
|
amdgpu::L2Cache l2("gfx1250_true16_lshrrev_l2");
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = ROCJITSU_CODE_ARCH_GFX1250;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
auto cu = amdgpu::ComputeUnitCore::create("gfx1250", cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
|
|
auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
|
|
ASSERT_NE(decoder, nullptr);
|
|
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
wf->set_exec(1);
|
|
|
|
const uint32_t vb = wf->vgpr_alloc().base;
|
|
cu->write_vgpr(vb + 16, 0, 0xAAAA5555u);
|
|
cu->write_vgpr(vb + 17, 0, 0xBEEF1234u);
|
|
|
|
// v_lshrrev_b16 v16.h, 8, v17.l op_sel:[0,0,1]
|
|
const uint32_t words[] = {0xD7394010U, 0x02022288U};
|
|
std::unique_ptr<Instruction> inst(decoder->decode(words));
|
|
ASSERT_NE(inst, nullptr);
|
|
ASSERT_EQ(std::string_view(inst->mnemonic()), "v_lshrrev_b16");
|
|
cu->execute_instruction(inst.get(), *wf);
|
|
|
|
EXPECT_EQ(cu->read_vgpr(vb + 16, 0), 0x00125555u);
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
|
|
TEST(Gfx1250True16Vop3Test, Bitop3B16UsesSelectedSourceHalfAndPreservesDestinationHalf) {
|
|
amdgpu::GpuMemory gpu_mem("gfx1250_true16_bitop_mem");
|
|
amdgpu::L2Cache l2("gfx1250_true16_bitop_l2");
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = ROCJITSU_CODE_ARCH_GFX1250;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
auto cu = amdgpu::ComputeUnitCore::create("gfx1250", cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
|
|
auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
|
|
ASSERT_NE(decoder, nullptr);
|
|
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
ASSERT_EQ(wf->wf_size(), 32u);
|
|
wf->set_exec((1ULL << wf->wf_size()) - 1ULL);
|
|
|
|
const uint32_t v2 = wf->vgpr_alloc().base + 2;
|
|
const uint32_t v3 = wf->vgpr_alloc().base + 3;
|
|
const uint32_t v8 = wf->vgpr_alloc().base + 8;
|
|
for (uint32_t lane = 0; lane < wf->wf_size(); ++lane) {
|
|
cu->write_vgpr(v2, lane, 0xA5A50020u);
|
|
cu->write_vgpr(v3, lane, 0x00F00000u);
|
|
cu->write_vgpr(v8, lane, 0x55550014u);
|
|
}
|
|
|
|
auto execute = [&](const uint32_t(&words)[2], std::string_view mnemonic) {
|
|
std::unique_ptr<Instruction> inst(decoder->decode(words));
|
|
ASSERT_NE(inst, nullptr);
|
|
ASSERT_EQ(std::string_view(inst->mnemonic()), mnemonic);
|
|
cu->execute_instruction(inst.get(), *wf);
|
|
};
|
|
|
|
// v_or_b16 v2.l, v8.l, 1
|
|
const uint32_t or_words[] = {0xD7630002U, 0x02010308U};
|
|
execute(or_words, "v_or_b16");
|
|
for (uint32_t lane = 0; lane < wf->wf_size(); ++lane)
|
|
EXPECT_EQ(cu->read_vgpr(v2, lane), 0xA5A50015u);
|
|
|
|
auto bitop3 = [](uint16_t a, uint16_t b, uint16_t c, uint8_t table) {
|
|
uint16_t result = 0;
|
|
for (int i = 0; i < 16; ++i) {
|
|
uint32_t idx = (((a >> i) & 1u) << 2) | (((b >> i) & 1u) << 1) | ((c >> i) & 1u);
|
|
result |= static_cast<uint16_t>(((table >> idx) & 1u) << i);
|
|
}
|
|
return result;
|
|
};
|
|
|
|
// v_bitop3_b16 v2.l, v2.l, v3.h, -2 bitop3:0xec op_sel:[0,1,0,0]
|
|
const uint32_t bitop_words[] = {0xD6331502U, 0x9B0A0702U};
|
|
execute(bitop_words, "v_bitop3_b16");
|
|
uint32_t expected = bitop3(0x0015u, 0x00F0u, 0xFFFEu, 0xECu);
|
|
for (uint32_t lane = 0; lane < wf->wf_size(); ++lane)
|
|
EXPECT_EQ(cu->read_vgpr(v2, lane), 0xA5A50000u | expected);
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
|
|
TEST(Gfx1250True16Vop1Test, MovB16HighDestinationPreservesLowHalf) {
|
|
amdgpu::GpuMemory gpu_mem("gfx1250_true16_vop1_mem");
|
|
amdgpu::L2Cache l2("gfx1250_true16_vop1_l2");
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = ROCJITSU_CODE_ARCH_GFX1250;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
auto cu = amdgpu::ComputeUnitCore::create("gfx1250", cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
|
|
auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
|
|
ASSERT_NE(decoder, nullptr);
|
|
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
wf->set_exec(1);
|
|
|
|
const uint32_t vb = wf->vgpr_alloc().base;
|
|
cu->write_vgpr(vb + 0, 0, 0x00001234u);
|
|
cu->write_vgpr(vb + 2, 0, 0xAAAA5555u);
|
|
|
|
// v_mov_b16_e32 v2.h, v0.l
|
|
const uint32_t words[] = {0x7F043900U, 0x00000000U};
|
|
std::unique_ptr<Instruction> inst(decoder->decode(words));
|
|
ASSERT_NE(inst, nullptr);
|
|
ASSERT_EQ(std::string_view(inst->mnemonic()), "v_mov_b16_e32");
|
|
EXPECT_NE(inst->disassemble().find("v2.h"), std::string::npos);
|
|
cu->execute_instruction(inst.get(), *wf);
|
|
|
|
EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0x12345555u);
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
|
|
TEST(Rdna3True16Vop1Test, MovB16HighDestinationPreservesLowHalf) {
|
|
amdgpu::GpuMemory gpu_mem("rdna3_true16_vop1_mem");
|
|
amdgpu::L2Cache l2("rdna3_true16_vop1_l2");
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = ROCJITSU_CODE_ARCH_RDNA3;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
auto cu = amdgpu::ComputeUnitCore::create("rdna3", cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
|
|
auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_RDNA3);
|
|
ASSERT_NE(decoder, nullptr);
|
|
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
wf->set_exec(1);
|
|
|
|
const uint32_t vb = wf->vgpr_alloc().base;
|
|
cu->write_vgpr(vb + 0, 0, 0x00001234u);
|
|
cu->write_vgpr(vb + 2, 0, 0xAAAA5555u);
|
|
|
|
// v_mov_b16_e32 v2.h, v0.l
|
|
const uint32_t words[] = {0x7F043900U, 0x00000000U};
|
|
std::unique_ptr<Instruction> inst(decoder->decode(words));
|
|
ASSERT_NE(inst, nullptr);
|
|
ASSERT_EQ(std::string_view(inst->mnemonic()), "v_mov_b16_e32");
|
|
EXPECT_NE(inst->disassemble().find("v2.h"), std::string::npos);
|
|
cu->execute_instruction(inst.get(), *wf);
|
|
|
|
EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0x12345555u);
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
|
|
TEST(Rdna3DppTest, VAddF32RowShrMatchesWaveReduceSequence) {
|
|
const float input[] = {-5.0f, 5.0f, 7.0f, -2.0f, -3.0f, -6.0f, 7.0f, -2.0f, -4.0f, 6.0f, 3.0f,
|
|
-2.0f, -7.0f, -1.0f, 7.0f, -7.0f, 2.0f, -3.0f, 0.0f, -6.0f, 7.0f, -3.0f,
|
|
1.0f, -3.0f, -1.0f, 4.0f, -6.0f, -4.0f, 3.0f, 4.0f, 5.0f, 2.0f};
|
|
|
|
const uint32_t dpp_adds[][2] = {
|
|
{0x060606FAU, 0xFF011103U}, // v_add_f32_dpp v3, v3, v3 row_shr:1
|
|
{0x060606FAU, 0xFF011203U}, // v_add_f32_dpp v3, v3, v3 row_shr:2
|
|
{0x060606FAU, 0xFF011403U}, // v_add_f32_dpp v3, v3, v3 row_shr:4
|
|
{0x060606FAU, 0xFF011803U}, // v_add_f32_dpp v3, v3, v3 row_shr:8
|
|
};
|
|
|
|
for (bool force_scalar : {false, true}) {
|
|
ForceScalarGuard guard(force_scalar);
|
|
amdgpu::GpuMemory gpu_mem(force_scalar ? "rdna3_dpp_scalar_mem" : "rdna3_dpp_simd_mem");
|
|
amdgpu::L2Cache l2(force_scalar ? "rdna3_dpp_scalar_l2" : "rdna3_dpp_simd_l2");
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = ROCJITSU_CODE_ARCH_RDNA3;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
auto cu = amdgpu::ComputeUnitCore::create("rdna3", cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
|
|
auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_RDNA3);
|
|
ASSERT_NE(decoder, nullptr);
|
|
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
ASSERT_EQ(wf->wf_size(), 32u);
|
|
wf->set_exec((1ULL << wf->wf_size()) - 1ULL);
|
|
|
|
const uint32_t vb = wf->vgpr_alloc().base;
|
|
for (uint32_t lane = 0; lane < wf->wf_size(); ++lane)
|
|
cu->write_vgpr(vb + 3, lane, std::bit_cast<uint32_t>(input[lane]));
|
|
|
|
for (const auto &words : dpp_adds) {
|
|
std::unique_ptr<Instruction> inst(decoder->decode(words));
|
|
ASSERT_NE(inst, nullptr);
|
|
ASSERT_EQ(std::string_view(inst->mnemonic()), "v_add_f32_e32");
|
|
cu->execute_instruction(inst.get(), *wf);
|
|
}
|
|
|
|
EXPECT_EQ(cu->read_vgpr(vb + 3, 15), std::bit_cast<uint32_t>(-4.0f));
|
|
EXPECT_EQ(cu->read_vgpr(vb + 3, 31), std::bit_cast<uint32_t>(2.0f));
|
|
|
|
const uint32_t swizzle_words[] = {0xD8D401E0U, 0x04000003U};
|
|
std::unique_ptr<Instruction> swizzle_inst(decoder->decode(swizzle_words));
|
|
ASSERT_NE(swizzle_inst, nullptr);
|
|
ASSERT_EQ(std::string_view(swizzle_inst->mnemonic()), "ds_swizzle_b32");
|
|
cu->execute_instruction(swizzle_inst.get(), *wf);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 4, 31), std::bit_cast<uint32_t>(-4.0f));
|
|
|
|
const uint32_t final_add_words[] = {0xD5030003U, 0x02020903U};
|
|
std::unique_ptr<Instruction> final_add_inst(decoder->decode(final_add_words));
|
|
ASSERT_NE(final_add_inst, nullptr);
|
|
ASSERT_EQ(std::string_view(final_add_inst->mnemonic()), "v_add_f32");
|
|
cu->execute_instruction(final_add_inst.get(), *wf);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 3, 31), std::bit_cast<uint32_t>(-2.0f));
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
}
|
|
|
|
TEST(Rdna3ScalarOperandTest, NullSdstDoesNotClobberM0) {
|
|
amdgpu::GpuMemory gpu_mem("rdna3_null_sdst_mem");
|
|
amdgpu::L2Cache l2("rdna3_null_sdst_l2");
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = ROCJITSU_CODE_ARCH_RDNA3;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
auto cu = amdgpu::ComputeUnitCore::create("rdna3", cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
|
|
auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_RDNA3);
|
|
ASSERT_NE(decoder, nullptr);
|
|
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
ASSERT_EQ(wf->wf_size(), 32u);
|
|
wf->set_exec(0x3);
|
|
wf->set_m0(0xDEADBEEFu);
|
|
|
|
const uint32_t vb = wf->vgpr_alloc().base;
|
|
const uint32_t sb = wf->sgpr_alloc().base;
|
|
cu->write_sgpr(sb + 9, 0x1u);
|
|
cu->write_sgpr(sb + 10, 0xA5A5A5A5u);
|
|
for (uint32_t lane = 0; lane < wf->wf_size(); ++lane)
|
|
cu->write_vgpr(vb + 5, lane, 0xCAFECAFEu);
|
|
|
|
// v_add_co_ci_u32_e64 v5, null, 0, 0, s9
|
|
const uint32_t words[] = {0xD5207C05U, 0x00250080U};
|
|
std::unique_ptr<Instruction> inst(decoder->decode(words));
|
|
ASSERT_NE(inst, nullptr);
|
|
ASSERT_EQ(std::string_view(inst->mnemonic()), "v_add_co_ci_u32");
|
|
EXPECT_NE(inst->disassemble().find("null"), std::string::npos);
|
|
cu->execute_instruction(inst.get(), *wf);
|
|
|
|
EXPECT_EQ(wf->m0(), 0xDEADBEEFu);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 5, 0), 1u);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 5, 1), 0u);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 5, 2), 0xCAFECAFEu);
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
|
|
TEST(Gfx1250True16Vop1Test, CvtF16F32HighDestinationPreservesLowHalf) {
|
|
amdgpu::GpuMemory gpu_mem("gfx1250_true16_cvt_vop1_mem");
|
|
amdgpu::L2Cache l2("gfx1250_true16_cvt_vop1_l2");
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = ROCJITSU_CODE_ARCH_GFX1250;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
auto cu = amdgpu::ComputeUnitCore::create("gfx1250", cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
|
|
auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
|
|
ASSERT_NE(decoder, nullptr);
|
|
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
wf->set_exec(1);
|
|
|
|
const uint32_t vb = wf->vgpr_alloc().base;
|
|
cu->write_vgpr(vb + 0, 0, 0xAAAA5555u);
|
|
cu->write_vgpr(vb + 2, 0, std::bit_cast<uint32_t>(1.0f));
|
|
|
|
// v_cvt_f16_f32_e32 v0.h, v2
|
|
const uint32_t words[] = {0x7F001502U, 0x00000000U};
|
|
std::unique_ptr<Instruction> inst(decoder->decode(words));
|
|
ASSERT_NE(inst, nullptr);
|
|
ASSERT_EQ(std::string_view(inst->mnemonic()), "v_cvt_f16_f32_e32");
|
|
EXPECT_NE(inst->disassemble().find("v0.h"), std::string::npos);
|
|
cu->execute_instruction(inst.get(), *wf);
|
|
|
|
EXPECT_EQ(cu->read_vgpr(vb + 0, 0), 0x3C005555u);
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
|
|
TEST(Gfx1250CvtFp8Test, F16DecodeVop3UsesSelectedByte) {
|
|
amdgpu::GpuMemory gpu_mem("gfx1250_cvt_fp8_bytesel_mem");
|
|
amdgpu::L2Cache l2("gfx1250_cvt_fp8_bytesel_l2");
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = ROCJITSU_CODE_ARCH_GFX1250;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
auto cu = amdgpu::ComputeUnitCore::create("gfx1250", cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
|
|
auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
|
|
ASSERT_NE(decoder, nullptr);
|
|
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
wf->set_exec(1);
|
|
|
|
const uint32_t vb = wf->vgpr_alloc().base;
|
|
cu->write_vgpr(vb + 0, 0, 0x00380000u);
|
|
cu->write_vgpr(vb + 1, 0, 0xBEEF5555u);
|
|
|
|
// v_cvt_f16_fp8 v1.l, v0 byte_sel:2
|
|
const uint32_t fp8_words[] = {0xD5F70801U, 0x00000100U};
|
|
std::unique_ptr<Instruction> fp8_inst(decoder->decode(fp8_words));
|
|
ASSERT_NE(fp8_inst, nullptr);
|
|
ASSERT_EQ(std::string_view(fp8_inst->mnemonic()), "v_cvt_f16_fp8");
|
|
cu->execute_instruction(fp8_inst.get(), *wf);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 1, 0), 0xBEEF3C00u);
|
|
|
|
cu->write_vgpr(vb + 0, 0, 0x3C000000u);
|
|
cu->write_vgpr(vb + 1, 0, 0xBEEF5555u);
|
|
|
|
// v_cvt_f16_bf8 v1.l, v0 byte_sel:3
|
|
const uint32_t bf8_words[] = {0xD5F81801U, 0x00000100U};
|
|
std::unique_ptr<Instruction> bf8_inst(decoder->decode(bf8_words));
|
|
ASSERT_NE(bf8_inst, nullptr);
|
|
ASSERT_EQ(std::string_view(bf8_inst->mnemonic()), "v_cvt_f16_bf8");
|
|
cu->execute_instruction(bf8_inst.get(), *wf);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 1, 0), 0xBEEF3C00u);
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
|
|
TEST(Gfx1250CvtFp8Test, F16E5M3OverflowHonorsFp16OvflMode) {
|
|
amdgpu::GpuMemory gpu_mem("gfx1250_cvt_fp8_f16_e5m3_overflow_mem");
|
|
amdgpu::L2Cache l2("gfx1250_cvt_fp8_f16_e5m3_overflow_l2");
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = ROCJITSU_CODE_ARCH_GFX1250;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
auto cu = amdgpu::ComputeUnitCore::create("gfx1250", cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
|
|
auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
|
|
ASSERT_NE(decoder, nullptr);
|
|
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
wf->set_exec(1);
|
|
|
|
const uint32_t vb = wf->vgpr_alloc().base;
|
|
cu->write_vgpr(vb + 0, 0, 0x00FE0000u);
|
|
|
|
// v_cvt_f16_fp8 v1.l, v0 byte_sel:2 clamp. On gfx1250, clamp selects the
|
|
// unsigned E5M3 FP8 decode; 0xFE is finite but overflows FP16.
|
|
const uint32_t fp8_words[] = {0xD5F78801U, 0x00000100U};
|
|
std::unique_ptr<Instruction> fp8_inst(decoder->decode(fp8_words));
|
|
ASSERT_NE(fp8_inst, nullptr);
|
|
ASSERT_EQ(std::string_view(fp8_inst->mnemonic()), "v_cvt_f16_fp8");
|
|
|
|
wf->set_mode_raw(0);
|
|
cu->write_vgpr(vb + 1, 0, 0xBEEF5555u);
|
|
cu->execute_instruction(fp8_inst.get(), *wf);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 1, 0), 0xBEEF7C00u);
|
|
|
|
wf->set_mode_raw(amdgpu::Wavefront::FP16_OVFL_BIT);
|
|
cu->write_vgpr(vb + 1, 0, 0xBEEF5555u);
|
|
cu->execute_instruction(fp8_inst.get(), *wf);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 1, 0), 0xBEEF7BFFu);
|
|
|
|
// Same conversion with OPSEL[3]=1 writes the high destination half.
|
|
const uint32_t fp8_high_words[] = {0xD5F7C801U, 0x00000100U};
|
|
std::unique_ptr<Instruction> fp8_high_inst(decoder->decode(fp8_high_words));
|
|
ASSERT_NE(fp8_high_inst, nullptr);
|
|
ASSERT_EQ(std::string_view(fp8_high_inst->mnemonic()), "v_cvt_f16_fp8");
|
|
cu->write_vgpr(vb + 1, 0, 0x5555BEEFu);
|
|
cu->execute_instruction(fp8_high_inst.get(), *wf);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 1, 0), 0x7BFFBEEFu);
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
|
|
TEST(Gfx1250CvtFp8Test, F32DecodeVop3UsesSelectedByte) {
|
|
amdgpu::GpuMemory gpu_mem("gfx1250_cvt_fp8_f32_bytesel_mem");
|
|
amdgpu::L2Cache l2("gfx1250_cvt_fp8_f32_bytesel_l2");
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = ROCJITSU_CODE_ARCH_GFX1250;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
auto cu = amdgpu::ComputeUnitCore::create("gfx1250", cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
|
|
auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
|
|
ASSERT_NE(decoder, nullptr);
|
|
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
wf->set_exec(1);
|
|
|
|
const uint32_t vb = wf->vgpr_alloc().base;
|
|
cu->write_vgpr(vb + 0, 0, 0x00380000u);
|
|
|
|
// v_cvt_f32_fp8 v1, v0 byte_sel:2
|
|
const uint32_t fp8_words[] = {0xD5EC0801U, 0x00000100U};
|
|
std::unique_ptr<Instruction> fp8_inst(decoder->decode(fp8_words));
|
|
ASSERT_NE(fp8_inst, nullptr);
|
|
ASSERT_EQ(std::string_view(fp8_inst->mnemonic()), "v_cvt_f32_fp8");
|
|
cu->execute_instruction(fp8_inst.get(), *wf);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 1, 0), std::bit_cast<uint32_t>(1.0f));
|
|
|
|
cu->write_vgpr(vb + 0, 0, 0x3C000000u);
|
|
|
|
// v_cvt_f32_bf8 v1, v0 byte_sel:3
|
|
const uint32_t bf8_words[] = {0xD5ED1801U, 0x00000100U};
|
|
std::unique_ptr<Instruction> bf8_inst(decoder->decode(bf8_words));
|
|
ASSERT_NE(bf8_inst, nullptr);
|
|
ASSERT_EQ(std::string_view(bf8_inst->mnemonic()), "v_cvt_f32_bf8");
|
|
cu->execute_instruction(bf8_inst.get(), *wf);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 1, 0), std::bit_cast<uint32_t>(1.0f));
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
|
|
TEST(Gfx1250CvtFp8Test, E4M3OverflowHonorsFp16OvflMode) {
|
|
amdgpu::GpuMemory gpu_mem("gfx1250_cvt_fp8_overflow_mem");
|
|
amdgpu::L2Cache l2("gfx1250_cvt_fp8_overflow_l2");
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = ROCJITSU_CODE_ARCH_GFX1250;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
auto cu = amdgpu::ComputeUnitCore::create("gfx1250", cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
|
|
auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
|
|
ASSERT_NE(decoder, nullptr);
|
|
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
wf->set_exec(1);
|
|
|
|
const uint32_t vb = wf->vgpr_alloc().base;
|
|
cu->write_vgpr(vb + 5, 0, std::bit_cast<uint32_t>(768.0f));
|
|
cu->write_vgpr(vb + 6, 0, std::bit_cast<uint32_t>(-768.0f));
|
|
cu->write_vgpr(vb + 2, 0, 0xA5A5BEEFu);
|
|
|
|
// v_cvt_pk_fp8_f32 v2.l, v5, v6
|
|
const auto words = encode_vop3(gfx1250::kVCvtPkFp8F32Vop3, 2, vgpr_src(5), vgpr_src(6), 0);
|
|
std::unique_ptr<Instruction> inst(decoder->decode(words.data()));
|
|
ASSERT_NE(inst, nullptr);
|
|
ASSERT_EQ(std::string_view(inst->mnemonic()), "v_cvt_pk_fp8_f32");
|
|
cu->execute_instruction(inst.get(), *wf);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0xA5A5FF7Fu);
|
|
|
|
wf->set_mode_raw(amdgpu::Wavefront::FP16_OVFL_BIT);
|
|
cu->write_vgpr(vb + 2, 0, 0xA5A5BEEFu);
|
|
cu->execute_instruction(inst.get(), *wf);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0xA5A5FE7Eu);
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
|
|
TEST(Gfx1250CvtFp8Test, Bf8OverflowHonorsFp16OvflMode) {
|
|
amdgpu::GpuMemory gpu_mem("gfx1250_cvt_bf8_overflow_mem");
|
|
amdgpu::L2Cache l2("gfx1250_cvt_bf8_overflow_l2");
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = ROCJITSU_CODE_ARCH_GFX1250;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
auto cu = amdgpu::ComputeUnitCore::create("gfx1250", cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
|
|
auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
|
|
ASSERT_NE(decoder, nullptr);
|
|
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
wf->set_exec(1);
|
|
|
|
const uint32_t vb = wf->vgpr_alloc().base;
|
|
cu->write_vgpr(vb + 5, 0, std::bit_cast<uint32_t>(std::numeric_limits<float>::infinity()));
|
|
cu->write_vgpr(vb + 6, 0, std::bit_cast<uint32_t>(-std::numeric_limits<float>::infinity()));
|
|
cu->write_vgpr(vb + 2, 0, 0xA5A5BEEFu);
|
|
|
|
// v_cvt_pk_bf8_f32 v2.l, v5, v6
|
|
const auto words = encode_vop3(gfx1250::kVCvtPkBf8F32Vop3, 2, vgpr_src(5), vgpr_src(6), 0);
|
|
std::unique_ptr<Instruction> inst(decoder->decode(words.data()));
|
|
ASSERT_NE(inst, nullptr);
|
|
ASSERT_EQ(std::string_view(inst->mnemonic()), "v_cvt_pk_bf8_f32");
|
|
cu->execute_instruction(inst.get(), *wf);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0xA5A5FC7Cu);
|
|
|
|
wf->set_mode_raw(amdgpu::Wavefront::FP16_OVFL_BIT);
|
|
cu->write_vgpr(vb + 2, 0, 0xA5A5BEEFu);
|
|
cu->execute_instruction(inst.get(), *wf);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0xA5A5FB7Bu);
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
|
|
TEST(Gfx1250CvtF16Test, SrPackedF16ConsumesAdvancedSeedAndFp16OvflMode) {
|
|
amdgpu::GpuMemory gpu_mem("gfx1250_cvt_sr_pk_f16_mem");
|
|
amdgpu::L2Cache l2("gfx1250_cvt_sr_pk_f16_l2");
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = ROCJITSU_CODE_ARCH_GFX1250;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
auto cu = amdgpu::ComputeUnitCore::create("gfx1250", cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
|
|
auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
|
|
ASSERT_NE(decoder, nullptr);
|
|
|
|
const auto words =
|
|
encode_vop3(gfx1250::kVCvtSrPkF16F32Vop3, 2, vgpr_src(5), vgpr_src(6), vgpr_src(7));
|
|
std::unique_ptr<Instruction> inst(decoder->decode(words.data()));
|
|
ASSERT_NE(inst, nullptr);
|
|
ASSERT_EQ(std::string_view(inst->mnemonic()), "v_cvt_sr_pk_f16_f32");
|
|
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
wf->set_exec(1);
|
|
|
|
const uint32_t vb = wf->vgpr_alloc().base;
|
|
constexpr uint32_t kSeed = 0x40000000u;
|
|
const float sr_value = std::bit_cast<float>(0x3F801000u);
|
|
const uint16_t expected_lo = util::f32_to_f16_sr_mode(sr_value, kSeed, false);
|
|
const uint16_t expected_hi = util::f32_to_f16_sr_mode(sr_value, util::prng_advance(kSeed), false);
|
|
ASSERT_NE(expected_lo, expected_hi);
|
|
|
|
wf->set_mode_raw(0);
|
|
cu->write_vgpr(vb + 5, 0, std::bit_cast<uint32_t>(sr_value));
|
|
cu->write_vgpr(vb + 6, 0, std::bit_cast<uint32_t>(sr_value));
|
|
cu->write_vgpr(vb + 7, 0, kSeed);
|
|
cu->execute_instruction(inst.get(), *wf);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 2, 0), pack16(expected_lo, expected_hi));
|
|
|
|
cu->write_vgpr(vb + 5, 0, std::bit_cast<uint32_t>(70000.0f));
|
|
cu->write_vgpr(vb + 6, 0, std::bit_cast<uint32_t>(-70000.0f));
|
|
cu->write_vgpr(vb + 7, 0, 0xFFFFFFFFu);
|
|
cu->execute_instruction(inst.get(), *wf);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 2, 0), pack16(0x7C00u, 0xFC00u));
|
|
|
|
wf->set_mode_raw(amdgpu::Wavefront::FP16_OVFL_BIT);
|
|
cu->execute_instruction(inst.get(), *wf);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 2, 0), pack16(0x7BFFu, 0xFBFFu));
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
|
|
TEST(Rdna4CvtFp8Test, E4M3OverflowHonorsFp16OvflMode) {
|
|
amdgpu::GpuMemory gpu_mem("rdna4_cvt_fp8_overflow_mem");
|
|
amdgpu::L2Cache l2("rdna4_cvt_fp8_overflow_l2");
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = ROCJITSU_CODE_ARCH_RDNA4;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
auto cu = amdgpu::ComputeUnitCore::create("rdna4", cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
|
|
auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_RDNA4);
|
|
ASSERT_NE(decoder, nullptr);
|
|
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
wf->set_exec(1);
|
|
|
|
const uint32_t vb = wf->vgpr_alloc().base;
|
|
cu->write_vgpr(vb + 5, 0, std::bit_cast<uint32_t>(768.0f));
|
|
cu->write_vgpr(vb + 6, 0, std::bit_cast<uint32_t>(-768.0f));
|
|
cu->write_vgpr(vb + 2, 0, 0xA5A5BEEFu);
|
|
|
|
// v_cvt_pk_fp8_f32 v2.l, v5, v6
|
|
const auto words = encode_vop3(rdna4::kVCvtPkFp8F32Vop3, 2, vgpr_src(5), vgpr_src(6), 0);
|
|
std::unique_ptr<Instruction> inst(decoder->decode(words.data()));
|
|
ASSERT_NE(inst, nullptr);
|
|
ASSERT_EQ(std::string_view(inst->mnemonic()), "v_cvt_pk_fp8_f32");
|
|
cu->execute_instruction(inst.get(), *wf);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0xA5A5FF7Fu);
|
|
|
|
wf->set_mode_raw(amdgpu::Wavefront::FP16_OVFL_BIT);
|
|
cu->write_vgpr(vb + 2, 0, 0xA5A5BEEFu);
|
|
cu->execute_instruction(inst.get(), *wf);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0xA5A5FE7Eu);
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
|
|
TEST(Rdna4CvtFp8Test, Bf8OverflowHonorsFp16OvflMode) {
|
|
amdgpu::GpuMemory gpu_mem("rdna4_cvt_bf8_overflow_mem");
|
|
amdgpu::L2Cache l2("rdna4_cvt_bf8_overflow_l2");
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = ROCJITSU_CODE_ARCH_RDNA4;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
auto cu = amdgpu::ComputeUnitCore::create("rdna4", cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
|
|
auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_RDNA4);
|
|
ASSERT_NE(decoder, nullptr);
|
|
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
wf->set_exec(1);
|
|
|
|
const uint32_t vb = wf->vgpr_alloc().base;
|
|
cu->write_vgpr(vb + 5, 0, std::bit_cast<uint32_t>(70000.0f));
|
|
cu->write_vgpr(vb + 6, 0, std::bit_cast<uint32_t>(-70000.0f));
|
|
cu->write_vgpr(vb + 2, 0, 0xA5A5BEEFu);
|
|
|
|
// v_cvt_pk_bf8_f32 v2.l, v5, v6
|
|
const auto packed_words = encode_vop3(rdna4::kVCvtPkBf8F32Vop3, 2, vgpr_src(5), vgpr_src(6), 0);
|
|
std::unique_ptr<Instruction> packed_inst(decoder->decode(packed_words.data()));
|
|
ASSERT_NE(packed_inst, nullptr);
|
|
ASSERT_EQ(std::string_view(packed_inst->mnemonic()), "v_cvt_pk_bf8_f32");
|
|
|
|
wf->set_mode_raw(0);
|
|
cu->execute_instruction(packed_inst.get(), *wf);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0xA5A5FC7Cu);
|
|
|
|
wf->set_mode_raw(amdgpu::Wavefront::FP16_OVFL_BIT);
|
|
cu->write_vgpr(vb + 2, 0, 0xA5A5BEEFu);
|
|
cu->execute_instruction(packed_inst.get(), *wf);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0xA5A5FB7Bu);
|
|
|
|
cu->write_vgpr(vb + 6, 0, 0u);
|
|
cu->write_vgpr(vb + 2, 0, 0xA5A5BEEFu);
|
|
|
|
// v_cvt_sr_bf8_f32 v2, v5, v6 byte_sel:0
|
|
const auto sr_words = encode_vop3(rdna4::kVCvtSrBf8F32Vop3, 2, vgpr_src(5), vgpr_src(6), 0);
|
|
std::unique_ptr<Instruction> sr_inst(decoder->decode(sr_words.data()));
|
|
ASSERT_NE(sr_inst, nullptr);
|
|
ASSERT_EQ(std::string_view(sr_inst->mnemonic()), "v_cvt_sr_bf8_f32");
|
|
|
|
wf->set_mode_raw(0);
|
|
cu->execute_instruction(sr_inst.get(), *wf);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0xA5A5BE7Cu);
|
|
|
|
wf->set_mode_raw(amdgpu::Wavefront::FP16_OVFL_BIT);
|
|
cu->write_vgpr(vb + 2, 0, 0xA5A5BEEFu);
|
|
cu->execute_instruction(sr_inst.get(), *wf);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0xA5A5BE7Bu);
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
|
|
TEST(Rdna4CvtF16Test, F32OverflowHonorsFp16OvflMode) {
|
|
amdgpu::GpuMemory gpu_mem("rdna4_cvt_f16_overflow_mem");
|
|
amdgpu::L2Cache l2("rdna4_cvt_f16_overflow_l2");
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = ROCJITSU_CODE_ARCH_RDNA4;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
auto cu = amdgpu::ComputeUnitCore::create("rdna4", cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
|
|
auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_RDNA4);
|
|
ASSERT_NE(decoder, nullptr);
|
|
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
wf->set_exec(1);
|
|
|
|
const uint32_t vb = wf->vgpr_alloc().base;
|
|
cu->write_vgpr(vb + 5, 0, std::bit_cast<uint32_t>(70000.0f));
|
|
cu->write_vgpr(vb + 2, 0, 0xA5A5BEEFu);
|
|
|
|
// v_cvt_f16_f32 v2.l, v5
|
|
const auto words = encode_vop3(rdna4::kVCvtF16F32Vop3, 2, vgpr_src(5), 0, 0);
|
|
std::unique_ptr<Instruction> inst(decoder->decode(words.data()));
|
|
ASSERT_NE(inst, nullptr);
|
|
ASSERT_EQ(std::string_view(inst->mnemonic()), "v_cvt_f16_f32");
|
|
cu->execute_instruction(inst.get(), *wf);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0xA5A57C00u);
|
|
|
|
wf->set_mode_raw(amdgpu::Wavefront::FP16_OVFL_BIT);
|
|
cu->write_vgpr(vb + 2, 0, 0xA5A5BEEFu);
|
|
cu->execute_instruction(inst.get(), *wf);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0xA5A57BFFu);
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
|
|
TEST(Rdna4CvtF16Test, U16OverflowHonorsFp16OvflModeInScalarAndSimdPaths) {
|
|
for (bool force_scalar : {false, true}) {
|
|
SCOPED_TRACE(force_scalar ? "scalar" : "simd");
|
|
ForceScalarGuard guard(force_scalar);
|
|
amdgpu::GpuMemory gpu_mem(force_scalar ? "rdna4_cvt_f16_u16_ovfl_scalar_mem"
|
|
: "rdna4_cvt_f16_u16_ovfl_simd_mem");
|
|
amdgpu::L2Cache l2(force_scalar ? "rdna4_cvt_f16_u16_ovfl_scalar_l2"
|
|
: "rdna4_cvt_f16_u16_ovfl_simd_l2");
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = ROCJITSU_CODE_ARCH_RDNA4;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
auto cu = amdgpu::ComputeUnitCore::create("rdna4", cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
|
|
auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_RDNA4);
|
|
ASSERT_NE(decoder, nullptr);
|
|
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
wf->set_exec(1);
|
|
|
|
const uint32_t vb = wf->vgpr_alloc().base;
|
|
cu->write_vgpr(vb + 5, 0, 0xffffu);
|
|
cu->write_vgpr(vb + 2, 0, 0xA5A5BEEFu);
|
|
|
|
// v_cvt_f16_u16 v2.l, v5
|
|
const auto words = encode_vop3(rdna4::kVCvtF16U16Vop3, 2, vgpr_src(5), 0, 0);
|
|
std::unique_ptr<Instruction> inst(decoder->decode(words.data()));
|
|
ASSERT_NE(inst, nullptr);
|
|
ASSERT_EQ(std::string_view(inst->mnemonic()), "v_cvt_f16_u16");
|
|
cu->execute_instruction(inst.get(), *wf);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0xA5A57C00u);
|
|
|
|
wf->set_mode_raw(amdgpu::Wavefront::FP16_OVFL_BIT);
|
|
cu->write_vgpr(vb + 2, 0, 0xA5A5BEEFu);
|
|
cu->execute_instruction(inst.get(), *wf);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0xA5A57BFFu);
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
}
|
|
|
|
TEST(Rdna4Fp16ValuTest, AddOverflowHonorsFp16OvflModeInScalarAndSimdPaths) {
|
|
for (bool force_scalar : {false, true}) {
|
|
SCOPED_TRACE(force_scalar ? "scalar" : "simd");
|
|
ForceScalarGuard guard(force_scalar);
|
|
amdgpu::GpuMemory gpu_mem(force_scalar ? "rdna4_add_f16_ovfl_scalar_mem"
|
|
: "rdna4_add_f16_ovfl_simd_mem");
|
|
amdgpu::L2Cache l2(force_scalar ? "rdna4_add_f16_ovfl_scalar_l2"
|
|
: "rdna4_add_f16_ovfl_simd_l2");
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = ROCJITSU_CODE_ARCH_RDNA4;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
auto cu = amdgpu::ComputeUnitCore::create("rdna4", cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
|
|
auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_RDNA4);
|
|
ASSERT_NE(decoder, nullptr);
|
|
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
wf->set_exec(1);
|
|
|
|
const uint32_t vb = wf->vgpr_alloc().base;
|
|
cu->write_vgpr(vb + 0, 0, pack16(0x7BFFu, 0));
|
|
cu->write_vgpr(vb + 1, 0, pack16(0x7BFFu, 0));
|
|
cu->write_vgpr(vb + 2, 0, 0xA5A5BEEFu);
|
|
|
|
const auto words = encode_vop3(rdna4::kVAddF16Vop3, 2, vgpr_src(0), vgpr_src(1), 0);
|
|
std::unique_ptr<Instruction> inst(decoder->decode(words.data()));
|
|
ASSERT_NE(inst, nullptr);
|
|
ASSERT_EQ(std::string_view(inst->mnemonic()), "v_add_f16");
|
|
cu->execute_instruction(inst.get(), *wf);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0xA5A57C00u);
|
|
|
|
wf->set_mode_raw(amdgpu::Wavefront::FP16_OVFL_BIT);
|
|
cu->write_vgpr(vb + 2, 0, 0xA5A5BEEFu);
|
|
cu->execute_instruction(inst.get(), *wf);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0xA5A57BFFu);
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
}
|
|
|
|
TEST(Cdna4CvtSrTest, SingleResultF16Bf16ConsumesSeedAndFp16OvflMode) {
|
|
amdgpu::GpuMemory gpu_mem("cdna4_cvt_sr_f16_bf16_mem");
|
|
amdgpu::L2Cache l2("cdna4_cvt_sr_f16_bf16_l2");
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = ROCJITSU_CODE_ARCH_CDNA4;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
auto cu = amdgpu::ComputeUnitCore::create("cdna4", cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
|
|
auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_CDNA4);
|
|
ASSERT_NE(decoder, nullptr);
|
|
|
|
const auto f16_words = encode_cdna_vop3(cdna4::kVCvtSrF16F32Vop3, 2, vgpr_src(5), vgpr_src(6), 0);
|
|
std::unique_ptr<Instruction> f16_inst(decoder->decode(f16_words.data()));
|
|
ASSERT_NE(f16_inst, nullptr);
|
|
ASSERT_EQ(std::string_view(f16_inst->mnemonic()), "v_cvt_sr_f16_f32");
|
|
|
|
const auto f16_hi_words =
|
|
encode_cdna_vop3(cdna4::kVCvtSrF16F32Vop3, 2, vgpr_src(5), vgpr_src(6), 0, 0x8);
|
|
std::unique_ptr<Instruction> f16_hi_inst(decoder->decode(f16_hi_words.data()));
|
|
ASSERT_NE(f16_hi_inst, nullptr);
|
|
ASSERT_EQ(std::string_view(f16_hi_inst->mnemonic()), "v_cvt_sr_f16_f32");
|
|
|
|
const auto bf16_words =
|
|
encode_cdna_vop3(cdna4::kVCvtSrBf16F32Vop3, 3, vgpr_src(5), vgpr_src(6), 0);
|
|
std::unique_ptr<Instruction> bf16_inst(decoder->decode(bf16_words.data()));
|
|
ASSERT_NE(bf16_inst, nullptr);
|
|
ASSERT_EQ(std::string_view(bf16_inst->mnemonic()), "v_cvt_sr_bf16_f32");
|
|
|
|
const auto bf16_hi_words =
|
|
encode_cdna_vop3(cdna4::kVCvtSrBf16F32Vop3, 3, vgpr_src(5), vgpr_src(6), 0, 0x8);
|
|
std::unique_ptr<Instruction> bf16_hi_inst(decoder->decode(bf16_hi_words.data()));
|
|
ASSERT_NE(bf16_hi_inst, nullptr);
|
|
ASSERT_EQ(std::string_view(bf16_hi_inst->mnemonic()), "v_cvt_sr_bf16_f32");
|
|
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
wf->set_exec(1);
|
|
|
|
const uint32_t vb = wf->vgpr_alloc().base;
|
|
constexpr uint32_t kSeedLo = 0u;
|
|
constexpr uint32_t kSeedHi = 0xFFFFFFFFu;
|
|
|
|
const float f16_value = std::bit_cast<float>(0x3F801000u);
|
|
const uint16_t f16_seed_lo = util::f32_to_f16_sr_mode(f16_value, kSeedLo, false);
|
|
const uint16_t f16_seed_hi = util::f32_to_f16_sr_mode(f16_value, kSeedHi, false);
|
|
ASSERT_NE(f16_seed_lo, f16_seed_hi);
|
|
|
|
wf->set_mode_raw(0);
|
|
cu->write_vgpr(vb + 5, 0, std::bit_cast<uint32_t>(f16_value));
|
|
cu->write_vgpr(vb + 6, 0, kSeedLo);
|
|
cu->write_vgpr(vb + 2, 0, 0xA5A50000u);
|
|
cu->execute_instruction(f16_inst.get(), *wf);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0xA5A50000u | f16_seed_lo);
|
|
cu->write_vgpr(vb + 6, 0, kSeedHi);
|
|
cu->write_vgpr(vb + 2, 0, 0x5A5A0000u);
|
|
cu->execute_instruction(f16_inst.get(), *wf);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0x5A5A0000u | f16_seed_hi);
|
|
|
|
cu->write_vgpr(vb + 6, 0, kSeedLo);
|
|
cu->write_vgpr(vb + 2, 0, 0x0000BEEFu);
|
|
cu->execute_instruction(f16_hi_inst.get(), *wf);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 2, 0), (static_cast<uint32_t>(f16_seed_lo) << 16) | 0xBEEFu);
|
|
|
|
constexpr uint32_t kOverflowSeed = 0xFFFFFFFFu;
|
|
const float f16_overflow = 70000.0f;
|
|
const uint16_t f16_clear = util::f32_to_f16_sr_mode(f16_overflow, kOverflowSeed, false);
|
|
const uint16_t f16_set = util::f32_to_f16_sr_mode(f16_overflow, kOverflowSeed, true);
|
|
ASSERT_NE(f16_clear, f16_set);
|
|
|
|
cu->write_vgpr(vb + 5, 0, std::bit_cast<uint32_t>(f16_overflow));
|
|
cu->write_vgpr(vb + 6, 0, kOverflowSeed);
|
|
wf->set_mode_raw(0);
|
|
cu->write_vgpr(vb + 2, 0, 0xA5A50000u);
|
|
cu->execute_instruction(f16_inst.get(), *wf);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0xA5A50000u | f16_clear);
|
|
wf->set_mode_raw(amdgpu::Wavefront::FP16_OVFL_BIT);
|
|
cu->write_vgpr(vb + 2, 0, 0x5A5A0000u);
|
|
cu->execute_instruction(f16_inst.get(), *wf);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0x5A5A0000u | f16_set);
|
|
|
|
const float bf16_value = std::bit_cast<float>(0x3F808000u);
|
|
const uint16_t bf16_seed_lo = util::f32_to_bf16_sr_mode(bf16_value, kSeedLo, false);
|
|
const uint16_t bf16_seed_hi = util::f32_to_bf16_sr_mode(bf16_value, kSeedHi, false);
|
|
ASSERT_NE(bf16_seed_lo, bf16_seed_hi);
|
|
|
|
cu->write_vgpr(vb + 5, 0, std::bit_cast<uint32_t>(bf16_value));
|
|
cu->write_vgpr(vb + 6, 0, kSeedLo);
|
|
cu->write_vgpr(vb + 3, 0, 0xA5A50000u);
|
|
cu->execute_instruction(bf16_inst.get(), *wf);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 3, 0), 0xA5A50000u | bf16_seed_lo);
|
|
cu->write_vgpr(vb + 6, 0, kSeedHi);
|
|
cu->write_vgpr(vb + 3, 0, 0x5A5A0000u);
|
|
cu->execute_instruction(bf16_inst.get(), *wf);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 3, 0), 0x5A5A0000u | bf16_seed_hi);
|
|
|
|
cu->write_vgpr(vb + 6, 0, kSeedLo);
|
|
cu->write_vgpr(vb + 3, 0, 0x0000BEEFu);
|
|
cu->execute_instruction(bf16_hi_inst.get(), *wf);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 3, 0), (static_cast<uint32_t>(bf16_seed_lo) << 16) | 0xBEEFu);
|
|
|
|
const float bf16_overflow = std::numeric_limits<float>::max();
|
|
const uint16_t bf16_clear = util::f32_to_bf16_sr_mode(bf16_overflow, kOverflowSeed, false);
|
|
const uint16_t bf16_set = util::f32_to_bf16_sr_mode(bf16_overflow, kOverflowSeed, true);
|
|
ASSERT_NE(bf16_clear, bf16_set);
|
|
|
|
cu->write_vgpr(vb + 5, 0, std::bit_cast<uint32_t>(bf16_overflow));
|
|
cu->write_vgpr(vb + 6, 0, kOverflowSeed);
|
|
wf->set_mode_raw(0);
|
|
cu->write_vgpr(vb + 3, 0, 0xA5A50000u);
|
|
cu->execute_instruction(bf16_inst.get(), *wf);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 3, 0), 0xA5A50000u | bf16_clear);
|
|
wf->set_mode_raw(amdgpu::Wavefront::FP16_OVFL_BIT);
|
|
cu->write_vgpr(vb + 3, 0, 0x5A5A0000u);
|
|
cu->execute_instruction(bf16_inst.get(), *wf);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 3, 0), 0x5A5A0000u | bf16_set);
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
|
|
TEST(Cdna4CvtPkTest, F16Bf16F32OverflowHonorsFp16OvflMode) {
|
|
amdgpu::GpuMemory gpu_mem("cdna4_cvt_pk_f16_bf16_f32_overflow_mem");
|
|
amdgpu::L2Cache l2("cdna4_cvt_pk_f16_bf16_f32_overflow_l2");
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = ROCJITSU_CODE_ARCH_CDNA4;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
auto cu = amdgpu::ComputeUnitCore::create("cdna4", cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
|
|
auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_CDNA4);
|
|
ASSERT_NE(decoder, nullptr);
|
|
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
wf->set_exec(1);
|
|
|
|
const uint32_t vb = wf->vgpr_alloc().base;
|
|
|
|
// v_cvt_pk_f16_f32 v2, v5, v6
|
|
const auto f16_words = encode_cdna_vop3(cdna4::kVCvtPkF16F32Vop3, 2, vgpr_src(5), vgpr_src(6), 0);
|
|
std::unique_ptr<Instruction> f16_inst(decoder->decode(f16_words.data()));
|
|
ASSERT_NE(f16_inst, nullptr);
|
|
ASSERT_EQ(std::string_view(f16_inst->mnemonic()), "v_cvt_pk_f16_f32");
|
|
|
|
cu->write_vgpr(vb + 5, 0, std::bit_cast<uint32_t>(70000.0f));
|
|
cu->write_vgpr(vb + 6, 0, std::bit_cast<uint32_t>(-70000.0f));
|
|
|
|
wf->set_mode_raw(0);
|
|
cu->execute_instruction(f16_inst.get(), *wf);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0xFC007C00u);
|
|
|
|
wf->set_mode_raw(amdgpu::Wavefront::FP16_OVFL_BIT);
|
|
cu->execute_instruction(f16_inst.get(), *wf);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0xFBFF7BFFu);
|
|
|
|
// v_cvt_pk_bf16_f32 v2, v5, v6
|
|
const auto bf16_words =
|
|
encode_cdna_vop3(cdna4::kVCvtPkBf16F32Vop3, 2, vgpr_src(5), vgpr_src(6), 0);
|
|
std::unique_ptr<Instruction> bf16_inst(decoder->decode(bf16_words.data()));
|
|
ASSERT_NE(bf16_inst, nullptr);
|
|
ASSERT_EQ(std::string_view(bf16_inst->mnemonic()), "v_cvt_pk_bf16_f32");
|
|
|
|
cu->write_vgpr(vb + 5, 0, std::bit_cast<uint32_t>(std::numeric_limits<float>::max()));
|
|
cu->write_vgpr(vb + 6, 0, std::bit_cast<uint32_t>(-std::numeric_limits<float>::max()));
|
|
|
|
wf->set_mode_raw(0);
|
|
cu->execute_instruction(bf16_inst.get(), *wf);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0xFF807F80u);
|
|
|
|
wf->set_mode_raw(amdgpu::Wavefront::FP16_OVFL_BIT);
|
|
cu->execute_instruction(bf16_inst.get(), *wf);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0xFF7F7F7Fu);
|
|
|
|
cu->write_vgpr(vb + 5, 0, std::bit_cast<uint32_t>(std::numeric_limits<float>::infinity()));
|
|
cu->write_vgpr(vb + 6, 0, std::bit_cast<uint32_t>(-std::numeric_limits<float>::infinity()));
|
|
cu->execute_instruction(bf16_inst.get(), *wf);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0xFF807F80u);
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
|
|
TEST(CdnaLegacyCvtF16Test, SetregImm32ModeControlsOverflowingF32ToF16) {
|
|
struct CdnaLegacyCase {
|
|
rj_code_arch_t arch;
|
|
const char *name;
|
|
uint32_t setreg_imm_op;
|
|
uint32_t cvt_op;
|
|
};
|
|
|
|
const CdnaLegacyCase cases[] = {
|
|
{ROCJITSU_CODE_ARCH_CDNA1, "cdna1", cdna1::kSSetregImm32B32Sopk, cdna1::kVCvtF16F32Vop3},
|
|
{ROCJITSU_CODE_ARCH_CDNA2, "cdna2", cdna2::kSSetregImm32B32Sopk, cdna2::kVCvtF16F32Vop3},
|
|
};
|
|
|
|
for (const CdnaLegacyCase &arch_case : cases) {
|
|
SCOPED_TRACE(arch_case.name);
|
|
amdgpu::GpuMemory gpu_mem(std::string("cdna_legacy_cvt_f16_mode_mem_") + arch_case.name);
|
|
amdgpu::L2Cache l2(std::string("cdna_legacy_cvt_f16_mode_l2_") + arch_case.name);
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = arch_case.arch;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
auto cu = amdgpu::ComputeUnitCore::create(arch_case.name, cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
|
|
auto decoder = Decoder::create(arch_case.arch);
|
|
ASSERT_NE(decoder, nullptr);
|
|
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
wf->set_exec(1);
|
|
|
|
const uint32_t vb = wf->vgpr_alloc().base;
|
|
cu->write_vgpr(vb + 5, 0, std::bit_cast<uint32_t>(70000.0f));
|
|
|
|
const auto cvt_words = encode_cdna_vop3(arch_case.cvt_op, 2, vgpr_src(5), 0, 0);
|
|
std::unique_ptr<Instruction> cvt_inst(decoder->decode(cvt_words.data()));
|
|
ASSERT_NE(cvt_inst, nullptr);
|
|
ASSERT_EQ(std::string_view(cvt_inst->mnemonic()), "v_cvt_f16_f32");
|
|
|
|
constexpr uint32_t kHwregFp16Ovfl = encode_hwreg(1, 23, 1);
|
|
const auto setreg_words = encode_sopk(arch_case.setreg_imm_op, 0, kHwregFp16Ovfl, 1);
|
|
std::unique_ptr<Instruction> setreg_inst(decoder->decode(setreg_words.data()));
|
|
ASSERT_NE(setreg_inst, nullptr);
|
|
ASSERT_EQ(std::string_view(setreg_inst->mnemonic()), "s_setreg_imm32_b32");
|
|
|
|
wf->set_mode_raw(0);
|
|
cu->execute_instruction(cvt_inst.get(), *wf);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 2, 0) & 0xFFFFu, 0x7C00u);
|
|
|
|
cu->write_vgpr(vb + 2, 0, 0);
|
|
cu->execute_instruction(setreg_inst.get(), *wf);
|
|
EXPECT_TRUE(wf->fp16_ovfl());
|
|
cu->execute_instruction(cvt_inst.get(), *wf);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 2, 0) & 0xFFFFu, 0x7BFFu);
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
}
|
|
|
|
TEST(Rdna4ScalarF16Test, AddDoesNotConsumeFp16OvflMode) {
|
|
amdgpu::GpuMemory gpu_mem("rdna4_scalar_f16_add_mode_mem");
|
|
amdgpu::L2Cache l2("rdna4_scalar_f16_add_mode_l2");
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = ROCJITSU_CODE_ARCH_RDNA4;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
auto cu = amdgpu::ComputeUnitCore::create("rdna4", cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
|
|
auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_RDNA4);
|
|
ASSERT_NE(decoder, nullptr);
|
|
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
|
|
const uint32_t sb = wf->sgpr_alloc().base;
|
|
cu->write_sgpr(sb + 0, util::f32_to_f16(65504.0f));
|
|
cu->write_sgpr(sb + 1, util::f32_to_f16(65504.0f));
|
|
|
|
const auto words = encode_sop2(rdna4::kSAddF16Sop2, 2, 0, 1);
|
|
std::unique_ptr<Instruction> inst(decoder->decode(words.data()));
|
|
ASSERT_NE(inst, nullptr);
|
|
ASSERT_EQ(std::string_view(inst->mnemonic()), "s_add_f16");
|
|
|
|
wf->set_mode_raw(amdgpu::Wavefront::FP16_OVFL_BIT);
|
|
cu->execute_instruction(inst.get(), *wf);
|
|
EXPECT_EQ(cu->read_sgpr(sb + 2) & 0xFFFFu, 0x7C00u);
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
|
|
TEST(Rdna4ScalarF16Test, CvtF16F32ConsumesFp16OvflMode) {
|
|
amdgpu::GpuMemory gpu_mem("rdna4_scalar_cvt_f16_mode_mem");
|
|
amdgpu::L2Cache l2("rdna4_scalar_cvt_f16_mode_l2");
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = ROCJITSU_CODE_ARCH_RDNA4;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
auto cu = amdgpu::ComputeUnitCore::create("rdna4", cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
|
|
auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_RDNA4);
|
|
ASSERT_NE(decoder, nullptr);
|
|
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
|
|
const uint32_t sb = wf->sgpr_alloc().base;
|
|
cu->write_sgpr(sb + 0, std::bit_cast<uint32_t>(70000.0f));
|
|
|
|
const auto words = encode_sop1(rdna4::kSCvtF16F32Sop1, 2, 0);
|
|
std::unique_ptr<Instruction> inst(decoder->decode(words.data()));
|
|
ASSERT_NE(inst, nullptr);
|
|
ASSERT_EQ(std::string_view(inst->mnemonic()), "s_cvt_f16_f32");
|
|
|
|
wf->set_mode_raw(0);
|
|
cu->execute_instruction(inst.get(), *wf);
|
|
EXPECT_EQ(cu->read_sgpr(sb + 2) & 0xFFFFu, 0x7C00u);
|
|
|
|
wf->set_mode_raw(amdgpu::Wavefront::FP16_OVFL_BIT);
|
|
cu->execute_instruction(inst.get(), *wf);
|
|
EXPECT_EQ(cu->read_sgpr(sb + 2) & 0xFFFFu, 0x7BFFu);
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
|
|
TEST(Cdna3CvtFp8Test, SetregImm32ModeChangeRecomputesFp16OvflRounding) {
|
|
amdgpu::GpuMemory gpu_mem("cdna3_cvt_fp8_mode_change_mem");
|
|
amdgpu::L2Cache l2("cdna3_cvt_fp8_mode_change_l2");
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = ROCJITSU_CODE_ARCH_CDNA3;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
auto cu = amdgpu::ComputeUnitCore::create("cdna3", cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
|
|
auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_CDNA3);
|
|
ASSERT_NE(decoder, nullptr);
|
|
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
wf->set_exec(1);
|
|
|
|
const uint32_t vb = wf->vgpr_alloc().base;
|
|
cu->write_vgpr(vb + 5, 0, std::bit_cast<uint32_t>(300.0f));
|
|
cu->write_vgpr(vb + 6, 0, std::bit_cast<uint32_t>(-300.0f));
|
|
cu->write_vgpr(vb + 2, 0, 0xA5A5BEEFu);
|
|
|
|
// v_cvt_pk_fp8_f32 v2.l, v5, v6. CDNA3 uses FNUZ FP8 encodings.
|
|
const auto words = encode_cdna_vop3(cdna3::kVCvtPkFp8F32Vop3, 2, vgpr_src(5), vgpr_src(6), 0);
|
|
std::unique_ptr<Instruction> inst(decoder->decode(words.data()));
|
|
ASSERT_NE(inst, nullptr);
|
|
ASSERT_EQ(std::string_view(inst->mnemonic()), "v_cvt_pk_fp8_f32");
|
|
|
|
constexpr uint32_t kHwregFp16Ovfl = encode_hwreg(1, 23, 1);
|
|
const auto setreg_words = encode_sopk(cdna3::kSSetregImm32B32Sopk, 0, kHwregFp16Ovfl, 1);
|
|
std::unique_ptr<Instruction> setreg_inst(decoder->decode(setreg_words.data()));
|
|
ASSERT_NE(setreg_inst, nullptr);
|
|
ASSERT_EQ(std::string_view(setreg_inst->mnemonic()), "s_setreg_imm32_b32");
|
|
|
|
// CDNA3 FNUZ rounded overflow toggles these deeper values between the FNUZ
|
|
// NaN code and signed max finite.
|
|
// Recompute through s_setreg_imm32_b32 so the conversion observes shader MODE updates end to end.
|
|
wf->set_mode_raw(0);
|
|
cu->execute_instruction(inst.get(), *wf);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0x00008080u);
|
|
|
|
cu->write_vgpr(vb + 2, 0, 0xA5A5BEEFu);
|
|
cu->execute_instruction(setreg_inst.get(), *wf);
|
|
EXPECT_EQ(wf->mode_raw(), amdgpu::Wavefront::FP16_OVFL_BIT);
|
|
EXPECT_TRUE(wf->fp16_ovfl());
|
|
cu->execute_instruction(inst.get(), *wf);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0x0000FF7Fu);
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
|
|
TEST(Gfx1250CvtFp8Test, F16SourceFp8Bf8ConversionsIgnoreFp16OvflMode) {
|
|
amdgpu::GpuMemory gpu_mem("gfx1250_cvt_fp8_f16_source_mode_mem");
|
|
amdgpu::L2Cache l2("gfx1250_cvt_fp8_f16_source_mode_l2");
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = ROCJITSU_CODE_ARCH_GFX1250;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
auto cu = amdgpu::ComputeUnitCore::create("gfx1250", cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
|
|
auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
|
|
ASSERT_NE(decoder, nullptr);
|
|
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
wf->set_exec(1);
|
|
|
|
const uint32_t vb = wf->vgpr_alloc().base;
|
|
const uint32_t f16_fp8_src = pack16(util::f32_to_f16(512.0f), util::f32_to_f16(-512.0f));
|
|
const uint32_t f16_bf8_src = pack16(util::f32_to_f16(65504.0f), util::f32_to_f16(-65504.0f));
|
|
|
|
auto run_packed_case = [&](uint32_t op, std::string_view mnemonic, uint32_t src,
|
|
uint32_t expected) {
|
|
const auto words = encode_vop3(op, 2, vgpr_src(5), 0, 0);
|
|
std::unique_ptr<Instruction> inst(decoder->decode(words.data()));
|
|
ASSERT_NE(inst, nullptr);
|
|
ASSERT_EQ(std::string_view(inst->mnemonic()), mnemonic);
|
|
|
|
cu->write_vgpr(vb + 5, 0, src);
|
|
wf->set_mode_raw(0);
|
|
cu->write_vgpr(vb + 2, 0, 0xA5A5BEEFu);
|
|
cu->execute_instruction(inst.get(), *wf);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 2, 0), expected);
|
|
|
|
wf->set_mode_raw(amdgpu::Wavefront::FP16_OVFL_BIT);
|
|
cu->write_vgpr(vb + 2, 0, 0x5A5ABEEFu);
|
|
cu->execute_instruction(inst.get(), *wf);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 2, 0), (0x5A5A0000u | (expected & 0xFFFFu)));
|
|
};
|
|
|
|
auto run_sr_case = [&](uint32_t op, std::string_view mnemonic, uint32_t src, uint32_t seed,
|
|
uint32_t expected_byte) {
|
|
const auto words = encode_vop3(op, 2, vgpr_src(5), vgpr_src(6), 0);
|
|
std::unique_ptr<Instruction> inst(decoder->decode(words.data()));
|
|
ASSERT_NE(inst, nullptr);
|
|
ASSERT_EQ(std::string_view(inst->mnemonic()), mnemonic);
|
|
|
|
cu->write_vgpr(vb + 5, 0, src);
|
|
cu->write_vgpr(vb + 6, 0, seed);
|
|
wf->set_mode_raw(0);
|
|
cu->write_vgpr(vb + 2, 0, 0xA5A5BEEFu);
|
|
cu->execute_instruction(inst.get(), *wf);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0xA5A5BE00u | expected_byte);
|
|
|
|
wf->set_mode_raw(amdgpu::Wavefront::FP16_OVFL_BIT);
|
|
cu->write_vgpr(vb + 2, 0, 0x5A5ABEEFu);
|
|
cu->execute_instruction(inst.get(), *wf);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0x5A5ABE00u | expected_byte);
|
|
};
|
|
|
|
run_packed_case(gfx1250::kVCvtPkFp8F16Vop3, "v_cvt_pk_fp8_f16", f16_fp8_src, 0xA5A5FF7Fu);
|
|
run_packed_case(gfx1250::kVCvtPkBf8F16Vop3, "v_cvt_pk_bf8_f16", f16_bf8_src, 0xA5A5FC7Cu);
|
|
run_sr_case(gfx1250::kVCvtSrFp8F16Vop3, "v_cvt_sr_fp8_f16", f16_fp8_src, 0, 0x7Fu);
|
|
run_sr_case(gfx1250::kVCvtSrBf8F16Vop3, "v_cvt_sr_bf8_f16", f16_bf8_src, 0xFFFFFFFFu, 0x7Cu);
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
|
|
TEST(Cdna3CvtFp8Test, Bf8AndStochasticPathsHonorFp16OvflMode) {
|
|
amdgpu::GpuMemory gpu_mem("cdna3_cvt_fp8_bf8_sr_overflow_mem");
|
|
amdgpu::L2Cache l2("cdna3_cvt_fp8_bf8_sr_overflow_l2");
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = ROCJITSU_CODE_ARCH_CDNA3;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
auto cu = amdgpu::ComputeUnitCore::create("cdna3", cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
|
|
auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_CDNA3);
|
|
ASSERT_NE(decoder, nullptr);
|
|
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
wf->set_exec(1);
|
|
|
|
const uint32_t vb = wf->vgpr_alloc().base;
|
|
|
|
cu->write_vgpr(vb + 5, 0, std::bit_cast<uint32_t>(70000.0f));
|
|
cu->write_vgpr(vb + 6, 0, std::bit_cast<uint32_t>(-70000.0f));
|
|
cu->write_vgpr(vb + 2, 0, 0xA5A5BEEFu);
|
|
|
|
// v_cvt_pk_bf8_f32 v2.l, v5, v6
|
|
const auto bf8_words = encode_cdna_vop3(cdna3::kVCvtPkBf8F32Vop3, 2, vgpr_src(5), vgpr_src(6), 0);
|
|
std::unique_ptr<Instruction> bf8_inst(decoder->decode(bf8_words.data()));
|
|
ASSERT_NE(bf8_inst, nullptr);
|
|
ASSERT_EQ(std::string_view(bf8_inst->mnemonic()), "v_cvt_pk_bf8_f32");
|
|
|
|
wf->set_mode_raw(0);
|
|
cu->execute_instruction(bf8_inst.get(), *wf);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0x00008080u);
|
|
|
|
wf->set_mode_raw(amdgpu::Wavefront::FP16_OVFL_BIT);
|
|
cu->write_vgpr(vb + 2, 0, 0xA5A5BEEFu);
|
|
cu->execute_instruction(bf8_inst.get(), *wf);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0x0000FF7Fu);
|
|
|
|
cu->write_vgpr(vb + 5, 0, std::bit_cast<uint32_t>(300.0f));
|
|
cu->write_vgpr(vb + 6, 0, 0u);
|
|
cu->write_vgpr(vb + 2, 0, 0xA5A5BEEFu);
|
|
|
|
// v_cvt_sr_fp8_f32 v2, v5, v6 byte_sel:0
|
|
const auto sr_fp8_words =
|
|
encode_cdna_vop3(cdna3::kVCvtSrFp8F32Vop3, 2, vgpr_src(5), vgpr_src(6), 0);
|
|
std::unique_ptr<Instruction> sr_fp8_inst(decoder->decode(sr_fp8_words.data()));
|
|
ASSERT_NE(sr_fp8_inst, nullptr);
|
|
ASSERT_EQ(std::string_view(sr_fp8_inst->mnemonic()), "v_cvt_sr_fp8_f32");
|
|
|
|
wf->set_mode_raw(0);
|
|
cu->execute_instruction(sr_fp8_inst.get(), *wf);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0xA5A5BE80u);
|
|
|
|
wf->set_mode_raw(amdgpu::Wavefront::FP16_OVFL_BIT);
|
|
cu->write_vgpr(vb + 2, 0, 0xA5A5BEEFu);
|
|
cu->execute_instruction(sr_fp8_inst.get(), *wf);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0xA5A5BE7Fu);
|
|
|
|
cu->write_vgpr(vb + 5, 0, std::bit_cast<uint32_t>(70000.0f));
|
|
cu->write_vgpr(vb + 6, 0, 0u);
|
|
cu->write_vgpr(vb + 2, 0, 0xA5A5BEEFu);
|
|
|
|
// v_cvt_sr_bf8_f32 v2, v5, v6 byte_sel:0
|
|
const auto sr_bf8_words =
|
|
encode_cdna_vop3(cdna3::kVCvtSrBf8F32Vop3, 2, vgpr_src(5), vgpr_src(6), 0);
|
|
std::unique_ptr<Instruction> sr_bf8_inst(decoder->decode(sr_bf8_words.data()));
|
|
ASSERT_NE(sr_bf8_inst, nullptr);
|
|
ASSERT_EQ(std::string_view(sr_bf8_inst->mnemonic()), "v_cvt_sr_bf8_f32");
|
|
|
|
wf->set_mode_raw(0);
|
|
cu->execute_instruction(sr_bf8_inst.get(), *wf);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0xA5A5BE80u);
|
|
|
|
wf->set_mode_raw(amdgpu::Wavefront::FP16_OVFL_BIT);
|
|
cu->write_vgpr(vb + 2, 0, 0xA5A5BEEFu);
|
|
cu->execute_instruction(sr_bf8_inst.get(), *wf);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0xA5A5BE7Fu);
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
|
|
TEST(Cdna4CvtScaleTest, ScaledFp8Bf8OverflowHonorsFp16OvflMode) {
|
|
amdgpu::GpuMemory gpu_mem("cdna4_cvt_scale_fp8_bf8_overflow_mem");
|
|
amdgpu::L2Cache l2("cdna4_cvt_scale_fp8_bf8_overflow_l2");
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = ROCJITSU_CODE_ARCH_CDNA4;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
auto cu = amdgpu::ComputeUnitCore::create("cdna4", cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
|
|
auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_CDNA4);
|
|
ASSERT_NE(decoder, nullptr);
|
|
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
wf->set_exec(1);
|
|
|
|
const uint32_t vb = wf->vgpr_alloc().base;
|
|
|
|
cu->write_vgpr(vb + 5, 0, std::bit_cast<uint32_t>(500.0f));
|
|
cu->write_vgpr(vb + 6, 0, std::bit_cast<uint32_t>(-500.0f));
|
|
cu->write_vgpr(vb + 7, 0, std::bit_cast<uint32_t>(1.0f));
|
|
cu->write_vgpr(vb + 2, 0, 0xA5A5BEEFu);
|
|
|
|
// v_cvt_scalef32_pk_fp8_f32 v2.l, v5, v6, v7
|
|
const auto scaled_fp8_words =
|
|
encode_cdna_vop3(cdna4::kVCvtScalef32PkFp8F32Vop3, 2, vgpr_src(5), vgpr_src(6), vgpr_src(7));
|
|
std::unique_ptr<Instruction> scaled_fp8_inst(decoder->decode(scaled_fp8_words.data()));
|
|
ASSERT_NE(scaled_fp8_inst, nullptr);
|
|
ASSERT_EQ(std::string_view(scaled_fp8_inst->mnemonic()), "v_cvt_scalef32_pk_fp8_f32");
|
|
|
|
wf->set_mode_raw(0);
|
|
cu->execute_instruction(scaled_fp8_inst.get(), *wf);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0xA5A5FF7Fu);
|
|
|
|
wf->set_mode_raw(amdgpu::Wavefront::FP16_OVFL_BIT);
|
|
cu->write_vgpr(vb + 2, 0, 0xA5A5BEEFu);
|
|
cu->execute_instruction(scaled_fp8_inst.get(), *wf);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0xA5A5FE7Eu);
|
|
|
|
cu->write_vgpr(vb + 5, 0, std::bit_cast<uint32_t>(std::numeric_limits<float>::infinity()));
|
|
cu->write_vgpr(vb + 6, 0, std::bit_cast<uint32_t>(-std::numeric_limits<float>::infinity()));
|
|
cu->write_vgpr(vb + 7, 0, std::bit_cast<uint32_t>(1.0f));
|
|
cu->write_vgpr(vb + 2, 0, 0xA5A5BEEFu);
|
|
|
|
// v_cvt_scalef32_pk_bf8_f32 v2.l, v5, v6, v7
|
|
const auto scaled_bf8_words =
|
|
encode_cdna_vop3(cdna4::kVCvtScalef32PkBf8F32Vop3, 2, vgpr_src(5), vgpr_src(6), vgpr_src(7));
|
|
std::unique_ptr<Instruction> scaled_bf8_inst(decoder->decode(scaled_bf8_words.data()));
|
|
ASSERT_NE(scaled_bf8_inst, nullptr);
|
|
ASSERT_EQ(std::string_view(scaled_bf8_inst->mnemonic()), "v_cvt_scalef32_pk_bf8_f32");
|
|
|
|
wf->set_mode_raw(0);
|
|
cu->execute_instruction(scaled_bf8_inst.get(), *wf);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0xA5A5FC7Cu);
|
|
|
|
wf->set_mode_raw(amdgpu::Wavefront::FP16_OVFL_BIT);
|
|
cu->write_vgpr(vb + 2, 0, 0xA5A5BEEFu);
|
|
cu->execute_instruction(scaled_bf8_inst.get(), *wf);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0xA5A5FB7Bu);
|
|
|
|
auto run_scaled_f16_dst_case = [&](uint32_t op, std::string_view mnemonic, uint32_t packed_src,
|
|
float scale) {
|
|
const auto low_words = encode_cdna_vop3(op, 2, vgpr_src(5), vgpr_src(7), 0);
|
|
std::unique_ptr<Instruction> low_inst(decoder->decode(low_words.data()));
|
|
ASSERT_NE(low_inst, nullptr);
|
|
ASSERT_EQ(std::string_view(low_inst->mnemonic()), mnemonic);
|
|
|
|
const auto high_words = encode_cdna_vop3(op, 2, vgpr_src(5), vgpr_src(7), 0, 0x8);
|
|
std::unique_ptr<Instruction> high_inst(decoder->decode(high_words.data()));
|
|
ASSERT_NE(high_inst, nullptr);
|
|
ASSERT_EQ(std::string_view(high_inst->mnemonic()), mnemonic);
|
|
|
|
cu->write_vgpr(vb + 5, 0, packed_src);
|
|
cu->write_vgpr(vb + 7, 0, std::bit_cast<uint32_t>(scale));
|
|
|
|
wf->set_mode_raw(0);
|
|
cu->write_vgpr(vb + 2, 0, 0xA5A5BEEFu);
|
|
cu->execute_instruction(low_inst.get(), *wf);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0xA5A57C00u);
|
|
|
|
wf->set_mode_raw(amdgpu::Wavefront::FP16_OVFL_BIT);
|
|
cu->write_vgpr(vb + 2, 0, 0xA5A5BEEFu);
|
|
cu->execute_instruction(low_inst.get(), *wf);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0xA5A57BFFu);
|
|
|
|
cu->write_vgpr(vb + 2, 0, 0x0000BEEFu);
|
|
cu->execute_instruction(high_inst.get(), *wf);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0x7BFFBEEFu);
|
|
};
|
|
|
|
run_scaled_f16_dst_case(cdna4::kVCvtScalef32F16Fp8Vop3, "v_cvt_scalef32_f16_fp8", 0x0000007Eu,
|
|
256.0f);
|
|
run_scaled_f16_dst_case(cdna4::kVCvtScalef32F16Bf8Vop3, "v_cvt_scalef32_f16_bf8", 0x0000007Bu,
|
|
2.0f);
|
|
|
|
auto run_scaled_packed_case = [&](uint32_t op, std::string_view mnemonic, uint32_t packed_src,
|
|
uint32_t expected_clear, uint32_t expected_set) {
|
|
const auto words = encode_cdna_vop3(op, 2, vgpr_src(5), vgpr_src(7), 0);
|
|
std::unique_ptr<Instruction> inst(decoder->decode(words.data()));
|
|
ASSERT_NE(inst, nullptr);
|
|
ASSERT_EQ(std::string_view(inst->mnemonic()), mnemonic);
|
|
|
|
cu->write_vgpr(vb + 5, 0, packed_src);
|
|
cu->write_vgpr(vb + 7, 0, std::bit_cast<uint32_t>(1.0f));
|
|
cu->write_vgpr(vb + 2, 0, 0xA5A5BEEFu);
|
|
|
|
wf->set_mode_raw(0);
|
|
cu->execute_instruction(inst.get(), *wf);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 2, 0), expected_clear);
|
|
|
|
wf->set_mode_raw(amdgpu::Wavefront::FP16_OVFL_BIT);
|
|
cu->write_vgpr(vb + 2, 0, 0xA5A5BEEFu);
|
|
cu->execute_instruction(inst.get(), *wf);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 2, 0), expected_set);
|
|
};
|
|
|
|
const uint32_t f16_fp8_src = pack16(util::f32_to_f16(512.0f), util::f32_to_f16(-512.0f));
|
|
run_scaled_packed_case(cdna4::kVCvtScalef32PkFp8F16Vop3, "v_cvt_scalef32_pk_fp8_f16", f16_fp8_src,
|
|
0xA5A5FF7Fu, 0xA5A5FE7Eu);
|
|
|
|
const uint32_t f16_bf8_src = pack16(util::f32_to_f16(65504.0f), util::f32_to_f16(-65504.0f));
|
|
run_scaled_packed_case(cdna4::kVCvtScalef32PkBf8F16Vop3, "v_cvt_scalef32_pk_bf8_f16", f16_bf8_src,
|
|
0xA5A5FC7Cu, 0xA5A5FB7Bu);
|
|
|
|
const uint32_t bf16_fp8_src = pack16(util::f32_to_bf16(512.0f), util::f32_to_bf16(-512.0f));
|
|
run_scaled_packed_case(cdna4::kVCvtScalef32PkFp8Bf16Vop3, "v_cvt_scalef32_pk_fp8_bf16",
|
|
bf16_fp8_src, 0xA5A5FF7Fu, 0xA5A5FE7Eu);
|
|
|
|
const uint32_t bf16_bf8_src = pack16(util::f32_to_bf16(65536.0f), util::f32_to_bf16(-65536.0f));
|
|
run_scaled_packed_case(cdna4::kVCvtScalef32PkBf8Bf16Vop3, "v_cvt_scalef32_pk_bf8_bf16",
|
|
bf16_bf8_src, 0xA5A5FC7Cu, 0xA5A5FB7Bu);
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
|
|
TEST(Cdna4CvtScaleTest, WideFp6ToF16ConsumesFp16OvflMode) {
|
|
amdgpu::GpuMemory gpu_mem("cdna4_cvt_scale_wide_fp6_f16_overflow_mem");
|
|
amdgpu::L2Cache l2("cdna4_cvt_scale_wide_fp6_f16_overflow_l2");
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = ROCJITSU_CODE_ARCH_CDNA4;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
auto cu = amdgpu::ComputeUnitCore::create("cdna4", cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
|
|
auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_CDNA4);
|
|
ASSERT_NE(decoder, nullptr);
|
|
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
wf->set_exec(1);
|
|
|
|
const uint32_t vb = wf->vgpr_alloc().base;
|
|
uint8_t fp6_values[32]{};
|
|
for (uint32_t i = 0; i < 32; ++i)
|
|
fp6_values[i] = (i & 1u) ? 0x3Fu : 0x1Fu;
|
|
|
|
uint32_t packed_fp6[6]{};
|
|
util::pack_6bit(fp6_values, packed_fp6);
|
|
for (uint32_t i = 0; i < 6; ++i)
|
|
cu->write_vgpr(vb + 20 + i, 0, packed_fp6[i]);
|
|
cu->write_vgpr(vb + 30, 0, std::bit_cast<uint32_t>(16384.0f));
|
|
|
|
// v_cvt_scalef32_pk32_f16_fp6 v[2:17], v[20:25], v30
|
|
const auto words =
|
|
encode_cdna_vop3(cdna4::kVCvtScalef32Pk32F16Fp6Vop3, 2, vgpr_src(20), vgpr_src(30), 0);
|
|
std::unique_ptr<Instruction> inst(decoder->decode(words.data()));
|
|
ASSERT_NE(inst, nullptr);
|
|
ASSERT_EQ(std::string_view(inst->mnemonic()), "v_cvt_scalef32_pk32_f16_fp6");
|
|
|
|
wf->set_mode_raw(0);
|
|
cu->execute_instruction(inst.get(), *wf);
|
|
for (uint32_t i = 0; i < 16; ++i)
|
|
EXPECT_EQ(cu->read_vgpr(vb + 2 + i, 0), 0xFC007C00u) << "dst dword " << i;
|
|
|
|
wf->set_mode_raw(amdgpu::Wavefront::FP16_OVFL_BIT);
|
|
cu->execute_instruction(inst.get(), *wf);
|
|
for (uint32_t i = 0; i < 16; ++i)
|
|
EXPECT_EQ(cu->read_vgpr(vb + 2 + i, 0), 0xFBFF7BFFu) << "dst dword " << i;
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
|
|
TEST(Gfx1250CvtFp8Test, E5M3ClampSelectsUnsignedFp8Format) {
|
|
amdgpu::GpuMemory gpu_mem("gfx1250_cvt_fp8_e5m3_mem");
|
|
amdgpu::L2Cache l2("gfx1250_cvt_fp8_e5m3_l2");
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = ROCJITSU_CODE_ARCH_GFX1250;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
auto cu = amdgpu::ComputeUnitCore::create("gfx1250", cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
|
|
auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
|
|
ASSERT_NE(decoder, nullptr);
|
|
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
wf->set_exec(1);
|
|
|
|
const uint32_t vb = wf->vgpr_alloc().base;
|
|
const uint32_t sb = wf->sgpr_alloc().base;
|
|
const uint32_t e5m3_0x38 = std::bit_cast<uint32_t>(std::ldexp(1.0f, -8));
|
|
const uint32_t e5m3_0x08 = std::bit_cast<uint32_t>(std::ldexp(1.0f, -14));
|
|
|
|
cu->write_vgpr(vb + 0, 0, 0x00380000u);
|
|
|
|
// v_cvt_f32_fp8 v1, v0 byte_sel:2 clamp
|
|
const uint32_t decode_words[] = {0xD5EC8801U, 0x00000100U};
|
|
std::unique_ptr<Instruction> decode_inst(decoder->decode(decode_words));
|
|
ASSERT_NE(decode_inst, nullptr);
|
|
ASSERT_EQ(std::string_view(decode_inst->mnemonic()), "v_cvt_f32_fp8");
|
|
cu->execute_instruction(decode_inst.get(), *wf);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 1, 0), e5m3_0x38);
|
|
|
|
cu->write_sgpr(sb + 5, e5m3_0x38);
|
|
cu->write_sgpr(sb + 6, e5m3_0x08);
|
|
cu->write_vgpr(vb + 2, 0, 0xA5A5BEEFu);
|
|
|
|
// v_cvt_pk_fp8_f32 v2.l, s5, s6 clamp
|
|
const uint32_t pack_lo_words[] = {0xD7698002U, 0x02000C05U};
|
|
std::unique_ptr<Instruction> pack_lo_inst(decoder->decode(pack_lo_words));
|
|
ASSERT_NE(pack_lo_inst, nullptr);
|
|
ASSERT_EQ(std::string_view(pack_lo_inst->mnemonic()), "v_cvt_pk_fp8_f32");
|
|
cu->execute_instruction(pack_lo_inst.get(), *wf);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0xA5A50838u);
|
|
|
|
cu->write_vgpr(vb + 2, 0, 0xDEADBEEFu);
|
|
|
|
// v_cvt_pk_fp8_f32 v2.h, s5, s6 op_sel:[0,0,1] clamp
|
|
const uint32_t pack_hi_words[] = {0xD769C002U, 0x02000C05U};
|
|
std::unique_ptr<Instruction> pack_hi_inst(decoder->decode(pack_hi_words));
|
|
ASSERT_NE(pack_hi_inst, nullptr);
|
|
ASSERT_EQ(std::string_view(pack_hi_inst->mnemonic()), "v_cvt_pk_fp8_f32");
|
|
cu->execute_instruction(pack_hi_inst.get(), *wf);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0x0838BEEFu);
|
|
|
|
cu->write_vgpr(vb + 2, 0, 0xDEADBEEFu);
|
|
|
|
// v_cvt_sr_fp8_f32 v2, s5, s6 byte_sel:2 clamp
|
|
const uint32_t sr_words[] = {0xD76BC002U, 0x02000C05U};
|
|
std::unique_ptr<Instruction> sr_inst(decoder->decode(sr_words));
|
|
ASSERT_NE(sr_inst, nullptr);
|
|
ASSERT_EQ(std::string_view(sr_inst->mnemonic()), "v_cvt_sr_fp8_f32");
|
|
cu->execute_instruction(sr_inst.get(), *wf);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0xDE38BEEFu);
|
|
|
|
cu->write_sgpr(sb + 5, util::f32_to_f16(std::ldexp(1.0f, -8)));
|
|
cu->write_sgpr(sb + 6, 0);
|
|
cu->write_vgpr(vb + 2, 0, 0xDEADBEEFu);
|
|
|
|
// v_cvt_sr_fp8_f16 v2, s5, s6 byte_sel:2 clamp
|
|
const uint32_t sr_f16_words[] = {0xD774C002U, 0x02000C05U};
|
|
std::unique_ptr<Instruction> sr_f16_inst(decoder->decode(sr_f16_words));
|
|
ASSERT_NE(sr_f16_inst, nullptr);
|
|
ASSERT_EQ(std::string_view(sr_f16_inst->mnemonic()), "v_cvt_sr_fp8_f16");
|
|
cu->execute_instruction(sr_f16_inst.get(), *wf);
|
|
EXPECT_EQ(cu->read_vgpr(vb + 2, 0), 0xDE38BEEFu);
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
|
|
TEST(Gfx1250CvtScaleTest, UnpackUsesSelectedE8M0ScaleByte) {
|
|
amdgpu::GpuMemory gpu_mem("gfx1250_cvt_scale_e8m0_mem");
|
|
amdgpu::L2Cache l2("gfx1250_cvt_scale_e8m0_l2");
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = ROCJITSU_CODE_ARCH_GFX1250;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
auto cu = amdgpu::ComputeUnitCore::create("gfx1250", cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
|
|
auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
|
|
ASSERT_NE(decoder, nullptr);
|
|
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
wf->set_exec(1);
|
|
|
|
const uint32_t vb = wf->vgpr_alloc().base;
|
|
const uint32_t e8m0_scales = 0x817E807Fu; // 1.0, 2.0, 0.5, 4.0
|
|
|
|
cu->write_vgpr(vb + 0, 0, 0x38383838u);
|
|
cu->write_vgpr(vb + 1, 0, 0x38383838u);
|
|
cu->write_vgpr(vb + 10, 0, e8m0_scales);
|
|
|
|
// v_cvt_scale_pk8_f32_fp8 v[2:9], v[0:1], v10 scale_sel:2
|
|
const uint32_t pk8_words[] = {0xD6AA1002U, 0x02021500U};
|
|
std::unique_ptr<Instruction> pk8_inst(decoder->decode(pk8_words));
|
|
ASSERT_NE(pk8_inst, nullptr);
|
|
ASSERT_EQ(std::string_view(pk8_inst->mnemonic()), "v_cvt_scale_pk8_f32_fp8");
|
|
cu->execute_instruction(pk8_inst.get(), *wf);
|
|
for (uint32_t i = 2; i <= 9; ++i)
|
|
EXPECT_EQ(cu->read_vgpr(vb + i, 0), std::bit_cast<uint32_t>(0.5f)) << "vgpr=" << i;
|
|
|
|
cu->write_vgpr(vb + 0, 0, 0x08208208u);
|
|
cu->write_vgpr(vb + 1, 0, 0x82082082u);
|
|
cu->write_vgpr(vb + 2, 0, 0x20820820u);
|
|
cu->write_vgpr(vb + 10, 0, e8m0_scales);
|
|
|
|
// v_cvt_scale_pk16_f32_fp6 v[18:33], v[0:2], v10 scale_sel:1
|
|
const uint32_t pk16_words[] = {0xD6C90812U, 0x02021500U};
|
|
std::unique_ptr<Instruction> pk16_inst(decoder->decode(pk16_words));
|
|
ASSERT_NE(pk16_inst, nullptr);
|
|
ASSERT_EQ(std::string_view(pk16_inst->mnemonic()), "v_cvt_scale_pk16_f32_fp6");
|
|
cu->execute_instruction(pk16_inst.get(), *wf);
|
|
for (uint32_t i = 18; i <= 33; ++i)
|
|
EXPECT_EQ(cu->read_vgpr(vb + i, 0), std::bit_cast<uint32_t>(2.0f)) << "vgpr=" << i;
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
|
|
TEST(Gfx1250DsSwizzleTest, VdsBroadcastReadsAddrSource) {
|
|
amdgpu::GpuMemory gpu_mem("gfx1250_ds_swizzle_vds_mem");
|
|
amdgpu::L2Cache l2("gfx1250_ds_swizzle_vds_l2");
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = ROCJITSU_CODE_ARCH_GFX1250;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
auto cu = amdgpu::ComputeUnitCore::create("gfx1250", cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
|
|
auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
|
|
ASSERT_NE(decoder, nullptr);
|
|
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
ASSERT_EQ(wf->wf_size(), 32u);
|
|
wf->set_exec((1ULL << wf->wf_size()) - 1ULL);
|
|
|
|
const uint32_t vb = wf->vgpr_alloc().base;
|
|
for (uint32_t lane = 0; lane < wf->wf_size(); ++lane) {
|
|
cu->write_vgpr(vb + 0, lane, 0u);
|
|
cu->write_vgpr(vb + 1, lane, 0x1000u + lane);
|
|
cu->write_vgpr(vb + 2, lane, 0xDEADBEEFu);
|
|
}
|
|
|
|
// ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
|
|
// gfx1250 encodes this as VDS, where the sole source VGPR is the addr field.
|
|
const uint32_t swizzle_words[] = {0xD8D401E0U, 0x02000001U};
|
|
std::unique_ptr<Instruction> swizzle_inst(decoder->decode(swizzle_words));
|
|
ASSERT_NE(swizzle_inst, nullptr);
|
|
ASSERT_EQ(std::string_view(swizzle_inst->mnemonic()), "ds_swizzle_b32");
|
|
cu->execute_instruction(swizzle_inst.get(), *wf);
|
|
|
|
for (uint32_t lane = 0; lane < wf->wf_size(); ++lane)
|
|
EXPECT_EQ(cu->read_vgpr(vb + 2, lane), 0x100Fu) << "lane=" << lane;
|
|
|
|
for (uint32_t lane = 0; lane < wf->wf_size(); ++lane)
|
|
cu->write_vgpr(vb + 2, lane, 0xDEADBEEFu);
|
|
|
|
// ds_swizzle_b32 v2, v1 offset:0x80b1, QDMode quad permutation (1, 0, 3, 2)
|
|
const uint32_t quad_perm_words[] = {0xD8D480B1U, 0x02000001U};
|
|
std::unique_ptr<Instruction> quad_perm_inst(decoder->decode(quad_perm_words));
|
|
ASSERT_NE(quad_perm_inst, nullptr);
|
|
ASSERT_EQ(std::string_view(quad_perm_inst->mnemonic()), "ds_swizzle_b32");
|
|
cu->execute_instruction(quad_perm_inst.get(), *wf);
|
|
|
|
constexpr uint32_t qd_src[4] = {1, 0, 3, 2};
|
|
for (uint32_t lane = 0; lane < wf->wf_size(); ++lane) {
|
|
const uint32_t src_lane = (lane & ~0x3u) | qd_src[lane & 0x3u];
|
|
EXPECT_EQ(cu->read_vgpr(vb + 2, lane), 0x1000u + src_lane) << "lane=" << lane;
|
|
}
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
|
|
TEST(Gfx1250AtomicReturnTest, GlobalAtomicAddF32ReturnsOldValue) {
|
|
amdgpu::GpuMemory gpu_mem("gfx1250_atomic_add_mem");
|
|
amdgpu::L2Cache l2("gfx1250_atomic_add_l2");
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = ROCJITSU_CODE_ARCH_GFX1250;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
Gfx1250MemoryTestCu cu("gfx1250", cfg, &gpu_mem, &l2);
|
|
|
|
auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
|
|
ASSERT_NE(decoder, nullptr);
|
|
|
|
auto *wf = cu.dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
wf->set_exec(1);
|
|
|
|
constexpr uint64_t kAddr = 0x1000;
|
|
constexpr float kOld = 2.0f;
|
|
constexpr float kAdd = 5.0f;
|
|
gpu_mem.write32(kAddr, std::bit_cast<uint32_t>(kOld));
|
|
cu.write_sgpr(wf->sgpr_alloc().base + 4, static_cast<uint32_t>(kAddr));
|
|
cu.write_sgpr(wf->sgpr_alloc().base + 5, static_cast<uint32_t>(kAddr >> 32));
|
|
cu.write_vgpr(wf->vgpr_alloc().base + 1, 0, std::bit_cast<uint32_t>(kAdd));
|
|
cu.write_vgpr(wf->vgpr_alloc().base + 2, 0, 0);
|
|
|
|
// global_atomic_add_f32 v0, v2, v1, s[4:5] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
|
|
const uint32_t words[] = {0xEE158004U, 0x00980000U, 0x00000002U};
|
|
std::unique_ptr<Instruction> inst(decoder->decode(words));
|
|
ASSERT_NE(inst, nullptr);
|
|
ASSERT_EQ(std::string_view(inst->mnemonic()), "global_atomic_add_f32");
|
|
cu.execute_and_route(inst.release(), *wf);
|
|
|
|
EXPECT_EQ(cu.read_vgpr(wf->vgpr_alloc().base + 0, 0), std::bit_cast<uint32_t>(kOld));
|
|
EXPECT_EQ(gpu_mem.read32(kAddr), std::bit_cast<uint32_t>(kOld + kAdd));
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
|
|
TEST(Gfx1250AtomicReturnTest, GlobalAtomicAddF32AllowsDstSrcAlias) {
|
|
amdgpu::GpuMemory gpu_mem("gfx1250_atomic_add_alias_mem");
|
|
amdgpu::L2Cache l2("gfx1250_atomic_add_alias_l2");
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = ROCJITSU_CODE_ARCH_GFX1250;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
Gfx1250MemoryTestCu cu("gfx1250", cfg, &gpu_mem, &l2);
|
|
|
|
auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
|
|
ASSERT_NE(decoder, nullptr);
|
|
|
|
auto *wf = cu.dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
wf->set_exec(1);
|
|
|
|
constexpr uint64_t kAddr = 0x1080;
|
|
constexpr float kOld = 0.0f;
|
|
constexpr float kAdd = 5.0f;
|
|
gpu_mem.write32(kAddr, std::bit_cast<uint32_t>(kOld));
|
|
cu.write_sgpr(wf->sgpr_alloc().base + 4, static_cast<uint32_t>(kAddr));
|
|
cu.write_sgpr(wf->sgpr_alloc().base + 5, static_cast<uint32_t>(kAddr >> 32));
|
|
cu.write_vgpr(wf->vgpr_alloc().base + 1, 0, std::bit_cast<uint32_t>(kAdd));
|
|
cu.write_vgpr(wf->vgpr_alloc().base + 2, 0, 0);
|
|
|
|
// global_atomic_add_f32 v1, v2, v1, s[4:5] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
|
|
const uint32_t words[] = {0xEE158004U, 0x00980001U, 0x00000002U};
|
|
std::unique_ptr<Instruction> inst(decoder->decode(words));
|
|
ASSERT_NE(inst, nullptr);
|
|
ASSERT_EQ(std::string_view(inst->mnemonic()), "global_atomic_add_f32");
|
|
cu.execute_and_route(inst.release(), *wf);
|
|
|
|
EXPECT_EQ(cu.read_vgpr(wf->vgpr_alloc().base + 1, 0), std::bit_cast<uint32_t>(kOld));
|
|
EXPECT_EQ(gpu_mem.read32(kAddr), std::bit_cast<uint32_t>(kOld + kAdd));
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
|
|
TEST(Gfx1250AtomicReturnTest, GlobalAtomicAddU64CopiesHighSourceDword) {
|
|
amdgpu::GpuMemory gpu_mem("gfx1250_atomic_add_u64_mem");
|
|
amdgpu::L2Cache l2("gfx1250_atomic_add_u64_l2");
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = ROCJITSU_CODE_ARCH_GFX1250;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
Gfx1250MemoryTestCu cu("gfx1250", cfg, &gpu_mem, &l2);
|
|
|
|
auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
|
|
ASSERT_NE(decoder, nullptr);
|
|
|
|
auto *wf = cu.dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
wf->set_exec(1);
|
|
|
|
constexpr uint64_t kAddr = 0x10C0;
|
|
constexpr uint64_t kOld = 0x0000000200000003ull;
|
|
constexpr uint64_t kAdd = 0x0000000400000005ull;
|
|
gpu_mem.write64(kAddr, kOld);
|
|
cu.write_sgpr(wf->sgpr_alloc().base + 4, static_cast<uint32_t>(kAddr));
|
|
cu.write_sgpr(wf->sgpr_alloc().base + 5, static_cast<uint32_t>(kAddr >> 32));
|
|
cu.write_vgpr(wf->vgpr_alloc().base + 0, 0, static_cast<uint32_t>(kAdd));
|
|
cu.write_vgpr(wf->vgpr_alloc().base + 1, 0, static_cast<uint32_t>(kAdd >> 32));
|
|
cu.write_vgpr(wf->vgpr_alloc().base + 2, 0, 0);
|
|
|
|
// global_atomic_add_u64 v2, v[0:1], s[4:5] scope:SCOPE_DEV
|
|
const uint32_t words[] = {0xEE10C004U, 0x00080000U, 0x00000002U};
|
|
std::unique_ptr<Instruction> inst(decoder->decode(words));
|
|
ASSERT_NE(inst, nullptr);
|
|
ASSERT_EQ(std::string_view(inst->mnemonic()), "global_atomic_add_u64");
|
|
cu.execute_and_route(inst.release(), *wf);
|
|
|
|
EXPECT_EQ(gpu_mem.read64(kAddr), kOld + kAdd);
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
|
|
TEST(Gfx1250AtomicReturnTest, GlobalAtomicCmpswapB32ReturnsOldValue) {
|
|
amdgpu::GpuMemory gpu_mem("gfx1250_atomic_cmpswap_mem");
|
|
amdgpu::L2Cache l2("gfx1250_atomic_cmpswap_l2");
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = ROCJITSU_CODE_ARCH_GFX1250;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
Gfx1250MemoryTestCu cu("gfx1250", cfg, &gpu_mem, &l2);
|
|
|
|
auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_GFX1250);
|
|
ASSERT_NE(decoder, nullptr);
|
|
|
|
auto *wf = cu.dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
wf->set_exec(1);
|
|
|
|
constexpr uint64_t kAddr = 0x1040;
|
|
constexpr uint32_t kOld = 0x11111111u;
|
|
constexpr uint32_t kNew = 0x22222222u;
|
|
gpu_mem.write32(kAddr, kOld);
|
|
cu.write_sgpr(wf->sgpr_alloc().base + 4, static_cast<uint32_t>(kAddr));
|
|
cu.write_sgpr(wf->sgpr_alloc().base + 5, static_cast<uint32_t>(kAddr >> 32));
|
|
cu.write_vgpr(wf->vgpr_alloc().base + 2, 0, 0);
|
|
cu.write_vgpr(wf->vgpr_alloc().base + 4, 0, kNew);
|
|
cu.write_vgpr(wf->vgpr_alloc().base + 5, 0, kOld);
|
|
|
|
// global_atomic_cmpswap_b32 v0, v2, v[4:5], s[4:5] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
|
|
const uint32_t words[] = {0xEE0D0004U, 0x02180000U, 0x00000002U};
|
|
std::unique_ptr<Instruction> inst(decoder->decode(words));
|
|
ASSERT_NE(inst, nullptr);
|
|
ASSERT_EQ(std::string_view(inst->mnemonic()), "global_atomic_cmpswap_b32");
|
|
cu.execute_and_route(inst.release(), *wf);
|
|
|
|
EXPECT_EQ(cu.read_vgpr(wf->vgpr_alloc().base + 0, 0), kOld);
|
|
EXPECT_EQ(gpu_mem.read32(kAddr), kNew);
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
|
|
TEST(HwregTest, GetregReadsModeAndStatusTargetIds) {
|
|
for (const HwregArchCase &arch_case : kHwregArchCases) {
|
|
SCOPED_TRACE(arch_case.name);
|
|
amdgpu::GpuMemory gpu_mem(std::string("hwreg_getreg_mem_") + arch_case.name);
|
|
amdgpu::L2Cache l2(std::string("hwreg_getreg_l2_") + arch_case.name);
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = arch_case.arch;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
auto cu = amdgpu::ComputeUnitCore::create(arch_case.name, cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
|
|
auto decoder = Decoder::create(arch_case.arch);
|
|
ASSERT_NE(decoder, nullptr);
|
|
|
|
constexpr uint32_t kHwregMode32 = encode_hwreg(1);
|
|
constexpr uint32_t kHwregStatus32 = encode_hwreg(2);
|
|
constexpr uint32_t kHwregUnknown32 = encode_hwreg(63);
|
|
const std::array<uint32_t, 2> mode_words = encode_sopk(arch_case.getreg_op, 4, kHwregMode32);
|
|
const std::array<uint32_t, 2> status_words =
|
|
encode_sopk(arch_case.getreg_op, 5, kHwregStatus32);
|
|
const std::array<uint32_t, 2> unknown_words =
|
|
encode_sopk(arch_case.getreg_op, 6, kHwregUnknown32);
|
|
|
|
std::unique_ptr<Instruction> mode_inst(decoder->decode(mode_words.data()));
|
|
ASSERT_NE(mode_inst, nullptr);
|
|
EXPECT_EQ(std::string_view(mode_inst->mnemonic()), "s_getreg_b32");
|
|
std::unique_ptr<Instruction> status_inst(decoder->decode(status_words.data()));
|
|
ASSERT_NE(status_inst, nullptr);
|
|
EXPECT_EQ(std::string_view(status_inst->mnemonic()), "s_getreg_b32");
|
|
std::unique_ptr<Instruction> unknown_inst(decoder->decode(unknown_words.data()));
|
|
ASSERT_NE(unknown_inst, nullptr);
|
|
EXPECT_EQ(std::string_view(unknown_inst->mnemonic()), "s_getreg_b32");
|
|
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
|
|
constexpr uint32_t kMode = 0x87654321u;
|
|
constexpr uint32_t kStatus = 0x12345678u;
|
|
wf->set_mode_raw(kMode);
|
|
wf->set_status_raw(kStatus);
|
|
|
|
cu->execute_instruction(mode_inst.get(), *wf);
|
|
cu->execute_instruction(status_inst.get(), *wf);
|
|
cu->execute_instruction(unknown_inst.get(), *wf);
|
|
|
|
const uint32_t sb = wf->sgpr_alloc().base;
|
|
EXPECT_EQ(cu->read_sgpr(sb + 4), kMode);
|
|
EXPECT_EQ(cu->read_sgpr(sb + 5), kStatus);
|
|
EXPECT_EQ(cu->read_sgpr(sb + 6), 0u);
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
}
|
|
|
|
TEST(HwregHelperTest, ReportsReadWriteResultContracts) {
|
|
amdgpu::GpuMemory gpu_mem("hwreg_helper_contract_mem");
|
|
amdgpu::L2Cache l2("hwreg_helper_contract_l2");
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = ROCJITSU_CODE_ARCH_RDNA4;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
auto cu = amdgpu::ComputeUnitCore::create("rdna4", cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
|
|
wf->set_mode_raw(amdgpu::Wavefront::FP16_OVFL_BIT);
|
|
uint32_t value = 0;
|
|
EXPECT_EQ(amdgpu::read_hwreg_field(*wf, encode_hwreg(1, 23, 1), value),
|
|
amdgpu::HwregAccessResult::Success);
|
|
EXPECT_EQ(value, 1u);
|
|
|
|
EXPECT_EQ(amdgpu::write_hwreg_field(*wf, encode_hwreg(2), 0xDEADBEEFu),
|
|
amdgpu::HwregAccessResult::ReadOnly);
|
|
EXPECT_EQ(amdgpu::write_hwreg_field(*wf, encode_hwreg(4), 1u),
|
|
amdgpu::HwregAccessResult::Privileged);
|
|
EXPECT_EQ(amdgpu::write_hwreg_field(*wf, encode_hwreg(18), 1u),
|
|
amdgpu::HwregAccessResult::Unsupported);
|
|
EXPECT_EQ(amdgpu::read_hwreg_field(*wf, encode_hwreg(63), value),
|
|
amdgpu::HwregAccessResult::Unsupported);
|
|
EXPECT_EQ(value, 0u);
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
|
|
TEST(HwregHelperTest, Gfx1250PreservesWaveSchedModeHwreg) {
|
|
amdgpu::GpuMemory gpu_mem("hwreg_helper_gfx1250_wave_sched_mem");
|
|
amdgpu::L2Cache l2("hwreg_helper_gfx1250_wave_sched_l2");
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = ROCJITSU_CODE_ARCH_GFX1250;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
auto cu = amdgpu::ComputeUnitCore::create("gfx1250", cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
|
|
constexpr uint32_t kWaveSchedMode = 0xA5A55A5Au;
|
|
uint32_t value = 0;
|
|
wf->set_wave_sched_mode_raw(kWaveSchedMode);
|
|
|
|
EXPECT_STREQ(amdgpu::hwreg_name(*wf, encode_hwreg(26)), "WAVE_SCHED_MODE");
|
|
EXPECT_EQ(amdgpu::read_hwreg_field(*wf, encode_hwreg(26), value),
|
|
amdgpu::HwregAccessResult::Success);
|
|
EXPECT_EQ(value, kWaveSchedMode);
|
|
EXPECT_EQ(amdgpu::read_hwreg_field(*wf, encode_hwreg(26, 8, 8), value),
|
|
amdgpu::HwregAccessResult::Success);
|
|
EXPECT_EQ(value, 0x5Au);
|
|
|
|
EXPECT_EQ(amdgpu::write_hwreg_field(*wf, encode_hwreg(26, 4, 4), 0x3u),
|
|
amdgpu::HwregAccessResult::Success);
|
|
EXPECT_EQ(wf->wave_sched_mode_raw(), 0xA5A55A3Au);
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
|
|
TEST(HwregHelperTest, Gfx1250UsesManualHwregIdsInsteadOfLegacyAliases) {
|
|
amdgpu::GpuMemory gpu_mem("hwreg_helper_gfx1250_hwreg_ids_mem");
|
|
amdgpu::L2Cache l2("hwreg_helper_gfx1250_hwreg_ids_l2");
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = ROCJITSU_CODE_ARCH_GFX1250;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
auto cu = amdgpu::ComputeUnitCore::create("gfx1250", cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
|
|
EXPECT_STREQ(amdgpu::hwreg_name(*wf, encode_hwreg(4)), "WAVE_STATE_PRIV");
|
|
EXPECT_EQ(amdgpu::write_hwreg_field(*wf, encode_hwreg(4), 1u),
|
|
amdgpu::HwregAccessResult::Privileged);
|
|
EXPECT_STREQ(amdgpu::hwreg_name(*wf, encode_hwreg(5)), "WAVE_GPR_ALLOC");
|
|
EXPECT_STREQ(amdgpu::hwreg_name(*wf, encode_hwreg(6)), "WAVE_LDS_ALLOC");
|
|
EXPECT_STREQ(amdgpu::hwreg_name(*wf, encode_hwreg(7)), "IB_STS");
|
|
EXPECT_EQ(amdgpu::write_hwreg_field(*wf, encode_hwreg(7), 1u),
|
|
amdgpu::HwregAccessResult::ReadOnly);
|
|
EXPECT_STREQ(amdgpu::hwreg_name(*wf, encode_hwreg(14)), "FLUSH_IB");
|
|
EXPECT_EQ(amdgpu::write_hwreg_field(*wf, encode_hwreg(14), 1u),
|
|
amdgpu::HwregAccessResult::Unsupported);
|
|
EXPECT_STREQ(amdgpu::hwreg_name(*wf, encode_hwreg(23)), "WAVE_HW_ID1");
|
|
EXPECT_STREQ(amdgpu::hwreg_name(*wf, encode_hwreg(24)), "WAVE_HW_ID2");
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
|
|
TEST(HwregHelperTest, Gfx1250ReadsModeledIbStsCounters) {
|
|
amdgpu::GpuMemory gpu_mem("hwreg_helper_gfx1250_ib_sts_mem");
|
|
amdgpu::L2Cache l2("hwreg_helper_gfx1250_ib_sts_l2");
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = ROCJITSU_CODE_ARCH_GFX1250;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
auto cu = amdgpu::ComputeUnitCore::create("gfx1250", cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
|
|
wf->wait_counters().dscnt = 5;
|
|
wf->wait_counters().vmcnt = 6;
|
|
wf->wait_counters().vscnt = 7;
|
|
|
|
uint32_t value = 0;
|
|
EXPECT_STREQ(amdgpu::hwreg_name(*wf, encode_hwreg(7)), "IB_STS");
|
|
EXPECT_EQ(amdgpu::read_hwreg_field(*wf, encode_hwreg(7), value),
|
|
amdgpu::HwregAccessResult::Success);
|
|
EXPECT_EQ(value, (5u << 3) | (6u << 9) | (7u << 24));
|
|
EXPECT_EQ(amdgpu::read_hwreg_field(*wf, encode_hwreg(7, 3, 6), value),
|
|
amdgpu::HwregAccessResult::Success);
|
|
EXPECT_EQ(value, 5u);
|
|
EXPECT_EQ(amdgpu::read_hwreg_field(*wf, encode_hwreg(7, 9, 6), value),
|
|
amdgpu::HwregAccessResult::Success);
|
|
EXPECT_EQ(value, 6u);
|
|
EXPECT_EQ(amdgpu::read_hwreg_field(*wf, encode_hwreg(7, 24, 6), value),
|
|
amdgpu::HwregAccessResult::Success);
|
|
EXPECT_EQ(value, 7u);
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
|
|
TEST(HwregHelperTest, Gfx1250ReadsModeledIbSts2Counters) {
|
|
amdgpu::GpuMemory gpu_mem("hwreg_helper_gfx1250_ib_sts2_mem");
|
|
amdgpu::L2Cache l2("hwreg_helper_gfx1250_ib_sts2_l2");
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = ROCJITSU_CODE_ARCH_GFX1250;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
auto cu = amdgpu::ComputeUnitCore::create("gfx1250", cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
|
|
wf->wait_counters().kmcnt = 17;
|
|
wf->wait_counters().asynccnt = 42;
|
|
wf->wait_counters().tensorcnt = 4;
|
|
wf->set_cluster_info(/*rank=*/6, /*size=*/8);
|
|
|
|
uint32_t value = 0;
|
|
EXPECT_STREQ(amdgpu::hwreg_name(*wf, encode_hwreg(28)), "IB_STS2");
|
|
EXPECT_EQ(amdgpu::read_hwreg_field(*wf, encode_hwreg(28), value),
|
|
amdgpu::HwregAccessResult::Success);
|
|
EXPECT_EQ(value, 17u | (1u << 6) | (42u << 12) | (3u << 18) | (6u << 21) | (3u << 28));
|
|
EXPECT_EQ(amdgpu::read_hwreg_field(*wf, encode_hwreg(28, 0, 5), value),
|
|
amdgpu::HwregAccessResult::Success);
|
|
EXPECT_EQ(value, 17u);
|
|
EXPECT_EQ(amdgpu::read_hwreg_field(*wf, encode_hwreg(28, 6, 4), value),
|
|
amdgpu::HwregAccessResult::Success);
|
|
EXPECT_EQ(value, 1u);
|
|
EXPECT_EQ(amdgpu::read_hwreg_field(*wf, encode_hwreg(28, 12, 6), value),
|
|
amdgpu::HwregAccessResult::Success);
|
|
EXPECT_EQ(value, 42u);
|
|
EXPECT_EQ(amdgpu::read_hwreg_field(*wf, encode_hwreg(28, 18, 2), value),
|
|
amdgpu::HwregAccessResult::Success);
|
|
EXPECT_EQ(value, 3u);
|
|
EXPECT_EQ(amdgpu::read_hwreg_field(*wf, encode_hwreg(28, 21, 4), value),
|
|
amdgpu::HwregAccessResult::Success);
|
|
EXPECT_EQ(value, 6u);
|
|
EXPECT_EQ(amdgpu::read_hwreg_field(*wf, encode_hwreg(28, 28, 2), value),
|
|
amdgpu::HwregAccessResult::Success);
|
|
EXPECT_EQ(value, 3u);
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
|
|
TEST(HwregHelperTest, ModeBit27OnlyEnablesGprIndexingWhereArchitected) {
|
|
struct GprIdxModeCase {
|
|
rj_code_arch_t arch;
|
|
const char *name;
|
|
bool mode_has_gpr_idx_en;
|
|
};
|
|
|
|
const GprIdxModeCase cases[] = {
|
|
{ROCJITSU_CODE_ARCH_CDNA1, "cdna1", true}, {ROCJITSU_CODE_ARCH_CDNA2, "cdna2", true},
|
|
{ROCJITSU_CODE_ARCH_CDNA3, "cdna3", true}, {ROCJITSU_CODE_ARCH_CDNA4, "cdna4", true},
|
|
{ROCJITSU_CODE_ARCH_RDNA1, "rdna1", false}, {ROCJITSU_CODE_ARCH_RDNA2, "rdna2", false},
|
|
{ROCJITSU_CODE_ARCH_RDNA3, "rdna3", false}, {ROCJITSU_CODE_ARCH_RDNA3_5, "rdna3_5", false},
|
|
{ROCJITSU_CODE_ARCH_RDNA4, "rdna4", false}, {ROCJITSU_CODE_ARCH_GFX1250, "gfx1250", true},
|
|
};
|
|
|
|
for (const GprIdxModeCase &arch_case : cases) {
|
|
SCOPED_TRACE(arch_case.name);
|
|
amdgpu::GpuMemory gpu_mem(std::string("hwreg_gpr_idx_mode_mem_") + arch_case.name);
|
|
amdgpu::L2Cache l2(std::string("hwreg_gpr_idx_mode_l2_") + arch_case.name);
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = arch_case.arch;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
auto cu = amdgpu::ComputeUnitCore::create(arch_case.name, cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
|
|
wf->set_mode_raw(amdgpu::Wavefront::GPR_IDX_EN_BIT);
|
|
EXPECT_EQ(wf->mode_raw() & amdgpu::Wavefront::GPR_IDX_EN_BIT,
|
|
amdgpu::Wavefront::GPR_IDX_EN_BIT);
|
|
EXPECT_EQ(wf->gpr_idx_en(), arch_case.mode_has_gpr_idx_en);
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
}
|
|
|
|
TEST(HwregHelperTest, ReadsGprAllocFromWaveState) {
|
|
enum class GprAllocLayout {
|
|
Gfx9_10,
|
|
Cdna3_4,
|
|
};
|
|
|
|
struct GprAllocCase {
|
|
rj_code_arch_t arch;
|
|
const char *name;
|
|
GprAllocLayout layout;
|
|
};
|
|
|
|
const GprAllocCase cases[] = {
|
|
{ROCJITSU_CODE_ARCH_CDNA1, "cdna1", GprAllocLayout::Gfx9_10},
|
|
{ROCJITSU_CODE_ARCH_CDNA2, "cdna2", GprAllocLayout::Gfx9_10},
|
|
{ROCJITSU_CODE_ARCH_CDNA3, "cdna3", GprAllocLayout::Cdna3_4},
|
|
{ROCJITSU_CODE_ARCH_CDNA4, "cdna4", GprAllocLayout::Cdna3_4},
|
|
{ROCJITSU_CODE_ARCH_RDNA1, "rdna1", GprAllocLayout::Gfx9_10},
|
|
{ROCJITSU_CODE_ARCH_RDNA2, "rdna2", GprAllocLayout::Gfx9_10},
|
|
};
|
|
|
|
auto blocks_minus_one = [](uint32_t count, uint32_t block_size) {
|
|
return ((count + block_size - 1u) / block_size) - 1u;
|
|
};
|
|
|
|
for (const GprAllocCase &arch_case : cases) {
|
|
SCOPED_TRACE(arch_case.name);
|
|
amdgpu::GpuMemory gpu_mem(std::string("hwreg_gpr_alloc_mem_") + arch_case.name);
|
|
amdgpu::L2Cache l2(std::string("hwreg_gpr_alloc_l2_") + arch_case.name);
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = arch_case.arch;
|
|
cfg.num_wf_slots = 2;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
auto cu = amdgpu::ComputeUnitCore::create(arch_case.name, cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
|
|
auto *first_wf = cu->dispatch_wf(0, 0, 32, 16);
|
|
ASSERT_NE(first_wf, nullptr);
|
|
auto *wf = cu->dispatch_wf(0, 1, 80, 40);
|
|
ASSERT_NE(wf, nullptr);
|
|
|
|
const amdgpu::RegAllocation &vgpr = wf->vgpr_alloc();
|
|
const amdgpu::RegAllocation &sgpr = wf->sgpr_alloc();
|
|
uint32_t expected = 0;
|
|
uint32_t expected_sgpr_size = 0;
|
|
switch (arch_case.layout) {
|
|
case GprAllocLayout::Gfx9_10:
|
|
expected_sgpr_size = blocks_minus_one(sgpr.count, 8);
|
|
expected = ((vgpr.base >> 2) & 0x3Fu) | ((blocks_minus_one(vgpr.count, 4) & 0x3Fu) << 8) |
|
|
(((sgpr.base >> 3) & 0x3Fu) << 16) | ((expected_sgpr_size & 0xFu) << 24);
|
|
break;
|
|
case GprAllocLayout::Cdna3_4:
|
|
expected_sgpr_size = blocks_minus_one(sgpr.count, 16);
|
|
expected = ((vgpr.base >> 2) & 0x3Fu) | ((blocks_minus_one(vgpr.count, 4) & 0x3Fu) << 6) |
|
|
(((sgpr.base >> 3) & 0x3Fu) << 18) | ((expected_sgpr_size & 0xFu) << 24);
|
|
break;
|
|
}
|
|
|
|
uint32_t value = 0;
|
|
if (arch_case.layout == GprAllocLayout::Gfx9_10) {
|
|
EXPECT_EQ(amdgpu::read_hwreg_field(*wf, encode_hwreg(5), value),
|
|
amdgpu::HwregAccessResult::Success);
|
|
EXPECT_EQ(value, expected);
|
|
} else {
|
|
EXPECT_EQ(amdgpu::read_hwreg_field(*wf, encode_hwreg(5), value),
|
|
amdgpu::HwregAccessResult::Unsupported);
|
|
EXPECT_EQ(value, 0u);
|
|
}
|
|
|
|
EXPECT_EQ(amdgpu::read_hwreg_field(*wf, encode_hwreg(5, 0, 6), value),
|
|
amdgpu::HwregAccessResult::Success);
|
|
EXPECT_EQ(value, (vgpr.base >> 2) & 0x3Fu);
|
|
|
|
EXPECT_EQ(amdgpu::read_hwreg_field(*wf, encode_hwreg(5, 24, 4), value),
|
|
amdgpu::HwregAccessResult::Success);
|
|
EXPECT_EQ(value, expected_sgpr_size);
|
|
if (arch_case.layout == GprAllocLayout::Cdna3_4) {
|
|
EXPECT_EQ(amdgpu::read_hwreg_field(*wf, encode_hwreg(5, 12, 6), value),
|
|
amdgpu::HwregAccessResult::Unsupported);
|
|
EXPECT_EQ(value, 0u);
|
|
}
|
|
EXPECT_EQ(amdgpu::write_hwreg_field(*wf, encode_hwreg(5), 0),
|
|
amdgpu::HwregAccessResult::ReadOnly);
|
|
|
|
if (!first_wf->is_halted())
|
|
first_wf->halt();
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
}
|
|
|
|
TEST(HwregHelperTest, Rdna3TablesNameXmlHwregIds) {
|
|
struct RdnaHwregCase {
|
|
rj_code_arch_t arch;
|
|
const char *name;
|
|
};
|
|
|
|
const RdnaHwregCase cases[] = {
|
|
{ROCJITSU_CODE_ARCH_RDNA3, "rdna3"},
|
|
{ROCJITSU_CODE_ARCH_RDNA3_5, "rdna3_5"},
|
|
};
|
|
|
|
for (const RdnaHwregCase &arch_case : cases) {
|
|
SCOPED_TRACE(arch_case.name);
|
|
amdgpu::GpuMemory gpu_mem(std::string("hwreg_helper_rdna3_mem_") + arch_case.name);
|
|
amdgpu::L2Cache l2(std::string("hwreg_helper_rdna3_l2_") + arch_case.name);
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = arch_case.arch;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
auto cu = amdgpu::ComputeUnitCore::create(arch_case.name, cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
|
|
EXPECT_STREQ(amdgpu::hwreg_name(*wf, encode_hwreg(3)), "TRAPSTS");
|
|
EXPECT_EQ(amdgpu::write_hwreg_field(*wf, encode_hwreg(3), 1u),
|
|
amdgpu::HwregAccessResult::Unsupported);
|
|
EXPECT_STREQ(amdgpu::hwreg_name(*wf, encode_hwreg(14)), "FLUSH_IB");
|
|
EXPECT_EQ(amdgpu::write_hwreg_field(*wf, encode_hwreg(14), 1u),
|
|
amdgpu::HwregAccessResult::Unsupported);
|
|
EXPECT_STREQ(amdgpu::hwreg_name(*wf, encode_hwreg(20)), "SHADER_FLAT_SCRATCH_LO");
|
|
EXPECT_EQ(amdgpu::write_hwreg_field(*wf, encode_hwreg(20), 1u),
|
|
amdgpu::HwregAccessResult::ReadOnly);
|
|
EXPECT_STREQ(amdgpu::hwreg_name(*wf, encode_hwreg(23)), "HW_ID1");
|
|
EXPECT_EQ(amdgpu::write_hwreg_field(*wf, encode_hwreg(23), 1u),
|
|
amdgpu::HwregAccessResult::ReadOnly);
|
|
EXPECT_STREQ(amdgpu::hwreg_name(*wf, encode_hwreg(63)), "unknown");
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
}
|
|
|
|
TEST(HwregTest, GetregExtractsPartialModeAndStatusFields) {
|
|
for (const HwregArchCase &arch_case : kHwregArchCases) {
|
|
SCOPED_TRACE(arch_case.name);
|
|
amdgpu::GpuMemory gpu_mem(std::string("hwreg_getreg_partial_mem_") + arch_case.name);
|
|
amdgpu::L2Cache l2(std::string("hwreg_getreg_partial_l2_") + arch_case.name);
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = arch_case.arch;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
auto cu = amdgpu::ComputeUnitCore::create(arch_case.name, cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
|
|
auto decoder = Decoder::create(arch_case.arch);
|
|
ASSERT_NE(decoder, nullptr);
|
|
|
|
constexpr uint32_t kHwregModeFp16Ovfl = encode_hwreg(1, 23, 1);
|
|
constexpr uint32_t kHwregStatusNibble = encode_hwreg(2, 8, 4);
|
|
const std::array<uint32_t, 2> mode_words =
|
|
encode_sopk(arch_case.getreg_op, 4, kHwregModeFp16Ovfl);
|
|
const std::array<uint32_t, 2> status_words =
|
|
encode_sopk(arch_case.getreg_op, 5, kHwregStatusNibble);
|
|
|
|
std::unique_ptr<Instruction> mode_inst(decoder->decode(mode_words.data()));
|
|
ASSERT_NE(mode_inst, nullptr);
|
|
EXPECT_EQ(std::string_view(mode_inst->mnemonic()), "s_getreg_b32");
|
|
std::unique_ptr<Instruction> status_inst(decoder->decode(status_words.data()));
|
|
ASSERT_NE(status_inst, nullptr);
|
|
EXPECT_EQ(std::string_view(status_inst->mnemonic()), "s_getreg_b32");
|
|
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
|
|
wf->set_mode_raw(amdgpu::Wavefront::FP16_OVFL_BIT);
|
|
wf->set_status_raw(0x12345678u);
|
|
|
|
cu->execute_instruction(mode_inst.get(), *wf);
|
|
cu->execute_instruction(status_inst.get(), *wf);
|
|
|
|
const uint32_t sb = wf->sgpr_alloc().base;
|
|
EXPECT_EQ(cu->read_sgpr(sb + 4), 1u);
|
|
EXPECT_EQ(cu->read_sgpr(sb + 5), 0x6u);
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
}
|
|
|
|
/// @brief Targeted test for s_setreg_imm32_b32: verifies that STATUS is not
|
|
/// writable through shader HWREG writes without modeled privileged semantics.
|
|
TEST(HwregTest, SetregImm32StatusIsReadOnly) {
|
|
for (const HwregArchCase &arch_case : kHwregArchCases) {
|
|
SCOPED_TRACE(arch_case.name);
|
|
amdgpu::GpuMemory gpu_mem(std::string("hwreg_status_readonly_mem_") + arch_case.name);
|
|
amdgpu::L2Cache l2(std::string("hwreg_status_readonly_l2_") + arch_case.name);
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = arch_case.arch;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
auto cu = amdgpu::ComputeUnitCore::create(arch_case.name, cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
|
|
auto decoder = Decoder::create(arch_case.arch);
|
|
ASSERT_NE(decoder, nullptr);
|
|
|
|
constexpr uint32_t kHwregStatus32 = encode_hwreg(2);
|
|
constexpr uint32_t kImm32Value = 0xDEADBEEF;
|
|
const std::array<uint32_t, 2> words =
|
|
encode_sopk(arch_case.setreg_imm_op, 0, kHwregStatus32, kImm32Value);
|
|
|
|
std::unique_ptr<Instruction> inst(decoder->decode(words.data()));
|
|
ASSERT_NE(inst, nullptr) << "Failed to decode s_setreg_imm32_b32";
|
|
EXPECT_EQ(std::string_view(inst->mnemonic()).substr(0, 16), "s_setreg_imm32_b");
|
|
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
|
|
wf->set_mode_raw(0);
|
|
wf->set_status_raw(0x12345678u);
|
|
cu->execute_instruction(inst.get(), *wf);
|
|
EXPECT_EQ(wf->status_raw(), 0x12345678u);
|
|
EXPECT_EQ(wf->mode_raw(), 0u);
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
}
|
|
|
|
TEST(HwregTest, SetregImm32WritesModeFp16Ovfl) {
|
|
for (const HwregArchCase &arch_case : kHwregArchCases) {
|
|
SCOPED_TRACE(arch_case.name);
|
|
amdgpu::GpuMemory gpu_mem(std::string("hwreg_mode_mem_") + arch_case.name);
|
|
amdgpu::L2Cache l2(std::string("hwreg_mode_l2_") + arch_case.name);
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = arch_case.arch;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
auto cu = amdgpu::ComputeUnitCore::create(arch_case.name, cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
|
|
auto decoder = Decoder::create(arch_case.arch);
|
|
ASSERT_NE(decoder, nullptr);
|
|
|
|
constexpr uint32_t kHwregMode32 = encode_hwreg(1);
|
|
constexpr uint32_t kImm32Value = amdgpu::Wavefront::FP16_OVFL_BIT;
|
|
const std::array<uint32_t, 2> words =
|
|
encode_sopk(arch_case.setreg_imm_op, 0, kHwregMode32, kImm32Value);
|
|
|
|
std::unique_ptr<Instruction> inst(decoder->decode(words.data()));
|
|
ASSERT_NE(inst, nullptr) << "Failed to decode s_setreg_imm32_b32";
|
|
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
|
|
wf->set_mode_raw(0);
|
|
wf->set_status_raw(0x12345678u);
|
|
cu->execute_instruction(inst.get(), *wf);
|
|
EXPECT_EQ(wf->mode_raw(), amdgpu::Wavefront::FP16_OVFL_BIT);
|
|
EXPECT_TRUE(wf->fp16_ovfl());
|
|
EXPECT_EQ(wf->status_raw(), 0x12345678u);
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
}
|
|
|
|
TEST(HwregTest, SetregB32WritesModeFp16OvflFromSgpr) {
|
|
for (const HwregArchCase &arch_case : kHwregArchCases) {
|
|
SCOPED_TRACE(arch_case.name);
|
|
amdgpu::GpuMemory gpu_mem(std::string("hwreg_setreg_mode_mem_") + arch_case.name);
|
|
amdgpu::L2Cache l2(std::string("hwreg_setreg_mode_l2_") + arch_case.name);
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = arch_case.arch;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
auto cu = amdgpu::ComputeUnitCore::create(arch_case.name, cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
|
|
auto decoder = Decoder::create(arch_case.arch);
|
|
ASSERT_NE(decoder, nullptr);
|
|
|
|
constexpr uint32_t kHwregMode32 = encode_hwreg(1);
|
|
const std::array<uint32_t, 2> words = encode_sopk(arch_case.setreg_op, 4, kHwregMode32);
|
|
std::unique_ptr<Instruction> inst(decoder->decode(words.data()));
|
|
ASSERT_NE(inst, nullptr) << "Failed to decode s_setreg_b32";
|
|
EXPECT_EQ(std::string_view(inst->mnemonic()), "s_setreg_b32");
|
|
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
|
|
const uint32_t sb = wf->sgpr_alloc().base;
|
|
wf->set_mode_raw(0);
|
|
wf->set_status_raw(0x12345678u);
|
|
cu->write_sgpr(sb + 4, amdgpu::Wavefront::FP16_OVFL_BIT);
|
|
cu->execute_instruction(inst.get(), *wf);
|
|
EXPECT_EQ(wf->mode_raw(), amdgpu::Wavefront::FP16_OVFL_BIT);
|
|
EXPECT_TRUE(wf->fp16_ovfl());
|
|
EXPECT_EQ(wf->status_raw(), 0x12345678u);
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
}
|
|
|
|
/// @brief Targeted test for s_setreg_imm32_b32 with partial MODE bitfield write.
|
|
TEST(HwregTest, SetregImm32PartialBitfield) {
|
|
for (const HwregArchCase &arch_case : kHwregArchCases) {
|
|
SCOPED_TRACE(arch_case.name);
|
|
amdgpu::GpuMemory gpu_mem(std::string("hwreg_partial_mem_") + arch_case.name);
|
|
amdgpu::L2Cache l2(std::string("hwreg_partial_l2_") + arch_case.name);
|
|
|
|
amdgpu::ComputeUnitCore::Config cfg{};
|
|
cfg.arch = arch_case.arch;
|
|
cfg.num_wf_slots = 1;
|
|
cfg.sgprs_per_wf = 106;
|
|
cfg.vgprs_per_wf = 256;
|
|
cfg.lds_size_kb = 64;
|
|
|
|
auto cu = amdgpu::ComputeUnitCore::create(arch_case.name, cfg, &gpu_mem, &l2);
|
|
ASSERT_NE(cu, nullptr);
|
|
|
|
auto decoder = Decoder::create(arch_case.arch);
|
|
ASSERT_NE(decoder, nullptr);
|
|
|
|
// Write MODE.FP_DENORM bits[7:4].
|
|
constexpr uint32_t kHwreg = encode_hwreg(1, 4, 4);
|
|
constexpr uint32_t kImm32 = 0xFFFFFFFF;
|
|
const std::array<uint32_t, 2> words = encode_sopk(arch_case.setreg_imm_op, 0, kHwreg, kImm32);
|
|
|
|
std::unique_ptr<Instruction> inst(decoder->decode(words.data()));
|
|
ASSERT_NE(inst, nullptr);
|
|
|
|
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf);
|
|
ASSERT_NE(wf, nullptr);
|
|
|
|
wf->set_mode_raw(0xAAAAAAAAu);
|
|
wf->set_status_raw(0x13579BDFu);
|
|
cu->execute_instruction(inst.get(), *wf);
|
|
EXPECT_EQ(wf->mode_raw(), 0xAAAAAAFAu) << "Partial bitfield write to MODE incorrect";
|
|
EXPECT_EQ(wf->status_raw(), 0x13579BDFu);
|
|
|
|
if (!wf->is_halted())
|
|
wf->halt();
|
|
}
|
|
}
|
|
|
|
} // namespace
|